feat(dev): local dev environment (#1102)

* feat(ci): allow override urls in metadata

* f

* f

* f

* f

* f

* f

* f

* f

* kots override

* kots override

* f

* use staging bucket

* fix unbound variable

* fix unbound vars

* Revert "kots override"

This reverts commit 290f257.

* Revert "kots override"

This reverts commit a46bed8.

* f

* f

---------

Co-authored-by: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>

.github/workflows/dependencies.yaml

@@ -23,7 +23,7 @@ jobs:
     - name: K0s
       run: |
         # Remove the '-ec.X' suffix and only update if the prefix (upstream k0s release) has changed.
-        export CURVERSION=$(awk '/^K0S_VERSION/{split($3,a,"-"); print a[1]}' Makefile)
+        export CURVERSION=$(make print-K0S_VERSION)
         export VERSION=`curl https://api.github.com/repos/k0sproject/k0s/releases/latest | jq -r .name`
         if [ "$CURVERSION" != "$VERSION" ]; then
           sed -i "/^K0S_VERSION/c\K0S_VERSION = $VERSION" Makefile

.github/workflows/image-scan.yaml

@@ -16,9 +16,11 @@ permissions:
 
 jobs:
 
-  build-images:
+  build-deps:
     outputs:
-      local-artifact-mirror: ${{ steps.local-artifact-mirror.outputs.image }}
+      local-artifact-mirror-image: ${{ steps.local-artifact-mirror.outputs.image }}
+      operator-image: ${{ steps.operator.outputs.image }}
+      operator-chart: ${{ steps.operator.outputs.chart }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -30,7 +32,8 @@ jobs:
         uses: actions/cache@v4
         with:
           path: |
-            build/.melange-cache
+            local-artifact-mirror/cache/.melange-cache
+            operator/cache/.melange-cache
           key: melange-cache
 
       - name: Setup Melange
@@ -42,9 +45,16 @@ jobs:
           make -C local-artifact-mirror apko build-ttl.sh
           echo "image=$(cat local-artifact-mirror/build/image)" >> $GITHUB_OUTPUT
 
+      - name: Build and push operator image
+        id: operator
+        run: |
+          make -C operator apko build-ttl.sh build-chart-ttl.sh
+          echo "image=$(cat operator/build/image)" >> $GITHUB_OUTPUT
+          echo "chart=$(cat operator/build/chart)" >> $GITHUB_OUTPUT
+
   output-matrix:
     runs-on: ubuntu-latest
-    needs: [build-images]
+    needs: [build-deps]
     outputs:
       matrix: ${{ steps.build-matrix.outputs.matrix }}
     steps:
@@ -57,9 +67,17 @@ jobs:
           go-version-file: go.mod
           cache-dependency-path: "**/*.sum"
 
+      - name: Update embedded-cluster-operator metadata.yaml
+        env:
+          IMAGES_REGISTRY_SERVER: ttl.sh
+          OPERATOR_CHART: ${{ needs.build-deps.outputs.operator-chart }}
+          OPERATOR_IMAGE: ${{ needs.build-deps.outputs.operator-image }}
+        run: |
+          ./scripts/ci-update-operator-metadata.sh
+
       - name: Build
         run: |
-          export LOCAL_ARTIFACT_MIRROR_IMAGE=${{ needs.build-images.outputs.local-artifact-mirror }}
+          export LOCAL_ARTIFACT_MIRROR_IMAGE=${{ needs.build-deps.outputs.local-artifact-mirror-image }}
           make embedded-cluster-linux-amd64
 
       - name: List images

.github/workflows/release-prod.yaml

@@ -63,7 +63,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: |
-            build/.melange-cache
+            operator/cache/.melange-cache
           key: melange-cache
 
       - name: Setup Melange
@@ -113,7 +113,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: |
-            build/.melange-cache
+            local-artifact-mirror/cache/.melange-cache
           key: melange-cache
 
       - name: Setup Melange
@@ -137,6 +137,13 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Cache embedded bins
+        uses: actions/cache@v4
+        with:
+          path: |
+            output/bins
+          key: bins-cache
+
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
@@ -161,14 +168,16 @@ jobs:
 
       - name: Build linux-amd64
         run: |
+          mkdir -p build
           make embedded-cluster-linux-amd64 \
             VERSION=${{ needs.get-tag.outputs.tag-name }} \
             LOCAL_ARTIFACT_MIRROR_IMAGE=proxy.replicated.com/anonymous/${{ needs.publish-images.outputs.local-artifact-mirror }}
-          tar -C output/bin -czvf embedded-cluster-linux-amd64.tgz embedded-cluster
+          tar -C output/bin -czvf build/embedded-cluster-linux-amd64.tgz embedded-cluster
 
       - name: Output Metadata
         run: |
-          ./output/bin/embedded-cluster version metadata > metadata.json
+          mkdir -p build
+          ./output/bin/embedded-cluster version metadata > build/metadata.json
 
       - name: Cache Staging Files
         env:
@@ -178,10 +187,9 @@ jobs:
           AWS_REGION: "us-east-1"
         run: |
           export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"
-          export OPERATOR_IMAGE=proxy.replicated.com/anonymous/${{ needs.publish-operator-image.outputs.image }}
-          export OPERATOR_VERSION=$(echo "${{ needs.get-tag.outputs.tag-name }}" | sed 's/^v//')
+          echo "${{ needs.publish-operator-image.outputs.image }}" > "operator/build/image-$EC_VERSION"
 
-          ./scripts/cache-files.sh
+          ./scripts/ci-cache-files.sh
 
       - name: Cache Prod Files
         env:
@@ -191,16 +199,15 @@ jobs:
           AWS_REGION: "us-east-1"
         run: |
           export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"
-          export OPERATOR_IMAGE=proxy.replicated.com/anonymous/${{ needs.publish-operator-image.outputs.image }}
-          export OPERATOR_VERSION=$(echo "${{ needs.get-tag.outputs.tag-name }}" | sed 's/^v//')
+          echo "${{ needs.publish-operator-image.outputs.image }}" > "operator/build/image-$EC_VERSION"
 
-          ./scripts/cache-files.sh
+          ./scripts/ci-cache-files.sh
 
       - name: Publish release
         uses: marvinpinto/action-automatic-releases@latest
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           prerelease: true
           files: |
-            *.tgz
-            metadata.json
+            build/*.tgz
+            build/metadata.json

.gitignore

@@ -1,4 +1,3 @@
-output
 bundle
 pkg/goods/bins/*
 !pkg/goods/bins/.placeholder
@@ -9,7 +8,8 @@ pkg/goods/images
 .pre-commit-config.yaml
 vendor
 e2e/kots-release-install/license.yaml
-/build/
 /bin/
+/build/
+/output/
 go.work
 go.work.sum

.shellcheckrc

@@ -0,0 +1 @@
+external-sources=true

Makefile

@@ -2,8 +2,6 @@ SHELL := /bin/bash
 
 include common.mk
 
-VERSION ?= $(shell git describe --tags --dirty --match='[0-9]*.[0-9]*.[0-9]*')
-CURRENT_USER := $(if $(GITHUB_USER),$(GITHUB_USER),$(shell id -u -n))
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
@@ -16,14 +14,23 @@ K0S_BINARY_SOURCE_OVERRIDE =
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.100.0
 KOTS_VERSION = v$(shell awk '/^version/{print $$2}' pkg/addons/adminconsole/static/metadata.yaml | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
+# When updating KOTS_BINARY_URL_OVERRIDE, also update the KOTS_VERSION above or
+# scripts/ci-cache-files.sh may find the version in the cache and not upload the overridden binary.
 KOTS_BINARY_URL_OVERRIDE =
 # TODO: move this to a manifest file
 LOCAL_ARTIFACT_MIRROR_IMAGE ?= proxy.replicated.com/anonymous/replicated/embedded-cluster-local-artifact-mirror:$(VERSION)
+# These are used to override the binary urls in dev and e2e tests
+METADATA_K0S_BINARY_URL_OVERRIDE =
+METADATA_KOTS_BINARY_URL_OVERRIDE =
+METADATA_OPERATOR_BINARY_URL_OVERRIDE =
 LD_FLAGS = \
 	-X github.com/replicatedhq/embedded-cluster/pkg/versions.K0sVersion=$(K0S_VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/versions.Version=$(VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/versions.TroubleshootVersion=$(TROUBLESHOOT_VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/versions.LocalArtifactMirrorImage=$(LOCAL_ARTIFACT_MIRROR_IMAGE) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/versions.K0sBinaryURLOverride=$(METADATA_K0S_BINARY_URL_OVERRIDE) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/versions.KOTSBinaryURLOverride=$(METADATA_KOTS_BINARY_URL_OVERRIDE) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/versions.OperatorBinaryURLOverride=$(METADATA_OPERATOR_BINARY_URL_OVERRIDE) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.ChartRepoOverride=$(ADMIN_CONSOLE_CHART_REPO_OVERRIDE) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.KurlProxyImageOverride=$(ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.KotsVersion=$(KOTS_VERSION) \
@@ -92,7 +99,7 @@ output/bins/kubectl-preflight-%:
 .PHONY: pkg/goods/bins/local-artifact-mirror
 pkg/goods/bins/local-artifact-mirror:
 	mkdir -p pkg/goods/bins
-	$(MAKE) -C local-artifact-mirror build GOOS=$(OS) GOARCH=$(ARCH)
+	$(MAKE) -C local-artifact-mirror build OS=$(OS) ARCH=$(ARCH)
 	cp local-artifact-mirror/bin/local-artifact-mirror-$(OS)-$(ARCH) $@
 	touch $@
 
@@ -109,34 +116,38 @@ endif
 
 .PHONY: pkg/goods/internal/bins/kubectl-kots
 pkg/goods/internal/bins/kubectl-kots:
-	$(MAKE) output/bins/kubectl-kots-$(KOTS_VERSION)-$(ARCH)
 	mkdir -p pkg/goods/internal/bins
-	cp output/bins/kubectl-kots-$(KOTS_VERSION)-$(ARCH) $@
+	if [ "$(KOTS_BINARY_URL_OVERRIDE)" != "" ]; then \
+		$(MAKE) output/bins/kubectl-kots-override ; \
+		cp output/bins/kubectl-kots-override $@ ; \
+	else \
+		$(MAKE) output/bins/kubectl-kots-$(KOTS_VERSION)-$(ARCH) ; \
+		cp output/bins/kubectl-kots-$(KOTS_VERSION)-$(ARCH) $@ ; \
+	fi
+	touch $@
 
 output/bins/kubectl-kots-%:
 	mkdir -p output/bins
 	mkdir -p output/tmp
-	if [ "$(KOTS_BINARY_URL_OVERRIDE)" != "" ]; then \
-	    curl --retry 5 --retry-all-errors -fL -o output/tmp/kots.tar.gz "$(KOTS_BINARY_URL_OVERRIDE)" ; \
-	else \
-	    curl --retry 5 --retry-all-errors -fL -o output/tmp/kots.tar.gz "https://github.com/replicatedhq/kots/releases/download/$(call split-hyphen,$*,1)/kots_$(OS)_$(call split-hyphen,$*,2).tar.gz" ; \
-	fi
+	curl --retry 5 --retry-all-errors -fL -o output/tmp/kots.tar.gz "https://github.com/replicatedhq/kots/releases/download/$(call split-hyphen,$*,1)/kots_$(OS)_$(call split-hyphen,$*,2).tar.gz"
 	tar -xzf output/tmp/kots.tar.gz -C output/tmp
 	mv output/tmp/kots $@
 	touch $@
 
-output/tmp/release.tar.gz: e2e/kots-release-install/*
+.PHONY: output/bins/kubectl-kots-override
+output/bins/kubectl-kots-override:
+	mkdir -p output/bins
 	mkdir -p output/tmp
-	tar -czf output/tmp/release.tar.gz -C e2e/kots-release-install .
+	curl --retry 5 --retry-all-errors -fL -o output/tmp/kots.tar.gz "$(KOTS_BINARY_URL_OVERRIDE)"
+	tar -xzf output/tmp/kots.tar.gz -C output/tmp
+	mv output/tmp/kots $@
+	touch $@
 
+.PHONY: output/bin/embedded-cluster-release-builder
 output/bin/embedded-cluster-release-builder:
 	mkdir -p output/bin
 	CGO_ENABLED=0 go build -o output/bin/embedded-cluster-release-builder e2e/embedded-cluster-release-builder/main.go
 
-.PHONY: embedded-release
-embedded-release: embedded-cluster-linux-amd64 output/tmp/release.tar.gz output/bin/embedded-cluster-release-builder
-	./output/bin/embedded-cluster-release-builder output/bin/embedded-cluster output/tmp/release.tar.gz output/bin/embedded-cluster
-
 .PHONY: go.mod
 go.mod: Makefile
 	go get github.com/k0sproject/k0s@$(K0S_GO_VERSION)
@@ -229,10 +240,3 @@ buildtools:
 	mkdir -p pkg/goods/bins pkg/goods/internal/bins
 	touch pkg/goods/bins/BUILD pkg/goods/internal/bins/BUILD # compilation will fail if no files are present
 	go build -o ./output/bin/buildtools ./cmd/buildtools
-
-.PHONY: cache-files
-cache-files:
-	./scripts/cache-files.sh
-
-print-%:
-	@echo -n $($*)

cmd/buildtools/addon.go

@@ -95,7 +95,7 @@ func (c *addonComponent) resolveApkoImageRepoAndTag(ctx context.Context, upstrea
 		return "", "", fmt.Errorf("failed to apko build and publish for %s: %w", c.name, err)
 	}
 
-	builtImage, err := GetImageNameFromBuildFile()
+	builtImage, err := GetImageNameFromBuildFile("build/image")
 	if err != nil {
 		return "", "", fmt.Errorf("failed to get digest from build file: %w", err)
 	}

cmd/buildtools/embeddedclusteroperator.go

@@ -34,28 +34,30 @@ var updateOperatorAddonCommand = &cli.Command{
 		if nextChartVersion != "" {
 			logrus.Infof("using input override from INPUT_OPERATOR_CHART_VERSION: %s", nextChartVersion)
 		} else {
-			logrus.Infof("fetching the latest embedded cluster operator release")
+			logrus.Infof("fetching the latest embedded cluster release")
 			latest, err := GetGitHubRelease(
-				c.Context, "replicatedhq", "embedded-cluster-operator",
+				c.Context, "replicatedhq", "embedded-cluster",
 				func(tag string) bool {
 					return !strings.Contains(tag, "build")
 				},
 			)
 			if err != nil {
-				return fmt.Errorf("failed to get embedded cluster operator release: %w", err)
+				return fmt.Errorf("failed to get embedded cluster release: %w", err)
 			}
 			nextChartVersion = strings.TrimPrefix(latest, "v")
-			logrus.Printf("latest embedded cluster operator release: %s", latest)
+			logrus.Printf("latest embedded cluster release: %s", latest)
 		}
 		nextChartVersion = strings.TrimPrefix(nextChartVersion, "v")
 
 		chartURL := os.Getenv("INPUT_OPERATOR_CHART_URL")
 		if chartURL != "" {
 			logrus.Infof("using input override from INPUT_OPERATOR_CHART_URL: %s", chartURL)
+			chartURL = strings.TrimPrefix(chartURL, "oci://")
+			chartURL = strings.TrimPrefix(chartURL, "proxy.replicated.com/anonymous/")
 		} else {
 			chartURL = "registry.replicated.com/library/embedded-cluster-operator"
-			chartURL = fmt.Sprintf("oci://proxy.replicated.com/anonymous/%s", chartURL)
 		}
+		chartURL = fmt.Sprintf("oci://proxy.replicated.com/anonymous/%s", chartURL)
 
 		imageOverride := os.Getenv("INPUT_OPERATOR_IMAGE")
 		if imageOverride != "" {

cmd/buildtools/utils.go

@@ -61,11 +61,15 @@ func ApkoBuildAndPublish(componentName, packageName, packageVersion string) erro
 }
 
 func ComponentImageName(componentName, packageName, packageVersion string) (string, error) {
+	registryServer := os.Getenv("IMAGES_REGISTRY_SERVER")
+	if registryServer == "" {
+		return "", fmt.Errorf("IMAGES_REGISTRY_SERVER not set")
+	}
 	tag, err := ComponentImageTag(componentName, packageName, packageVersion)
 	if err != nil {
 		return "", fmt.Errorf("component image tag: %w", err)
 	}
-	return fmt.Sprintf("%s/replicated/ec-%s:%s", os.Getenv("IMAGES_REGISTRY_SERVER"), componentName, tag), nil
+	return fmt.Sprintf("%s/replicated/ec-%s:%s", registryServer, componentName, tag), nil
 }
 
 func ComponentImageTag(componentName, packageName, packageVersion string) (string, error) {
@@ -96,8 +100,8 @@ func ResolveApkoPackageVersion(componentName, packageName, packageVersion string
 	return strings.TrimSpace(string(out)), nil
 }
 
-func GetImageNameFromBuildFile() (string, error) {
-	contents, err := os.ReadFile("build/image")
+func GetImageNameFromBuildFile(imageBuildFile string) (string, error) {
+	contents, err := os.ReadFile(imageBuildFile)
 	if err != nil {
 		return "", fmt.Errorf("read build file: %w", err)
 	}

cmd/embedded-cluster/metadata.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"runtime"
 	"sort"
 
 	k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
@@ -70,10 +71,20 @@ func gatherVersionMetadata(k0sCfg *k0sconfig.ClusterConfig) (*types.ReleaseMetad
 	}
 
 	artifacts := map[string]string{
-		"kots":                        fmt.Sprintf("kots-binaries/%s.tar.gz", adminconsole.KotsVersion),
-		"operator":                    fmt.Sprintf("operator-binaries/%s.tar.gz", embeddedclusteroperator.Metadata.Version),
+		"k0s":                         fmt.Sprintf("k0s-binaries/%s-%s", versions.K0sVersion, runtime.GOARCH),
+		"kots":                        fmt.Sprintf("kots-binaries/%s-%s.tar.gz", adminconsole.KotsVersion, runtime.GOARCH),
+		"operator":                    fmt.Sprintf("operator-binaries/%s-%s.tar.gz", embeddedclusteroperator.Metadata.Version, runtime.GOARCH),
 		"local-artifact-mirror-image": versions.LocalArtifactMirrorImage,
 	}
+	if versions.K0sBinaryURLOverride != "" {
+		artifacts["k0s"] = versions.K0sBinaryURLOverride
+	}
+	if versions.KOTSBinaryURLOverride != "" {
+		artifacts["kots"] = versions.KOTSBinaryURLOverride
+	}
+	if versions.OperatorBinaryURLOverride != "" {
+		artifacts["operator"] = versions.OperatorBinaryURLOverride
+	}
 
 	meta := types.ReleaseMetadata{
 		Versions:  versionsMap,