diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b38bfaea0..15f1d1b65 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -473,6 +473,77 @@ jobs: echo "K0S_VERSION=\"$K0S_VERSION\"" echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT" + build-upgrade-v2: + name: Build upgrade v2 + runs-on: embedded-cluster-2 + needs: + - git-sha + outputs: + k0s_version: ${{ steps.export.outputs.k0s_version }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Cache embedded bins + uses: actions/cache@v4 + with: + path: | + output/bins + key: bins-cache + + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache-dependency-path: "**/*.sum" + + - name: Install dagger + run: | + curl -fsSL https://dl.dagger.io/dagger/install.sh | DAGGER_VERSION=v0.14.0 sh + sudo mv ./bin/dagger /usr/local/bin/dagger + + - name: Build + env: + APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP + APP_CHANNEL_SLUG: ci + RELEASE_YAML_DIR: e2e/kots-release-upgrade + S3_BUCKET: "tf-staging-embedded-cluster-bin" + USES_DEV_BUCKET: "0" + AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }} + AWS_REGION: "us-east-1" + USE_CHAINGUARD: "1" + UPLOAD_BINARIES: "1" + SKIP_RELEASE: "1" + MANGLE_METADATA: "1" + V2_ENABLED: "1" + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + export K0S_VERSION=$(make print-K0S_VERSION) + export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-v2upgrade + export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-v2upgrade + # avoid rate limiting + export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2) + + ./scripts/build-and-release.sh + cp output/bin/embedded-cluster output/bin/embedded-cluster-v2upgrade + + - name: Upload release + uses: actions/upload-artifact@v4 + with: + name: upgrade-v2-release + path: | + output/bin/embedded-cluster-v2upgrade + + - name: Export k0s version + id: export + run: | + K0S_VERSION="$(make print-K0S_VERSION)" + echo "K0S_VERSION=\"$K0S_VERSION\"" + echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT" + check-images: name: Check images runs-on: ubuntu-latest @@ -537,6 +608,7 @@ jobs: - build-legacydr - build-previous-k0s - build-upgrade + - build-upgrade-v2 - find-previous-stable steps: - name: Checkout @@ -600,6 +672,14 @@ jobs: export RELEASE_YAML_DIR=e2e/kots-release-install ./scripts/ci-release-app.sh + # an app upgrade to v2 + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-v2upgrade" + export APP_VERSION="appver-${SHORT_SHA}-v2upgrade" + export V2_ENABLED="1" + export RELEASE_YAML_DIR=e2e/kots-release-upgrade + ./scripts/ci-release-app.sh + unset V2_ENABLED + # and finally an app upgrade export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade" export APP_VERSION="appver-${SHORT_SHA}-upgrade" @@ -640,6 +720,14 @@ jobs: export RELEASE_YAML_DIR=e2e/kots-release-install-legacydr ./scripts/ci-release-app.sh + # an app upgrade to v2 + export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-v2upgrade" + export APP_VERSION="appver-${SHORT_SHA}-v2upgrade" + export V2_ENABLED="1" + export RELEASE_YAML_DIR=e2e/kots-release-upgrade + ./scripts/ci-release-app.sh + unset V2_ENABLED + # and finally an app upgrade export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade" export APP_VERSION="appver-${SHORT_SHA}-upgrade" @@ -682,6 +770,7 @@ jobs: - build-legacydr - build-previous-k0s - build-upgrade + - build-upgrade-v2 - find-previous-stable - release-app - export-version-specifier @@ -701,6 +790,7 @@ jobs: - TestSingleNodeUpgradePreviousStable - TestInstallFromReplicatedApp - TestUpgradeFromReplicatedApp + - TestMigrateV2FromReplicatedApp - TestInstallWithoutEmbed - TestUpgradeEC18FromReplicatedApp - TestResetAndReinstall @@ -782,6 +872,7 @@ jobs: - build-legacydr - build-previous-k0s - build-upgrade + - build-upgrade-v2 - find-previous-stable - release-app - export-version-specifier @@ -850,6 +941,7 @@ jobs: - build-legacydr - build-previous-k0s - build-upgrade + - build-upgrade-v2 - find-previous-stable - release-app - export-version-specifier diff --git a/cmd/local-artifact-mirror/artifact.go b/cmd/local-artifact-mirror/artifact.go index eb510007f..7b640c5fe 100644 --- a/cmd/local-artifact-mirror/artifact.go +++ b/cmd/local-artifact-mirror/artifact.go @@ -2,129 +2,35 @@ package main import ( "context" - "crypto/tls" - "encoding/json" "fmt" - "net/http" "os" + "github.com/replicatedhq/embedded-cluster/pkg/registry" "github.com/sirupsen/logrus" "go.uber.org/multierr" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content/file" - "oras.land/oras-go/v2/registry" - "oras.land/oras-go/v2/registry/remote" - "oras.land/oras-go/v2/registry/remote/auth" - "oras.land/oras-go/v2/registry/remote/credentials" ) -var ( - insecureTransport *http.Transport -) - -func init() { - insecureTransport = http.DefaultTransport.(*http.Transport).Clone() - insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} -} - -// DockerConfig represents the content of the '.dockerconfigjson' secret. -type DockerConfig struct { - Auths map[string]DockerConfigEntry `json:"auths"` -} - -// DockerConfigEntry represents the content of the '.dockerconfigjson' secret. -type DockerConfigEntry struct { - Username string `json:"username"` - Password string `json:"password"` -} - -// registryAuth returns the authentication store to be used when reaching the -// registry. The authentication store is read from the cluster secret named -// 'registry-creds' in the 'kotsadm' namespace. -func registryAuth(ctx context.Context) (credentials.Store, error) { - nsn := types.NamespacedName{Name: "registry-creds", Namespace: "kotsadm"} - var sct corev1.Secret - if err := kubecli.Get(ctx, nsn, &sct); err != nil { - if !errors.IsNotFound(err) { - return nil, fmt.Errorf("unable to get secret: %w", err) - } - - // if we can't locate a secret then returns an empty credentials - // store so we attempt to fetch the assets without auth. - logrus.Infof("no registry auth found, trying anonymous access") - return credentials.NewMemoryStore(), nil - } - - data, ok := sct.Data[".dockerconfigjson"] - if !ok { - return nil, fmt.Errorf("unable to find secret .dockerconfigjson") - } - - var cfg DockerConfig - if err := json.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("unable to unmarshal secret: %w", err) - } - - creds := credentials.NewMemoryStore() - for addr, entry := range cfg.Auths { - creds.Put(ctx, addr, auth.Credential{ - Username: entry.Username, - Password: entry.Password, - }) - } - return creds, nil -} - // pullArtifact fetches an artifact from the registry pointed by 'from'. The artifact // is stored in a temporary directory and the path to this directory is returned. // Callers are responsible for removing the temporary directory when it is no longer // needed. In case of error, the temporary directory is removed here. func pullArtifact(ctx context.Context, from string) (string, error) { - store, err := registryAuth(ctx) - if err != nil { - return "", fmt.Errorf("unable to get registry auth: %w", err) - } - - imgref, err := registry.ParseReference(from) - if err != nil { - return "", fmt.Errorf("unable to parse image reference: %w", err) - } - tmpdir, err := os.MkdirTemp("", "embedded-cluster-artifact-*") if err != nil { return "", fmt.Errorf("unable to create temp dir: %w", err) } - repo, err := remote.NewRepository(from) - if err != nil { - return "", fmt.Errorf("unable to create repository: %w", err) - } - - fs, err := file.New(tmpdir) - if err != nil { - return "", fmt.Errorf("unable to create file store: %w", err) - } - defer fs.Close() - - repo.Client = &auth.Client{ - Client: &http.Client{Transport: insecureTransport}, - Credential: store.Get, - } - - tag := imgref.Reference - _, tlserr := oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions) + opts := registry.PullArtifactOptions{} + tlserr := registry.PullArtifact(ctx, kubecli, from, tmpdir, opts) if tlserr == nil { return tmpdir, nil } // if we fail to fetch the artifact using https we gonna try once more using plain // http as some versions of the registry were deployed without tls. - repo.PlainHTTP = true + opts.PlainHTTP = true logrus.Infof("unable to fetch artifact using tls, retrying with http") - if _, err := oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions); err != nil { + if err := registry.PullArtifact(ctx, kubecli, from, tmpdir, opts); err != nil { os.RemoveAll(tmpdir) err = multierr.Combine(tlserr, err) return "", fmt.Errorf("unable to fetch artifacts with or without tls: %w", err) diff --git a/dev/dockerfiles/operator/Dockerfile.ttlsh b/dev/dockerfiles/operator/Dockerfile.ttlsh index 3f99de89a..e01c41dd9 100644 --- a/dev/dockerfiles/operator/Dockerfile.ttlsh +++ b/dev/dockerfiles/operator/Dockerfile.ttlsh @@ -18,6 +18,7 @@ ENV VERSION=${VERSION} ARG K0S_VERSION ENV K0S_VERSION=${K0S_VERSION} +ENV GOCACHE=/root/.cache/go-build RUN --mount=type=cache,target="/root/.cache/go-build" make -C operator build FROM debian:bookworm-slim diff --git a/e2e/install_test.go b/e2e/install_test.go index aea21a58d..81dd9aca4 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -487,6 +487,91 @@ func TestInstallFromReplicatedApp(t *testing.T) { t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } +func TestMigrateV2FromReplicatedApp(t *testing.T) { + t.Parallel() + + RequireEnvVars(t, []string{"SHORT_SHA"}) + + tc := docker.NewCluster(&docker.ClusterInput{ + T: t, + Nodes: 1, + Distro: "debian-bookworm", + }) + defer tc.Cleanup() + + t.Logf("%s: downloading embedded-cluster on node 0", time.Now().Format(time.RFC3339)) + line := []string{"vandoor-prepare.sh", fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), os.Getenv("LICENSE_ID"), "false"} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to download embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) + } + + t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) + line = []string{"single-node-install.sh", "ui", os.Getenv("SHORT_SHA")} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to install embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) + } + + if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) + } + + t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339)) + line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) + } + + // upgrade the cluster and migrate to v2 + + appUpgradeVersion := fmt.Sprintf("appver-%s-v2upgrade", os.Getenv("SHORT_SHA")) + testArgs := []string{appUpgradeVersion} + + t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339)) + if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) + } + + t.Logf("%s: checking installation state after upgrade and v2 migration", time.Now().Format(time.RFC3339)) + line = []string{"check-postupgrade-state.sh", k8sVersion(), ecUpgradeTargetVersion(), "true"} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check postupgrade state: %v: %s: %s", err, stdout, stderr) + } + + t.Logf("%s: checking installation state after migration to v2", time.Now().Format(time.RFC3339)) + line = []string{"check-postv2migration-state.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check post v2 migration state: %v: %s: %s", err, stdout, stderr) + } + + // upgrade the cluster one more time post v2 migration + + appUpgradeVersion = fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")) + testArgs = []string{appUpgradeVersion} + + t.Logf("%s: upgrading cluster again", time.Now().Format(time.RFC3339)) + if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) + } + + t.Logf("%s: checking postuprgrade state for an install2 cluster", time.Now().Format(time.RFC3339)) + line = []string{"check-postupgrade-state2.sh", appUpgradeVersion, k8sVersion()} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) + } + + t.Logf("%s: resetting admin console password", time.Now().Format(time.RFC3339)) + newPassword := "newpass" + line = []string{"embedded-cluster", "admin-console", "reset-password", newPassword} + _, _, err := tc.RunCommandOnNode(0, line) + require.NoError(t, err, "unable to reset admin console password") + + t.Logf("%s: logging in with the new password", time.Now().Format(time.RFC3339)) + _, _, err = tc.RunPlaywrightTest("login-with-custom-password", newPassword) + require.NoError(t, err, "unable to login with the new password") + + t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) +} + func TestSingleNodeUpgradePreviousStable(t *testing.T) { t.Parallel() diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts index 7f76d05f1..9b90afab3 100644 --- a/e2e/playwright/tests/deploy-upgrade/test.spec.ts +++ b/e2e/playwright/tests/deploy-upgrade/test.spec.ts @@ -22,7 +22,8 @@ async function fillConfigForm(iframe: FrameLocator) { await expect(iframe.locator('h3')).toContainText('The First Config Group', { timeout: 60 * 1000 }); // can take time to download the kots binary const hostnameInput = iframe.locator('#hostname-group').locator('input[type="text"]'); - await expect(hostnameInput).toHaveValue(process.env.APP_INITIAL_HOSTNAME ? process.env.APP_INITIAL_HOSTNAME : 'initial-hostname.com'); + // the hostname can be either 'initial-hostname.com' or 'updated-hostname.com' if we have run the upgrade multiple times + await expect(hostnameInput).toHaveValue(process.env.APP_INITIAL_HOSTNAME ? process.env.APP_INITIAL_HOSTNAME : /(initial|updated)-hostname\.com/); await hostnameInput.click(); await hostnameInput.fill('updated-hostname.com'); diff --git a/e2e/playwright/tests/shared/deploy-ec18-app-version.ts b/e2e/playwright/tests/shared/deploy-ec18-app-version.ts index 505a98865..2bed17a68 100644 --- a/e2e/playwright/tests/shared/deploy-ec18-app-version.ts +++ b/e2e/playwright/tests/shared/deploy-ec18-app-version.ts @@ -11,8 +11,7 @@ export const deployEC18AppVersion = async (page: Page, expect: Expect) => { await page.getByRole('button', { name: 'Continue' }).click(); await expect(page.getByText('Preflight checks', { exact: true })).toBeVisible({ timeout: 10 * 1000 }); await expect(page.getByRole('button', { name: 'Re-run' })).toBeVisible({ timeout: 10 * 1000 }); - await expect(page.locator('#app')).toContainText('Embedded Cluster Installation CRD exists'); - await expect(page.locator('#app')).toContainText('Embedded Cluster Config CRD exists'); + await expect(page.locator('#app')).toContainText('K0s ClusterConfig CRD exists'); await page.getByRole('button', { name: 'Deploy' }).click(); await expect(page.locator('#app')).toContainText('Currently deployed version', { timeout: 90000 }); await expect(page.locator('#app')).toContainText('Ready', { timeout: 45000 }); diff --git a/e2e/scripts/check-postupgrade-state.sh b/e2e/scripts/check-postupgrade-state.sh index b1ce631f6..cb449b551 100755 --- a/e2e/scripts/check-postupgrade-state.sh +++ b/e2e/scripts/check-postupgrade-state.sh @@ -14,16 +14,21 @@ function check_nginx_version { main() { local k8s_version="$1" local ec_version="$2" + local v2_enabled="${3:-}" echo "ensure that installation is installed" - wait_for_installation + if [ "$v2_enabled" != "true" ]; then + wait_for_installation + else + wait_for_installation_v2 + fi echo "pods" kubectl get pods -A echo "charts" kubectl get charts -A echo "installations" - kubectl get installations + kubectl get installations || true # ensure that memcached exists if ! kubectl get ns memcached; then @@ -47,36 +52,52 @@ main() { exit 1 fi - # ensure that nginx-ingress has been updated - kubectl describe chart -n kube-system k0s-addon-chart-ingress-nginx - # ensure new values are present - if ! kubectl describe chart -n kube-system k0s-addon-chart-ingress-nginx | grep -q "test-upgrade-value"; then - echo "test-upgrade-value not found in ingress-nginx chart" - exit 1 + if [ "$v2_enabled" != "true" ]; then + # ensure that nginx-ingress has been updated + kubectl describe chart -n kube-system k0s-addon-chart-ingress-nginx + # ensure new values are present + if ! kubectl describe chart -n kube-system k0s-addon-chart-ingress-nginx | grep -q "test-upgrade-value"; then + echo "test-upgrade-value not found in ingress-nginx chart" + exit 1 + fi + # ensure new version is present + if ! kubectl describe chart -n kube-system k0s-addon-chart-ingress-nginx | grep -q "4.12.0-beta.0"; then + echo "4.12.0-beta.0 not found in ingress-nginx chart" + exit 1 + fi + + # ensure that the embedded-cluster-operator has been updated + kubectl describe chart -n kube-system k0s-addon-chart-embedded-cluster-operator + kubectl describe chart -n kube-system k0s-addon-chart-embedded-cluster-operator | grep "embeddedClusterVersion:" | grep -q -e "$ec_version" + kubectl describe pod -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator + # ensure the new value made it into the pod + if ! kubectl describe pod -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator | grep "EMBEDDEDCLUSTER_VERSION" | grep -q -e "$ec_version" ; then + echo "Upgrade version not present in embedded-cluster-operator environment variable" + kubectl logs -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator --tail=100 + exit 1 + fi + + echo "ensure that the default chart order remained 110" + if ! kubectl describe clusterconfig -n kube-system k0s | grep -q -e 'Order:\W*110' ; then + kubectl describe clusterconfig -n kube-system k0s + echo "no charts had an order of '110'" + exit 1 + fi fi - # ensure new version is present - if ! kubectl describe chart -n kube-system k0s-addon-chart-ingress-nginx | grep -q "4.12.0-beta.0"; then - echo "4.12.0-beta.0 not found in ingress-nginx chart" + + # ensure new values are present in the nginx service + if ! kubectl -n ingress-nginx get service -oyaml ingress-nginx-controller | grep -q "test-upgrade-value"; then + echo "test-upgrade-value not found in ingress-nginx service" + kubectl describe service -n ingress-nginx exit 1 fi - # ensure the new version made it into the pod + # ensure the new version made it into the nginx pod if ! retry 5 check_nginx_version ; then echo "4.12.0-beta.0 not found in ingress-nginx pod" kubectl describe pod -n ingress-nginx exit 1 fi - # ensure that the embedded-cluster-operator has been updated - kubectl describe chart -n kube-system k0s-addon-chart-embedded-cluster-operator - kubectl describe chart -n kube-system k0s-addon-chart-embedded-cluster-operator | grep "embeddedClusterVersion:" | grep -q -e "$ec_version" - kubectl describe pod -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator - # ensure the new value made it into the pod - if ! kubectl describe pod -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator | grep "EMBEDDEDCLUSTER_VERSION" | grep -q -e "$ec_version" ; then - echo "Upgrade version not present in embedded-cluster-operator environment variable" - kubectl logs -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator --tail=100 - exit 1 - fi - # TODO: validate that labels are added after upgrading from an older version echo "ensure that the admin console branding is available" kubectl get cm -n kotsadm kotsadm-application-metadata @@ -98,13 +119,6 @@ main() { exit 1 fi - echo "ensure that the default chart order remained 110" - if ! kubectl describe clusterconfig -n kube-system k0s | grep -q -e 'Order:\W*110' ; then - kubectl describe clusterconfig -n kube-system k0s - echo "no charts had an order of '110'" - exit 1 - fi - echo "ensure that all nodes are running k8s $k8s_version" if ! ensure_nodes_match_kube_version "$k8s_version"; then echo "not all nodes are running k8s $k8s_version" diff --git a/e2e/scripts/check-postv2migration-state.sh b/e2e/scripts/check-postv2migration-state.sh new file mode 100755 index 000000000..3dcf3bf3e --- /dev/null +++ b/e2e/scripts/check-postv2migration-state.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euox pipefail + +DIR=/usr/local/bin +. $DIR/common.sh + +main() { + echo "ensure that the embedded-cluster-operator is removed" + if kubectl get deployment -n embedded-cluster embedded-cluster-operator 2>/dev/null ; then + kubectl get deployment -n embedded-cluster embedded-cluster-operator + echo "embedded-cluster-operator found" + exit 1 + fi + + echo "ensure that IS_EC2_INSTALL is set to true" + if ! kubectl -n kotsadm get deployment kotsadm -o jsonpath='{.spec.template.spec.containers[0].env}' | grep -q IS_EC2_INSTALL ; then + kubectl -n kotsadm get deployment kotsadm -o yaml + echo "IS_EC2_INSTALL not found in kotsadm deployment" + exit 1 + fi + + echo "ensure that there are no helm chart extensions in the cluster config" + if kubectl get clusterconfig -n kube-system k0s -o jsonpath='{.spec.extensions.helm.charts}' | grep -q 'chartname:' ; then + kubectl get clusterconfig -n kube-system k0s -o yaml + echo "helm chart extensions found in cluster config" + exit 1 + fi + + echo "ensure that there are no chart custom resources in the cluster" + if kubectl get charts -n kube-system 2>/dev/null | grep -qe '.*' ; then + kubectl get charts -n kube-system + echo "chart custom resources found in cluster" + exit 1 + fi +} + +main "$@" diff --git a/e2e/scripts/common.sh b/e2e/scripts/common.sh index b69488289..b9a675353 100644 --- a/e2e/scripts/common.sh +++ b/e2e/scripts/common.sh @@ -89,6 +89,26 @@ wait_for_installation() { done } +wait_for_installation_v2() { + ready=$(kubectl -n embedded-cluster get cm -l replicated.com/installation=embedded-cluster -o yaml | grep -c '"state":"Installed"' || true) + counter=0 + while [ "$ready" -lt "1" ]; do + if [ "$counter" -gt 84 ]; then + echo "installation did not become ready" + kubectl -n embedded-cluster get cm -l replicated.com/installation=embedded-cluster -o yaml 2>&1 || true + kubectl get secrets -A + kubectl describe clusterconfig -A + kubectl get pods -A + return 1 + fi + sleep 5 + counter=$((counter+1)) + echo "Waiting for installation" + ready=$(kubectl -n embedded-cluster get cm -l replicated.com/installation=embedded-cluster -o yaml | grep -c '"state":"Installed"' || true) + kubectl get installations 2>&1 || true + done +} + wait_for_nginx_pods() { ready=$(kubectl get pods -n "$APP_NAMESPACE" | grep "nginx" | grep -c Running || true) counter=0 diff --git a/operator/pkg/cli/migrate_v2_installmanager.go b/operator/pkg/cli/migrate_v2_installmanager.go index fa3f8b959..8f393d82c 100644 --- a/operator/pkg/cli/migrate_v2_installmanager.go +++ b/operator/pkg/cli/migrate_v2_installmanager.go @@ -2,9 +2,11 @@ package cli import ( "fmt" + "os" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/operator/pkg/cli/migratev2" + "github.com/replicatedhq/embedded-cluster/operator/pkg/k8sutil" "github.com/replicatedhq/embedded-cluster/pkg/helpers" "github.com/replicatedhq/embedded-cluster/pkg/manager" "github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig" @@ -43,6 +45,8 @@ func MigrateV2InstallManagerCmd() *cobra.Command { // set the runtime config from the installation spec runtimeconfig.Set(installation.Spec.RuntimeConfig) + os.Setenv("TMPDIR", runtimeconfig.EmbeddedClusterTmpSubDir()) + manager.SetServiceName(appSlug) return nil @@ -50,9 +54,14 @@ func MigrateV2InstallManagerCmd() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - err := migratev2.InstallAndStartManager( - ctx, - license.Spec.LicenseID, license.Spec.Endpoint, appVersionLabel, + cli, err := k8sutil.KubeClient() + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %w", err) + } + + err = migratev2.InstallAndStartManager( + ctx, cli, + installation, license.Spec.LicenseID, license.Spec.Endpoint, appVersionLabel, ) if err != nil { return fmt.Errorf("failed to run manager migration: %w", err) diff --git a/operator/pkg/cli/migratev2/adminconsole.go b/operator/pkg/cli/migratev2/adminconsole.go index eaee0e175..92f2718bc 100644 --- a/operator/pkg/cli/migratev2/adminconsole.go +++ b/operator/pkg/cli/migratev2/adminconsole.go @@ -7,6 +7,7 @@ import ( k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/replicatedhq/embedded-cluster/pkg/helpers" "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" apitypes "k8s.io/apimachinery/pkg/types" @@ -54,7 +55,12 @@ func updateAdminConsoleClusterConfig(ctx context.Context, cli client.Client) err } } - err = cli.Update(ctx, &clusterConfig) + unstructured, err := helpers.K0sClusterConfigTo129Compat(&clusterConfig) + if err != nil { + return fmt.Errorf("convert cluster config to 1.29 compat: %w", err) + } + + err = cli.Update(ctx, unstructured) if err != nil { return fmt.Errorf("update k0s cluster config: %w", err) } diff --git a/operator/pkg/cli/migratev2/managerpod.go b/operator/pkg/cli/migratev2/managerpod.go index 19070f407..358cdef2c 100644 --- a/operator/pkg/cli/migratev2/managerpod.go +++ b/operator/pkg/cli/migratev2/managerpod.go @@ -12,6 +12,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" ) var _managerInstallPodSpec = corev1.Pod{ @@ -128,16 +129,27 @@ var _managerInstallPodSpec = corev1.Pod{ // InstallAndStartManager installs and starts the manager service on the host. This is run in a pod // on all nodes in the cluster. -func InstallAndStartManager(ctx context.Context, licenseID string, licenseEndpoint string, appVersionLabel string) error { +func InstallAndStartManager(ctx context.Context, cli client.Client, in *ecv1beta1.Installation, licenseID string, licenseEndpoint string, appVersionLabel string) error { binPath := runtimeconfig.PathToEmbeddedClusterBinary("manager") - // TODO: airgap - err := manager.DownloadBinaryOnline(ctx, binPath, licenseID, licenseEndpoint, appVersionLabel) - if err != nil { - return fmt.Errorf("download manager binary: %w", err) + if in.Spec.AirGap { + srcImage := in.Spec.Artifacts.AdditionalArtifacts["manager"] + if srcImage == "" { + return fmt.Errorf("missing manager binary in airgap artifacts") + } + + err := manager.DownloadBinaryAirgap(ctx, cli, binPath, srcImage) + if err != nil { + return fmt.Errorf("pull manager binary from registry: %w", err) + } + } else { + err := manager.DownloadBinaryOnline(ctx, binPath, licenseID, licenseEndpoint, appVersionLabel) + if err != nil { + return fmt.Errorf("download manager binary: %w", err) + } } - err = manager.Install(ctx, logrus.Infof) + err := manager.Install(ctx, logrus.Infof) if err != nil { return fmt.Errorf("install manager: %w", err) } diff --git a/operator/pkg/cli/migratev2/migrate.go b/operator/pkg/cli/migratev2/migrate.go index 2b02f4339..5e84e81e8 100644 --- a/operator/pkg/cli/migratev2/migrate.go +++ b/operator/pkg/cli/migratev2/migrate.go @@ -3,6 +3,7 @@ package migratev2 import ( "context" "fmt" + "time" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/helm" @@ -51,6 +52,9 @@ func Run( return fmt.Errorf("disable operator: %w", err) } + // allow some time for the operator to be disabled + time.Sleep(5 * time.Second) + err = enableV2AdminConsole(ctx, logf, cli, in) if err != nil { return fmt.Errorf("enable v2 admin console: %w", err) @@ -61,7 +65,7 @@ func Run( return fmt.Errorf("set installation state to installed: %w", err) } - err = cleanupV1(ctx, logf, cli) + err = cleanupV1(ctx, logf, cli, helmCLI) if err != nil { return fmt.Errorf("cleanup v1: %w", err) } diff --git a/operator/pkg/cli/migratev2/operator.go b/operator/pkg/cli/migratev2/operator.go index 60b9afc80..80942b3a1 100644 --- a/operator/pkg/cli/migratev2/operator.go +++ b/operator/pkg/cli/migratev2/operator.go @@ -35,7 +35,7 @@ func disableOperator(ctx context.Context, logf LogFunc, cli client.Client, in *e // cleanupV1 removes control of the Helm Charts from the k0s controller and uninstalls the Embedded // Cluster operator. -func cleanupV1(ctx context.Context, logf LogFunc, cli client.Client) error { +func cleanupV1(ctx context.Context, logf LogFunc, cli client.Client, helmCLI helm.Client) error { logf("Force deleting Chart custom resources") // forceDeleteChartCRs is necessary because the k0s controller will otherwise uninstall the // Helm releases and we don't want that. @@ -52,6 +52,13 @@ func cleanupV1(ctx context.Context, logf LogFunc, cli client.Client) error { } logf("Successfully removed Helm Charts from ClusterConfig") + logf("Uninstalling operator") + err = helmUninstallOperator(ctx, helmCLI) + if err != nil { + return fmt.Errorf("helm uninstall operator: %w", err) + } + logf("Successfully uninstalled operator") + return nil } diff --git a/pkg/helpers/fs.go b/pkg/helpers/fs.go index 13afe1c2f..1d09baa01 100644 --- a/pkg/helpers/fs.go +++ b/pkg/helpers/fs.go @@ -35,7 +35,27 @@ func MoveFile(src, dst string) error { } if srcinfo.IsDir() { - return fmt.Errorf("move directory %s", src) + err := os.MkdirAll(dst, srcinfo.Mode()) + if err != nil { + return fmt.Errorf("mkdir: %s", err) + } + + err = os.Chmod(dst, srcinfo.Mode()) + if err != nil { + return fmt.Errorf("chmod dir: %s", err) + } + + entries, err := os.ReadDir(src) + if err != nil { + return fmt.Errorf("read source dir: %s", err) + } + for _, entry := range entries { + err = MoveFile(filepath.Join(src, entry.Name()), filepath.Join(dst, entry.Name())) + if err != nil { + return fmt.Errorf("move file %s to %s: %s", entry.Name(), dst, err) + } + } + return nil } srcfp, err := os.Open(src) @@ -59,6 +79,11 @@ func MoveFile(src, dst string) error { return fmt.Errorf("sync file: %s", err) } + err = os.Chmod(dst, srcinfo.Mode()) + if err != nil { + return fmt.Errorf("chmod file: %s", err) + } + if err := os.Remove(src); err != nil { return fmt.Errorf("remove source file: %s", err) } diff --git a/pkg/helpers/fs_test.go b/pkg/helpers/fs_test.go index 909af289f..57bc29b80 100644 --- a/pkg/helpers/fs_test.go +++ b/pkg/helpers/fs_test.go @@ -62,8 +62,39 @@ func TestMoveFile_Directory(t *testing.T) { srcDir, err := os.MkdirTemp("", "sourcedir-*") assert.NoError(t, err) defer os.RemoveAll(srcDir) - err = MoveFile(srcDir, "destination") - assert.Error(t, err) + + tmpDst, err := os.MkdirTemp("", "destination-*") + assert.NoError(t, err) + defer os.RemoveAll(tmpDst) + + dstDir := filepath.Join(tmpDst, "destination") + + err = os.Chmod(srcDir, 0777) + assert.NoError(t, err) + + srcContent := []byte("test") + srcFile, err := os.Create(filepath.Join(srcDir, "test.txt")) + assert.NoError(t, err) + defer os.Remove(srcFile.Name()) + defer srcFile.Close() + + _, err = srcFile.Write(srcContent) + assert.NoError(t, err) + + err = MoveFile(srcDir, dstDir) + assert.NoError(t, err) + + info, err := os.Stat(dstDir) + assert.NoError(t, err) + assert.Equal(t, info.IsDir(), true, "expected directory") + assert.Equal(t, os.FileMode(0777|os.ModeDir).String(), info.Mode().String(), "unexpected file mode") + + _, err = os.Stat(filepath.Join(dstDir, "test.txt")) + assert.NoError(t, err) + + content, err := os.ReadFile(filepath.Join(dstDir, "test.txt")) + assert.NoError(t, err) + assert.Equal(t, srcContent, content, "unexpected content") } func TestMoveFile_Symlink(t *testing.T) { diff --git a/pkg/manager/binary.go b/pkg/manager/binary.go index 3b0242c8a..1421ec193 100644 --- a/pkg/manager/binary.go +++ b/pkg/manager/binary.go @@ -10,8 +10,10 @@ import ( "path/filepath" "github.com/replicatedhq/embedded-cluster/pkg/helpers" + "github.com/replicatedhq/embedded-cluster/pkg/registry" "github.com/replicatedhq/embedded-cluster/pkg/tgzutils" "github.com/sirupsen/logrus" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -82,3 +84,27 @@ func DownloadBinaryOnline( return nil } + +func DownloadBinaryAirgap(ctx context.Context, cli client.Client, dstPath string, srcImage string) error { + tmpdir, err := os.MkdirTemp("", "embedded-cluster-artifact-*") + if err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(tmpdir) + + err = registry.PullArtifact(ctx, cli, srcImage, tmpdir, registry.PullArtifactOptions{}) + if err != nil { + return fmt.Errorf("pull manager binary from registry: %w", err) + } + + // NOTE: We do not try to pull the image using plain http as we do with LAM. This is untested + // and it is possible that this will not work for very old installations. + + src := filepath.Join(tmpdir, BinaryName) + err = helpers.MoveFile(src, dstPath) + if err != nil { + return fmt.Errorf("move file: %w", err) + } + + return nil +} diff --git a/pkg/registry/artifact.go b/pkg/registry/artifact.go new file mode 100644 index 000000000..8372bc706 --- /dev/null +++ b/pkg/registry/artifact.go @@ -0,0 +1,56 @@ +package registry + +import ( + "context" + "fmt" + + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/registry" + "oras.land/oras-go/v2/registry/remote" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// PullArtifactOptions are options for pulling an artifact from a registry. +type PullArtifactOptions struct { + PlainHTTP bool +} + +// PullArtifact fetches an artifact from the registry pointed by 'from' and stores it in the 'dstDir' directory. +func PullArtifact(ctx context.Context, cli client.Client, from string, dstDir string, opts PullArtifactOptions) error { + imgref, err := registry.ParseReference(from) + if err != nil { + return fmt.Errorf("parse image reference: %w", err) + } + + repo, err := remote.NewRepository(from) + if err != nil { + return fmt.Errorf("new repository: %w", err) + } + + authClient := newInsecureAuthClient() + + store, err := registryAuth(ctx, cli) + if err != nil { + return fmt.Errorf("get registry auth: %w", err) + } + authClient.Credential = store.Get + + repo.Client = authClient + + repo.PlainHTTP = opts.PlainHTTP + + fs, err := file.New(dstDir) + if err != nil { + return fmt.Errorf("create file store: %w", err) + } + defer fs.Close() + + tag := imgref.Reference + _, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions) + if err != nil { + return fmt.Errorf("registry copy: %w", err) + } + + return nil +} diff --git a/pkg/registry/auth.go b/pkg/registry/auth.go new file mode 100644 index 000000000..d8d6611f2 --- /dev/null +++ b/pkg/registry/auth.go @@ -0,0 +1,57 @@ +package registry + +import ( + "context" + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/credentials" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// dockerConfig represents the content of the '.dockerconfigjson' secret. +type dockerConfig struct { + Auths map[string]dockerConfigEntry `json:"auths"` +} + +// dockerConfigEntry represents the content of the '.dockerconfigjson' secret. +type dockerConfigEntry struct { + Username string `json:"username"` + Password string `json:"password"` +} + +// registryAuth returns the authentication store to be used when reaching the +// registry. The authentication store is read from the cluster secret named +// 'registry-creds' in the 'kotsadm' namespace. +func registryAuth(ctx context.Context, cli client.Client) (credentials.Store, error) { + nsn := types.NamespacedName{Name: "registry-creds", Namespace: "kotsadm"} + var sct corev1.Secret + if err := cli.Get(ctx, nsn, &sct); err != nil { + return nil, fmt.Errorf("get registry-creds secret: %w", err) + } + + data, ok := sct.Data[".dockerconfigjson"] + if !ok { + return nil, fmt.Errorf("no .dockerconfigjson entry found in secret") + } + + var cfg dockerConfig + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("unmarshal secret: %w", err) + } + + creds := credentials.NewMemoryStore() + for addr, entry := range cfg.Auths { + err := creds.Put(ctx, addr, auth.Credential{ + Username: entry.Username, + Password: entry.Password, + }) + if err != nil { + return nil, fmt.Errorf("put credential for %s: %w", addr, err) + } + } + return creds, nil +} diff --git a/pkg/registry/client.go b/pkg/registry/client.go new file mode 100644 index 000000000..10ef32613 --- /dev/null +++ b/pkg/registry/client.go @@ -0,0 +1,29 @@ +package registry + +import ( + "crypto/tls" + "net/http" + + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/retry" +) + +var ( + insecureHTTPClient *http.Client +) + +func init() { + insecureTransport := http.DefaultTransport.(*http.Transport).Clone() + insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + insecureHTTPClient = &http.Client{Transport: retry.NewTransport(insecureTransport)} +} + +func newInsecureAuthClient() *auth.Client { + return &auth.Client{ + Client: insecureHTTPClient, + Header: http.Header{ + "User-Agent": {"oras-go"}, + }, + Cache: auth.DefaultCache, + } +}