From 6619e2176da82863eb01e4cf06b2af568e8a290f Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 19 Jul 2024 15:09:56 -0700 Subject: [PATCH] f --- .github/workflows/image-deps-updater.yaml | 18 +-- Makefile | 8 +- cmd/buildtools/openebs.go | 26 ++-- cmd/buildtools/utils.go | 132 ++++++++++++++----- cmd/buildtools/utils_test.go | 112 ++++++++++++++++ deploy/images/openebs-kubectl/apko.tmpl.yaml | 2 +- 6 files changed, 240 insertions(+), 58 deletions(-) create mode 100644 cmd/buildtools/utils_test.go diff --git a/.github/workflows/image-deps-updater.yaml b/.github/workflows/image-deps-updater.yaml index b6f434c11c..c036154e62 100644 --- a/.github/workflows/image-deps-updater.yaml +++ b/.github/workflows/image-deps-updater.yaml @@ -10,7 +10,7 @@ on: required: false push: branches: - - emosbaugh/sc-108755/use-chainguard-openebs + - emosbaugh/sc-108755/use-chainguard-openebs # TODO remove jobs: update-k0s-images: @@ -27,16 +27,16 @@ jobs: - name: Compile buildtools run: make buildtools - # - name: Update k0s images - # env: - # REGISTRY_SERVER: docker.io - # REGISTRY_USER: ${{ secrets.DOCKERHUB_USER }} - # REGISTRY_PASS: ${{ secrets.DOCKERHUB_PASSWORD }} - # run: ./output/bin/buildtools update images k0s + - name: Update k0s images + env: + REGISTRY_SERVER: index.docker.io + REGISTRY_USER: ${{ secrets.DOCKERHUB_USER }} + REGISTRY_PASS: ${{ secrets.DOCKERHUB_PASSWORD }} + run: ./output/bin/buildtools update images k0s - name: Update OpenEBS images env: - REGISTRY_SERVER: docker.io + REGISTRY_SERVER: index.docker.io REGISTRY_USER: ${{ secrets.DOCKERHUB_USER }} REGISTRY_PASS: ${{ secrets.DOCKERHUB_PASSWORD }} run: ./output/bin/buildtools update images openebs @@ -55,7 +55,7 @@ jobs: images type::security draft: false - base: "main" + base: "emosbaugh/sc-108755/use-chainguard-openebs" # TODO main body: "Automated changes by the [image-deps-updater](https://github.com/replicatedhq/embedded-cluster/blob/main/.github/workflows/image-deps-updater.yaml) GitHub action" - name: Check outputs diff --git a/Makefile b/Makefile index b379edcf96..c2688ecf0e 100644 --- a/Makefile +++ b/Makefile @@ -23,11 +23,11 @@ OPENEBS_CHART_VERSION = 4.1.0 OPENEBS_IMAGE_VERSION = 4.1.0 OPENEBS_KUBECTL_IMAGE_VERSION = 1.29 OPENEBS_PROVISIONER_LOCALPV_IMAGE_REPO_OVERRIDE = ttl.sh/replicated/ec-openebs-provisioner-localpv -OPENEBS_PROVISIONER_LOCALPV_IMAGE_TAG = 4.1.0 +OPENEBS_PROVISIONER_LOCALPV_IMAGE_TAG = 4.1.0@sha256:bef7f29b98802343d5c8fb8a075c84bf9741e6bb69faed6165985442b71af49b OPENEBS_LINUX_UTILS_IMAGE_REPO_OVERRIDE = ttl.sh/replicated/ec-openebs-linux-utils -OPENEBS_LINUX_UTILS_IMAGE_TAG = 4.1.0 +OPENEBS_LINUX_UTILS_IMAGE_TAG = 4.1.0@sha256:f1c3fefb69356ef559ceb95682511a5fd3bd0fbdb0c9a008f89942045dd7051e OPENEBS_KUBECTL_IMAGE_REPO_OVERRIDE = ttl.sh/replicated/ec-openebs-kubectl -OPENEBS_KUBECTL_IMAGE_TAG = 1.29 +OPENEBS_KUBECTL_IMAGE_TAG = 1.29.5@sha256:9a77ad64c2cc97ed0f4b68b1a59d93bfd21c2299892bf0ac374a13c1127a7e0e SEAWEEDFS_CHART_VERSION = 4.0.0 REGISTRY_CHART_VERSION = 2.2.3 REGISTRY_IMAGE_VERSION = 2.8.3 @@ -294,7 +294,7 @@ melange-template: check-env-MELANGE_CONFIG check-env-PACKAGE_VERSION .PHONY: apko-template apko-template: check-env-APKO_CONFIG check-env-PACKAGE_VERSION mkdir -p build - envsubst '$${PACKAGE_VERSION}' < ${APKO_CONFIG} > build/apko.yaml + envsubst '$${PACKAGE_NAME} $${PACKAGE_VERSION}' < ${APKO_CONFIG} > build/apko.yaml .PHONY: buildtools buildtools: diff --git a/cmd/buildtools/openebs.go b/cmd/buildtools/openebs.go index 2e05c06103..9b7a444fee 100644 --- a/cmd/buildtools/openebs.go +++ b/cmd/buildtools/openebs.go @@ -85,6 +85,7 @@ var updateOpenEBSAddonCommand = &cli.Command{ type addonComponent struct { name string wolfiPackageName string + getWolfiPackageName func(upstreamVersion string) string upstreamVersionMakefileVar string upstreamVersionFlagOverride string } @@ -103,7 +104,7 @@ var openebsComponents = []addonComponent{ }, { name: "openebs-kubectl", - wolfiPackageName: "kubectl", + getWolfiPackageName: func(upstreamVersion string) string { return "kubectl-" + upstreamVersion + "-default" }, upstreamVersionMakefileVar: "OPENEBS_KUBECTL_IMAGE_VERSION", upstreamVersionFlagOverride: "kubectl-version", }, @@ -141,15 +142,23 @@ var updateOpenEBSImagesCommand = &cli.Command{ return fmt.Errorf("failed to get upstream version for %s: %w", component.name, err) } + packageName := component.wolfiPackageName + if component.getWolfiPackageName != nil { + packageName = component.getWolfiPackageName(upstreamVersion) + } packageVersion := upstreamVersion - if component.wolfiPackageName != "" { - packageVersion, err = GetWolfiPackageVersion(wolfiAPKIndex, component.name, upstreamVersion) + if packageName != "" { + packageVersion, err = GetWolfiPackageVersion(wolfiAPKIndex, packageName, upstreamVersion) if err != nil { return fmt.Errorf("failed to get package version for %s: %w", component.name, err) } } - if err := ApkoBuildAndPublish(component.name, packageVersion); err != nil { + extraArgs := []string{} + if packageName != "" { + extraArgs = append(extraArgs, fmt.Sprintf("PACKAGE_NAME=%s", packageName)) + } + if err := ApkoBuildAndPublish(component.name, packageVersion, extraArgs...); err != nil { return fmt.Errorf("failed to apko build and publish for %s: %w", component.name, err) } @@ -201,14 +210,15 @@ func ApkoLogin() error { return nil } -func ApkoBuildAndPublish(componentName string, packageVersion string) error { - if err := RunCommand( - "make", +func ApkoBuildAndPublish(componentName string, packageVersion string, extraArgs ...string) error { + args := []string{ "apko-build-and-publish", fmt.Sprintf("IMAGE=%s/replicated/ec-%s:%s", os.Getenv("REGISTRY_SERVER"), componentName, packageVersion), fmt.Sprintf("APKO_CONFIG=%s", filepath.Join("deploy", "images", componentName, "apko.tmpl.yaml")), fmt.Sprintf("PACKAGE_VERSION=%s", packageVersion), - ); err != nil { + } + args = append(args, extraArgs...) + if err := RunCommand("make", args...); err != nil { return err } return nil diff --git a/cmd/buildtools/utils.go b/cmd/buildtools/utils.go index 5c2bda2141..5b686cbb6c 100644 --- a/cmd/buildtools/utils.go +++ b/cmd/buildtools/utils.go @@ -12,10 +12,10 @@ import ( "os" "path/filepath" "regexp" - "sort" "strconv" "strings" + "github.com/Masterminds/semver/v3" "github.com/google/go-github/v62/github" "github.com/sirupsen/logrus" ) @@ -43,21 +43,48 @@ func GetWolfiAPKIndex() ([]byte, error) { return contents, nil } -func GetWolfiPackageVersion(wolfiAPKIndex []byte, pkgName string, pinnedVersion string) (string, error) { - // example APKINDEX content: - // P:calico-node - // V:3.26.1-r1 - // ... - // - // P:calico-node - // V:3.26.1-r10 - // ... - // - // P:calico-node - // V:3.26.1-r9 - // ... - - var revisions []int +type PackageVersion struct { + semver semver.Version + revision int +} + +func (v *PackageVersion) matches(version *semver.Version) bool { + parts := strings.SplitN(version.Original(), "-", 2) + parts = strings.SplitN(parts[0], "+", 2) + parts = strings.Split(parts[0], ".") + switch len(parts) { + case 1: + return v.semver.Major() == version.Major() + case 2: + return v.semver.Major() == version.Major() && v.semver.Minor() == version.Minor() + default: + return v.semver.Major() == version.Major() && v.semver.Minor() == version.Minor() && v.semver.Patch() == version.Patch() + } +} + +func (v *PackageVersion) String() string { + return fmt.Sprintf("%s-r%d", v.semver.Original(), v.revision) +} + +func ParsePackageVersion(version string) (*PackageVersion, error) { + parts := strings.Split(version, "-r") + if len(parts) != 2 { + return nil, fmt.Errorf("incorrect number of parts in version %s", version) + } + sv, err := semver.NewVersion(parts[0]) + if err != nil { + return nil, fmt.Errorf("parse version: %w", err) + } + revision, err := strconv.Atoi(parts[1]) + if err != nil { + return nil, fmt.Errorf("parse revision: %w", err) + } + return &PackageVersion{semver: *sv, revision: revision}, nil +} + +// listWolfiPackageVersions returns a list of all versions for a given package name +func istWolfiPackageVersions(wolfiAPKIndex []byte, pkgName string) ([]*PackageVersion, error) { + var versions []*PackageVersion scanner := bufio.NewScanner(bytes.NewReader(wolfiAPKIndex)) for scanner.Scan() { line := scanner.Text() @@ -67,37 +94,70 @@ func GetWolfiPackageVersion(wolfiAPKIndex []byte, pkgName string, pinnedVersion } scanner.Scan() line = scanner.Text() - // filter by package version - if !strings.HasPrefix(line, "V:"+pinnedVersion+"-r") { - continue - } - // find the revision number - parts := strings.Split(line, "-r") - if len(parts) != 2 { - return "", fmt.Errorf("incorrect number of parts in APKINDEX line: %s", line) + if !strings.HasPrefix(line, "V:") { + return nil, fmt.Errorf("incorrect APKINDEX version line: %s", line) } - revision, err := strconv.Atoi(parts[1]) + // extract the version + pv, err := ParsePackageVersion(line[2:]) if err != nil { - return "", fmt.Errorf("parse revision: %w", err) + return nil, fmt.Errorf("parse package version from line %s: %w", line, err) } - revisions = append(revisions, revision) + versions = append(versions, pv) } - if err := scanner.Err(); err != nil { - return "", fmt.Errorf("scan APKINDEX: %w", err) + return nil, fmt.Errorf("scan APKINDEX: %w", err) } + return versions, nil +} - if len(revisions) == 0 { - return "", fmt.Errorf("package %q not found", pkgName) +// listMatchingWolfiPackageVersions returns a list of all versions for a given package name that +// match the pinned version based on the number of version segments. +func listMatchingWolfiPackageVersions(wolfiAPKIndex []byte, pkgName, pinnedVersion string) ([]*PackageVersion, error) { + pinnedSV, err := semver.NewVersion(pinnedVersion) + if err != nil { + return nil, fmt.Errorf("parse pinned version: %w", err) + } + + versions, err := istWolfiPackageVersions(wolfiAPKIndex, pkgName) + if err != nil { + return nil, fmt.Errorf("list package versions: %w", err) + } + + var matchingVersions []*PackageVersion + for _, version := range versions { + // filter by package version + if !version.matches(pinnedSV) { + continue + } + matchingVersions = append(matchingVersions, version) } + return matchingVersions, nil +} - // get the latest revision +// GetWolfiPackageVersion returns the latest version and revision of a package in the wolfi APK +// index that matches the pinned version based on the number of version segments. +func GetWolfiPackageVersion(wolfiAPKIndex []byte, pkgName, pinnedVersion string) (string, error) { + versions, err := listMatchingWolfiPackageVersions(wolfiAPKIndex, pkgName, pinnedVersion) + if err != nil { + return "", fmt.Errorf("list package versions: %w", err) + } - sort.Slice(revisions, func(i, j int) bool { - return revisions[i] > revisions[j] - }) + var maxVersion *PackageVersion + for _, version := range versions { + if maxVersion == nil { + maxVersion = version + } else if version.semver.GreaterThan(&maxVersion.semver) { + maxVersion = version + } else if version.semver.Equal(&maxVersion.semver) && version.revision > maxVersion.revision { + maxVersion = version + } + } + + if maxVersion == nil { + return "", fmt.Errorf("package %q not found", pkgName) + } - return fmt.Sprintf("%s-r%d", pinnedVersion, revisions[0]), nil + return maxVersion.semver.Original(), nil } func GetLatestGitHubRelease(ctx context.Context, owner, repo string) (string, error) { diff --git a/cmd/buildtools/utils_test.go b/cmd/buildtools/utils_test.go new file mode 100644 index 0000000000..853ca75272 --- /dev/null +++ b/cmd/buildtools/utils_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "testing" + + "github.com/Masterminds/semver/v3" +) + +func TestPackageVersion_matches(t *testing.T) { + type fields struct { + semver semver.Version + revision int + } + type args struct { + version *semver.Version + } + tests := []struct { + name string + fields fields + args args + want bool + }{ + { + name: "matches exact", + fields: fields{ + semver: *semver.MustParse("1.2.3"), + revision: 1, + }, + args: args{ + version: semver.MustParse("1.2.3"), + }, + want: true, + }, + { + name: "not matches exact", + fields: fields{ + semver: *semver.MustParse("1.2.3"), + revision: 1, + }, + args: args{ + version: semver.MustParse("1.2.4"), + }, + want: false, + }, + { + name: "matches minor", + fields: fields{ + semver: *semver.MustParse("1.2.3"), + revision: 1, + }, + args: args{ + version: semver.MustParse("1.2"), + }, + want: true, + }, + { + name: "not matches minor", + fields: fields{ + semver: *semver.MustParse("1.2.3"), + revision: 1, + }, + args: args{ + version: semver.MustParse("1.3"), + }, + want: false, + }, + { + name: "matches major", + fields: fields{ + semver: *semver.MustParse("1.2.3"), + revision: 1, + }, + args: args{ + version: semver.MustParse("1"), + }, + want: true, + }, + { + name: "not matches major", + fields: fields{ + semver: *semver.MustParse("1.2.3"), + revision: 1, + }, + args: args{ + version: semver.MustParse("2"), + }, + want: false, + }, + { + name: "ignores prerelease and metadata", + fields: fields{ + semver: *semver.MustParse("1.2.3"), + revision: 1, + }, + args: args{ + version: semver.MustParse("1.2.3-alpha.1+metadata.2"), + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := &PackageVersion{ + semver: tt.fields.semver, + revision: tt.fields.revision, + } + if got := v.matches(tt.args.version); got != tt.want { + t.Errorf("PackageVersion.matches() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/deploy/images/openebs-kubectl/apko.tmpl.yaml b/deploy/images/openebs-kubectl/apko.tmpl.yaml index d5270be6af..221d2fb263 100644 --- a/deploy/images/openebs-kubectl/apko.tmpl.yaml +++ b/deploy/images/openebs-kubectl/apko.tmpl.yaml @@ -8,7 +8,7 @@ contents: - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub packages: - busybox - - kubectl-${PACKAGE_VERSION}-default + - ${PACKAGE_NAME}=${PACKAGE_VERSION} accounts: groups: