From d395a0c043d111b8aa2da0cc5827987f3e1d7d77 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 25 Jul 2024 12:53:16 -0700 Subject: [PATCH] f --- Makefile | 27 ++-- cmd/buildtools/metadata.go | 130 ++++++++++++++++++ cmd/embedded-cluster/install.go | 4 +- cmd/embedded-cluster/list_images.go | 16 +-- cmd/embedded-cluster/metadata.go | 15 +- pkg/addons/adminconsole/adminconsole.go | 8 ++ pkg/addons/applier.go | 14 ++ .../embeddedclusteroperator.go | 8 ++ pkg/addons/openebs/openebs.go | 8 ++ pkg/addons/registry/registry.go | 8 ++ pkg/addons/seaweedfs/seaweedfs.go | 8 ++ pkg/addons/velero/velero.go | 8 ++ pkg/config/helm.go | 45 ------ 13 files changed, 210 insertions(+), 89 deletions(-) create mode 100644 cmd/buildtools/metadata.go delete mode 100644 pkg/config/helm.go diff --git a/Makefile b/Makefile index 5bee8e61d3..3683183b40 100644 --- a/Makefile +++ b/Makefile @@ -124,14 +124,17 @@ static: pkg/goods/bins/k0s \ pkg/goods/internal/bins/kubectl-kots .PHONY: embedded-cluster-linux-amd64 -embedded-cluster-linux-amd64: buildtools embedded-cluster version-metadata -embedded-cluster-linux-amd64: - $(MAKE) embedded-cluster version-metadata-embed GOOS=linux GOARCH=amd64 +embedded-cluster-linux-amd64: GOOS = linux +embedded-cluster-linux-amd64: GOARCH = amd64 +embedded-cluster-linux-amd64: embedded-cluster + cp ./build/$(APP_NAME)-$(GOOS)-$(GOARCH) ./output/bin/$(APP_NAME) # for testing .PHONY: embedded-cluster-darwin-arm64 -embedded-cluster-darwin-arm64: buildtools embedded-cluster version-metadata-embed - $(MAKE) embedded-cluster version-metadata-embed GOOS=darwin GOARCH=arm64 +embedded-cluster-darwin-arm64: GOOS = darwin +embedded-cluster-darwin-arm64: GOARCH = arm64 +embedded-cluster-darwin-arm64: embedded-cluster + cp ./build/$(APP_NAME)-$(GOOS)-$(GOARCH) ./output/bin/$(APP_NAME) .PHONY: embedded-cluster embedded-cluster: @@ -139,20 +142,6 @@ embedded-cluster: go build -ldflags "$(LD_FLAGS)" -o ./build/$(APP_NAME)-$(GOOS)-$(GOARCH) \ ./cmd/embedded-cluster -.PHONY: version-metadata -version-metadata: export GOOS = $(shell go env GOOS) -version-metadata: export GOARCH = $(shell go env GOARCH) -version-metadata: embedded-cluster - ./build/$(APP_NAME)-$(GOOS)-$(GOARCH) version metadata > ./build/$(APP_NAME)-metadata.yaml - -.PHONY: version-metadata-embed -version-metadata-embed: - ./output/bin/buildtools embed \ - --binary-path ./build/$(APP_NAME)-$(GOOS)-$(GOARCH) \ - --binary-destination-path ./output/bin/$(APP_NAME) \ - --embed-path ./build/$(APP_NAME)-metadata.yaml \ - --embed-delimiter "RELEASE METADATA" - .PHONY: unit-tests unit-tests: go test -v ./pkg/... ./cmd/... diff --git a/cmd/buildtools/metadata.go b/cmd/buildtools/metadata.go new file mode 100644 index 0000000000..24639c135e --- /dev/null +++ b/cmd/buildtools/metadata.go @@ -0,0 +1,130 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "sort" + + embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1" + "github.com/replicatedhq/embedded-cluster-kinds/types" + "github.com/replicatedhq/embedded-cluster/pkg/helm" + "github.com/replicatedhq/embedded-cluster/pkg/utils" + "github.com/urfave/cli/v2" + "gopkg.in/yaml.v2" +) + +var metadataCommand = &cli.Command{ + Name: "metadata", + Usage: "Perform operations on the version-metadata.json file", + Subcommands: []*cli.Command{ + metadataListImagesCommand, + metadataExtractHelmChartImagesCommand, + }, +} + +var metadataListImagesCommand = &cli.Command{ + Name: "extract-images", + Usage: "List images", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "metadata-path", + Usage: "Path to the metadata file", + }, + }, + Action: func(c *cli.Context) error { + metadata, err := readMetadataFromFile(c.String("metadata-path")) + if err != nil { + return fmt.Errorf("failed to read metadata from file: %w", err) + } + + for _, image := range metadata.Images { + fmt.Println(image) + } + + return nil + }, +} + +var metadataExtractHelmChartImagesCommand = &cli.Command{ + Name: "extract-images", + Usage: "Extract images from Helm charts", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "metadata-path", + Usage: "Path to the metadata file", + }, + }, + Action: func(c *cli.Context) error { + metadata, err := readMetadataFromFile(c.String("metadata-path")) + if err != nil { + return fmt.Errorf("failed to read metadata from file: %w", err) + } + + repos := metadata.Configs.Repositories + charts := metadata.Configs.Charts + for _, chart := range metadata.BuiltinConfigs { + repos = append(repos, chart.Repositories...) + charts = append(charts, chart.Charts...) + } + + images, err := extractImagesFromHelmExtensions(repos, charts) + if err != nil { + return fmt.Errorf("failed to extract images from helm extensions: %w", err) + } + + for _, image := range images { + fmt.Println(image) + } + + return nil + }, +} + +func readMetadataFromFile(path string) (*types.ReleaseMetadata, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read file: %w", err) + } + + var metadata types.ReleaseMetadata + if err := json.Unmarshal(b, &metadata); err != nil { + return nil, fmt.Errorf("unmarshal metadata: %w", err) + } + + return &metadata, nil +} + +func extractImagesFromHelmExtensions(repos []embeddedclusterv1beta1.Repository, charts []embeddedclusterv1beta1.Chart) ([]string, error) { + hcli, err := helm.NewHelm(helm.HelmOptions{ + K0sVersion: "1.29.6", // TODO + }) + if err != nil { + return nil, fmt.Errorf("create helm client: %w", err) + } + defer hcli.Close() + + var images []string + for _, ext := range charts { + ims, err := extractImagesFromChart(hcli, ext) + if err != nil { + return nil, fmt.Errorf("extract images from chart %s: %w", ext.Name, err) + } + images = append(images, ims...) + } + images = utils.UniqueStringSlice(images) + sort.Strings(images) + return images, nil +} + +func extractImagesFromChart(hcli *helm.Helm, chart embeddedclusterv1beta1.Chart) ([]string, error) { + values := map[string]interface{}{} + if chart.Values != "" { + err := yaml.Unmarshal([]byte(chart.Values), &values) + if err != nil { + return nil, fmt.Errorf("unmarshal values: %w", err) + } + } + + return helm.ExtractImagesFromOCIChart(hcli, chart.ChartName, chart.Name, chart.Version, values) +} diff --git a/cmd/embedded-cluster/install.go b/cmd/embedded-cluster/install.go index ce6faac891..92733dde9e 100644 --- a/cmd/embedded-cluster/install.go +++ b/cmd/embedded-cluster/install.go @@ -494,9 +494,9 @@ func runOutro(c *cli.Context, cfg *k0sconfig.ClusterConfig, adminConsolePwd stri os.Setenv("KUBECONFIG", defaults.PathToKubeConfig()) opts := []addons.Option{} - metadata, err := goods.ExtractReleaseMetadataFromSelf() + metadata, err := gatherVersionMetadata() if err != nil { - return fmt.Errorf("extract release metadata from self: %w", err) + return fmt.Errorf("unable to gather release metadata: %w", err) } opts = append(opts, addons.WithVersionMetadata(metadata)) diff --git a/cmd/embedded-cluster/list_images.go b/cmd/embedded-cluster/list_images.go index 55ef673d06..aa3378205a 100644 --- a/cmd/embedded-cluster/list_images.go +++ b/cmd/embedded-cluster/list_images.go @@ -3,7 +3,6 @@ package main import ( "fmt" - "github.com/replicatedhq/embedded-cluster/pkg/goods" "github.com/urfave/cli/v2" ) @@ -11,21 +10,10 @@ var listImagesCommand = &cli.Command{ Name: "list-images", Usage: "List images embedded in the cluster", Hidden: true, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "license", - Aliases: []string{"l"}, - Usage: "Path to the license file", - }, - &cli.StringFlag{ - Name: "airgap-bundle", - Usage: "Path to the air gap bundle. Required to list images without internet access.", - }, - }, Action: func(c *cli.Context) error { - metadata, err := goods.ExtractReleaseMetadataFromSelf() + metadata, err := gatherVersionMetadata() if err != nil { - return fmt.Errorf("failed to extract release metadata from self: %w", err) + return fmt.Errorf("failed to gather version metadata: %w", err) } for _, image := range metadata.Images { diff --git a/cmd/embedded-cluster/metadata.go b/cmd/embedded-cluster/metadata.go index 319ccdd1aa..9b99cefac5 100644 --- a/cmd/embedded-cluster/metadata.go +++ b/cmd/embedded-cluster/metadata.go @@ -125,17 +125,14 @@ func gatherVersionMetadata() (*types.ReleaseMetadata, error) { } meta.K0sImages = append(meta.K0sImages, additionalImages...) - meta.Images = append([]string{}, meta.K0sImages...) + meta.K0sImages = utils.UniqueStringSlice(meta.K0sImages) + sort.Strings(meta.K0sImages) - repos := append([]eckinds.Repository{}, repconfig...) - charts := append([]eckinds.Chart{}, chtconfig...) - for _, chart := range builtinCharts { - repos = append(repos, chart.Repositories...) - charts = append(charts, chart.Charts...) - } - images, err := config.ExtractImagesFromHelmExtensions(repos, charts) + meta.Images = config.ListK0sImages(cfg) + + images, err := applier.GetImages() if err != nil { - return nil, fmt.Errorf("unable to extract images from helm extensions: %w", err) + return nil, fmt.Errorf("unable to get images: %w", err) } meta.Images = append(meta.Images, images...) diff --git a/pkg/addons/adminconsole/adminconsole.go b/pkg/addons/adminconsole/adminconsole.go index 0d951292e6..50b1691c11 100644 --- a/pkg/addons/adminconsole/adminconsole.go +++ b/pkg/addons/adminconsole/adminconsole.go @@ -166,6 +166,14 @@ func (a *AdminConsole) GenerateHelmConfig(onlyDefaults bool) ([]eckinds.Chart, [ return []eckinds.Chart{chartConfig}, nil, nil } +func (a *AdminConsole) GetImages() []string { + var images []string + for image, tag := range Metadata.Images { + images = append(images, fmt.Sprintf("%s:%s", image, tag)) + } + return images +} + func (a *AdminConsole) GetAdditionalImages() []string { return nil } diff --git a/pkg/addons/applier.go b/pkg/addons/applier.go index 850988c804..13c089c0a5 100644 --- a/pkg/addons/applier.go +++ b/pkg/addons/applier.go @@ -35,6 +35,7 @@ type AddOn interface { GenerateHelmConfig(onlyDefaults bool) ([]embeddedclusterv1beta1.Chart, []embeddedclusterv1beta1.Repository, error) Outro(context.Context, client.Client) error GetProtectedFields() map[string][]string + GetImages() []string GetAdditionalImages() []string } @@ -212,6 +213,19 @@ func (a *Applier) GetBuiltinCharts() (map[string]embeddedclusterv1beta1.Helm, er return builtinCharts, nil } +func (a *Applier) GetImages() ([]string, error) { + additionalImages := []string{} + addons, err := a.load() + if err != nil { + return nil, fmt.Errorf("unable to load addons: %w", err) + } + for _, addon := range addons { + additionalImages = append(additionalImages, addon.GetImages()...) + } + + return additionalImages, nil +} + func (a *Applier) GetAdditionalImages() ([]string, error) { additionalImages := []string{} addons, err := a.load() diff --git a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go index caf638addf..00cd07557d 100644 --- a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go +++ b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go @@ -149,6 +149,14 @@ func (e *EmbeddedClusterOperator) GenerateHelmConfig(onlyDefaults bool) ([]embed return []embeddedclusterv1beta1.Chart{chartConfig}, nil, nil } +func (a *EmbeddedClusterOperator) GetImages() []string { + var images []string + for image, tag := range Metadata.Images { + images = append(images, fmt.Sprintf("%s:%s", image, tag)) + } + return images +} + func (e *EmbeddedClusterOperator) GetAdditionalImages() []string { var images []string if UtilsImage != "" { diff --git a/pkg/addons/openebs/openebs.go b/pkg/addons/openebs/openebs.go index ff760beb2d..9b156c2e00 100644 --- a/pkg/addons/openebs/openebs.go +++ b/pkg/addons/openebs/openebs.go @@ -88,6 +88,14 @@ func (o *OpenEBS) GenerateHelmConfig(onlyDefaults bool) ([]eckinds.Chart, []ecki return []eckinds.Chart{chartConfig}, nil, nil } +func (a *OpenEBS) GetImages() []string { + var images []string + for image, tag := range Metadata.Images { + images = append(images, fmt.Sprintf("%s:%s", image, tag)) + } + return images +} + func (o *OpenEBS) GetAdditionalImages() []string { var images []string if tag, ok := Metadata.Images["openebs-linux-utils"]; ok { diff --git a/pkg/addons/registry/registry.go b/pkg/addons/registry/registry.go index 48d05d32af..e13117ccef 100644 --- a/pkg/addons/registry/registry.go +++ b/pkg/addons/registry/registry.go @@ -146,6 +146,14 @@ func (o *Registry) GenerateHelmConfig(onlyDefaults bool) ([]eckinds.Chart, []eck return []eckinds.Chart{chartConfig}, nil, nil } +func (a *Registry) GetImages() []string { + var images []string + for image, tag := range Metadata.Images { + images = append(images, fmt.Sprintf("%s:%s", image, tag)) + } + return images +} + func (o *Registry) GetAdditionalImages() []string { return nil } diff --git a/pkg/addons/seaweedfs/seaweedfs.go b/pkg/addons/seaweedfs/seaweedfs.go index d9ea9654be..699d35564a 100644 --- a/pkg/addons/seaweedfs/seaweedfs.go +++ b/pkg/addons/seaweedfs/seaweedfs.go @@ -93,6 +93,14 @@ func (o *SeaweedFS) GenerateHelmConfig(onlyDefaults bool) ([]eckinds.Chart, []ec return []eckinds.Chart{chartConfig}, nil, nil } +func (a *SeaweedFS) GetImages() []string { + var images []string + for image, tag := range Metadata.Images { + images = append(images, fmt.Sprintf("%s:%s", image, tag)) + } + return images +} + func (o *SeaweedFS) GetAdditionalImages() []string { return nil } diff --git a/pkg/addons/velero/velero.go b/pkg/addons/velero/velero.go index 778a819497..1ec06e65a8 100644 --- a/pkg/addons/velero/velero.go +++ b/pkg/addons/velero/velero.go @@ -105,6 +105,14 @@ func (o *Velero) GenerateHelmConfig(onlyDefaults bool) ([]eckinds.Chart, []eckin return []eckinds.Chart{chartConfig}, nil, nil } +func (a *Velero) GetImages() []string { + var images []string + for image, tag := range Metadata.Images { + images = append(images, fmt.Sprintf("%s:%s", image, tag)) + } + return images +} + func (o *Velero) GetAdditionalImages() []string { var images []string if tag, ok := Metadata.Images["velero-restore-helper"]; ok { diff --git a/pkg/config/helm.go b/pkg/config/helm.go deleted file mode 100644 index c0d48ba9cd..0000000000 --- a/pkg/config/helm.go +++ /dev/null @@ -1,45 +0,0 @@ -package config - -import ( - "fmt" - "sort" - - embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1" - "github.com/replicatedhq/embedded-cluster/pkg/helm" - "github.com/replicatedhq/embedded-cluster/pkg/utils" - "gopkg.in/yaml.v3" -) - -func ExtractImagesFromHelmExtensions(repos []embeddedclusterv1beta1.Repository, charts []embeddedclusterv1beta1.Chart) ([]string, error) { - hcli, err := helm.NewHelm(helm.HelmOptions{ - K0sVersion: "1.29.6", // TODO - }) - if err != nil { - return nil, fmt.Errorf("create helm client: %w", err) - } - defer hcli.Close() - - var images []string - for _, ext := range charts { - ims, err := extractImagesFromChart(hcli, ext) - if err != nil { - return nil, fmt.Errorf("extract images from chart %s: %w", ext.Name, err) - } - images = append(images, ims...) - } - images = utils.UniqueStringSlice(images) - sort.Strings(images) - return images, nil -} - -func extractImagesFromChart(hcli *helm.Helm, chart embeddedclusterv1beta1.Chart) ([]string, error) { - values := map[string]interface{}{} - if chart.Values != "" { - err := yaml.Unmarshal([]byte(chart.Values), &values) - if err != nil { - return nil, fmt.Errorf("unmarshal values: %w", err) - } - } - - return helm.ExtractImagesFromOCIChart(hcli, chart.ChartName, chart.Name, chart.Version, values) -}