From 809589e735c9278c3ded28d6f1fb6836325d7c35 Mon Sep 17 00:00:00 2001 From: Gerd Oberlechner Date: Wed, 9 Oct 2024 14:08:46 +0200 Subject: [PATCH] manifest work Signed-off-by: Gerd Oberlechner --- cmd/install/assets/hypershift_operator.go | 4 + cmd/install/install.go | 10 +- cmd/install/install_helm.go | 82 +++++++++++++ cmd/install/install_render.go | 143 ++++++++-------------- cmd/install/render.go | 75 ++++++++++++ 5 files changed, 220 insertions(+), 94 deletions(-) create mode 100644 cmd/install/install_helm.go create mode 100644 cmd/install/render.go diff --git a/cmd/install/assets/hypershift_operator.go b/cmd/install/assets/hypershift_operator.go index 5ca06770958..b8454980d85 100644 --- a/cmd/install/assets/hypershift_operator.go +++ b/cmd/install/assets/hypershift_operator.go @@ -377,6 +377,7 @@ type HyperShiftOperatorDeployment struct { ManagedService string EnableSizeTagging bool EnableEtcdRecovery bool + RegistryOverrides string } func (o HyperShiftOperatorDeployment) Build() *appsv1.Deployment { @@ -390,6 +391,9 @@ func (o HyperShiftOperatorDeployment) Build() *appsv1.Deployment { fmt.Sprintf("--enable-ci-debug-output=%t", o.EnableCIDebugOutput), fmt.Sprintf("--private-platform=%s", o.PrivatePlatform), } + if o.RegistryOverrides != "" { + args = append(args, fmt.Sprintf("--registry-overrides=%s", o.RegistryOverrides)) + } var volumeMounts []corev1.VolumeMount var initVolumeMounts []corev1.VolumeMount diff --git a/cmd/install/install.go b/cmd/install/install.go index caa0b5e539f..f761bc7a9b7 100644 --- a/cmd/install/install.go +++ b/cmd/install/install.go @@ -104,6 +104,8 @@ type Options struct { ManagedService string EnableSizeTagging bool EnableEtcdRecovery bool + RegistryOverrides string + RenderNamespace bool } func (o *Options) Validate() error { @@ -159,6 +161,7 @@ func (o *Options) Validate() error { if len(o.ManagedService) > 0 && o.ManagedService != hyperv1.AroHCP { errs = append(errs, fmt.Errorf("not a valid managed service type: %s", o.ManagedService)) } + // todo: validate opts.RegistryOverrides return errors.NewAggregate(errs) } @@ -235,6 +238,7 @@ func NewCommand() *cobra.Command { cmd.PersistentFlags().StringVar(&opts.ManagedService, "managed-service", opts.ManagedService, "The type of managed service the HyperShift Operator is installed on; this is used to configure different HostedCluster options depending on the managed service. Examples: ARO-HCP, ROSA-HCP") cmd.PersistentFlags().BoolVar(&opts.EnableSizeTagging, "enable-size-tagging", opts.EnableSizeTagging, "If true, HyperShift will tag the HostedCluster with a size label corresponding to the number of worker nodes") cmd.PersistentFlags().BoolVar(&opts.EnableEtcdRecovery, "enable-etcd-recovery", opts.EnableEtcdRecovery, "If true, the HyperShift operator checks for failed etcd pods and attempts a recovery if possible") + cmd.PersistentFlags().StringVar(&opts.RegistryOverrides, "registry-overrides", "", "registry-overrides contains the source registry string as a key and the destination registry string as value. Images before being applied are scanned for the source registry string and if found the string is replaced with the destination registry string. Format is: sr1=dr1,sr2=dr2") cmd.RunE = func(cmd *cobra.Command, args []string) error { opts.ApplyDefaults() @@ -274,6 +278,7 @@ func NewCommand() *cobra.Command { } cmd.AddCommand(NewRenderCommand(&opts)) + cmd.AddCommand(NewHelmRenderCommand(&opts)) return cmd } @@ -472,7 +477,9 @@ func hyperShiftOperatorManifests(opts Options) ([]crclient.Object, []crclient.Ob Name: opts.Namespace, EnableOCPClusterMonitoring: opts.PlatformMonitoring.IsEnabled(), }.Build() - objects = append(objects, operatorNamespace) + if opts.RenderNamespace { + objects = append(objects, operatorNamespace) + } // Setup RBAC resources operatorServiceAccount, rbacObjs := setupRBAC(opts, operatorNamespace) @@ -681,6 +688,7 @@ func setupOperatorResources(opts Options, userCABundleCM *corev1.ConfigMap, trus ManagedService: opts.ManagedService, EnableSizeTagging: opts.EnableSizeTagging, EnableEtcdRecovery: opts.EnableEtcdRecovery, + RegistryOverrides: opts.RegistryOverrides, }.Build() operatorService := assets.HyperShiftOperatorService{ Namespace: operatorNamespace, diff --git a/cmd/install/install_helm.go b/cmd/install/install_helm.go new file mode 100644 index 00000000000..9a8d91f5099 --- /dev/null +++ b/cmd/install/install_helm.go @@ -0,0 +1,82 @@ +package install + +import ( + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + + +var helmTemplateParams = TemplateParams{ + HyperShiftImage: ".Values.operator.image", + HyperShiftImageTag: ".Values.operator.imageTag", + Namespace: ".Release.Namespace", + OIDCS3Name: ".Values.oidc.s3.name", + OIDCS3Region: ".Values.oidc.s3.region", + OIDCS3CredsSecret: ".Values.oidc.s3.credsSecret", + OIDCS3CredsSecretKey: ".Values.oidc.s3.credsSecretKey", + AWSPrivateRegion: ".Values.aws.private.region", + AWSPrivateCredsSecret: ".Values.aws.private.credsSecret", + AWSPrivateCredsSecretKey: ".Values.aws.private.credsSecretKey", + ExternalDNSCredsSecret: ".Values.externaldns.credsSecret", + ExternalDNSDomainFilter: ".Values.externaldns.domainFilter", + ExternalDNSTxtOwnerID: ".Values.externaldns.txtOwnerId", + ExternalDNSAzureWorkloadIdentity: ".Values.externaldns.azureWorkloadIdentity", + ExternalDNSImage: ".Values.externaldns.image", + RegistryOverrides: "registryOverrides", + TemplateNamespace: false, + TemplateParamWrapper: func(name string) string { + return fmt.Sprintf("{{ %s }}", name) + }, +} + +func NewHelmRenderCommand(opts *Options) *cobra.Command { + cmd := &cobra.Command{ + Use: "helm", + Short: "Generate a Helm chart for the HyperShift Operator", + SilenceUsage: true, + } + + cmd.Flags().StringVar(&opts.OutputFile, "output-dir", "", "File to write the rendered manifests to. Writes to STDOUT if not specified.") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + opts.ApplyDefaults() + + crds, manifests, err := hyperShiftOperatorTemplateManifest(opts, helmTemplateParams) + if err != nil { + return err + } + + if opts.OutputFile == "" { + opts.OutputFile = "./chart" + } + WriteManifestsToDir(crds, fmt.Sprintf("%s/crds", opts.OutputFile)) + WriteManifestsToDir(manifests, fmt.Sprintf("%s/templates", opts.OutputFile)) + return nil + } + + return cmd +} + +func WriteManifestsToDir(manifests []crclient.Object, dir string) error { + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + return err + } + + for _, manifest := range manifests { + file, err := os.Create(fmt.Sprintf("%s/%s-%s.yaml", dir, strings.ToLower(manifest.GetObjectKind().GroupVersionKind().Kind), manifest.GetName())) + if err != nil { + return err + } + defer file.Close() + err = render([]crclient.Object{manifest}, RenderFormatYaml, file) + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/install/install_render.go b/cmd/install/install_render.go index c50f315a059..4af1f87d28c 100644 --- a/cmd/install/install_render.go +++ b/cmd/install/install_render.go @@ -9,7 +9,6 @@ import ( hyperapi "github.com/openshift/hypershift/support/api" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" crclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -25,23 +24,31 @@ const ( var ( RenderFormatYaml = "yaml" RenderFormatJson = "json" - - TemplateParamHyperShiftImage = "OPERATOR_IMG" - TemplateParamHyperShiftImageTag = "IMAGE_TAG" - TemplateParamNamespace = "NAMESPACE" - TemplateParamOIDCS3Name = "OIDC_S3_NAME" - TemplateParamOIDCS3Region = "OIDC_S3_REGION" - TemplateParamOIDCS3CredsSecret = "OIDC_S3_CREDS_SECRET" - TemplateParamOIDCS3CredsSecretKey = "OIDC_S3_CREDS_SECRET_KEY" - TemplateParamAWSPrivateRegion = "AWS_PRIVATE_REGION" - TemplateParamAWSPrivateCredsSecret = "AWS_PRIVATE_CREDS_SECRET" - TemplateParamAWSPrivateCredsSecretKey = "AWS_PRIVATE_CREDS_SECRET_KEY" - TemplateParamOperatorReplicas = "OPERATOR_REPLICAS" - TemplateParamExternalDNSCredsSecret = "EXTERNAL_DNS_CREDS_SECRET" - TemplateParamExternalDNSDomainFilter = "EXTERNAL_DNS_DOMAIN_FILTER" - TemplateParamExternalDNSTxtOwnerID = "EXTERNAL_DNS_TXT_OWNER_ID" ) +var openshiftTemplateParams = TemplateParams{ + HyperShiftImage: "OPERATOR_IMG", + HyperShiftImageTag: "IMAGE_TAG", + Namespace: "NAMESPACE", + OIDCS3Name: "OIDC_S3_NAME", + OIDCS3Region: "OIDC_S3_REGION", + OIDCS3CredsSecret: "OIDC_S3_CREDS_SECRET", + OIDCS3CredsSecretKey: "OIDC_S3_CREDS_SECRET_KEY", + AWSPrivateRegion: "AWS_PRIVATE_REGION", + AWSPrivateCredsSecret: "AWS_PRIVATE_CREDS_SECRET", + AWSPrivateCredsSecretKey: "AWS_PRIVATE_CREDS_SECRET_KEY", + ExternalDNSCredsSecret: "EXTERNAL_DNS_CREDS_SECRET", + ExternalDNSDomainFilter: "EXTERNAL_DNS_DOMAIN_FILTER", + ExternalDNSTxtOwnerID: "EXTERNAL_DNS_TXT_OWNER_ID", + ExternalDNSAzureWorkloadIdentity: "EXTERNAL_DNS_AZURE_WORKLOAD_IDENTITY", + ExternalDNSImage: "EXTERNAL_DNS_IMAGE", + RegistryOverrides: "REGISTRY_OVERRIDES", + TemplateNamespace: true, + TemplateParamWrapper: func(name string) string { + return fmt.Sprintf("${%s}", name) + }, +} + func NewRenderCommand(opts *Options) *cobra.Command { cmd := &cobra.Command{ Use: "render", @@ -67,7 +74,7 @@ func NewRenderCommand(opts *Options) *cobra.Command { var objects []crclient.Object if opts.Template { - templateObject, err := hyperShiftOperatorTemplateManifest(opts) + templateObject, err := openshiftTemplate(opts) if err != nil { return err } @@ -127,89 +134,48 @@ func (o *Options) ValidateRender() error { return nil } -func hyperShiftOperatorTemplateManifest(opts *Options) (crclient.Object, error) { +func openshiftTemplate(opts *Options) (crclient.Object, error) { templateParameters := []map[string]string{} - - // image parameter - templateParameters = append( - templateParameters, - map[string]string{"name": TemplateParamHyperShiftImage, "value": version.HypershiftImageBase}, - map[string]string{"name": TemplateParamHyperShiftImageTag, "value": version.HypershiftImageTag}, - ) - opts.HyperShiftImage = fmt.Sprintf("${%s}:${%s}", TemplateParamHyperShiftImage, TemplateParamHyperShiftImageTag) - - // namespace parameter templateParameters = append( templateParameters, - map[string]string{"name": TemplateParamNamespace, "value": opts.Namespace}, + map[string]string{"name": openshiftTemplateParams.HyperShiftImage, "value": version.HypershiftImageBase}, + map[string]string{"name": openshiftTemplateParams.HyperShiftImageTag, "value": version.HypershiftImageTag}, + map[string]string{"name": openshiftTemplateParams.Namespace, "value": opts.Namespace}, ) - opts.Namespace = fmt.Sprintf("${%s}", TemplateParamNamespace) - - // oidc S3 parameter if opts.OIDCStorageProviderS3BucketName != "" { templateParameters = append( templateParameters, - map[string]string{"name": TemplateParamOIDCS3Name}, - map[string]string{"name": TemplateParamOIDCS3Region}, - map[string]string{"name": TemplateParamOIDCS3CredsSecret, "value": opts.OIDCStorageProviderS3CredentialsSecret}, - map[string]string{"name": TemplateParamOIDCS3CredsSecretKey, "value": opts.OIDCStorageProviderS3CredentialsSecretKey}, + map[string]string{"name": openshiftTemplateParams.OIDCS3Name}, + map[string]string{"name": openshiftTemplateParams.OIDCS3Region}, + map[string]string{"name": openshiftTemplateParams.OIDCS3CredsSecret, "value": opts.OIDCStorageProviderS3CredentialsSecret}, + map[string]string{"name": openshiftTemplateParams.OIDCS3CredsSecretKey, "value": opts.OIDCStorageProviderS3CredentialsSecretKey}, ) - opts.OIDCStorageProviderS3BucketName = fmt.Sprintf("${%s}", TemplateParamOIDCS3Name) - opts.OIDCStorageProviderS3Region = fmt.Sprintf("${%s}", TemplateParamOIDCS3Region) - opts.OIDCStorageProviderS3CredentialsSecret = fmt.Sprintf("${%s}", TemplateParamOIDCS3CredsSecret) - opts.OIDCStorageProviderS3CredentialsSecretKey = fmt.Sprintf("${%s}", TemplateParamOIDCS3CredsSecretKey) } - - // aws private credentials if opts.AWSPrivateCredentialsSecret != "" { templateParameters = append( templateParameters, - map[string]string{"name": TemplateParamAWSPrivateRegion, "value": opts.AWSPrivateRegion}, - map[string]string{"name": TemplateParamAWSPrivateCredsSecret, "value": opts.AWSPrivateCredentialsSecret}, - map[string]string{"name": TemplateParamAWSPrivateCredsSecretKey, "value": opts.AWSPrivateCredentialsSecretKey}, + map[string]string{"name": openshiftTemplateParams.AWSPrivateRegion, "value": opts.AWSPrivateRegion}, + map[string]string{"name": openshiftTemplateParams.AWSPrivateCredsSecret, "value": opts.AWSPrivateCredentialsSecret}, + map[string]string{"name": openshiftTemplateParams.AWSPrivateCredsSecretKey, "value": opts.AWSPrivateCredentialsSecretKey}, ) - opts.AWSPrivateRegion = fmt.Sprintf("${%s}", TemplateParamAWSPrivateRegion) - opts.AWSPrivateCredentialsSecret = fmt.Sprintf("${%s}", TemplateParamAWSPrivateCredsSecret) - opts.AWSPrivateCredentialsSecretKey = fmt.Sprintf("${%s}", TemplateParamAWSPrivateCredsSecretKey) } - // external DNS if opts.ExternalDNSProvider != "" && opts.ExternalDNSDomainFilter != "" && opts.ExternalDNSCredentialsSecret != "" { templateParameters = append( templateParameters, - map[string]string{"name": TemplateParamExternalDNSDomainFilter, "value": opts.ExternalDNSDomainFilter}, - map[string]string{"name": TemplateParamExternalDNSCredsSecret, "value": opts.ExternalDNSCredentialsSecret}, + map[string]string{"name": openshiftTemplateParams.ExternalDNSDomainFilter, "value": opts.ExternalDNSDomainFilter}, + map[string]string{"name": openshiftTemplateParams.ExternalDNSCredsSecret, "value": opts.ExternalDNSCredentialsSecret}, ) - opts.ExternalDNSDomainFilter = fmt.Sprintf("${%s}", TemplateParamExternalDNSDomainFilter) - opts.ExternalDNSCredentialsSecret = fmt.Sprintf("${%s}", TemplateParamExternalDNSCredsSecret) - if opts.ExternalDNSTxtOwnerId != "" { templateParameters = append( templateParameters, - map[string]string{"name": TemplateParamExternalDNSTxtOwnerID, "value": opts.ExternalDNSTxtOwnerId}, + map[string]string{"name": openshiftTemplateParams.ExternalDNSTxtOwnerID, "value": opts.ExternalDNSTxtOwnerId}, ) - opts.ExternalDNSTxtOwnerId = fmt.Sprintf("${%s}", TemplateParamExternalDNSTxtOwnerID) } - - } - - // create manifests - crds, objects, err := hyperShiftOperatorManifests(*opts) - if err != nil { - return nil, err } - objects = append(objects, crds...) - - // patch those manifests, where the template parameter placeholder was not injectable with opts (e.g. type mistmatch) - patches := []ObjectPatch{ - {Kind: "Deployment", Name: "operator", Path: []string{"spec", "replicas"}, Value: fmt.Sprintf("${{%s}}", TemplateParamOperatorReplicas)}, - } - templateParameters = append( - templateParameters, - map[string]string{"name": TemplateParamOperatorReplicas, "value": "1"}, - ) - patchedObjects, err := applyPatchesToObjects(objects, patches) + crds, manifests, err := hyperShiftOperatorTemplateManifest(opts, openshiftTemplateParams) + manifests = append(manifests, crds...) if err != nil { return nil, err } @@ -222,31 +188,13 @@ func hyperShiftOperatorTemplateManifest(opts *Options) (crclient.Object, error) "metadata": map[string]interface{}{ "name": "hypershift-operator-template", }, - "objects": patchedObjects, + "objects": manifests, "parameters": templateParameters, }, } return template, nil } -func applyPatchesToObjects(objects []crclient.Object, patches []ObjectPatch) ([]crclient.Object, error) { - patchedObjects := make([]crclient.Object, len(objects)) - for i, obj := range objects { - content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) - if err != nil { - return nil, err - } - patchedObject := &unstructured.Unstructured{Object: content} - for _, p := range patches { - if p.CanBeAppliedTo(patchedObject) { - unstructured.SetNestedField(patchedObject.Object, p.Value, p.Path...) - } - } - patchedObjects[i] = patchedObject - } - return patchedObjects, nil -} - func render(objects []crclient.Object, format string, out io.Writer) error { switch format { case RenderFormatYaml: @@ -302,3 +250,12 @@ func (p *ObjectPatch) CanBeAppliedTo(obj crclient.Object) bool { } return true } + +func getByKindAndName(objects []crclient.Object, kind, name string) (crclient.Object, error) { + for _, obj := range objects { + if obj.GetObjectKind().GroupVersionKind().Kind == kind && obj.GetName() == name { + return obj, nil + } + } + return nil, fmt.Errorf("object not found: kind=%s, name=%s", kind, name) +} diff --git a/cmd/install/render.go b/cmd/install/render.go new file mode 100644 index 00000000000..8a599a778ed --- /dev/null +++ b/cmd/install/render.go @@ -0,0 +1,75 @@ +package install + +import ( + "fmt" + + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +type TemplateParams struct { + HyperShiftImage string + HyperShiftImageTag string + Namespace string + OIDCS3Name string + OIDCS3Region string + OIDCS3CredsSecret string + OIDCS3CredsSecretKey string + AWSPrivateRegion string + AWSPrivateCredsSecret string + AWSPrivateCredsSecretKey string + ExternalDNSCredsSecret string + ExternalDNSDomainFilter string + ExternalDNSTxtOwnerID string + ExternalDNSAzureWorkloadIdentity string + ExternalDNSImage string + RegistryOverrides string + TemplateNamespace bool + TemplateParamWrapper func(string) string +} + +func hyperShiftOperatorTemplateManifest(opts *Options, templateParamConfig TemplateParams) ([]crclient.Object, []crclient.Object, error) { + // validate options + if err := opts.ValidateRender(); err != nil { + return nil, nil, err + } + + opts.HyperShiftImage = fmt.Sprintf("%s:%s", templateParamConfig.TemplateParamWrapper(templateParamConfig.HyperShiftImage), templateParamConfig.TemplateParamWrapper(templateParamConfig.HyperShiftImageTag)) + + // namespace parameter + opts.Namespace = templateParamConfig.TemplateParamWrapper(templateParamConfig.Namespace) + + // oidc S3 parameter + if opts.OIDCStorageProviderS3BucketName != "" { + opts.OIDCStorageProviderS3BucketName = templateParamConfig.TemplateParamWrapper(templateParamConfig.OIDCS3Name) + opts.OIDCStorageProviderS3Region = templateParamConfig.TemplateParamWrapper(templateParamConfig.OIDCS3Region) + opts.OIDCStorageProviderS3CredentialsSecret = templateParamConfig.TemplateParamWrapper(templateParamConfig.OIDCS3CredsSecret) + opts.OIDCStorageProviderS3CredentialsSecretKey = templateParamConfig.TemplateParamWrapper(templateParamConfig.OIDCS3CredsSecretKey) + } + + // aws private credentials + if opts.AWSPrivateCredentialsSecret != "" { + opts.AWSPrivateRegion = templateParamConfig.TemplateParamWrapper(templateParamConfig.AWSPrivateRegion) + opts.AWSPrivateCredentialsSecret = templateParamConfig.TemplateParamWrapper(templateParamConfig.AWSPrivateCredsSecret) + opts.AWSPrivateCredentialsSecretKey = templateParamConfig.TemplateParamWrapper(templateParamConfig.AWSPrivateCredsSecretKey) + } + + // external DNS + if opts.ExternalDNSProvider != "" { + opts.ExternalDNSImage = templateParamConfig.TemplateParamWrapper(templateParamConfig.ExternalDNSImage) + opts.ExternalDNSDomainFilter = templateParamConfig.TemplateParamWrapper(templateParamConfig.ExternalDNSDomainFilter) + opts.ExternalDNSCredentialsSecret = templateParamConfig.TemplateParamWrapper(templateParamConfig.ExternalDNSCredsSecret) + opts.ExternalDNSTxtOwnerId = templateParamConfig.TemplateParamWrapper(templateParamConfig.ExternalDNSTxtOwnerID) + } + + // registry overrides + opts.RegistryOverrides = templateParamConfig.TemplateParamWrapper(templateParamConfig.RegistryOverrides) + + // create manifests + opts.RenderNamespace = templateParamConfig.TemplateNamespace + crds, objects, err := hyperShiftOperatorManifests(*opts) + if err != nil { + return nil, nil, err + } + return crds, objects, nil + +}