pipeline preprocess steps on load (#1426)

we use the `pipeline.Variable` struct for both jsonschema `variable` and `variableRef` types. the only difference between them is `name` being required by `variable` while it's optional by `variableRef`. to simplify codepaths on the EV2 generator, we will keep the optional nature in `variableRef` but fill the name with a default during pipeline loading and preprocessing.

Signed-off-by: Gerd Oberlechner <goberlec@redhat.com>

tooling/templatize/Makefile

@@ -5,7 +5,7 @@ SHELL = /bin/bash
 BINARY = templatize
 
 # Define the source files
-SOURCES = $(shell find . -name '*.go')
+SOURCES = $(shell find . -name '*.go' -o -name '*.json')
 
 # Build the binary
 $(BINARY): $(SOURCES) $(MAKEFILE_LIST)

tooling/templatize/pkg/pipeline/pipeline.schema.v1.json

@@ -18,7 +18,11 @@
         },
         "variableRef": {
             "type": "object",
+            "additionalProperties": false,
             "properties": {
+                "name": {
+                    "type": "string"
+                },
                 "input": {
                     "type": "object",
                     "additionalProperties": false,
@@ -62,6 +66,7 @@
         },
         "variable": {
             "type": "object",
+            "additionalProperties": false,
             "properties": {
                 "name": {
                     "type": "string"

tooling/templatize/pkg/pipeline/types.go

@@ -11,6 +11,7 @@ import (
 type subsciptionLookup func(context.Context, string) (string, error)
 
 type Pipeline struct {
+	schema           string `yaml:"$schema,omitempty"`
 	pipelineFilePath string
 	ServiceGroup     string           `yaml:"serviceGroup"`
 	RolloutName      string           `yaml:"rolloutName"`
@@ -26,20 +27,33 @@ type ResourceGroup struct {
 
 func NewPlainPipelineFromBytes(filepath string, bytes []byte) (*Pipeline, error) {
 	rawPipeline := &struct {
+		Schema         string `yaml:"$schema,omitempty"`
 		ServiceGroup   string `yaml:"serviceGroup"`
 		RolloutName    string `yaml:"rolloutName"`
 		ResourceGroups []struct {
-			Name         string `yaml:"name"`
-			Subscription string `yaml:"subscription"`
-			AKSCluster   string `yaml:"aksCluster,omitempty"`
-			Steps        []any  `yaml:"steps"`
+			Name         string           `yaml:"name"`
+			Subscription string           `yaml:"subscription"`
+			AKSCluster   string           `yaml:"aksCluster,omitempty"`
+			Steps        []map[string]any `yaml:"steps"`
 		} `yaml:"resourceGroups"`
 	}{}
 	err := yaml.Unmarshal(bytes, rawPipeline)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to unmarshal pipeline: %w", err)
 	}
+
+	// find step properties that are variableRefs
+	pipelineSchema, _, err := getSchemaForRef(rawPipeline.Schema)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get schema for pipeline: %w", err)
+	}
+	variableRefStepProperties, err := getVariableRefStepProperties(pipelineSchema)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get variableRef step properties: %w", err)
+	}
+
 	pipeline := &Pipeline{
+		schema:           rawPipeline.Schema,
 		pipelineFilePath: filepath,
 		ServiceGroup:     rawPipeline.ServiceGroup,
 		RolloutName:      rawPipeline.RolloutName,
@@ -54,6 +68,14 @@ func NewPlainPipelineFromBytes(filepath string, bytes []byte) (*Pipeline, error)
 		rg.AKSCluster = rawRg.AKSCluster
 		rg.Steps = make([]Step, len(rawRg.Steps))
 		for i, rawStep := range rawRg.Steps {
+			// preprocess variableRef step properties
+			for propName := range rawStep {
+				if _, ok := variableRefStepProperties[propName]; ok {
+					variableRef := rawStep[propName].(map[string]any)
+					variableRef["name"] = propName
+				}
+			}
+
 			// unmarshal the map into a StepMeta
 			stepMeta := &StepMeta{}
 			err := mapToStruct(rawStep, stepMeta)
@@ -75,6 +97,12 @@ func NewPlainPipelineFromBytes(filepath string, bytes []byte) (*Pipeline, error)
 		}
 	}
 
+	// another round of validation after postprocessing
+	err = ValidatePipelineSchemaForStruct(pipeline)
+	if err != nil {
+		return nil, fmt.Errorf("pipeline schema validation failed after postprocessing: %w", err)
+	}
+
 	return pipeline, nil
 }
 
@@ -218,13 +246,7 @@ type DryRun struct {
 }
 
 type Variable struct {
-	Name      string `yaml:"name"`
-	ConfigRef string `yaml:"configRef,omitempty"`
-	Value     string `yaml:"value,omitempty"`
-	Input     *Input `yaml:"input,omitempty"`
-}
-
-type VariableRef struct {
+	Name      string `yaml:"name,omitempty"`
 	ConfigRef string `yaml:"configRef,omitempty"`
 	Value     string `yaml:"value,omitempty"`
 	Input     *Input `yaml:"input,omitempty"`

tooling/templatize/pkg/pipeline/validation.go

@@ -4,6 +4,7 @@ import (
 	_ "embed"
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	"github.com/santhosh-tekuri/jsonschema/v6"
 	"gopkg.in/yaml.v3"
@@ -36,19 +37,42 @@ func ValidatePipelineSchema(pipelineContent []byte) error {
 	return nil
 }
 
+func ValidatePipelineSchemaForStruct(pipeline *Pipeline) error {
+	pipelineBytes, err := yaml.Marshal(pipeline)
+	if err != nil {
+		return fmt.Errorf("failed to marshal pipeline: %w", err)
+	}
+
+	return ValidatePipelineSchema(pipelineBytes)
+}
+
 func getSchemaForPipeline(pipelineMap map[string]interface{}) (pipelineSchema *jsonschema.Schema, schemaRef string, err error) {
-	schemaRef, ok := pipelineMap["$schema"].(string)
-	if !ok {
+	schemaRef, _ = pipelineMap["$schema"].(string)
+	return getSchemaForRef(schemaRef)
+}
+
+func getSchemaForRef(schemaRef string) (*jsonschema.Schema, string, error) {
+	if schemaRef == "" {
 		schemaRef = defaultSchemaRef
 	}
-
 	switch schemaRef {
 	case pipelineSchemaV1Ref:
-		pipelineSchema, err = compileSchema(schemaRef, pipelineSchemaV1Content)
+		pipelineSchema, err := compileSchema(schemaRef, pipelineSchemaV1Content)
+		return pipelineSchema, schemaRef, err
 	default:
 		return nil, "", fmt.Errorf("unsupported schema reference: %s", schemaRef)
 	}
-	return
+}
+
+func getVariableRefStepProperties(pipelineSchema *jsonschema.Schema) (map[string]*jsonschema.Schema, error) {
+	stepProperties := pipelineSchema.Properties["resourceGroups"].Items.(*jsonschema.Schema).Properties["steps"].Items.(*jsonschema.Schema).Properties
+	variableRefProperties := make(map[string]*jsonschema.Schema)
+	for propName, propValue := range stepProperties {
+		if propValue.Ref != nil && strings.HasSuffix(propValue.Ref.Location, "#/definitions/variableRef") {
+			variableRefProperties[propName] = propValue
+		}
+	}
+	return variableRefProperties, nil
 }
 
 func compileSchema(schemaRef string, schemaBytes []byte) (*jsonschema.Schema, error) {

tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2pipeline.yaml

@@ -29,53 +29,69 @@ resourceGroups:
             - deploy
           childZone:
             configRef: childZone
+            name: childZone
           parentZone:
             configRef: parentZone
+            name: parentZone
         - name: issuerTest
           action: SetCertificateIssuer
           dependsOn:
             - deploy
           issuer:
             configRef: provider
+            name: issuer
           vaultBaseUrl:
             configRef: vaultBaseUrl
+            name: vaultBaseUrl
         - name: issuerTestOutputChaining
           action: SetCertificateIssuer
           dependsOn:
             - deploy
           issuer:
+            name: issuer
             value: provider
           vaultBaseUrl:
             input:
                 name: kvUrl
                 step: deploy
+            name: vaultBaseUrl
         - name: cert
           action: CreateCertificate
           certificateName:
+            name: certificateName
             value: hcp-mdsd
           contentType:
+            name: contentType
             value: x-pem-file
           issuer:
+            name: issuer
             value: OneCertV2-PrivateCA
           san:
+            name: san
             value: hcp-mdsd.geneva.keyvault.aro-int.azure.com
           vaultBaseUrl:
+            name: vaultBaseUrl
             value: https://arohcp-svc-ln.vault.azure.net
         - name: rpRegistration
           action: ResourceProviderRegistration
           resourceProviderNamespaces:
+            name: resourceProviderNamespaces
             value:
                 - Microsoft.Storage
                 - Microsoft.EventHub
                 - Microsoft.Insights
         - name: clusterAccount
           action: LogsAccount
           certdescription:
+            name: certdescription
             value: HCP Management Cluster
           certsan:
+            name: certsan
             value: MGMT.GENEVA.KEYVAULT.ARO-HCP-INT.AZURE.COM
           namespace:
+            name: namespace
             value: HCPManagementLogs
           subscriptionId:
+            name: subscriptionId
             value:
                 - abc