diff --git a/install.sh b/install.sh index c37be2db..f7dfa036 100755 --- a/install.sh +++ b/install.sh @@ -5,6 +5,9 @@ function main() { cwd="${1}" pushd "${cwd}" > /dev/null + export CGO_ENABLED + CGO_ENABLED=0 + go install \ -ldflags "-X main.version=$(git rev-parse HEAD)" \ -gcflags=-trimpath="${cwd}" \ diff --git a/internal/acceptance/workflows/baking_a_tile.feature b/internal/acceptance/workflows/baking_a_tile.feature index 8dc06936..834d0d6d 100644 --- a/internal/acceptance/workflows/baking_a_tile.feature +++ b/internal/acceptance/workflows/baking_a_tile.feature @@ -15,3 +15,10 @@ Feature: As a developer, I want to bake a tile And "bake_records/0.2.0-dev.json" contains substring: "version": "0.2.0-dev" And "bake_records/0.2.0-dev.json" contains substring: "source_revision": And "bake_records/0.2.0-dev.json" contains substring: "kiln_version": "0.0.0+acceptance-tests" + + Scenario: it reads directory configuration from Kilnfile + Given I have a tile source directory "testdata/tiles/non-standard-paths" + When I invoke kiln + | bake | + | --stub-releases | + Then a Tile is created diff --git a/internal/acceptance/workflows/testdata/tiles/non-standard-paths/Kilnfile b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/Kilnfile new file mode 100644 index 00000000..77b8188c --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/Kilnfile @@ -0,0 +1,10 @@ +bake_configurations: + - tile_name: hello + metadata_filepath: product_template.yml + variables_filepaths: + - text.yml + icon_filepath: "gopher.png" + instance_groups_directories: + - job_types + properties_directories: + - configuration \ No newline at end of file diff --git a/internal/acceptance/workflows/testdata/tiles/non-standard-paths/Kilnfile.lock b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/Kilnfile.lock new file mode 100644 index 00000000..c015566a --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/Kilnfile.lock @@ -0,0 +1,3 @@ +stemcell_criteria: + os: ubuntu-jammy + version: "1.329" \ No newline at end of file diff --git a/internal/acceptance/workflows/testdata/tiles/non-standard-paths/configuration/networking.yml b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/configuration/networking.yml new file mode 100644 index 00000000..eccbf65a --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/configuration/networking.yml @@ -0,0 +1,5 @@ +--- +- name: port + type: port + configurable: true + default: 8080 \ No newline at end of file diff --git a/internal/acceptance/workflows/testdata/tiles/non-standard-paths/gopher.png b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/gopher.png new file mode 100644 index 00000000..3d878e52 Binary files /dev/null and b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/gopher.png differ diff --git a/internal/acceptance/workflows/testdata/tiles/non-standard-paths/job_types/hello.yml b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/job_types/hello.yml new file mode 100644 index 00000000..2dc8e682 --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/job_types/hello.yml @@ -0,0 +1,55 @@ +name: hello-server +label: Server +resource_label: Server +description: HTTP Server + +templates: [] + +static_ip: 1 +dynamic_ip: 0 + +max_in_flight: 1 +single_az_only: true + +instance_definition: + name: instances + type: integer + label: Instances + configurable: true + default: 1 + constraints: + min: 0 + max: 1 + +resource_definitions: + - name: ram + type: integer + label: RAM + configurable: true + default: 1024 + constraints: + min: 1024 + + - name: ephemeral_disk + type: integer + label: Ephemeral Disk + configurable: true + default: 4000 + constraints: + min: 2000 + + - name: persistent_disk + type: integer + label: Persistent Disk + configurable: false + default: 4000 + constraints: + min: 2000 + + - name: cpu + type: integer + label: CPU + configurable: true + default: 1 + constraints: + min: 1 diff --git a/internal/acceptance/workflows/testdata/tiles/non-standard-paths/product_template.yml b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/product_template.yml new file mode 100644 index 00000000..78b616b8 --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/product_template.yml @@ -0,0 +1,29 @@ +--- +name: hello +label: "some label" +description: "some description" +icon_image: $( icon ) + +metadata_version: "2.7.0" +minimum_version_for_upgrade: 0.1.0 +product_version: $( version ) +provides_product_versions: + - name: hello + version: $( version ) + +rank: 90 +serial: false + +releases: [] + +stemcell_criteria: $( stemcell ) + +job_types: + - $( instance_group "hello-server" ) + +runtime_configs: [] + +property_blueprints: + - $( property "port" ) + +form_types: [] diff --git a/internal/acceptance/workflows/testdata/tiles/non-standard-paths/text.yml b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/text.yml new file mode 100644 index 00000000..2339a1c0 --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/text.yml @@ -0,0 +1,2 @@ +label: "some label" +description: "some description" \ No newline at end of file diff --git a/internal/acceptance/workflows/testdata/tiles/non-standard-paths/version b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/version new file mode 100644 index 00000000..e2cac26c --- /dev/null +++ b/internal/acceptance/workflows/testdata/tiles/non-standard-paths/version @@ -0,0 +1 @@ +1.2.3 \ No newline at end of file diff --git a/internal/commands/bake.go b/internal/commands/bake.go index 0c6a881b..eb69a02b 100644 --- a/internal/commands/bake.go +++ b/internal/commands/bake.go @@ -1,6 +1,7 @@ package commands import ( + "bytes" "errors" "fmt" "io" @@ -8,6 +9,7 @@ import ( "os" "path" "path/filepath" + "slices" "strings" "github.com/go-git/go-billy/v5" @@ -17,6 +19,7 @@ import ( "github.com/pivotal-cf/kiln/internal/builder" "github.com/pivotal-cf/kiln/internal/commands/flags" "github.com/pivotal-cf/kiln/internal/helper" + "github.com/pivotal-cf/kiln/pkg/cargo" "github.com/pivotal-cf/kiln/pkg/source" ) @@ -164,32 +167,34 @@ type Bake struct { metadata metadataService fetcher jhanda.Command - Options struct { - flags.Standard - flags.FetchBakeOptions - - IsFinal bool `long:"final" description:"this flag causes build metadata to be written to bake_records"` - - Metadata string `short:"m" long:"metadata" default:"base.yml" description:"path to the metadata file"` - ReleaseDirectories []string `short:"rd" long:"releases-directory" default:"releases" description:"path to a directory containing release tarballs"` - FormDirectories []string `short:"f" long:"forms-directory" default:"forms" description:"path to a directory containing forms"` - IconPath string `short:"i" long:"icon" default:"icon.png" description:"path to icon file"` - InstanceGroupDirectories []string `short:"ig" long:"instance-groups-directory" default:"instance_groups" description:"path to a directory containing instance groups"` - JobDirectories []string `short:"j" long:"jobs-directory" default:"jobs" description:"path to a directory containing jobs"` - MigrationDirectories []string `short:"md" long:"migrations-directory" default:"migrations" description:"path to a directory containing migrations"` - PropertyDirectories []string `short:"pd" long:"properties-directory" default:"properties" description:"path to a directory containing property blueprints"` - RuntimeConfigDirectories []string `short:"rcd" long:"runtime-configs-directory" default:"runtime_configs" description:"path to a directory containing runtime configs"` - BOSHVariableDirectories []string `short:"vd" long:"bosh-variables-directory" default:"bosh_variables" description:"path to a directory containing BOSH variables"` - StemcellTarball string `short:"st" long:"stemcell-tarball" description:"deprecated -- path to a stemcell tarball (NOTE: mutually exclusive with --kilnfile)"` - StemcellsDirectories []string `short:"sd" long:"stemcells-directory" description:"path to a directory containing stemcells (NOTE: mutually exclusive with --kilnfile or --stemcell-tarball)"` - EmbedPaths []string `short:"e" long:"embed" description:"path to files to include in the tile /embed directory"` - OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output"` - MetadataOnly bool `short:"mo" long:"metadata-only" description:"don't build a tile, output the metadata to stdout"` - Sha256 bool ` long:"sha256" description:"calculates a SHA256 checksum of the output file"` - StubReleases bool `short:"sr" long:"stub-releases" description:"skips importing release tarballs into the tile"` - Version string `short:"v" long:"version" description:"version of the tile"` - SkipFetchReleases bool `short:"sfr" long:"skip-fetch" description:"skips the automatic release fetch for all release directories" alias:"skip-fetch-directories"` - } + Options BakeOptions +} + +type BakeOptions struct { + flags.Standard + flags.FetchBakeOptions + + Metadata string `short:"m" long:"metadata" default:"base.yml" description:"path to the metadata file"` + ReleaseDirectories []string `short:"rd" long:"releases-directory" default:"releases" description:"path to a directory containing release tarballs"` + FormDirectories []string `short:"f" long:"forms-directory" default:"forms" description:"path to a directory containing forms"` + IconPath string `short:"i" long:"icon" default:"icon.png" description:"path to icon file"` + InstanceGroupDirectories []string `short:"ig" long:"instance-groups-directory" default:"instance_groups" description:"path to a directory containing instance groups"` + JobDirectories []string `short:"j" long:"jobs-directory" default:"jobs" description:"path to a directory containing jobs"` + MigrationDirectories []string `short:"md" long:"migrations-directory" default:"migrations" description:"path to a directory containing migrations"` + PropertyDirectories []string `short:"pd" long:"properties-directory" default:"properties" description:"path to a directory containing property blueprints"` + RuntimeConfigDirectories []string `short:"rcd" long:"runtime-configs-directory" default:"runtime_configs" description:"path to a directory containing runtime configs"` + BOSHVariableDirectories []string `short:"vd" long:"bosh-variables-directory" default:"bosh_variables" description:"path to a directory containing BOSH variables"` + StemcellTarball string `short:"st" long:"stemcell-tarball" description:"deprecated -- path to a stemcell tarball (NOTE: mutually exclusive with --kilnfile)"` + StemcellsDirectories []string `short:"sd" long:"stemcells-directory" description:"path to a directory containing stemcells (NOTE: mutually exclusive with --kilnfile or --stemcell-tarball)"` + EmbedPaths []string `short:"e" long:"embed" description:"path to files to include in the tile /embed directory"` + OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output"` + MetadataOnly bool `short:"mo" long:"metadata-only" description:"don't build a tile, output the metadata to stdout"` + Sha256 bool ` long:"sha256" description:"calculates a SHA256 checksum of the output file"` + StubReleases bool `short:"sr" long:"stub-releases" description:"skips importing release tarballs into the tile"` + Version string `short:"v" long:"version" description:"version of the tile"` + SkipFetchReleases bool `short:"sfr" long:"skip-fetch" description:"skips the automatic release fetch for all release directories" alias:"skip-fetch-directories"` + + IsFinal bool `long:"final" description:"this flag causes build metadata to be written to bake_records"` } func NewBakeWithInterfaces(interpolator interpolator, tileWriter tileWriter, outLogger *log.Logger, errLogger *log.Logger, templateVariablesService templateVariablesService, boshVariablesService metadataTemplatesParser, releasesService fromDirectories, stemcellService stemcellService, formsService metadataTemplatesParser, instanceGroupsService metadataTemplatesParser, jobsService metadataTemplatesParser, propertiesService metadataTemplatesParser, runtimeConfigsService metadataTemplatesParser, iconService iconService, metadataService metadataService, checksummer checksummer, fetcher jhanda.Command, fs FileSystem, homeDir flags.HomeDirFunc, writeBakeRecordFn writeBakeRecordSignature) Bake { @@ -392,30 +397,6 @@ func (b Bake) Execute(args []string) error { } } - if b.Options.Metadata == "" { - return errors.New("missing required flag \"--metadata\"") - } - - if len(b.Options.InstanceGroupDirectories) == 0 && len(b.Options.JobDirectories) > 0 { - return errors.New("--jobs-directory flag requires --instance-groups-directory to also be specified") - } - - if b.Options.Kilnfile != "" && b.Options.StemcellTarball != "" { - return errors.New("--kilnfile cannot be provided when using --stemcell-tarball") - } - - if b.Options.Kilnfile != "" && len(b.Options.StemcellsDirectories) > 0 { - return errors.New("--kilnfile cannot be provided when using --stemcells-directory") - } - - if b.Options.StemcellTarball != "" && len(b.Options.StemcellsDirectories) > 0 { - return errors.New("--stemcell-tarball cannot be provided when using --stemcells-directory") - } - - if b.Options.OutputFile != "" && b.Options.MetadataOnly { - return errors.New("--output-file cannot be provided when using --metadata-only") - } - // TODO: Remove check after deprecation of --stemcell-tarball if b.Options.StemcellTarball != "" { b.errLogger.Println("warning: --stemcell-tarball is being deprecated in favor of --stemcells-directory") @@ -437,6 +418,14 @@ func (b Bake) Execute(args []string) error { // TODO remove when stemcell tarball is deprecated stemcellManifest, err = b.stemcell.FromTarball(b.Options.StemcellTarball) } else if b.Options.Kilnfile != "" { + if err := bakeArgumentsFromKilnfileConfiguration(&b.Options, templateVariables); err != nil { + return fmt.Errorf("failed to parse releases: %s", err) + } + templateVariables, err = b.templateVariables.FromPathsAndPairs(b.Options.VariableFiles, b.Options.Variables) + if err != nil { + return fmt.Errorf("failed to parse template variables: %s", err) + } + stemcellManifests, err = b.stemcell.FromKilnfile(b.Options.Kilnfile) } else if len(b.Options.StemcellsDirectories) > 0 { stemcellManifests, err = b.stemcell.FromDirectories(b.Options.StemcellsDirectories) @@ -445,6 +434,30 @@ func (b Bake) Execute(args []string) error { return fmt.Errorf("failed to parse stemcell: %s", err) } + if b.Options.Metadata == "" { + return errors.New("missing required flag \"--metadata\"") + } + + if len(b.Options.InstanceGroupDirectories) == 0 && len(b.Options.JobDirectories) > 0 { + return errors.New("--jobs-directory flag requires --instance-groups-directory to also be specified") + } + + if b.Options.Kilnfile != "" && b.Options.StemcellTarball != "" { + return errors.New("--kilnfile cannot be provided when using --stemcell-tarball") + } + + if b.Options.Kilnfile != "" && len(b.Options.StemcellsDirectories) > 0 { + return errors.New("--kilnfile cannot be provided when using --stemcells-directory") + } + + if b.Options.StemcellTarball != "" && len(b.Options.StemcellsDirectories) > 0 { + return errors.New("--stemcell-tarball cannot be provided when using --stemcells-directory") + } + + if b.Options.OutputFile != "" && b.Options.MetadataOnly { + return errors.New("--output-file cannot be provided when using --metadata-only") + } + boshVariables, err := b.boshVariables.ParseMetadataTemplates(b.Options.BOSHVariableDirectories, templateVariables) if err != nil { return fmt.Errorf("failed to parse bosh variables: %s", err) @@ -551,3 +564,84 @@ func (b Bake) Usage() jhanda.Usage { Flags: b.Options, } } + +func bakeArgumentsFromKilnfileConfiguration(options *BakeOptions, variables map[string]any) error { + if options.Kilnfile == "" { + return nil + } + if variables == nil { + variables = make(map[string]any) + } + buf, err := os.ReadFile(options.Kilnfile) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + kf, err := cargo.InterpolateAndParseKilnfile(bytes.NewReader(buf), variables) + if err != nil { + return err + } + if tileName, ok := variables[builder.TileNameVariable]; ok { + name, ok := tileName.(string) + if ok { + return fmt.Errorf("%s value must be a string got value %#[2]v with type %[2]T", builder.TileNameVariable, tileName) + } + if index := slices.IndexFunc(kf.BakeConfigurations, func(configuration cargo.BakeConfiguration) bool { + return configuration.TileName == name + }); index >= 0 { + fromConfiguration(options, kf.BakeConfigurations[index]) + } + } else if len(kf.BakeConfigurations) == 1 { + configuration := kf.BakeConfigurations[0] + fromConfiguration(options, configuration) + if configuration.TileName != "" { + variables[builder.TileNameVariable] = configuration.TileName + } + } + return nil +} + +func fromConfiguration(b *BakeOptions, configuration cargo.BakeConfiguration) { + if len(configuration.Metadata) > 0 { + b.Metadata = configuration.Metadata + } + if len(configuration.FormDirectories) > 0 { + b.FormDirectories = configuration.FormDirectories + } + if len(configuration.IconPath) > 0 { + b.IconPath = configuration.IconPath + } + if len(configuration.InstanceGroupDirectories) > 0 { + b.InstanceGroupDirectories = configuration.InstanceGroupDirectories + } + if len(configuration.JobDirectories) > 0 { + b.JobDirectories = configuration.JobDirectories + } + if len(configuration.MigrationDirectories) > 0 { + b.MigrationDirectories = configuration.MigrationDirectories + } + if len(configuration.PropertyDirectories) > 0 { + b.PropertyDirectories = configuration.PropertyDirectories + } + if len(configuration.RuntimeConfigDirectories) > 0 { + b.RuntimeConfigDirectories = configuration.RuntimeConfigDirectories + } + if len(configuration.BOSHVariableDirectories) > 0 { + b.BOSHVariableDirectories = configuration.BOSHVariableDirectories + } + if len(configuration.EmbedPaths) > 0 { + b.EmbedPaths = configuration.EmbedPaths + } + if len(configuration.VariableFiles) > 0 { + // simplify when go1.22 comes out https://pkg.go.dev/slices@master#Concat + variableFiles := make([]string, 0, len(configuration.VariableFiles)+len(b.VariableFiles)) + variableFiles = append(variableFiles, configuration.VariableFiles...) + variableFiles = append(variableFiles, b.VariableFiles...) + + slices.Sort(variableFiles) + variableFiles = slices.Compact(variableFiles) + b.VariableFiles = variableFiles + } +} diff --git a/internal/test/integration_test.go b/internal/test/integration_test.go index 1b9f36e2..a072654f 100644 --- a/internal/test/integration_test.go +++ b/internal/test/integration_test.go @@ -71,6 +71,9 @@ func TestDockerIntegration(t *testing.T) { func checkDaemonVersion(t *testing.T) { t.Helper() + if testing.Short() { + t.Skip() + } constraints, err := semver.NewConstraint(test.MinimumDockerServerVersion) require.NoError(t, err) diff --git a/pkg/cargo/kilnfile.go b/pkg/cargo/kilnfile.go index 0c6fe852..6594797c 100644 --- a/pkg/cargo/kilnfile.go +++ b/pkg/cargo/kilnfile.go @@ -17,12 +17,13 @@ const ( ) type Kilnfile struct { - ReleaseSources []ReleaseSourceConfig `yaml:"release_sources,omitempty"` - Slug string `yaml:"slug,omitempty"` - PreGaUserGroups []string `yaml:"pre_ga_user_groups,omitempty"` - Releases []BOSHReleaseTarballSpecification `yaml:"releases,omitempty"` - TileNames []string `yaml:"tile_names,omitempty"` - Stemcell Stemcell `yaml:"stemcell_criteria,omitempty"` + ReleaseSources []ReleaseSourceConfig `yaml:"release_sources,omitempty"` + Slug string `yaml:"slug,omitempty"` + PreGaUserGroups []string `yaml:"pre_ga_user_groups,omitempty"` + Releases []BOSHReleaseTarballSpecification `yaml:"releases,omitempty"` + TileNames []string `yaml:"tile_names,omitempty"` + Stemcell Stemcell `yaml:"stemcell_criteria,omitempty"` + BakeConfigurations []BakeConfiguration `yaml:"bake_configurations"` } func (kf *Kilnfile) BOSHReleaseTarballSpecification(name string) (BOSHReleaseTarballSpecification, error) { @@ -355,3 +356,18 @@ func deGlazeBOSHReleaseTarballSpecification(spec BOSHReleaseTarballSpecification spec.Version, err = spec.DeGlazeBehavior.createConstraint(lock.Version) return spec, err } + +type BakeConfiguration struct { + TileName string `yaml:"tile_name,omitempty" json:"tile_name,omitempty"` + Metadata string `yaml:"metadata_filepath,omitempty" json:"metadata_filepath,omitempty"` + FormDirectories []string `yaml:"forms_directories,omitempty" json:"forms_directories,omitempty"` + IconPath string `yaml:"icon_filepath,omitempty" json:"icon_filepath,omitempty"` + InstanceGroupDirectories []string `yaml:"instance_groups_directories,omitempty" json:"instance_groups_directories,omitempty"` + JobDirectories []string `yaml:"jobs_directories,omitempty" json:"jobs_directories,omitempty"` + MigrationDirectories []string `yaml:"migrations_directories,omitempty" json:"migrations_directories,omitempty"` + PropertyDirectories []string `yaml:"properties_directories,omitempty" json:"properties_directories,omitempty"` + RuntimeConfigDirectories []string `yaml:"runtime_configurations_directories,omitempty" json:"runtime_configurations_directories,omitempty"` + BOSHVariableDirectories []string `yaml:"bosh_variables_directories,omitempty" json:"bosh_variables_directories,omitempty"` + EmbedPaths []string `yaml:"embed_paths_directories,omitempty" json:"embed_paths_directories,omitempty"` + VariableFiles []string `yaml:"variable_files,omitempty" json:"variable_files,omitempty"` +} diff --git a/test.sh b/test.sh index 36746f45..3c5e7b56 100755 --- a/test.sh +++ b/test.sh @@ -7,6 +7,9 @@ function main() { cwd="${1}" pushd "${cwd}" > /dev/null + export CGO_ENABLED + CGO_ENABLED=0 + go test -cover -short ./... go vet ./...