diff --git a/README.md b/README.md index 2d59ee8..8887165 100644 --- a/README.md +++ b/README.md @@ -126,12 +126,14 @@ bundle exec kdt generate # Additional options bundle exec kdt generate -m -i -o + e.g: + bundle exec kdt generate -m spec/resources/deploy.yaml -# Can be tested only via running Unit tests -bundle exec kdt publish +# Requires mock aritifactory to run. Test changes using `rake test` cmd instead +bundle exec kdt publish -m spec/resources/deploy.yaml # Publish with extra argument capable of parallel runs in Jenkins - bundle exec kdt publish -o + bundle exec kdt publish -m spec/resources/deploy.yaml -o -e -a ``` ## Parallel Generate/Publish in Jenkins Pipeline @@ -144,8 +146,8 @@ could be overwritten by another parallel step. Following steps can be added in the jenkins pipeline for parallel generation/ publishing the manifest files. ```bash String uniqOutputPath = "build/kubernetes/${BRANCH_NAME}/build/${BUILD_ID}/${env}/${app}/" -bundle exec kdt generate -i kubernetes/${env}/${app} -o ${uniqOutputPath} -bundle exec kdt publish -o ${uniqOutputPath} +bundle exec kdt generate -i kubernetes/${env}/${app} -o ${uniqOutputPath} +bundle exec kdt publish -o ${uniqOutputPath} -e ${env} -a ${app} ``` ## FAQ diff --git a/bin/publish b/bin/publish index e0fa946..036c394 100755 --- a/bin/publish +++ b/bin/publish @@ -21,8 +21,19 @@ KubeDeployTools::Shellrunner.shellrunner = KubeDeployTools::Shellrunner.new config = KubeDeployTools::DeployConfigFile.new(options.manifest_file) artifact_registry = config.artifact_registries[config.artifact_registry] -KubeDeployTools::Publish.new( - manifest: options.manifest_file, - artifact_registry: artifact_registry, - output_dir: options.output_path, -).publish + +if options.env && options.app + KubeDeployTools::Publish.new( + manifest: options.manifest_file, + artifact_registry: artifact_registry, + output_dir: options.output_path, + ).publish_with_env_app(options.env, options.app) + +else + KubeDeployTools::Publish.new( + manifest: options.manifest_file, + artifact_registry: artifact_registry, + output_dir: options.output_path, + ).publish + +end \ No newline at end of file diff --git a/lib/kube_deploy_tools/artifact_registry/driver_artifactory.rb b/lib/kube_deploy_tools/artifact_registry/driver_artifactory.rb index 7ba2b00..bfb1458 100644 --- a/lib/kube_deploy_tools/artifact_registry/driver_artifactory.rb +++ b/lib/kube_deploy_tools/artifact_registry/driver_artifactory.rb @@ -43,6 +43,11 @@ def get_registry_artifact_path(name:, flavor:, project:, build_number:) "#{project}/#{build_number}/#{get_artifact_name(name: name, flavor: flavor)}" end + def get_registry_artifact_path_env_app(name:, flavor:, project:, build_number:, env:, app:) + "#{project}/#{build_number}/#{env}/#{app}/#{get_artifact_name(name: name, flavor: flavor)}" + end + + def upload(local_dir:, name:, flavor:, project:, build_number:) # Pack up contents of each flavor_dir to a correctly named artifact. flavor_dir = File.join(local_dir, "#{name}_#{flavor}") @@ -67,6 +72,39 @@ def upload(local_dir:, name:, flavor:, project:, build_number:) build_number: build_number, ) artifactory_url = "#{Artifactory.endpoint}/#{@repo}/#{registry_artifact_path}" + + Logger.info("Uploading #{local_artifact_path} to #{artifactory_url}") + artifact = Artifactory::Resource::Artifact.new(local_path: local_artifact_path) + artifact.upload(@repo, registry_artifact_path) + end + + def upload_with_env_app(local_dir:, name:, flavor:, project:, build_number:, env:, app:) + # Pack up contents of each flavor_dir to a correctly named artifact. + flavor_dir = File.join(local_dir, "#{name}_#{flavor}") + + package( + name: name, + flavor: flavor, + input_dir: flavor_dir, + output_dir: local_dir, + ) + + local_artifact_path = get_local_artifact_path( + local_dir: local_dir, + name: name, + flavor: flavor, + ) + + registry_artifact_path = get_registry_artifact_path_env_app( + project: project, + name: name, + flavor: flavor, + build_number: build_number, + env: env, + app: app, + ) + artifactory_url = "#{Artifactory.endpoint}/#{@repo}/#{registry_artifact_path}" + Logger.info("Uploading #{local_artifact_path} to #{artifactory_url}") artifact = Artifactory::Resource::Artifact.new(local_path: local_artifact_path) artifact.upload(@repo, registry_artifact_path) diff --git a/lib/kube_deploy_tools/artifact_registry/driver_base.rb b/lib/kube_deploy_tools/artifact_registry/driver_base.rb index 362cf56..1ee185b 100644 --- a/lib/kube_deploy_tools/artifact_registry/driver_base.rb +++ b/lib/kube_deploy_tools/artifact_registry/driver_base.rb @@ -18,6 +18,11 @@ def upload(local_dir:, name:, flavor:, project:, build_number:) raise "#{self.class}#publish not implemented" end + # Same as above but with 2 more parameters: env, app + def upload_with_env_app(local_dir:, name:, flavor:, project:, build_number:, env:, app:) + raise "#{self.class}#publish not implemented" + end + # download should retrieve the artifact namespaced with the given # project and build number and identified by the name and flavor. # The artifact should be put into the output directory. diff --git a/lib/kube_deploy_tools/artifact_registry/driver_gcs.rb b/lib/kube_deploy_tools/artifact_registry/driver_gcs.rb index 38659fc..27d835f 100644 --- a/lib/kube_deploy_tools/artifact_registry/driver_gcs.rb +++ b/lib/kube_deploy_tools/artifact_registry/driver_gcs.rb @@ -46,6 +46,10 @@ def get_registry_artifact_path(name:, flavor:, project:, build_number:) "#{get_registry_build_path(project: project)}/#{build_number}/artifact/#{get_artifact_name(name: name, flavor: flavor)}" end + def get_registry_artifact_path_env_app(name:, flavor:, project:, build_number:, env:, app:) + "#{get_registry_build_path(project: project)}/#{build_number}/artifact/#{env}/#{app}/#{get_artifact_name(name: name, flavor: flavor)}" + end + def get_artifact_name(name:, flavor:) "manifests_#{name}_#{flavor}.yaml" end @@ -137,5 +141,40 @@ def upload(local_dir:, name:, flavor:, project:, build_number:) registry_artifact_path end + + def upload_with_env_app(local_dir:, name:, flavor:, project:, build_number:, env:, app:) + # Pack up contents of each flavor_dir to a correctly named artifact. + flavor_dir = File.join(local_dir, "#{name}_#{flavor}") + + package( + name: name, + flavor: flavor, + input_dir: flavor_dir, + output_dir: local_dir, + ) + + local_artifact_path = get_local_artifact_path( + local_dir: local_dir, + name: name, + flavor: flavor, + ) + + registry_artifact_path = get_registry_artifact_path_env_app( + project: project, + name: name, + flavor: flavor, + build_number: build_number, + env: env, + app: app, + ) + + Logger.info("Uploading #{local_artifact_path} to #{registry_artifact_path}") + out, err, status = Shellrunner.run_call('gsutil', '-m', 'cp', local_artifact_path, registry_artifact_path) + if !status.success? + raise "Failed to upload remote deploy artifact from #{local_artifact_path} to #{registry_artifact_path}" + end + + registry_artifact_path + end end end diff --git a/lib/kube_deploy_tools/publish.rb b/lib/kube_deploy_tools/publish.rb index d8cf6b8..3310446 100644 --- a/lib/kube_deploy_tools/publish.rb +++ b/lib/kube_deploy_tools/publish.rb @@ -36,5 +36,28 @@ def publish() end end end + + + def publish_with_env_app(env, app) + + @config.artifacts.each do |c| + name = c.fetch('name') + + # Allow deploy.yaml to gate certain flavors to certain targets. + cluster_flavors = @config.flavors.select { |key, value| c['flavors'].nil? || c['flavors'].include?(key) } + + cluster_flavors.each do |flavor, _| + @artifact_registry.upload_with_env_app( + local_dir: @output_dir, + name: name, + flavor: flavor, + project: @project, + build_number: @build_number, + env: env, + app: app, + ) + end + end + end end end diff --git a/lib/kube_deploy_tools/publish/options.rb b/lib/kube_deploy_tools/publish/options.rb index df9caf3..e5e332b 100644 --- a/lib/kube_deploy_tools/publish/options.rb +++ b/lib/kube_deploy_tools/publish/options.rb @@ -3,7 +3,15 @@ module KubeDeployTools class Publish::Optparser class Options - attr_accessor :manifest_file, :output_path + attr_accessor :manifest_file, :output_path, :env, :app + + def env + @env || nil + end + + def app + @app || nil + end def initialize self.output_path = File.join('build', 'kubernetes') @@ -18,6 +26,14 @@ def define_options(parser) self.output_path = p end + parser.on('-e', '--env-name NAME', 'Env name') do |p| + self.env = p + end + + parser.on('-a', '--app-name NAME', 'App name') do |p| + self.app = p + end + parser.on('-') end end diff --git a/spec/unit/publish_spec.rb b/spec/unit/publish_spec.rb index 7b947df..f42cb58 100644 --- a/spec/unit/publish_spec.rb +++ b/spec/unit/publish_spec.rb @@ -76,6 +76,48 @@ all_uploads = expected_uploads expect(uploads).to contain_exactly(*all_uploads) end + + it 'publishes artifacts according to deploy.yaml and given env & app name' do + KubeDeployTools::Logger.logger = logger + + # Mock artifact upload + uploads = Set.new + allow_any_instance_of(Artifactory::Resource::Artifact).to receive(:upload) do |artifact, repo, path| + # Expect to upload to kubernetes-snapshots-local/ + expect(path).to start_with(PROJECT) + + # add only the basenames of the files to the set as the BUILD_ID + # will vary on each build + uploads.add(File.basename(path)) + end + + expected_uploads = [ + 'manifests_colo-service-prod_default.tar.gz', + 'manifests_colo-service-staging_default.tar.gz', + 'manifests_local_default.tar.gz', + 'manifests_us-east-1-prod_default.tar.gz', + 'manifests_us-east-1-staging_default.tar.gz', + 'manifests_ingestion-prod_default.tar.gz', + 'manifests_pippio-production_default.tar.gz', + 'manifests_platforms-prod_default.tar.gz', + 'manifests_filtered-artifact_default.tar.gz', + ] + + Dir.mktmpdir do |dir| + expected_uploads.each do |f| + FileUtils.touch File.join(dir, f) + end + + KubeDeployTools::Publish.new( + manifest: MANIFEST_FILE, + artifact_registry: artifact_registry, + output_dir: dir, + ).publish_with_env_app("env", "app") + end + + all_uploads = expected_uploads + expect(uploads).to contain_exactly(*all_uploads) + end end context 'GCS driver' do @@ -105,6 +147,31 @@ expect(local_artifact_resources.length).to eq(2) end end + + it 'publishes artifacts according to deploy.yaml and given env & app name' do + KubeDeployTools::Logger.logger = logger + + Dir.tmpdir do |dir| + FileUtils.cp_r INPUT_DIR, dir, :verbose => true + KubeDeployTools::Publish.new( + manifest: MANIFEST_GCS_FILE, + artifact_registry: artifact_registry, + output_dir: dir, + ).publish_with_env_app("env", "app") + + artifacts = Find.find(dir). + select { |path| path =~ /.*manifests_.*\.yaml$/ } + + expect(artifacts.length).to eq(1) + + # Check that the local artifact contains 2 concatenated resources + local_artifact = artifacts.select { |path| path =~ /local/ }.first + local_artifact_contents = YAML.load_file(local_artifact) + local_artifact_resources = [] + YAML.load_stream(File.read local_artifact) { |doc| local_artifact_resources << doc } + expect(local_artifact_resources.length).to eq(2) + end + end end end