[ DEVOI-638 ] Changes for publish step to add env and app in upload path

README.md

@@ -126,12 +126,14 @@ bundle exec kdt generate
 
     # Additional options 
     bundle exec kdt generate -m <deploy_yaml_manifest_file> -i <tmp_input_directory> -o <tmp_directory> 
+    e.g:
+    bundle exec kdt generate -m spec/resources/deploy.yaml 
 
-# Can be tested only via running Unit tests
-bundle exec kdt publish
+# Requires mock aritifactory to run. Test changes using `rake test` cmd instead
+bundle exec kdt publish -m spec/resources/deploy.yaml
 
     # Publish with extra argument capable of parallel runs in Jenkins
-    bundle exec kdt publish -o <tmp_directory>
+    bundle exec kdt publish -m spec/resources/deploy.yaml -o <tmp_directory> -e <env> -a <app>
 ```
 
 ## Parallel Generate/Publish in Jenkins Pipeline
@@ -144,8 +146,8 @@ could be overwritten by another parallel step.
 Following steps can be added in the jenkins pipeline for parallel generation/ publishing the manifest files.
 ```bash
 String uniqOutputPath = "build/kubernetes/${BRANCH_NAME}/build/${BUILD_ID}/${env}/${app}/"
-bundle exec kdt generate -i kubernetes/${env}/${app} -o ${uniqOutputPath}
-bundle exec kdt publish -o ${uniqOutputPath}
+bundle exec kdt generate -i kubernetes/${env}/${app} -o ${uniqOutputPath} 
+bundle exec kdt publish -o ${uniqOutputPath} -e ${env} -a ${app}
 ```
 
 ## FAQ

bin/publish

@@ -21,8 +21,19 @@ KubeDeployTools::Shellrunner.shellrunner = KubeDeployTools::Shellrunner.new
 config = KubeDeployTools::DeployConfigFile.new(options.manifest_file)
 artifact_registry = config.artifact_registries[config.artifact_registry]
 
-KubeDeployTools::Publish.new(
-  manifest: options.manifest_file,
-  artifact_registry: artifact_registry,
-  output_dir: options.output_path,
-).publish
+
+if options.env && options.app
+  KubeDeployTools::Publish.new(
+    manifest: options.manifest_file,
+    artifact_registry: artifact_registry,
+    output_dir: options.output_path,
+  ).publish_with_env_app(options.env, options.app)
+
+else
+  KubeDeployTools::Publish.new(
+    manifest: options.manifest_file,
+    artifact_registry: artifact_registry,
+    output_dir: options.output_path,
+  ).publish
+
+end

lib/kube_deploy_tools/artifact_registry/driver_artifactory.rb

@@ -43,6 +43,11 @@ def get_registry_artifact_path(name:, flavor:, project:, build_number:)
       "#{project}/#{build_number}/#{get_artifact_name(name: name, flavor: flavor)}"
     end
 
+    def get_registry_artifact_path_env_app(name:, flavor:, project:, build_number:, env:, app:)
+      "#{project}/#{build_number}/#{env}/#{app}/#{get_artifact_name(name: name, flavor: flavor)}"
+    end
+
+
     def upload(local_dir:, name:, flavor:, project:, build_number:)
       # Pack up contents of each flavor_dir to a correctly named artifact.
       flavor_dir = File.join(local_dir, "#{name}_#{flavor}")
@@ -67,6 +72,39 @@ def upload(local_dir:, name:, flavor:, project:, build_number:)
         build_number: build_number,
       )
       artifactory_url = "#{Artifactory.endpoint}/#{@repo}/#{registry_artifact_path}"
+
+      Logger.info("Uploading #{local_artifact_path} to #{artifactory_url}")
+      artifact = Artifactory::Resource::Artifact.new(local_path: local_artifact_path)
+      artifact.upload(@repo, registry_artifact_path)
+    end
+
+    def upload_with_env_app(local_dir:, name:, flavor:, project:, build_number:, env:, app:)
+      # Pack up contents of each flavor_dir to a correctly named artifact.
+      flavor_dir = File.join(local_dir, "#{name}_#{flavor}")
+
+      package(
+        name: name,
+        flavor: flavor,
+        input_dir: flavor_dir,
+        output_dir: local_dir,
+      )
+
+      local_artifact_path = get_local_artifact_path(
+        local_dir: local_dir,
+        name: name,
+        flavor: flavor,
+      )
+
+      registry_artifact_path = get_registry_artifact_path_env_app(
+        project: project,
+        name: name,
+        flavor: flavor,
+        build_number: build_number,
+        env: env,
+        app: app,
+      )
+      artifactory_url = "#{Artifactory.endpoint}/#{@repo}/#{registry_artifact_path}"
+
       Logger.info("Uploading #{local_artifact_path} to #{artifactory_url}")
       artifact = Artifactory::Resource::Artifact.new(local_path: local_artifact_path)
       artifact.upload(@repo, registry_artifact_path)

lib/kube_deploy_tools/artifact_registry/driver_base.rb

@@ -18,6 +18,11 @@ def upload(local_dir:, name:, flavor:, project:, build_number:)
           raise "#{self.class}#publish not implemented"
         end
 
+        # Same as above but with 2 more parameters: env, app
+        def upload_with_env_app(local_dir:, name:, flavor:, project:, build_number:, env:, app:)
+          raise "#{self.class}#publish not implemented"
+        end
+
         # download should retrieve the artifact namespaced with the given
         # project and build number and identified by the name and flavor.
         # The artifact should be put into the output directory.

lib/kube_deploy_tools/artifact_registry/driver_gcs.rb

@@ -46,6 +46,10 @@ def get_registry_artifact_path(name:, flavor:, project:, build_number:)
       "#{get_registry_build_path(project: project)}/#{build_number}/artifact/#{get_artifact_name(name: name, flavor: flavor)}"
     end
 
+    def get_registry_artifact_path_env_app(name:, flavor:, project:, build_number:, env:, app:)
+      "#{get_registry_build_path(project: project)}/#{build_number}/artifact/#{env}/#{app}/#{get_artifact_name(name: name, flavor: flavor)}"
+    end
+
     def get_artifact_name(name:, flavor:)
       "manifests_#{name}_#{flavor}.yaml"
     end
@@ -137,5 +141,40 @@ def upload(local_dir:, name:, flavor:, project:, build_number:)
 
       registry_artifact_path
     end
+
+    def upload_with_env_app(local_dir:, name:, flavor:, project:, build_number:, env:, app:)
+      # Pack up contents of each flavor_dir to a correctly named artifact.
+      flavor_dir = File.join(local_dir, "#{name}_#{flavor}")
+
+      package(
+        name: name,
+        flavor: flavor,
+        input_dir: flavor_dir,
+        output_dir: local_dir,
+      )
+
+      local_artifact_path = get_local_artifact_path(
+        local_dir: local_dir,
+        name: name,
+        flavor: flavor,
+      )
+
+      registry_artifact_path = get_registry_artifact_path_env_app(
+        project: project,
+        name: name,
+        flavor: flavor,
+        build_number: build_number,
+        env: env,
+        app: app,
+      )
+
+      Logger.info("Uploading #{local_artifact_path} to #{registry_artifact_path}")
+      out, err, status = Shellrunner.run_call('gsutil',  '-m', 'cp', local_artifact_path, registry_artifact_path)
+      if !status.success?
+        raise "Failed to upload remote deploy artifact from #{local_artifact_path} to #{registry_artifact_path}"
+      end
+
+      registry_artifact_path
+    end
   end
 end

lib/kube_deploy_tools/publish.rb

@@ -36,5 +36,28 @@ def publish()
         end
       end
     end
+
+
+    def publish_with_env_app(env, app)
+
+      @config.artifacts.each do |c|
+        name = c.fetch('name')
+
+        # Allow deploy.yaml to gate certain flavors to certain targets.
+        cluster_flavors = @config.flavors.select { |key, value| c['flavors'].nil? || c['flavors'].include?(key) }
+
+        cluster_flavors.each do |flavor, _|
+          @artifact_registry.upload_with_env_app(
+            local_dir: @output_dir,
+            name: name,
+            flavor: flavor,
+            project: @project,
+            build_number: @build_number,
+            env: env,
+            app: app,
+          )
+        end
+      end
+    end
   end
 end

lib/kube_deploy_tools/publish/options.rb

@@ -3,7 +3,15 @@
 module KubeDeployTools
   class Publish::Optparser
     class Options
-      attr_accessor :manifest_file, :output_path
+      attr_accessor :manifest_file, :output_path, :env, :app
+
+      def env
+          @env || nil
+        end
+
+      def app
+          @app || nil
+        end
 
       def initialize
         self.output_path = File.join('build', 'kubernetes')
@@ -18,6 +26,14 @@ def define_options(parser)
           self.output_path = p
         end
 
+        parser.on('-e', '--env-name NAME', 'Env name') do |p|
+          self.env = p
+        end
+
+        parser.on('-a', '--app-name NAME', 'App name') do |p|
+          self.app = p
+        end
+
         parser.on('-')
       end
     end

spec/unit/publish_spec.rb

@@ -76,6 +76,48 @@
       all_uploads = expected_uploads
       expect(uploads).to contain_exactly(*all_uploads)
     end
+
+    it 'publishes artifacts according to deploy.yaml and given env & app name' do
+      KubeDeployTools::Logger.logger = logger
+
+      # Mock artifact upload
+      uploads = Set.new
+      allow_any_instance_of(Artifactory::Resource::Artifact).to receive(:upload) do |artifact, repo, path|
+        # Expect to upload to kubernetes-snapshots-local/<project>
+        expect(path).to start_with(PROJECT)
+
+        # add only the basenames of the files to the set as the BUILD_ID
+        # will vary on each build
+        uploads.add(File.basename(path))
+      end
+
+      expected_uploads = [
+        'manifests_colo-service-prod_default.tar.gz',
+        'manifests_colo-service-staging_default.tar.gz',
+        'manifests_local_default.tar.gz',
+        'manifests_us-east-1-prod_default.tar.gz',
+        'manifests_us-east-1-staging_default.tar.gz',
+        'manifests_ingestion-prod_default.tar.gz',
+        'manifests_pippio-production_default.tar.gz',
+        'manifests_platforms-prod_default.tar.gz',
+        'manifests_filtered-artifact_default.tar.gz',
+      ]
+
+      Dir.mktmpdir do |dir|
+        expected_uploads.each do |f|
+          FileUtils.touch File.join(dir, f)
+        end
+
+        KubeDeployTools::Publish.new(
+          manifest: MANIFEST_FILE,
+          artifact_registry: artifact_registry,
+          output_dir: dir,
+        ).publish_with_env_app("env", "app")
+      end
+
+      all_uploads = expected_uploads
+      expect(uploads).to contain_exactly(*all_uploads)
+    end
   end
 
   context 'GCS driver' do
@@ -105,6 +147,31 @@
         expect(local_artifact_resources.length).to eq(2)
       end
     end
+
+    it 'publishes artifacts according to deploy.yaml and given env & app name' do
+      KubeDeployTools::Logger.logger = logger
+
+      Dir.tmpdir do |dir|
+        FileUtils.cp_r INPUT_DIR, dir, :verbose => true
+        KubeDeployTools::Publish.new(
+          manifest: MANIFEST_GCS_FILE,
+          artifact_registry: artifact_registry,
+          output_dir: dir,
+        ).publish_with_env_app("env", "app")
+
+        artifacts = Find.find(dir).
+          select { |path| path =~ /.*manifests_.*\.yaml$/ }
+
+        expect(artifacts.length).to eq(1)
+
+        # Check that the local artifact contains 2 concatenated resources
+        local_artifact = artifacts.select { |path| path =~ /local/ }.first
+        local_artifact_contents = YAML.load_file(local_artifact)
+        local_artifact_resources = []
+        YAML.load_stream(File.read local_artifact) { |doc| local_artifact_resources << doc }
+        expect(local_artifact_resources.length).to eq(2)
+      end
+    end
   end
 end