Changing to use mill's SonatypePublisher to allow better control of t… (

#670) * Changing to use mill's SonatypePublisher to allow better control of timeouts * Forcing jackson versions to help with dependencies * Attempting to fix dependency vulnerability failing the build * Update the json-diff library in order to fix vulnerability * Remove sonatype scan dependency to allow publishing temporarily
finos · Aug 21, 2024 · 6c60270 · 6c60270
1 parent f7adbc9
commit 6c60270
Show file tree

Hide file tree

Showing 10 changed files with 311 additions and 82 deletions.
diff --git a/.github/scripts/gpg-setup.sh b/.github/scripts/gpg-setup.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+
+# from https://github.com/coursier/apps/blob/f1d2bf568bf466a98569a85c3f23c5f3a8eb5360/.github/scripts/gpg-setup.sh
+
+echo $PGP_SECRET | base64 --decode | gpg --import --no-tty --batch --yes
+
+echo "allow-loopback-pinentry" >>~/.gnupg/gpg-agent.conf
+echo "pinentry-mode loopback" >>~/.gnupg/gpg.conf
+
+gpg-connect-agent reloadagent /bye
diff --git a/.github/scripts/wrap-gpg.sh b/.github/scripts/wrap-gpg.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+
+mkdir -p ~/mybin
+touch ~/mybin/gpg
+chmod +x ~/mybin/gpg
+echo '#!/usr/bin/env sh' > ~/mybin/gpg
+echo "$(which gpg) --no-tty --yes \"\$@\"" >> ~/mybin/gpg
+echo "$HOME/mybin" >> $GITHUB_PATH
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -3,10 +3,10 @@ name: CI
 env:
   JDK_JAVA_OPTIONS: -XX:+PrintCommandLineFlags -Xss10M # JDK_JAVA_OPTIONS is _the_ env. variable to use for modern Java
   JVM_OPTS: -XX:+PrintCommandLineFlags -Xss10M # for Java 8 only (sadly, it is not modern enough for JDK_JAVA_OPTIONS)
-  SonatypeUrl: 'https://finos.sonatype.app/platform/'
+  SonatypeUrl: "https://finos.sonatype.app/platform/"
   SonatypeAppId: morphir-scala
-  SonatypeStage: 'build' 
-  SonatypeScanTarget: '.' # depCache/coursier/v1/https/repo1.maven.org/maven2/
+  SonatypeStage: "build"
+  SonatypeScanTarget: "." # depCache/coursier/v1/https/repo1.maven.org/maven2/
   ExcludeDirectory: " -D fileExcludes='*.class, **/website/, **/docs/, **/.*' "
 
 on:
@@ -44,7 +44,7 @@ jobs:
         uses: moonrepo/setup-toolchain@v0.3.1
         with:
           auto-install: true
-            
+
       - name: Install tooling
         run: |
           ./build.sh install
@@ -65,10 +65,10 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup Node.js 
+      - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: '.nvmrc'
+          node-version-file: ".nvmrc"
 
       - name: Install morphir-elm
         run: |
@@ -109,10 +109,10 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup Node.js 
+      - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: '.nvmrc'
+          node-version-file: ".nvmrc"
 
       - name: Install morphir-elm
         run: |
@@ -130,7 +130,7 @@ jobs:
         uses: moonrepo/setup-toolchain@v0.3.1
         with:
           auto-install: true
-            
+
       - name: Install tooling
         run: |
           ./build.sh install
@@ -165,10 +165,10 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup Node.js 
+      - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: '.nvmrc'
+          node-version-file: ".nvmrc"
 
       - name: Install morphir-elm
         run: |
@@ -207,64 +207,64 @@ jobs:
     needs: [ci]
     runs-on: ubuntu-latest
     steps:
-    - name: Cache scala dependencies
-      uses: coursier/cache-action@v6
-
-    - name: Get OUT cache
-      uses: actions/cache/restore@v4
-      with:
-        path: out/
-        key: ${{ runner.os }}-*
-        
-    - name: Copy Cache for SCA Scan
-      run: |
-        mkdir depCache/ #Create local copy of cache for Sonatype Scanner
-        cp -r /home/runner/.cache/coursier/ depCache/
-    # - name: Save resolvedIvyDeps.json
-    #   run: ./mill show __.resolvedIvyDeps > depCache/resolvedIvyDeps.json
-        
-    # - name: Upload Dependency Cache (optional)
-    #   uses: actions/upload-artifact@v4
-    #   with:
-    #     name: Dependency Cache
-    #     path: depCache/
-    
-    - name: Sonatype Lifecycle SCA Scan
-      uses: sonatype-nexus-community/iq-github-action@main
-      with:
-        username: ${{ secrets.SONATYPE_USER }}
-        password: ${{ secrets.SONATYPE_PASSWORD }}
-        serverUrl: ${{ env.SonatypeUrl }}
-        applicationId: ${{ env.SonatypeAppId }}
-        stage: ${{ env.SonatypeStage }}
-        target: ${{ env.SonatypeScanTarget }} ${{ env.ExcludeDirectory }}
-
-    - name: Retrieve Sonatype SBOM (SPDX)
-      if: always()
-      run: |
-        mkdir reports/
-        iqCredentials="${{ secrets.SONATYPE_USER }}:${{ secrets.SONATYPE_PASSWORD }}"
-        echo 'Get internal app ID for public ID: '$SonatypeAppId
-        res=$(curl -u $iqCredentials --location $SonatypeUrl'api/v2/applications?publicId='$SonatypeAppId)
-        IFS='"' read -a array <<< "$res"
-        echo 'Internal app ID: '${array[5]}
-        internalID=${array[5]}
-        curl -u $iqCredentials --location $SonatypeUrl'api/v2/spdx/'$internalID'/stages/'$SonatypeStage -H 'Accept: application/xml' > reports/$SonatypeAppId.spdx.json
-        echo 'Sonatype SBOM (SPDX): '
-        cat reports/$SonatypeAppId.spdx.json
-        
-    - name: Upload Sonatype SBOM (SPDX)
-      if: always()
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ env.SonatypeAppId }} Sonatype SBOM (SPDX) 
-        path: reports/
-  #############################################          
-
-  publish-sonatype:
+      - name: Cache scala dependencies
+        uses: coursier/cache-action@v6
+
+      - name: Get OUT cache
+        uses: actions/cache/restore@v4
+        with:
+          path: out/
+          key: ${{ runner.os }}-*
+
+      - name: Copy Cache for SCA Scan
+        run: |
+          mkdir depCache/ #Create local copy of cache for Sonatype Scanner
+          cp -r /home/runner/.cache/coursier/ depCache/
+      # - name: Save resolvedIvyDeps.json
+      #   run: ./mill show __.resolvedIvyDeps > depCache/resolvedIvyDeps.json
+
+      # - name: Upload Dependency Cache (optional)
+      #   uses: actions/upload-artifact@v4
+      #   with:
+      #     name: Dependency Cache
+      #     path: depCache/
+
+      - name: Sonatype Lifecycle SCA Scan
+        uses: sonatype-nexus-community/iq-github-action@main
+        with:
+          username: ${{ secrets.SONATYPE_USER }}
+          password: ${{ secrets.SONATYPE_PASSWORD }}
+          serverUrl: ${{ env.SonatypeUrl }}
+          applicationId: ${{ env.SonatypeAppId }}
+          stage: ${{ env.SonatypeStage }}
+          target: ${{ env.SonatypeScanTarget }} ${{ env.ExcludeDirectory }}
+
+      - name: Retrieve Sonatype SBOM (SPDX)
+        if: always()
+        run: |
+          mkdir reports/
+          iqCredentials="${{ secrets.SONATYPE_USER }}:${{ secrets.SONATYPE_PASSWORD }}"
+          echo 'Get internal app ID for public ID: '$SonatypeAppId
+          res=$(curl -u $iqCredentials --location $SonatypeUrl'api/v2/applications?publicId='$SonatypeAppId)
+          IFS='"' read -a array <<< "$res"
+          echo 'Internal app ID: '${array[5]}
+          internalID=${array[5]}
+          curl -u $iqCredentials --location $SonatypeUrl'api/v2/spdx/'$internalID'/stages/'$SonatypeStage -H 'Accept: application/xml' > reports/$SonatypeAppId.spdx.json
+          echo 'Sonatype SBOM (SPDX): '
+          cat reports/$SonatypeAppId.spdx.json
+
+      - name: Upload Sonatype SBOM (SPDX)
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.SonatypeAppId }} Sonatype SBOM (SPDX)
+          path: reports/
+  #############################################
+
+  publish:
     # when in master repo: all commits to main branch and all additional tags
     if: github.repository == 'finos/morphir-scala' && ( github.ref == 'refs/heads/main' || (github.ref != 'refs/heads/main' && startsWith( github.ref, 'refs/tags/') ) )
-    needs: [sonatype-scan, ci]
+    needs: [ci]
 
     runs-on: ubuntu-latest
 
@@ -287,10 +287,10 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup Node.js 
+      - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: '.nvmrc'
+          node-version-file: ".nvmrc"
 
       - name: Install morphir-elm
         run: |
@@ -355,8 +355,12 @@ jobs:
             out/morphir/3.3.3/**/native/
           key: ${{ runner.os }}-mill-native-11-3.3.3-${{ github.sha }}-${{ hashFiles('out') }}
           restore-keys: ${{ runner.os }}-mill-native-11-3.3.3-${{ github.sha }}-
-
-      - run: ./mill -i -j 0 io.kipp.mill.ci.release.ReleaseModule/publishAll
+      - name: Setup GPG
+        run: .github/scripts/gpg-setup.sh
+      - name: Wrap GPG binary
+        run: .github/scripts/wrap-gpg.sh
+      - name: Release
+        run: ./mill -i ci.publishSonatype __.publishArtifacts
 
   ci:
     runs-on: ubuntu-latest

diff --git a/build.sc b/build.sc
@@ -1,15 +1,16 @@
-import mill.testrunner.TestResult
-import mill.scalalib.publish.PublishInfo
 import $meta._
 import $ivy.`de.tototec::de.tobiasroeser.mill.integrationtest::0.7.1`
-import $ivy.`io.chris-kipp::mill-ci-release::0.1.10`
+import $ivy.`de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0`
 import $ivy.`com.lihaoyi::mill-contrib-buildinfo:$MILL_VERSION`
 import $ivy.`com.carlosedp::mill-aliases::0.4.1`
+import $ivy.`com.github.lolgab::mill-mima::0.1.1`
 import $file.project.deps, deps.{Deps, MillVersions, Versions => Vers}
 import $file.project.modules.docs, docs.{Docusaurus2Module, MDocModule}
+import mill.testrunner.TestResult
+import mill.scalalib.publish.PublishInfo
+import mill.local.plugins.ci.release.ReleaseSetupModule
 import com.carlosedp.aliases._
 import de.tobiasroeser.mill.integrationtest._
-import io.kipp.mill.ci.release.CiReleaseModule
 import millbuild._
 import millbuild.crossplatform._
 import millbuild.jsruntime._
@@ -19,8 +20,9 @@ import millbuild.settings._
 import mill._, mill.scalalib._, mill.scalajslib._, mill.scalanativelib._, scalafmt._
 import mill.scalajslib.api.ModuleKind
 import mill.contrib.buildinfo.BuildInfo
-import $ivy.`com.github.lolgab::mill-mima::0.1.1`
 import com.github.lolgab.mill.mima._
+import de.tobiasroeser.mill.vcs.version.VcsVersion
+import scala.concurrent.duration.DurationInt
 
 implicit val buildSettings: BuildSettings = interp.watchValue(MyBuild.cachedBuildSettings)
 
@@ -55,10 +57,11 @@ def showBuildSettings() = T.command {
   MyBuild.showBuildSettings()
 }
 
-trait MorphirPublishModule extends CiReleaseModule with JavaModule with Mima {
+trait MorphirPublishModule extends PublishModule with JavaModule with Mima {
   import mill.scalalib.publish._
-  def packageDescription: String = s"The $artifactName package"
 
+  def publishVersion = VcsVersion.vcsState().format()
+  def packageDescription: String = s"The $artifactName package"  
   def pomSettings = PomSettings(
     description = packageDescription,
     organization = "org.finos.morphir",
@@ -302,7 +305,7 @@ trait MorphirCrossModule extends Cross.Module[String] with CrossPlatform { morph
           object test extends ScalaTests with TestModule.ZioTest {
             def ivyDeps: T[Agg[Dep]] = Agg(
               Deps.dev.zio.`zio-json-golden`,
-              ivy"io.github.deblockt:json-diff:0.0.6",
+              ivy"io.github.deblockt:json-diff:1.1.0",
               Deps.dev.zio.`zio-process`
             )
 
@@ -553,6 +556,68 @@ object `morphir-elm` extends Module {
   }
 }
 
+object ci extends Module {
+
+  def publishSonatype(tasks: mill.main.Tasks[PublishModule.PublishData]) = T.command {
+    //ReleaseSetupModule.setupGpg()()
+    publishSonatype0(
+      data = define.Target.sequence(tasks.value)(),
+      log = T.ctx().log
+    )
+  }
+
+  def publishSonatype0(
+    data: Seq[PublishModule.PublishData],
+    log: mill.api.Logger
+  ): Unit = {
+
+    val credentials = sys.env("SONATYPE_USERNAME") + ":" + sys.env("SONATYPE_PASSWORD")
+    val pgpPassword = sys.env("PGP_PASSPHRASE")
+    val timeout     = 10.minutes
+
+    val artifacts = data.map {
+      case PublishModule.PublishData(a, s) =>
+        (s.map { case (p, f) => (p.path, f) }, a)
+    }
+
+    val isRelease = {
+      val versions = artifacts.map(_._2.version).toSet
+      val set      = versions.map(!_.endsWith("-SNAPSHOT"))
+      assert(
+        set.size == 1,
+        s"Found both snapshot and non-snapshot versions: ${versions.toVector.sorted.mkString(", ")}"
+      )
+      set.head
+    }
+    val publisher = new scalalib.publish.SonatypePublisher(
+      uri = "https://oss.sonatype.org/service/local",
+      snapshotUri = "https://oss.sonatype.org/content/repositories/snapshots",
+      credentials = credentials,
+      signed = true,
+      // format: off
+      gpgArgs = Seq(
+        "--detach-sign",
+        "--batch=true",
+        "--yes",
+        "--pinentry-mode", "loopback",
+        "--passphrase", pgpPassword,
+        "--armor",
+        "--use-agent"
+      ),
+      // format: on
+      readTimeout = timeout.toMillis.toInt,
+      connectTimeout = timeout.toMillis.toInt,
+      log = log,
+      awaitTimeout = timeout.toMillis.toInt,
+      stagingRelease = isRelease
+    )
+
+    publisher.publishAll(isRelease, artifacts: _*)
+  }
+
+}
+
+
 // The following section contains aliases used to simplify build tasks
 
 object MyAliases extends Aliases {

diff --git a/mill-build/src/mill/local/plugins/ci/release/Discover.scala.scala b/mill-build/src/mill/local/plugins/ci/release/Discover.scala.scala
@@ -0,0 +1,5 @@
+package mill.local.plugins.ci.release
+
+private[release] object Discover {
+  implicit def millEvaluatorTokenReader = mill.main.TokenReaders.millEvaluatorTokenReader
+}