Debug quiltcore (#194)

Add debug build of quiltcore

Ensure errors are thrown if bucket lacks write permission

.github/workflows/test.yml

@@ -51,10 +51,14 @@ jobs:
 
     - name: Run Gradle Tests
       run: make test
+      env:
+        LOG4J_DEBUG: true
 
     - name: Archive production artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
+      if: ${{ always() }}
       with:
         name: nf-quilt-test-reports
         path: |
           /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/
+        overwrite: true

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [0.7.10] 2024-05-10 UNOFFICIAL
+
+- Debug build of QuiltCore-Java
+- Fail when pushing to read-only buckets
+
 ## [0.7.9] 2024-01-30
 
 - Auto-install (needed for < 23.12?)

README.md

@@ -75,8 +75,9 @@ From the command-line, do, e.g.:
 
 ```bash
 # export NXF_VER=23.04.3
-export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.9/nf-quilt-0.7.9-meta.json
-nextflow run main.nf -plugins nf-quilt@0.7.9
+export LOG4J_DEBUG=true  # for verbose logging
+export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.10/nf-quilt-0.7.10-meta.json
+nextflow run main.nf -plugins nf-quilt@0.7.10
 ```
 
 For Tower, you can use the "Pre-run script" to set the environment variables.

plugins/nf-quilt/build.gradle

@@ -60,7 +60,7 @@ ext {
 
 dependencies {
     // quiltcore
-    implementation 'com.quiltdata:quiltcore:0.1'
+    implementation 'com.quiltdata:quiltcore:0.1.2'
 
     // This dependency is exported to consumers, that is to say found on their compile classpath.
     compileOnly "io.nextflow:nextflow:$nextflowVersion"

plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy

@@ -139,7 +139,9 @@ ${nextflow}
         List<Map> quilt_summarize = setupSummarize()
         log.debug("setupSummarize: $quilt_summarize")
         try {
-            pkg.push(msg, meta)
+            log.info("publish.pushing: ${pkg}")
+            def m = pkg.push(msg, meta)
+            log.info("publish.pushed: ${m}")
         }
         catch (Exception e) {
             log.error("Exception: ${e}")

plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy

@@ -227,7 +227,7 @@ class QuiltPackage {
         })
     }
     // https://docs.quiltdata.com/v/version-5.0.x/examples/gitlike#install-a-package
-    void push(String msg = 'update', Map meta = [:]) {
+    Manifest push(String msg = 'update', Map meta = [:]) {
         S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null)
         Registry registry = new Registry(registryPath)
         Namespace namespace = registry.getNamespace(packageName)
@@ -250,8 +250,19 @@ class QuiltPackage {
         builder.setMetadata((ObjectNode)mapper.valueToTree(fullMeta))
 
         Manifest m = builder.build()
-        log.debug('QuiltPackage.push', m)
-        m.push(namespace, "nf-quilt:${today()}-${msg}", parsed.workflowName)
+        log.debug("push[${this.parsed}]: ${m}")
+        try {
+            Manifest manifest = m.push(namespace, "nf-quilt:${today()}-${msg}", parsed.workflowName)
+            log.debug("pushed[${this.parsed}]: ${manifest}")
+            return manifest
+        } catch (Exception e) {
+            log.error('ERROR: Failed to push manifest', e)
+            print("FAILED: ${this.parsed}\n")
+            e.printStackTrace()
+            /* groovylint-disable-next-line ThrowRuntimeException */
+            throw new RuntimeException(e)
+        }
+        return m
     }
 
     @Override

plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF

@@ -1,7 +1,7 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.quilt.QuiltPlugin
 Plugin-Id: nf-quilt
-Plugin-Version: 0.7.9
+Plugin-Version: 0.7.10
 Plugin-Provider: Quilt Data
 Plugin-Requires: >=22.10.6
 

plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy

@@ -41,6 +41,7 @@ class QuiltPackageTest extends QuiltSpecification {
 
     private final static String PACKAGE_URL = 'quilt+s3://quilt-example#package=examples%2fsmart-report@d68a7e9'
     private final static String TEST_URL = PACKAGE_URL + '&path=README.md'
+    private final static String READONLY_URL = 'quilt+s3://allencell#package=test%2ftmp&path=foo%2fbar.txt'
 
     private QuiltPathFactory factory
     private QuiltPath qpath
@@ -170,22 +171,26 @@ class QuiltPackageTest extends QuiltSpecification {
         !Files.isDirectory(qpath)
     }
 
-    @IgnoreIf({ System.getProperty('os.name').contains('indows') })
+    // @IgnoreIf({ System.getProperty('os.name').contains('indows') })
     void 'should fail pushing new files to read-only bucket '() {
         given:
-        def qout = factory.parseUri(TEST_URL)
+        println("read-only-bucket:TEST_URL: ${READONLY_URL}")
+        def qout = factory.parseUri(READONLY_URL)
         def opkg = qout.pkg()
-        def outPath = Paths.get(opkg.packageDest().toString(), QuiltProduct.README_FILE)
+        println("opkg: ${opkg}")
+        def outPath = Paths.get(opkg.packageDest().toString(), 'foo/bar.txt')
+        println("outPath: ${outPath}")
         Files.writeString(outPath, "Time: ${timestamp}")
         expect:
         Files.exists(outPath)
         when:
+        println('Pushing to read-only bucket')
         opkg.push()
         then:
-        thrown(IOException)
+        thrown(RuntimeException)
     }
 
-    @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET ==  null })
+    @IgnoreIf({ env.WRITE_BUCKET ==  null })
     void 'should get writeable package '() {
         given:
         QuiltPackage opkg = writeablePackage('observer')
@@ -196,7 +201,7 @@ class QuiltPackageTest extends QuiltSpecification {
         opkg.packageName.contains('test/observer')
     }
 
-    @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET ==  null })
+    @IgnoreIf({ env.WRITE_BUCKET ==  null })
     void 'should succeed pushing new files to writeable bucket '() {
         given:
         QuiltPackage opkg = writeablePackage('observer')