Feature/collect versions

.github/workflows/main.yml

@@ -0,0 +1,171 @@
+name: CI
+
+on: [push]
+
+jobs:
+  docker:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        params:
+          - { mapper: minimap2, mode: dna2dna }
+          - { mapper: hisat2, mode: rna2dna }
+          - { mapper: kallisto, mode: rna2rna }
+    steps:
+      - name: Install Nextflow
+        env:
+          NXF_VERSION: 19.10.0
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Setup test-data cache
+        id: cache-data
+        uses: actions/cache@v1
+        with:
+          path: downloaded
+          key: data-${{ github.sha }}
+          restore-keys: |
+            data-${{ github.sha }}-
+            data-
+
+      - name: Setup work-dir cache
+        id: cache-work
+        uses: actions/cache@v1
+        with:
+          path: work
+          key: work-docker-${{ matrix.params.mode }}-${{ github.sha }}
+          restore-keys: |
+            work-docker-${{ matrix.params.mode }}-${{ github.sha }}-
+            work-docker-${{ matrix.params.mode }}-
+
+      - name: Setup NF cache
+        id: cache-NF
+        uses: actions/cache@v1
+        with:
+          path: .nextflow
+          key: nf-docker-${{ matrix.params.mode }}-${{ github.sha }}
+          restore-keys: |
+            nf-docker-${{ matrix.params.mode }}-${{ github.sha }}-
+            nf-docker-${{ matrix.params.mode }}-
+
+      - name: Test workflow with docker - ${{ matrix.params.mapper }}
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile CI,docker --subset 1 --mappers ${{ matrix.params.mapper }} --mapmode ${{ matrix.params.mode }} --max_cpus 2 --max_memory 6.GB -ansi-log false -with-dag dag.dot -resume
+      - name: sysinfo
+        run: |
+          df -h
+          lscpu | egrep 'Model name|Socket|Thread|NUMA|CPU\(s\)'
+          awk '/MemTotal/ {print "RAM : "$2/1E6" GB"}' /proc/meminfo
+          docker image ls
+      # - name: garaph-easy dag
+      #   run: |
+      #     cat dag.dot | docker run -i panguolin/grapheasy:latest graph-easy
+
+  singularity:
+    runs-on: ubuntu-18.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        params:
+          - { mapper: 'minimap2|hisat2|kallisto', mode: 'dna2dna|rna2dnarna2rna' }
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+        id: go
+      - name: Install Dependencies for Singularity
+        run: |
+          sudo apt-get update && sudo apt-get install -y \
+            build-essential \
+            libssl-dev \
+            uuid-dev \
+            libgpgme11-dev \
+            squashfs-tools \
+            libseccomp-dev \
+            pkg-config
+      - name: Install Singularity
+        env:
+          SINGULARITY_VERSION: 3.5.2
+        run: |
+          export GOPATH=/tmp/go
+          mkdir -p $GOPATH
+          sudo mkdir -p /usr/local/var/singularity/mnt && \
+          mkdir -p $GOPATH/src/github.com/sylabs && \
+          cd $GOPATH/src/github.com/sylabs && \
+          wget -qO- https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-${SINGULARITY_VERSION}.tar.gz | \
+          tar xzv && \
+          cd singularity && \
+          ./mconfig -p /usr/local && \
+          make -C builddir && \
+          sudo make -C builddir install
+      - name: Install Nextflow
+        env:
+          NXF_VERSION: 19.10.0
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Setup test-data cache
+        id: cache-data
+        uses: actions/cache@v1
+        with:
+          path: downloaded
+          key: data-${{ github.sha }}
+          restore-keys: |
+            data-${{ github.sha }}-
+            data-
+
+      - name: Setup work cache
+        id: cache-work
+        uses: actions/cache@v1
+        with:
+          path: work
+          key: work-sing-${{ github.sha }}
+          restore-keys: |
+            work-sing-${{ github.sha }}-
+            work-sing-
+
+      - name: Setup NF cache
+        id: cache-NF
+        uses: actions/cache@v1
+        with:
+          path: .nextflow
+          key: nf-sing-${{ github.sha }}
+          restore-keys: |
+            nf-sing-${{ github.sha }}-
+            nf-sing-
+
+      - name: Setup singularity images cache
+        id: cache-singularity-images
+        uses: actions/cache@v1
+        with:
+          path: singularity-images
+          key: simg-${{ github.sha }}
+          restore-keys: |
+            simg-${{ github.sha }}-
+            simg-
+
+      - name: Pull containers
+        run: |
+          nextflow run ${GITHUB_WORKSPACE}/pull_containers.nf --mappers '${{ matrix.params.mapper }}' -ansi-log false -with-dag false
+      - name: Test workflow with singularity - '${{ matrix.params.mapper }}' - '${{ matrix.params.mode }}'
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile CI,singularity --subset 1 --mappers '${{ matrix.params.mapper }}' --mapmode '${{ matrix.params.mode }}' --max_cpus 2 --max_memory 6.GB -ansi-log false -with-dag dag.dot -resume
+      - name: sysinfo
+        run: |
+          df -h
+          lscpu | egrep 'Model name|Socket|Thread|NUMA|CPU\(s\)'
+          awk '/MemTotal/ {print "RAM : "$2/1E6" GB"}' /proc/meminfo
+          ls -lhS singularity-images/

README.md

@@ -433,18 +433,52 @@ There are several ways for rendering of the report outside the pipeline, with do
 
 ### Using docker
 
+1. Put all requited files in one place
+
 ```sh
-docker run --rm --user $(id -u):$(id -g) \
-  --volume $(pwd)/report:/report \
-  --workdir /report rsuchecki/renderer:0.2 ./render.R
+mkdir -p localrender
+cp report/report.Rmd  localrender/
+cp results/* localrender/
+cp flowinfo/*.{json,tsv} localrender/
 ```
 
+2. Docker run rendering
+
+```sh
+docker run \
+  --rm \
+  --user $(id -u):$(id -g) \
+  --volume $(pwd)/localrender:/render \
+  --volume $(pwd)/bin:/binr \
+  --workdir /render \
+  rsuchecki/renderer:0.4.1_81ab6b5d71509d48e3a37b5eafb4bca5b117b5fc /binr/render.R
+```
+
+3. Rendered report should be available under `./localrender`
+
+
 ### Using singularity
 
+1. Put all requited files in one place
+
+```sh
+mkdir -p localrender \
+ && cp report/report.Rmd  localrender/ \
+ && cp results/* localrender/ \
+ && cp flowinfo/*.{json,tsv} localrender/ 
+```
+
+2. Docker run rendering
+
 ```sh
-singularity exec --pwd $(pwd)/report docker://rsuchecki/renderer:0.1 ./render.R
+singularity exec \
+  --bind $(pwd)/bin:/binr \
+  --pwd $(pwd)/localrender \
+  docker://rsuchecki/renderer:0.4.1_81ab6b5d71509d48e3a37b5eafb4bca5b117b5fc /binr/render.R
 ```
 
+3. Rendered report should be available under `./localrender`
+
 ### Natively
 
 If you'd like to render the report without docker/singularity, you will need the following:
@@ -456,13 +490,20 @@ If you'd like to render the report without docker/singularity, you will need the
   * `rmarkdown`
   * `rticles`
   * `bookdown`
+  * `tidyverse`
+  * `jsonlite`
+  * `kableExtra`
 
 Then:
 
 ```
-cd report && ./render.R
-```
+mkdir -p localrender \
+ && cp report/report.Rmd  localrender/ \
+ && cp results/* localrender/ \
+ && cp flowinfo/*.{json,tsv} localrender/ 
 
+cd localrender && ../bin/render.R
+```
 
 
 # Manuscript

bin/eval_rnf.groovy

@@ -5,7 +5,8 @@ import java.util.zip.GZIPInputStream
 import java.util.zip.GZIPOutputStream
 
 
-@Grab('info.picocli:picocli:4.0.0-alpha-3') //command line interface
+@Grab('info.picocli:picocli-groovy:4.1.2') //command line interface
+
 @Command(header = [
        //Font Name: Calvin S (Caps)
        $/@|bold,blue  ╔═╗╦  ╦╔═╗╦    ╦═╗╔╗╔╔═╗   |@/$,

bin/render.R

@@ -3,5 +3,8 @@
 library(rmarkdown)
 library(rticles)
 library(bookdown)
+library(tidyverse)
+library(jsonlite)
+library(kableExtra)
 
 rmarkdown::render(Sys.glob("*.Rmd"))

bin/tct_rnf.groovy

@@ -4,7 +4,8 @@
 import java.util.zip.GZIPInputStream
 import java.util.zip.GZIPOutputStream
 
-@Grab('info.picocli:picocli:4.0.0-alpha-3') //command line interface
+@Grab('info.picocli:picocli-groovy:4.1.2') //command line interface
+
 @Command(header = [
 
         $/@|bold,blue  ╔╦╗╔═╗╔╦╗  ╦═╗╔╗╔╔═╗   |@/$,

conf/containers.config

@@ -1,38 +1,32 @@
 //Containers for processess other then indexing and alignment/mapping
 process {
-  //default container, standard-ish linux toool set: wget, gawk... //replaced by individual containers where applicable
+  //default container, standard-ish linux tool set: wget, gawk... //replaced by individual containers where applicable
   container = 'rsuchecki/tools:0.2' // container = {docker.enabled ? 'rsuchecki/tools:0.2@sha256:....' : 'rsuchecki/tools:0.2'}
 
-    withLabel: benchmark {
-      container = 'rsuchecki/biokanga_benchmark:0.7'
-    }
     withLabel: gffread {
       container = 'rsuchecki/gffread:0.11.4_32ec4f0a9d4c9acbbe9b93c0e6c2da3e9f60e097'
     }
     withLabel: groovy {
-      container = 'rsuchecki/groovy:3.0_868da92992a46b74552abbbf72b76c8aba3fbc9c'
+      container = 'rsuchecki/groovy-samtools:3.0jre-1.9-alpine_879f6ced4ecb8faac3e6e42e32342b6fbe5ac9bd'
     }
     withLabel: groovy_samtools { //now redundant, simply switch to 'samtools' label with our samtools container
-      // container = 'rsuchecki/groovy-conda-samtools:0.1_67d8f0d93333fa0511ae1d23064beba2b544ea0c'
-      container = 'rsuchecki/samtools:1.9_358fa2a91e7feaf5f30e46818d5c9d81ad3975f9'
+      container = 'rsuchecki/groovy-samtools:3.0jre-1.9-alpine_879f6ced4ecb8faac3e6e42e32342b6fbe5ac9bd'
     }
     withLabel: rnftools {
-      // container = 'rsuchecki/rnftools:0.3.1.3_3123fca68e14580a453deea77a0549929ed44715' //WORKS, more recent builds fail
-      container = 'rsuchecki/rnftools:0.3.1.3_3d09a45044213bfbc96a3e0e70924f68812c718b'
-      // container = 'rsuchecki/rnftools:0.3.1.3_a7e1f831b716c12385636652dc999e988b3c8af4'
-      //FAILS container = 'quay.io/biocontainers/rnftools@sha256:8b588055977bbf83116f394d755c088c885b37b2ccce0b81d50b2d87ba0d2f29'
-      //FAILS container = 'quay.io/biocontainers/rnftools@sha256:f3b8bedc40416bd40de0f1b5ef5096b9ab47a079ba1735a130825989de20f4d9'
+      //container = 'rsuchecki/rnftools:0.3.1.3_3d09a45044213bfbc96a3e0e70924f68812c718b' //OK
+      // container = 'quay.io/biocontainers/rnftools:0.3.1.3--py36_0'  //OK
+      container = 'rsuchecki/rnftools:0.3.1.2_ce8211d50e5223ec1cbe03faaccf5060273f34f9' 
     }
     withLabel: rrender {
       container = 'rsuchecki/renderer:0.4.1_81ab6b5d71509d48e3a37b5eafb4bca5b117b5fc'
     }
-    withLabel: rscript {
-      container = 'rsuchecki/rscripts:0.7_c66407a6a160a64a5f973868bd095e28bade0094'
-    }
+    // withLabel: rscript {
+    //   container = 'rsuchecki/rscripts:0.7_c66407a6a160a64a5f973868bd095e28bade0094'
+    // }
     withLabel: samtools {
-      container = 'rsuchecki/samtools:1.9_358fa2a91e7feaf5f30e46818d5c9d81ad3975f9'
-    }
-    withLabel: sra {
-      container = 'ncbi/sra-toolkit:2.9.2' //ncbi/sra-toolkit@sha256:0e6ff2d6560ad7e59821dad53488bdcf09a37c6ccdeab0caced9bbf76837278d
+      container = 'rsuchecki/samtools:1.10_9870ff5feb92e24fca1b6bf69afed30c1d7684f2'
     }
+    // withLabel: sra {
+    //   container = 'ncbi/sra-toolkit:2.9.2' //ncbi/sra-toolkit@sha256:0e6ff2d6560ad7e59821dad53488bdcf09a37c6ccdeab0caced9bbf76837278d
+    // }
 }