WDL meta, versioning, tests in caper (#78)

ENCODE-DCC · Feb 26, 2021 · 3dee9c7 · 3dee9c7
1 parent b7b257c
commit 3dee9c7
Show file tree

Hide file tree

Showing 9 changed files with 94 additions and 348 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -35,6 +35,53 @@ get_kallisto_index: &get_kallisto_index
   command: |
     curl https://storage.googleapis.com/circle_ci_test_data/rna-seq-pipeline/Homo_sapiens.GRCh38.cdna.all.chr19_ERCC_phix_k31_kallisto.idx -o test_data/Homo_sapiens.GRCh38.cdna.all.chr19_ERCC_phix_k31_kallisto.idx
 
+commands:
+  set_env_variables:
+    description: "Set environment ready for docker running and pushing"
+    steps:
+      - run:
+          command: |
+            echo "export TAG=encodedcc/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID}" >> ${BASH_ENV}
+            echo "export CROMWELL=./cromwell-56.jar" >> ${BASH_ENV}
+            echo "export WOMTOOL=./womtool-56.jar" >> ${BASH_ENV}
+
+  install_python_requirements:
+    description: "Install pytest workflow requirements"
+    steps:
+      - run: pyenv global 3.7.8
+      - run: pip install --upgrade pip
+      - run: pip install -r test_env_requirements.txt
+
+  download_file:
+    description: "Download a file using wget"
+    parameters:
+      file_url:
+        type: string
+    steps:
+      - run: wget <<parameters.file_url>>
+
+  prepare_pytest_environment:
+    description: "Get machine ready to run caper and pytest-workflow"
+    steps:
+      - set_env_variables
+      - install_python_requirements
+      - download_file:
+          file_url: "https://github.com/broadinstitute/cromwell/releases/download/56/cromwell-56.jar"
+      - download_file:
+          file_url: "https://github.com/broadinstitute/cromwell/releases/download/56/womtool-56.jar"
+
+  run_test_tag:
+    description: "Run tagged test"
+    parameters:
+      tag:
+        type: string
+      wt:
+        type: string
+    steps:
+      - run:
+          command: pytest -v --tag <<parameters.tag>> --wt <<parameters.wt>> --symlink --kwd
+          no_output_timeout: 60m
+
 jobs:
   unittests:
     <<: *python_defaults
@@ -50,11 +97,10 @@ jobs:
     steps:
       - checkout
       - setup_remote_docker
-      - run: *make_tag
+      - set_env_variables
       - run:
           name: build image
           command: |
-            source ${BASH_ENV}
             echo "pulling template!"
             docker pull encodedcc/rna-seq-pipeline:template
             docker login -u=${DOCKERHUB_USER} -p=${DOCKERHUB_PASS}
@@ -68,10 +114,9 @@ jobs:
       steps:
         - checkout
         - setup_remote_docker
-        - run: *make_tag
+        - set_env_variables
         - run:
             command: |
-              source ${BASH_ENV}
               docker pull $TAG
               docker login -u=${DOCKERHUB_USER} -p=${DOCKERHUB_PASS}
               docker tag $TAG encodedcc/rna-seq-pipeline:template
@@ -83,14 +128,13 @@ jobs:
     <<: *machine_defaults
     steps:
       - checkout
-      - run: *make_tag
+      - prepare_pytest_environment
       - run: *get_star_index
       - run: *get_kallisto_index
       - run:
           command: |
-            source ${BASH_ENV}
-            test/test_workflow/test.sh rna-seq-pipeline.wdl test/test_workflow/PE_unstranded_input.json $TAG docker
-            python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_unstranded rna.bam_to_signals.unique_unstranded --metadata_json PE_unstranded_input.metadata.json --reference_json test/test_workflow/PE_unstranded_reference_md5.json --outfile PE_unstranded_input.result.json
+            test/caper_run.sh rna-seq-pipeline.wdl test/test_workflow/PE_unstranded_input.json
+            python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_unstranded rna.bam_to_signals.unique_unstranded --metadata_json metadata.json --reference_json test/test_workflow/PE_unstranded_reference_md5.json --outfile PE_unstranded_input.result.json
             cat PE_unstranded_input.result.json
             python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < PE_unstranded_input.result.json
           no_output_timeout: 30m
@@ -99,14 +143,13 @@ jobs:
     <<: *machine_defaults
     steps:
       - checkout
-      - run: *make_tag
+      - prepare_pytest_environment
       - run: *get_star_index
       - run: *get_kallisto_index
       - run:
           command: |
-            source ${BASH_ENV}
-            test/test_workflow/test.sh rna-seq-pipeline.wdl test/test_workflow/SE_unstranded_input.json $TAG docker
-            python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.mad_qc.madQCmetrics rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_unstranded rna.bam_to_signals.unique_unstranded --metadata_json SE_unstranded_input.metadata.json --reference_json test/test_workflow/SE_unstranded_reference_md5.json --outfile SE_unstranded_input.result.json
+            test/caper_run.sh rna-seq-pipeline.wdl test/test_workflow/SE_unstranded_input.json
+            python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.mad_qc.madQCmetrics rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_unstranded rna.bam_to_signals.unique_unstranded --metadata_json metadata.json --reference_json test/test_workflow/SE_unstranded_reference_md5.json --outfile SE_unstranded_input.result.json
             cat SE_unstranded_input.result.json
             python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < SE_unstranded_input.result.json
           no_output_timeout: 30m
@@ -115,14 +158,13 @@ jobs:
     <<: *machine_defaults
     steps:
       - checkout
-      - run: *make_tag
+      - prepare_pytest_environment
       - run: *get_star_index
       - run: *get_kallisto_index
       - run:
           command: |
-            source ${BASH_ENV}
-            test/test_workflow/test.sh rna-seq-pipeline.wdl test/test_workflow/PE_stranded_input.json $TAG docker
-            python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.mad_qc.madQCmetrics rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_plus rna.bam_to_signals.all_minus rna.bam_to_signals.unique_plus rna.bam_to_signals.unique_minus --metadata_json PE_stranded_input.metadata.json --reference_json test/test_workflow/PE_stranded_reference_md5.json --outfile PE_stranded_input.result.json
+            test/caper_run.sh rna-seq-pipeline.wdl test/test_workflow/PE_stranded_input.json
+            python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.mad_qc.madQCmetrics rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_plus rna.bam_to_signals.all_minus rna.bam_to_signals.unique_plus rna.bam_to_signals.unique_minus --metadata_json metadata.json --reference_json test/test_workflow/PE_stranded_reference_md5.json --outfile PE_stranded_input.result.json
             cat PE_stranded_input.result.json
             python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < PE_stranded_input.result.json
           no_output_timeout: 30m
@@ -131,13 +173,12 @@ jobs:
     <<: *machine_defaults
     steps:
       - checkout
-      - run: *make_tag
+      - prepare_pytest_environment
       - run: *get_star_index
       - run:
           command: |
-            source ${BASH_ENV}
-            test/test_workflow/test.sh test/test_task/test_align.wdl test/test_task/test_align_PE_input.json $TAG docker
-            python3 src/compare_md5.py --keys_to_inspect test_align.align.genome_flagstat_json test_align.align.anno_flagstat_json --metadata_json test_align_PE_input.metadata.json --reference_json test/test_task/test_align_PE_reference_md5.json --outfile test_align_PE_input.result.json
+            test/caper_run.sh test/test_task/test_align.wdl test/test_task/test_align_PE_input.json
+            python3 src/compare_md5.py --keys_to_inspect test_align.align.genome_flagstat_json test_align.align.anno_flagstat_json --metadata_json metadata.json --reference_json test/test_task/test_align_PE_reference_md5.json --outfile test_align_PE_input.result.json
             cat test_align_PE_input.result.json
             python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_align_PE_input.result.json
           no_output_timeout: 30m
@@ -146,13 +187,12 @@ jobs:
     <<: *machine_defaults
     steps:
       - checkout
-      - run: *make_tag
+      - prepare_pytest_environment
       - run: *get_star_index
       - run:
           command: |
-            source ${BASH_ENV}
-            test/test_workflow/test.sh test/test_task/test_align.wdl test/test_task/test_align_SE_input.json $TAG docker
-            python3 src/compare_md5.py --keys_to_inspect test_align.align.genome_flagstat_json test_align.align.anno_flagstat_json --metadata_json test_align_SE_input.metadata.json --reference_json test/test_task/test_align_SE_reference_md5.json --outfile test_align_SE_input.result.json
+            test/caper_run.sh test/test_task/test_align.wdl test/test_task/test_align_SE_input.json
+            python3 src/compare_md5.py --keys_to_inspect test_align.align.genome_flagstat_json test_align.align.anno_flagstat_json --metadata_json metadata.json --reference_json test/test_task/test_align_SE_reference_md5.json --outfile test_align_SE_input.result.json
             cat test_align_SE_input.result.json
             python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_align_SE_input.result.json
           no_output_timeout: 30m

diff --git a/.isort.cfg b/.isort.cfg
@@ -3,4 +3,4 @@ multi_line_output=3
 include_trailing_comma=True
 force_grid_wrap=0
 use_parentheses=True
-known_third_party = compare_md5,merge_annotation,pandas,pysam,qc_utils,rna_qc,rsem_quant
+known_third_party = align,compare_md5,merge_annotation,pandas,pysam,qc_utils,rna_qc,rsem_quant
diff --git a/Dockerfile b/Dockerfile
@@ -1,7 +1,8 @@
 # Dockerfile for ENCODE-DCC rna-seq-pipeline
-FROM ubuntu@sha256:97b54e5692c27072234ff958a7442dde4266af21e7b688e7fca5dc5acc8ed7d9
+FROM ubuntu@sha256:2695d3e10e69cc500a16eae6d6629c803c43ab075fa5ce60813a0fc49c47e859
 MAINTAINER Otto Jolanki
 
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y \
     python3-dev \
     python3-pip \