PIP-1310-dockerhub (#65)

ENCODE-DCC · Aug 4, 2020 · 7164f0b · 7164f0b
1 parent 8a29a99
commit 7164f0b
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 155 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -12,7 +12,7 @@ defaults: &defaults
 
 python_defaults: &python_defaults
   docker:
-    - image: quay.io/encode-dcc/rna-seq-pipeline:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID}
+    - image: encodedcc/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID}
   working_directory: ~/rna-seq-pipeline
 
 machine_defaults: &machine_defaults
@@ -23,7 +23,7 @@ machine_defaults: &machine_defaults
 make_tag: &make_tag
   name: make docker image tag
   command: |
-    echo "export TAG=quay.io/encode-dcc/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID}" > ${BASH_ENV}
+    echo "export TAG=encodedcc/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID}" > ${BASH_ENV}
 
 get_star_index: &get_star_index
   name: get star index for test
@@ -35,26 +35,6 @@ get_kallisto_index: &get_kallisto_index
   command: |
     curl https://storage.googleapis.com/circle_ci_test_data/rna-seq-pipeline/Homo_sapiens.GRCh38.cdna.all.chr19_ERCC_phix_k31_kallisto.idx -o test_data/Homo_sapiens.GRCh38.cdna.all.chr19_ERCC_phix_k31_kallisto.idx
 
-install_singularity: &install_singularity
-  name: install singularity
-  command: |
-    sudo apt-get update
-    sudo apt-get install \
-    python \
-    dh-autoreconf \
-    build-essential \
-    libarchive-dev \
-    squashfs-tools
-    wget https://github.com/singularityware/singularity/releases/download/2.6.0/singularity-2.6.0.tar.gz
-    tar xvf singularity-2.6.0.tar.gz
-    cd singularity-2.6.0
-    ./configure --prefix=/usr/local --sysconfdir=/etc
-    make
-    sudo make install
-    singularity --version
-# Jobs
-orbs:
-  singularity: singularity/singularity@1.0.3
 jobs:
   unittests:
     <<: *python_defaults
@@ -76,9 +56,9 @@ jobs:
           command: |
             source ${BASH_ENV}
             echo "pulling template!"
-            docker pull quay.io/encode-dcc/rna-seq-pipeline:template
-            docker login -u=${QUAY_ROBOT_USER} -p=${QUAY_ROBOT_USER_TOKEN} quay.io
-            docker build --cache-from quay.io/encode-dcc/rna-seq-pipeline:template --build-arg GIT_COMMIT_HASH=${CIRCLE_SHA1} --build-arg BRANCH=${CIRCLE_BRANCH} --build-arg BUILD_TAG=${TAG} -t $TAG .
+            docker pull encodedcc/rna-seq-pipeline:template
+            docker login -u=${DOCKERHUB_USER} -p=${DOCKERHUB_PASS}
+            docker build --cache-from encodedcc/rna-seq-pipeline:template --build-arg GIT_COMMIT_HASH=${CIRCLE_SHA1} --build-arg BRANCH=${CIRCLE_BRANCH} --build-arg BUILD_TAG=${TAG} -t $TAG .
             docker push $TAG
             docker logout
           no_output_timeout: 30m
@@ -93,9 +73,9 @@ jobs:
             command: |
               source ${BASH_ENV}
               docker pull $TAG
-              docker login -u=${QUAY_ROBOT_USER} -p=${QUAY_ROBOT_USER_TOKEN} quay.io
-              docker tag $TAG quay.io/encode-dcc/rna-seq-pipeline:template
-              docker push quay.io/encode-dcc/rna-seq-pipeline:template
+              docker login -u=${DOCKERHUB_USER} -p=${DOCKERHUB_PASS}
+              docker tag $TAG encodedcc/rna-seq-pipeline:template
+              docker push encodedcc/rna-seq-pipeline:template
               docker logout
             no_output_timeout: 30m
 
@@ -150,66 +130,6 @@ jobs:
             python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < PE_stranded_input.result.json
           no_output_timeout: 30m
 
-  test_PE_unstranded_singularity:
-    <<: *machine_defaults
-    steps:
-      - checkout
-      - run: *make_tag
-      - run: sudo apt-get update
-      - singularity/install-go
-      - singularity/debian-install-3
-      - run: *get_star_index
-      - run: *get_kallisto_index
-      - run:
-          command: |
-            pyenv global 3.5.2
-            source ${BASH_ENV}
-            test/test_workflow/test.sh rna-seq-pipeline.wdl test/test_workflow/PE_unstranded_input.json $TAG singularity
-            python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_unstranded rna.bam_to_signals.unique_unstranded --metadata_json PE_unstranded_input.metadata.json --reference_json test/test_workflow/PE_unstranded_reference_md5.json --outfile PE_unstranded_input.result.json
-            cat PE_unstranded_input.result.json
-            python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < PE_unstranded_input.result.json
-          no_output_timeout: 30m
-
-  test_SE_unstranded_singularity:
-    <<: *machine_defaults
-    steps:
-      - checkout
-      - run: *make_tag
-      - run: sudo apt-get update
-      - singularity/install-go
-      - singularity/debian-install-3
-      - run: *get_star_index
-      - run: *get_kallisto_index
-      - run:
-          command: |
-            pyenv global 3.5.2
-            source ${BASH_ENV}
-            test/test_workflow/test.sh rna-seq-pipeline.wdl test/test_workflow/SE_unstranded_input.json $TAG singularity
-            python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.mad_qc.madQCmetrics rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_unstranded rna.bam_to_signals.unique_unstranded --metadata_json SE_unstranded_input.metadata.json --reference_json test/test_workflow/SE_unstranded_reference_md5.json --outfile SE_unstranded_input.result.json
-            cat SE_unstranded_input.result.json
-            python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < SE_unstranded_input.result.json
-          no_output_timeout: 30m
-
-  test_PE_stranded_singularity:
-    <<: *machine_defaults
-    steps:
-      - checkout
-      - run: *make_tag
-      - run: sudo apt-get update
-      - singularity/install-go
-      - singularity/debian-install-3
-      - run: *get_star_index
-      - run: *get_kallisto_index
-      - run:
-          command: |
-            pyenv global 3.5.2
-            source ${BASH_ENV}
-            test/test_workflow/test.sh rna-seq-pipeline.wdl test/test_workflow/PE_stranded_input.json $TAG singularity
-            python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.mad_qc.madQCmetrics rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_plus rna.bam_to_signals.all_minus rna.bam_to_signals.unique_plus rna.bam_to_signals.unique_minus --metadata_json PE_stranded_input.metadata.json --reference_json test/test_workflow/PE_stranded_reference_md5.json --outfile PE_stranded_input.result.json
-            cat PE_stranded_input.result.json
-            python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < PE_stranded_input.result.json
-          no_output_timeout: 30m
-
   test_PE_align_docker:
     <<: *machine_defaults
     steps:
@@ -226,25 +146,6 @@ jobs:
             python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_align_PE_input.result.json
           no_output_timeout: 30m
 
-  test_PE_align_singularity:
-    <<: *machine_defaults
-    steps:
-      - checkout
-      - run: *make_tag
-      - run: sudo apt-get update
-      - singularity/install-go
-      - singularity/debian-install-3
-      - run: *get_star_index
-      - run:
-          command: |
-            pyenv global 3.5.2
-            source ${BASH_ENV}
-            test/test_workflow/test.sh test/test_task/test_align.wdl test/test_task/test_align_PE_input.json $TAG docker
-            python3 src/compare_md5.py --keys_to_inspect test_align.align.genome_flagstat_json test_align.align.anno_flagstat_json --metadata_json test_align_PE_input.metadata.json --reference_json test/test_task/test_align_PE_reference_md5.json --outfile test_align_PE_input.result.json
-            cat test_align_PE_input.result.json
-            python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_align_PE_input.result.json
-          no_output_timeout: 30m
-
   test_SE_align_docker:
     <<: *machine_defaults
     steps:
@@ -261,25 +162,6 @@ jobs:
             python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_align_SE_input.result.json
           no_output_timeout: 30m
 
-  test_SE_align_singularity:
-    <<: *machine_defaults
-    steps:
-      - checkout
-      - run: *make_tag
-      - run: sudo apt-get update
-      - singularity/install-go
-      - singularity/debian-install-3
-      - run: *get_star_index
-      - run:
-          command: |
-            pyenv global 3.5.2
-            source ${BASH_ENV}
-            test/test_workflow/test.sh test/test_task/test_align.wdl test/test_task/test_align_SE_input.json $TAG docker
-            python3 src/compare_md5.py --keys_to_inspect test_align.align.genome_flagstat_json test_align.align.anno_flagstat_json --metadata_json test_align_SE_input.metadata.json --reference_json test/test_task/test_align_SE_reference_md5.json --outfile test_align_SE_input.result.json
-            cat test_align_SE_input.result.json
-            python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_align_SE_input.result.json
-          no_output_timeout: 30m
-
 # Workflow
 workflows:
   build_workflow:
@@ -297,37 +179,17 @@ workflows:
       - test_PE_stranded_docker:
           requires:
             - build
-      - test_PE_unstranded_singularity:
-          requires:
-            - build
-      - test_SE_unstranded_singularity:
-          requires:
-            - build
-      - test_PE_stranded_singularity:
-          requires:
-            - build
       - test_PE_align_docker:
           requires:
             - build
-      - test_PE_align_singularity:
-          requires:
-            - build
       - test_SE_align_docker:
           requires:
             - build
-      - test_SE_align_singularity:
-          requires:
-            - build
       - push_template:
           requires:
             - unittests
             - test_PE_unstranded_docker
             - test_SE_unstranded_docker
             - test_PE_stranded_docker
-            - test_PE_unstranded_singularity
-            - test_SE_unstranded_singularity
-            - test_PE_stranded_singularity
             - test_PE_align_docker
-            - test_PE_align_singularity
             - test_SE_align_docker
-            - test_SE_align_singularity
diff --git a/make_index_wdl/build_genome_index.wdl b/make_index_wdl/build_genome_index.wdl
@@ -2,13 +2,13 @@ version 1.0
 
 # ENCODE DCC RNA-Seq pipeline build_genome_index
 
-#CAPER docker quay.io/encode-dcc/rna-seq-pipeline:v1.2.0
-#CAPER singularity docker://quay.io/encode-dcc/rna-seq-pipeline:v1.2.0
+#CAPER docker encodedcc/rna-seq-pipeline:v1.2.1
+#CAPER singularity docker://encodedcc/rna-seq-pipeline:v1.2.1
 
 workflow build_index {
     meta {
         author: "Otto Jolanki"
-        version: "v1.2.0"
+        version: "v1.2.1"
     }
 
     input {

diff --git a/make_index_wdl/merge_anno.wdl b/make_index_wdl/merge_anno.wdl
@@ -2,13 +2,13 @@ version 1.0
 
 #ENCODE DCC RNA-Seq pipeline merge-annotation
 
-#CAPER docker quay.io/encode-dcc/rna-seq-pipeline:v1.2.0
-#CAPER singularity docker://quay.io/encode-dcc/rna-seq-pipeline:v1.2.0
+#CAPER docker encodedcc/rna-seq-pipeline:v1.2.1
+#CAPER singularity docker://encodedcc/rna-seq-pipeline:v1.2.1
 
 workflow merge_anno {
     meta {
         author: "Otto Jolanki"
-        version: "v1.2.0"
+        version: "v1.2.1"
     }
 
     input {

diff --git a/rna-seq-pipeline.wdl b/rna-seq-pipeline.wdl
@@ -2,14 +2,14 @@ version 1.0
 
 # ENCODE DCC RNA-seq pipeline
 
-#CAPER docker quay.io/encode-dcc/rna-seq-pipeline:v1.2.0
-#CAPER singularity docker://quay.io/encode-dcc/rna-seq-pipeline:v1.2.0
+#CAPER docker encodedcc/rna-seq-pipeline:v1.2.1
+#CAPER singularity docker://encodedcc/rna-seq-pipeline:v1.2.1
 #CROO out_def https://storage.googleapis.com/encode-pipeline-output-definition/bulkrna.output_definition.json
 
 workflow rna {
     meta {
         author: "Otto Jolanki"
-        version: "v1.2.0"
+        version: "v1.2.1"
     }
 
     input {