From 7164f0be05304cfff0c84966be381fba8bfacb8f Mon Sep 17 00:00:00 2001 From: ottojolanki Date: Tue, 4 Aug 2020 01:16:25 -0700 Subject: [PATCH] PIP-1310-dockerhub (#65) --- .circleci/config.yml | 154 ++------------------------ make_index_wdl/build_genome_index.wdl | 6 +- make_index_wdl/merge_anno.wdl | 6 +- rna-seq-pipeline.wdl | 6 +- 4 files changed, 17 insertions(+), 155 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fa4dde3..60cfd4a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ defaults: &defaults python_defaults: &python_defaults docker: - - image: quay.io/encode-dcc/rna-seq-pipeline:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID} + - image: encodedcc/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID} working_directory: ~/rna-seq-pipeline machine_defaults: &machine_defaults @@ -23,7 +23,7 @@ machine_defaults: &machine_defaults make_tag: &make_tag name: make docker image tag command: | - echo "export TAG=quay.io/encode-dcc/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID}" > ${BASH_ENV} + echo "export TAG=encodedcc/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID}" > ${BASH_ENV} get_star_index: &get_star_index name: get star index for test @@ -35,26 +35,6 @@ get_kallisto_index: &get_kallisto_index command: | curl https://storage.googleapis.com/circle_ci_test_data/rna-seq-pipeline/Homo_sapiens.GRCh38.cdna.all.chr19_ERCC_phix_k31_kallisto.idx -o test_data/Homo_sapiens.GRCh38.cdna.all.chr19_ERCC_phix_k31_kallisto.idx -install_singularity: &install_singularity - name: install singularity - command: | - sudo apt-get update - sudo apt-get install \ - python \ - dh-autoreconf \ - build-essential \ - libarchive-dev \ - squashfs-tools - wget https://github.com/singularityware/singularity/releases/download/2.6.0/singularity-2.6.0.tar.gz - tar xvf singularity-2.6.0.tar.gz - cd singularity-2.6.0 - ./configure --prefix=/usr/local --sysconfdir=/etc - make - sudo make install - singularity --version -# Jobs -orbs: - singularity: singularity/singularity@1.0.3 jobs: unittests: <<: *python_defaults @@ -76,9 +56,9 @@ jobs: command: | source ${BASH_ENV} echo "pulling template!" - docker pull quay.io/encode-dcc/rna-seq-pipeline:template - docker login -u=${QUAY_ROBOT_USER} -p=${QUAY_ROBOT_USER_TOKEN} quay.io - docker build --cache-from quay.io/encode-dcc/rna-seq-pipeline:template --build-arg GIT_COMMIT_HASH=${CIRCLE_SHA1} --build-arg BRANCH=${CIRCLE_BRANCH} --build-arg BUILD_TAG=${TAG} -t $TAG . + docker pull encodedcc/rna-seq-pipeline:template + docker login -u=${DOCKERHUB_USER} -p=${DOCKERHUB_PASS} + docker build --cache-from encodedcc/rna-seq-pipeline:template --build-arg GIT_COMMIT_HASH=${CIRCLE_SHA1} --build-arg BRANCH=${CIRCLE_BRANCH} --build-arg BUILD_TAG=${TAG} -t $TAG . docker push $TAG docker logout no_output_timeout: 30m @@ -93,9 +73,9 @@ jobs: command: | source ${BASH_ENV} docker pull $TAG - docker login -u=${QUAY_ROBOT_USER} -p=${QUAY_ROBOT_USER_TOKEN} quay.io - docker tag $TAG quay.io/encode-dcc/rna-seq-pipeline:template - docker push quay.io/encode-dcc/rna-seq-pipeline:template + docker login -u=${DOCKERHUB_USER} -p=${DOCKERHUB_PASS} + docker tag $TAG encodedcc/rna-seq-pipeline:template + docker push encodedcc/rna-seq-pipeline:template docker logout no_output_timeout: 30m @@ -150,66 +130,6 @@ jobs: python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < PE_stranded_input.result.json no_output_timeout: 30m - test_PE_unstranded_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: *get_star_index - - run: *get_kallisto_index - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test_workflow/test.sh rna-seq-pipeline.wdl test/test_workflow/PE_unstranded_input.json $TAG singularity - python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_unstranded rna.bam_to_signals.unique_unstranded --metadata_json PE_unstranded_input.metadata.json --reference_json test/test_workflow/PE_unstranded_reference_md5.json --outfile PE_unstranded_input.result.json - cat PE_unstranded_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < PE_unstranded_input.result.json - no_output_timeout: 30m - - test_SE_unstranded_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: *get_star_index - - run: *get_kallisto_index - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test_workflow/test.sh rna-seq-pipeline.wdl test/test_workflow/SE_unstranded_input.json $TAG singularity - python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.mad_qc.madQCmetrics rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_unstranded rna.bam_to_signals.unique_unstranded --metadata_json SE_unstranded_input.metadata.json --reference_json test/test_workflow/SE_unstranded_reference_md5.json --outfile SE_unstranded_input.result.json - cat SE_unstranded_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < SE_unstranded_input.result.json - no_output_timeout: 30m - - test_PE_stranded_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: *get_star_index - - run: *get_kallisto_index - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test_workflow/test.sh rna-seq-pipeline.wdl test/test_workflow/PE_stranded_input.json $TAG singularity - python3 src/compare_md5.py --keys_to_inspect rna.rna_qc.rnaQC rna.rsem_quant.number_of_genes rna.mad_qc.madQCmetrics rna.kallisto.quants rna.align.genome_flagstat rna.align.anno_flagstat rna.bam_to_signals.all_plus rna.bam_to_signals.all_minus rna.bam_to_signals.unique_plus rna.bam_to_signals.unique_minus --metadata_json PE_stranded_input.metadata.json --reference_json test/test_workflow/PE_stranded_reference_md5.json --outfile PE_stranded_input.result.json - cat PE_stranded_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < PE_stranded_input.result.json - no_output_timeout: 30m - test_PE_align_docker: <<: *machine_defaults steps: @@ -226,25 +146,6 @@ jobs: python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_align_PE_input.result.json no_output_timeout: 30m - test_PE_align_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: *get_star_index - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test_workflow/test.sh test/test_task/test_align.wdl test/test_task/test_align_PE_input.json $TAG docker - python3 src/compare_md5.py --keys_to_inspect test_align.align.genome_flagstat_json test_align.align.anno_flagstat_json --metadata_json test_align_PE_input.metadata.json --reference_json test/test_task/test_align_PE_reference_md5.json --outfile test_align_PE_input.result.json - cat test_align_PE_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_align_PE_input.result.json - no_output_timeout: 30m - test_SE_align_docker: <<: *machine_defaults steps: @@ -261,25 +162,6 @@ jobs: python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_align_SE_input.result.json no_output_timeout: 30m - test_SE_align_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: *get_star_index - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test_workflow/test.sh test/test_task/test_align.wdl test/test_task/test_align_SE_input.json $TAG docker - python3 src/compare_md5.py --keys_to_inspect test_align.align.genome_flagstat_json test_align.align.anno_flagstat_json --metadata_json test_align_SE_input.metadata.json --reference_json test/test_task/test_align_SE_reference_md5.json --outfile test_align_SE_input.result.json - cat test_align_SE_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_align_SE_input.result.json - no_output_timeout: 30m - # Workflow workflows: build_workflow: @@ -297,37 +179,17 @@ workflows: - test_PE_stranded_docker: requires: - build - - test_PE_unstranded_singularity: - requires: - - build - - test_SE_unstranded_singularity: - requires: - - build - - test_PE_stranded_singularity: - requires: - - build - test_PE_align_docker: requires: - build - - test_PE_align_singularity: - requires: - - build - test_SE_align_docker: requires: - build - - test_SE_align_singularity: - requires: - - build - push_template: requires: - unittests - test_PE_unstranded_docker - test_SE_unstranded_docker - test_PE_stranded_docker - - test_PE_unstranded_singularity - - test_SE_unstranded_singularity - - test_PE_stranded_singularity - test_PE_align_docker - - test_PE_align_singularity - test_SE_align_docker - - test_SE_align_singularity diff --git a/make_index_wdl/build_genome_index.wdl b/make_index_wdl/build_genome_index.wdl index 0432d72..7971727 100644 --- a/make_index_wdl/build_genome_index.wdl +++ b/make_index_wdl/build_genome_index.wdl @@ -2,13 +2,13 @@ version 1.0 # ENCODE DCC RNA-Seq pipeline build_genome_index -#CAPER docker quay.io/encode-dcc/rna-seq-pipeline:v1.2.0 -#CAPER singularity docker://quay.io/encode-dcc/rna-seq-pipeline:v1.2.0 +#CAPER docker encodedcc/rna-seq-pipeline:v1.2.1 +#CAPER singularity docker://encodedcc/rna-seq-pipeline:v1.2.1 workflow build_index { meta { author: "Otto Jolanki" - version: "v1.2.0" + version: "v1.2.1" } input { diff --git a/make_index_wdl/merge_anno.wdl b/make_index_wdl/merge_anno.wdl index c593fdd..9e0a2f5 100644 --- a/make_index_wdl/merge_anno.wdl +++ b/make_index_wdl/merge_anno.wdl @@ -2,13 +2,13 @@ version 1.0 #ENCODE DCC RNA-Seq pipeline merge-annotation -#CAPER docker quay.io/encode-dcc/rna-seq-pipeline:v1.2.0 -#CAPER singularity docker://quay.io/encode-dcc/rna-seq-pipeline:v1.2.0 +#CAPER docker encodedcc/rna-seq-pipeline:v1.2.1 +#CAPER singularity docker://encodedcc/rna-seq-pipeline:v1.2.1 workflow merge_anno { meta { author: "Otto Jolanki" - version: "v1.2.0" + version: "v1.2.1" } input { diff --git a/rna-seq-pipeline.wdl b/rna-seq-pipeline.wdl index c80d94d..8998151 100644 --- a/rna-seq-pipeline.wdl +++ b/rna-seq-pipeline.wdl @@ -2,14 +2,14 @@ version 1.0 # ENCODE DCC RNA-seq pipeline -#CAPER docker quay.io/encode-dcc/rna-seq-pipeline:v1.2.0 -#CAPER singularity docker://quay.io/encode-dcc/rna-seq-pipeline:v1.2.0 +#CAPER docker encodedcc/rna-seq-pipeline:v1.2.1 +#CAPER singularity docker://encodedcc/rna-seq-pipeline:v1.2.1 #CROO out_def https://storage.googleapis.com/encode-pipeline-output-definition/bulkrna.output_definition.json workflow rna { meta { author: "Otto Jolanki" - version: "v1.2.0" + version: "v1.2.1" } input {