Merge branch 'dev' of https://github.com/fmalmeida/bacannot into 30-a…

…dd-a-sourmash-module

.github/workflows/test_pr_docker.yml

@@ -1,39 +1,46 @@
 name: Testing new PR with docker
 on:
   pull_request:
-    branches: master
-    types: [ ready_for_review, synchronize, reopened ]
+    branches: [master, dev]
+    types: [ opened, synchronize, reopened ]
 
 jobs:
   run_nextflow:
     name: Run pipeline for the upcoming PR
     runs-on: ubuntu-latest
-    env:
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-      DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }}
-
+
     steps:
-
-      - name: Check out pipeline code
-        uses: actions/checkout@v2
-
-      - name: Install Nextflow
-        env:
-          CAPSULE_LOG: none
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
-      
-      - name: Clean environment
-        run: |
-          sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android
-          sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET
-      
-      - name: Build bacannot database
-        run: |
-          nextflow run main.nf -profile docker --get_dbs --output bacannot_dbs --max_cpus 2 --max_memory '6.GB' --max_time '6.h'
-          rm -rf bacannot_dbs/antismash_db bacannot_dbs/kofamscan_db bacannot_dbs/prokka_db/PGAP_NCBI.hmm # remove unused in quicktest to diminish size
-
-      - name: Run the pipeline
-        run: |
-          nextflow run main.nf -profile docker,quicktest --bacannot_db bacannot_dbs
+
+    - name: Check out pipeline code
+      uses: actions/checkout@v2
+
+    - name: Install Nextflow
+      env:
+        CAPSULE_LOG: none
+      run: |
+        wget -qO- get.nextflow.io | bash
+        sudo mv nextflow /usr/local/bin/
+
+    - name: Clean environment
+      run: |
+        sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android
+        sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET
+
+    - name: Get database
+      run: |
+        nextflow run main.nf -profile docker --get_zenodo_db --output ./ --max_memory '6.GB' --max_cpus 2
+        nextflow run main.nf -profile docker --get_zenodo_db --output ./ --max_memory '6.GB' --max_cpus 2 -resume
+        nextflow run main.nf -profile docker --get_zenodo_db --output ./ --max_memory '6.GB' --max_cpus 2 -resume
+        sudo rm -r work .nextflow*
+        yes | docker system prune
+
+    - name: Run quicktest profile
+      run: |
+        nextflow run main.nf -profile docker,quicktest --bacannot_db $( realpath ./bac* ) --output ./results --max_memory '6.GB' --max_cpus 2
+        sudo rm -r work .nextflow*
+        yes | docker system prune
+
+    - name: View results
+      run: |
+        sudo apt-get install -y tree
+        tree ./results

.zenodo.json

@@ -2,7 +2,7 @@
     "description": "<p>The pipeline</p>\n\n<p>bacannot, is a customisable, easy to use, pipeline that uses state-of-the-art software for comprehensively annotating prokaryotic genomes having only Docker and Nextflow as dependencies. It is able to annotate and detect virulence and resistance genes, plasmids, secondary metabolites, genomic islands, prophages, ICEs, KO, and more, while providing nice an beautiful interactive documents for results exploration.</p>", 
     "license": "other-open", 
     "title": "fmalmeida/bacannot: A generic but comprehensive bacterial annotation pipeline", 
-    "version": "v3.3", 
+    "version": "v3.3.3", 
     "upload_type": "software",
     "creators": [
         {

README.md

@@ -1,13 +1,14 @@
 <img src="images/lOGO_3.png" width="300px">
 
-[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.3627669-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.3627669)
+[![F1000 Paper](https://img.shields.io/badge/Citation%20F1000-10.12688/f1000research.139488.1-orange)](https://doi.org/10.12688/f1000research.139488.1)
 [![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/fmalmeida/bacannot?include_prereleases&label=Latest%20release)](https://github.com/fmalmeida/bacannot/releases)
 [![Documentation](https://img.shields.io/badge/Documentation-readthedocs-brightgreen)](https://bacannot.readthedocs.io/en/latest/?badge=latest)
 [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg?labelColor=000000)](https://www.nextflow.io/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)
 [![License](https://img.shields.io/badge/License-GPL%203-black)](https://github.com/fmalmeida/bacannot/blob/master/LICENSE)
 [![Follow on Twitter](http://img.shields.io/badge/twitter-%40fmarquesalmeida-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/fmarquesalmeida)
+[![Zenodo Archive](https://img.shields.io/badge/Zenodo-Archive-blue)](https://doi.org/10.5281/zenodo.3627669)
 
 [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/github.com/fmalmeida/bacannot)
 
@@ -87,16 +88,7 @@ These images have been kept separate to not create massive Docker image and to a
 
 ## Installation
 
-1. If you don't have it already install [Docker](https://docs.docker.com/) in your computer.
-    * After installed, you need to download the required Docker images
-
-    ```bash
-    docker pull fmalmeida/bacannot:v3.3_misc ;
-    docker pull fmalmeida/bacannot:v3.3_renv ;
-    docker pull fmalmeida/bacannot:jbrowse   ;
-    ```
-
-🔥 Nextflow can also automatically handle images download on the fly when executed. All the other docker images from **biocontainers** are downloaded automatically. If docker has exceeded its download limit rates, please try again in a few hours.
+1. If you don't have it already install either [Docker](https://docs.docker.com/) or [Singularity](https://docs.sylabs.io/guides/3.5/user-guide/index.html) in your computer.
 
 2. Install Nextflow (version 20.10 or higher):
 
@@ -110,54 +102,7 @@ These images have been kept separate to not create massive Docker image and to a
 
 🔥 Users can get let the pipeline always updated with: `nextflow pull fmalmeida/bacannot`
 
-### Downloading and updating databases
-
-Bacannot databases are not inside the docker images anymore to avoid huge images and problems with connections and limit rates with dockerhub.
-
-#### Pre-formatted
-
-Users can directly download pre-formatted databases from Zenodo: https://doi.org/10.5281/zenodo.7615811
-
-Useful for standardization and also overcoming known issues that may arise when formatting databases with `singularity` profile.
-
-A module to download the latest pre-formatted database has also been made available:
-```bash
-# Download pipeline pre-built databases
-nextflow run fmalmeida/bacannot --get_zenodo_db --output ./ -profile <docker/singularity>
-```
-
-#### I want to generate a new formatted database
-
-To download and format a copy of required bacannot databases users can execute the following:
-
-```bash
-# Download pipeline databases
-nextflow run fmalmeida/bacannot --get_dbs --output bacannot_dbs -profile <docker/singularity>
-```
-
-This will produce a directory like this:
-
-```bash
-bacannot_dbs
-├── amrfinder_db
-├── antismash_db
-├── argminer_db
-├── card_db
-├── iceberg_db
-├── kofamscan_db
-├── mlst_db
-├── phast_db
-├── phigaro_db
-├── pipeline_info
-├── plasmidfinder_db
-├── platon_db
-├── prokka_db
-├── resfinder_db
-├── vfdb_db
-└── victors_db
-```
-
-> To update databases you can either download a new one to a new directory. Remove the database you want to get a new one from the root bacannot dir and use the same command above to save in the same directory (the pipeline will only try to download missing databases). Or, you can use the parameter `--force_update` to download everything again.
+<a href="https://bacannot.readthedocs.io/en/latest/installation"><strong>Please refer to the installation page, for a complete guide on required images and databases. »</strong></a>
 
 ## Quickstart
 
@@ -242,11 +187,19 @@ It will result in the following:
 2. The JBrowse wrapper in the shiny server is not capable of displaying the GC content and methylation plots when available. It can only display the simpler tracks. If the user wants to visualise and interrogate the GC or methylation tracks it must open the JBrowse outside from the shiny server. For that, two options are available:
     * You can navigate to the `jbrowse` directory under your sample's output folder and simply execute `http-server`. This command can be found at: https://www.npmjs.com/package/http-server
     * Or, you can download the [JBrowse Desktop app](https://jbrowse.org/docs/jbrowse_desktop.html) and, from inside the app, select the folder `jbrowse/data` that is available in your sample's output directory.
-3. If you face some weird error using v3.1, please, before opening a flag, try updating your docker image, we had some inconsistencies lately and this may be the source of the issue.
+3. If you face some weird error using v3.1 or v3.2, please, before opening a ticket, try updating your docker images, we had some inconsistencies lately and this may be the source of the issue.
+4. If facing an issue with the `BACANNOT:SUMMARY` module, identical or similar to the one reported in issue [[#96]](https://github.com/fmalmeida/bacannot/issues/96), please, before opening a ticket, try updating the python env docker image: `docker pull fmalmeida/bacannot:v3.2_pyenv`. The image has been recently updated to have the latest version of my python scripts, and that may solve the issue. If not, please open another.
+5. Sometimes, the `BACANNOT:UNICYCLER` may fail with different, random issues, that does not seem correct, or seem really very random. For example, saying that a read is not available, even though it is there. After some tracing, we realised that the unicycler 0.4.8 installation from conda, and the biocontainer form quay.io is causing this random problem. To solve this issue, please run with a newer version of the tool. This solves the issue in most cases: `--unicycler_version 0.5.0--py310h6cc9453_3`.
+    * Because `v3.2` is already tagged and frozen with Zenodo, we will not update it, thus, for this version, using the parameter to overwrite the tool version should be used.
+    * In `v3.3`, unicycler version will be defaulted to `0.5.0--py310h6cc9453_3`
 
 ## Citation
 
-To cite this tool please refer to our [Zenodo tag](https://doi.org/10.5281/zenodo.3627669).
+In order to cite this pipeline, please refer to:
+
+> Almeida FMd, Campos TAd and Pappas Jr GJ. Scalable and versatile container-based pipelines for de novo genome assembly and bacterial annotation. [version 1; peer review: awaiting peer review]. F1000Research 2023, 12:1205 (https://doi.org/10.12688/f1000research.139488.1)
+
+Additionally, archived versions of the pipeline are also found in [Zenodo](https://doi.org/10.5281/zenodo.3627669).
 
 This pipeline uses code and infrastructure developed and maintained by the [nf-core](https://nf-co.re) community, reused here under the [GPLv3](https://github.com/fmalmeida/bacannot/blob/master/LICENSE).
 

conf/defaults.config

@@ -32,6 +32,9 @@ params {
 // It is also documented in the main manual: https://bacannot.readthedocs.io/en/latest/samplesheet
   input = null
 
+// Enable reads deduplication for assembly? (If input has reads)
+  enable_deduplication = false
+
 // path to directory containing databases used by bacannot
 // you can download databases with: 
 // nextflow run fmalmeida/bacannot --get_dbs --output bacannot_dbs -profile <docker/conda/singularity>
@@ -122,6 +125,12 @@ params {
 // (NOT RUN?) sourmash
   skip_sourmash = false
 
+// (NOT RUN?) integron finder tool
+  skip_integron_finder = false
+
+// (NOT RUN?) CIRCOS tool
+  skip_circos = false
+
           /*
            * Custom databases can be used to annotate additional genes in the genome.
            * It runs a BLAST alignment against the genome, therefore, the custom database
@@ -199,4 +208,4 @@ params {
   max_cpus                   = 16
   max_time                   = '40.h'
 
-}
+}

conf/docker.config

@@ -4,6 +4,7 @@ docker {
   enabled      = true
   runOptions   = '--platform linux/amd64 -u root:$(id -g)'
 }
+params.running_engine = 'docker'
 
 
 /*
@@ -18,21 +19,27 @@ process {
     // Custom pipeline's containers with various tools for general purposes
     //
     withLabel: 'db_download|db_tools|misc' {
-      container = 'fmalmeida/bacannot:v3.3_misc'
+      container = 'fmalmeida/bacannot@sha256:bdb31637cacf99736656ab3b69f1f01ba1b5eb026771d5c266b4c84e96153057'
     }
 
     // container for R tools
     withLabel: 'renv' {
-      container = 'fmalmeida/bacannot:v3.3_renv'
+      container = 'fmalmeida/bacannot@sha256:952f58a2c03e50f8a376073346fb1ccda28d6249e3fdfea07a3286a6ff1adf0c'
     }
 
     // container for bacannot server
     withLabel: 'server' {
-      container = 'fmalmeida/bacannot:server'
+      container = 'fmalmeida/bacannot@sha256:0ec3b289d6e0c624556d125b2ed9b63499178e266a315175fd87cf020a402898'
     }
 
+    // container for jbrowser
     withLabel: 'jbrowse' {
-      container = 'fmalmeida/bacannot:jbrowse'
+      container = 'fmalmeida/bacannot@sha256:6afdca17b561bf212c1f976422aee3fe047563c32a15112a6262556d1f75201e'
+    }
+
+    // container for antismash
+    withName: 'ANTISMASH|ANTISMASH_DB' {
+      container = 'fmalmeida/bacannot@sha256:fe42fbbfb7d4a026dafb146cb533ee7f1d9a97b25ec6df64840796c343707ebb'
     }
 
     //
@@ -95,7 +102,7 @@ process {
     }
 
     withName: PHIGARO {
-      container = "quay.io/biocontainers/phigaro:2.3.0--pyh7b7c402_0"
+      container = "quay.io/biocontainers/phigaro:2.4.0--pyhdfd78af_0"
     }
 
     withName: PHISPY {