Add GTNet paper files

gtnet_paper/sota/.gitattributes

@@ -0,0 +1 @@
+metadata_r207.tsv.gz filter=lfs diff=lfs merge=lfs -text

gtnet_paper/sota/README.md

@@ -0,0 +1,221 @@
+Setting up data
+===============
+You must install the [deep-taxon repository](https://github.com/exabiome/deep-taxon). Its recommended to 
+first create a fresh conda environment for installing this.
+
+
+```bash
+git checkout git@github.com:exabiome/deep-taxon.git deep-taxon.git
+cd deep-taxon.git
+conda env create --name deep-taxon --file=environments.yml
+conda activate deep-taxon
+python setup.py install
+```
+
+Get test accessions from metadata file and make file-of-files
+```bash
+gunzip metadata_r207.tsv.gz
+deep-taxon get-accessions -t metadata_r207.tsv > metadata_r207.test.accs.tsv
+deep-taxon make-fof --genomic $CFS/m3513/endurable/ncbi/genomes metadata_r207.test.accs.tsv > metadata_r207.test.fof
+```
+
+Load balance by splitting files by size
+
+```bash
+deep-taxon split-fof metadata_r207.test.fof 2000 contigs/metadata_r207.test > contigs/split_file.log 
+```
+
+Append accessions to contig names for post-processing of predictions
+```bash
+sbatch append_all_chunks.sh
+```
+
+This script should be backed up with this file, but in case it is not, here is what it should look like:
+```bash
+#!/bin/bash
+#SBATCH -J append
+#SBATCH -A m2865
+#SBATCH -C cpu
+#SBATCH -q regular
+#SBATCH -t 2
+#SBATCH -n 1
+#SBATCH -o contigs/slurm/append.%A_%a.log
+#SBATCH -e contigs/slurm/append.%A_%a.log
+#SBATCH -n 1
+#SBATCH --array=0-1999
+
+module load parallel
+date +%s
+i=`printf "%04d\n" $SLURM_ARRAY_TASK_ID`
+fof=contigs/metadata_r207.test.$i.fof
+out=contigs/metadata_r207.test.$i.fna
+echo $fof $out
+cat $fof | parallel -j 128 "deep-taxon append-accessions {}" > $out
+date +%s
+```
+
+
+Make bin directories for classifying bins
+
+```bash
+cd bins
+ls ../contigs/metadata_r207.test.*.fof | parallel -j 50 bash make_dir.sh > make_dirs.log 2>&1 &
+```
+
+`make_dir.sh` should be backed up with this file, but in case it is not, here is what it should look like:
+
+```bash
+fof=$1
+dir=`basename $fof | cut -f3 -d.`
+mkdir $dir
+for f in `cat $fof`; do
+    zcat $f > $dir/`basename $f .gz`
+done
+```
+
+Running CAT
+===========
+Create and activate the `cat-gtnet-paper` environment:
+
+```bash
+conda create --file=cat-gtnet-paper.yaml
+conda activate cat-gtnet-paper
+```
+
+Run from `./cat` directory and activate `cat-gtnet-paper` environment
+
+```bash
+conda activate cat-gtnet-paper
+```
+
+Contigs
+-------
+Run CAT on chunks created in "Settup up data" above. Do this using a Slurm Job Array
+
+```bash
+sbatch cmd_contigs.array.sh
+```
+
+Bins
+----
+Run CAT on bins created in "Settup up data" above. Do this using a Slurm Job Array
+
+```bash
+sbatch cmd_bins.array.sh
+```
+
+
+Running Sourmash
+================
+Create and activate the `sourmash-gtnet-paper` environment:
+
+```bash
+conda create --file=sourmash-gtnet-paper.yaml
+conda activate sourmash-gtnet-paper
+```
+
+Run from `./sourmash` and activate `sourmash-gtnet-paper` environment
+
+Contigs
+-------
+Run Sourmash on chunks created in "Settup up data" above. Do this using a Slurm Job Array
+
+```bash
+sbatch cmd_contigs.array.sh
+```
+
+This relies on the script `cmd_contigs.sh`. It should be backed up with this file, but in case it is not, here is what it should look like:
+
+```bash
+i=${1:?"Missing partition argument"}
+
+K="51"
+INDIR="../contigs"
+DB="gtdb-rs207.genomic-reps.dna.k$K.lca.json.gz"
+FNA="$INDIR/metadata_r207.test.$i.fna"
+SIG="$INDIR/metadata_r207.test.$i.sig"
+CSV="$INDIR/metadata_r207.test.$i.sourmash.csv"
+
+sourmash sketch dna $FNA --singleton -p scaled=10000,k=$K -o $SIG
+sourmash lca classify --db $DB --query $SIG -o $CSV
+```
+
+Bins
+----
+Run Sourmash on chunks created in "Settup up data" above. Do this using a Slurm Job Array
+
+```bash
+sbatch cmd_bins.array.sh
+```
+
+This relies on the script `cmd_bins.sh`. It should be backed up with this file, but in case it is not, here is what it should look like:
+
+```bash
+i=${1:?"Missing partition argument"}
+
+K="51"
+INDIR="../bins"
+DB="gtdb-rs207.genomic-reps.dna.k$K.lca.json.gz"
+FNA_DIR="$INDIR/$i"
+SIG="$INDIR/metadata_r207.test.$i.sig"
+CSV="$INDIR/metadata_r207.test.$i.sourmash.csv"
+
+sourmash sketch dna $FNA_DIR/*.fna -p scaled=10000,k=51 -o $SIG
+sourmash lca classify --db $DB --query $SIG -o $CSV
+```
+
+
+
+Running GTNet
+=============
+Create and activate the `gtnet-gtnet-paper` environment:
+
+```bash
+conda create --file=gtnet-gtnet-paper.yaml
+conda activate gtnet-gtnet-paper
+```
+
+Run from `./gtnet` and activate `gtnet-gtnet-paper` environment
+
+Contigs
+-------
+Run GTNet on chunks created in "Settup up data" above. Do this using a Slurm Job Array
+
+```bash
+sbatch cmd_contigs.array.sh
+```
+
+This relies on the script `cmd_contigs.sh`. It should be backed up with this file, but in case it is not, here is what it should look like:
+
+```bash
+arg=$((SLURM_LOCALID + 1))
+part=${!arg}
+
+INDIR="../contigs"
+FNA="$INDIR/metadata_r207.test.$part.fna"
+RAW="$INDIR/metadata_r207.test.$part.gtnet.raw.csv"
+CSV="$INDIR/metadata_r207.test.$part.gtnet.csv"
+
+gtnet predict -s -g -D $SLURM_LOCALID -o $RAW $FNA
+gtnet filter -o $CSV $RAW
+```
+
+Bins
+----
+Run GTNet on chunks created in "Settup up data" above. Do this using a Slurm Job Array
+
+```bash
+sbatch cmd_bins.array.sh
+```
+
+This relies on the script `cmd_bins.sh`. It should be backed up with this file, but in case it is not, here is what it should look like:
+
+```bash
+part=${1:?"Missing partition argument"}
+
+INDIR="../bins"
+FNA_DIR="$INDIR/$part"
+CSV="$INDIR/metadata_r207.test.$part.gtnet.$SLURM_LOCALID.csv"
+
+gtnet classify -g -D $SLURM_LOCALID -o $CSV `ls $FNA_DIR/*.fna | awk "NR % 4 == $SLURM_LOCALID"`
+```

gtnet_paper/sota/append_all_chunks.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#SBATCH -J append
+#SBATCH -A m2865
+#SBATCH -C cpu
+#SBATCH -q regular
+#SBATCH -t 2
+#SBATCH -n 1
+#SBATCH -o contigs/slurm/append.%A_%a.log
+#SBATCH -e contigs/slurm/append.%A_%a.log
+#SBATCH -n 1
+#SBATCH --array=0-1999
+
+module load parallel
+date +%s
+i=`printf "%04d\n" $SLURM_ARRAY_TASK_ID`
+fof=contigs/metadata_r207.test.$i.fof
+out=contigs/metadata_r207.test.$i.fna
+echo $fof $out
+cat $fof | parallel -j 128 "deep-taxon append-accessions {}" > $out
+date +%s

gtnet_paper/sota/bins/make_dir.sh

@@ -0,0 +1,6 @@
+fof=$1
+dir=`basename $fof | cut -f3 -d.`
+mkdir $dir
+for f in `cat $fof`; do
+    zcat $f > $dir/`basename $f .gz`
+done

gtnet_paper/sota/cat-gtnet-paper.yaml

@@ -0,0 +1,73 @@
+name: cat-gtnet-paper
+channels:
+  - bioconda
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - blast=2.14.0=h7d5a4b4_1
+  - bzip2=1.0.8=h7f98852_4
+  - c-ares=1.19.0=hd590300_0
+  - ca-certificates=2023.5.7=hbcca054_0
+  - cat=5.2.3=hdfd78af_1
+  - curl=8.1.0=h409715c_0
+  - diamond=2.1.6=h5b5514e_1
+  - entrez-direct=16.2=he881be0_1
+  - gettext=0.21.1=h27087fc_0
+  - keyutils=1.6.1=h166bdaf_0
+  - krb5=1.20.1=h81ceb04_0
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - libcurl=8.1.0=h409715c_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=h516909a_1
+  - libexpat=2.5.0=hcb278e6_1
+  - libffi=3.4.2=h7f98852_5
+  - libgcc-ng=12.2.0=h65d4601_19
+  - libgomp=12.2.0=h65d4601_19
+  - libidn2=2.3.4=h166bdaf_0
+  - libnghttp2=1.52.0=h61bc06f_0
+  - libnsl=2.0.0=h7f98852_0
+  - libsqlite=3.42.0=h2797004_0
+  - libssh2=1.10.0=hf14f497_3
+  - libstdcxx-ng=12.2.0=h46fd767_19
+  - libunistring=0.9.10=h7f98852_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libzlib=1.2.13=h166bdaf_4
+  - ncurses=6.3=h27087fc_1
+  - openssl=3.1.0=hd590300_3
+  - pcre=8.45=h9c3ff4c_0
+  - perl=5.32.1=2_h7f98852_perl5
+  - perl-archive-tar=2.40=pl5321hdfd78af_0
+  - perl-carp=1.50=pl5321hd8ed1ab_0
+  - perl-common-sense=3.75=pl5321hd8ed1ab_0
+  - perl-compress-raw-bzip2=2.201=pl5321h166bdaf_0
+  - perl-compress-raw-zlib=2.202=pl5321h166bdaf_0
+  - perl-encode=3.19=pl5321h166bdaf_0
+  - perl-exporter=5.74=pl5321hd8ed1ab_0
+  - perl-exporter-tiny=1.002002=pl5321hd8ed1ab_0
+  - perl-extutils-makemaker=7.70=pl5321hd8ed1ab_0
+  - perl-io-compress=2.201=pl5321hdbdd923_2
+  - perl-io-zlib=1.14=pl5321hdfd78af_0
+  - perl-json=4.10=pl5321hdfd78af_0
+  - perl-json-xs=2.34=pl5321h4ac6f70_6
+  - perl-list-moreutils=0.430=pl5321hdfd78af_0
+  - perl-list-moreutils-xs=0.430=pl5321h031d066_2
+  - perl-parent=0.241=pl5321hd8ed1ab_0
+  - perl-pathtools=3.75=pl5321h166bdaf_0
+  - perl-scalar-list-utils=1.63=pl5321h166bdaf_0
+  - perl-storable=3.15=pl5321h166bdaf_0
+  - perl-types-serialiser=1.01=pl5321hdfd78af_0
+  - pip=23.1.2=pyhd8ed1ab_0
+  - prodigal=2.6.3=h031d066_6
+  - python=3.11.3=h2755cc3_0_cpython
+  - readline=8.2=h8228510_1
+  - setuptools=67.7.2=pyhd8ed1ab_0
+  - tk=8.6.12=h27826a3_0
+  - tzdata=2023c=h71feb2d_0
+  - wget=1.20.3=ha35d2d1_1
+  - wheel=0.40.0=pyhd8ed1ab_0
+  - xz=5.2.6=h166bdaf_0
+  - zlib=1.2.13=h166bdaf_4
+  - zstd=1.5.2=h3eb15da_6
+prefix: /pscratch/sd/a/ajtritt/.conda/envs/cat-gtnet-paper

gtnet_paper/sota/cat/cmd_bins.array.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#SBATCH -J cat_bins
+#SBATCH -A m2865
+#SBATCH -C cpu
+#SBATCH -q preempt
+#SBATCH -t 70
+#SBATCH -o /pscratch/sd/a/ajtritt/exabiome/deep-taxon/gtnet/sota/bins/cat_log/%A_%a.log
+#SBATCH -e /pscratch/sd/a/ajtritt/exabiome/deep-taxon/gtnet/sota/bins/cat_log/%A_%a.log
+#SBATCH -n 1
+#SBATCH --array=1-1999
+
+i=`printf "%04d\n" $SLURM_ARRAY_TASK_ID`
+
+BINS_DIR="/pscratch/sd/a/ajtritt/exabiome/deep-taxon/gtnet/sota/bins"
+BIN_DIR="$BINS_DIR/$i"
+OUT_PFX="$BINS_DIR/metadata_r207.test.$i.BAT"
+
+echo "T_START `date +%s`"
+CAT.git/CAT_pack/CAT bins --force \
+--bin_folder $BIN_DIR \
+--database_folder gtdb_prepared/db \
+--taxonomy_folder gtdb_prepared/tax \
+--out_prefix $OUT_PFX
+echo "T_END `date +%s`"

gtnet_paper/sota/cat/cmd_contigs.array.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#SBATCH -J cat_ctgs
+#SBATCH -A m2865
+#SBATCH -C cpu
+#SBATCH -q preempt
+#SBATCH -t 70
+#SBATCH -o /pscratch/sd/a/ajtritt/exabiome/deep-taxon/gtnet/sota/contigs/cat_log/%A_%a.log
+#SBATCH -e /pscratch/sd/a/ajtritt/exabiome/deep-taxon/gtnet/sota/contigs/cat_log/%A_%a.log
+#SBATCH -n 1
+#SBATCH --array=1-1999
+
+i=`printf "%04d\n" $SLURM_ARRAY_TASK_ID`
+
+CHUNK_DIR="/pscratch/sd/a/ajtritt/exabiome/deep-taxon/gtnet/sota/contigs"
+BASE="$CHUNK_DIR/metadata_r207.test.$i"
+FNA="$BASE.fna"
+OUT_PFX="$BASE.CAT"
+
+echo "T_START `date +%s`"
+CAT.git/CAT_pack/CAT contigs --force \
+--contigs_fasta $FNA \
+--database_folder gtdb_prepared/db \
+--taxonomy_folder gtdb_prepared/tax \
+--out_prefix $OUT_PFX
+echo "T_END `date +%s`"