Merge pull request #174 from wtsi-npg/devel

prep for release 0.14.0 (merge devel to master)

.github/workflows/testing_and_building_repo.yml

@@ -0,0 +1,50 @@
+name: testing_and_building_repo
+on: [push, pull_request]
+jobs:
+  build: 
+    strategy:
+      matrix:
+        os: ['ubuntu-18.04']
+
+    runs-on: ${{matrix.os}} 
+    name: Workflow on ${{ matrix.os }} 
+    steps:
+    - uses: actions/checkout@v2 
+      with:
+        fetch-depth: '0'
+
+    - name: install libgd-dev and liblzma-dev
+      run:
+          sudo apt-get install -y libgd-dev liblzma-dev  libcurl4-openssl-dev
+
+    - name: clone samtools branch
+      run: |
+           export HTSLIB=${{ github.workspace }}/htslib
+           git clone --branch $HTSLIB_BRANCH git://github.com/samtools/htslib.git $HTSLIB
+      env:
+        HTSLIB_BRANCH: develop
+
+    - name: make HTSLIB
+      run: |
+          export HTSLIB=${{ github.workspace }}/htslib
+          export HTSDIR=${{ github.workspace }}/htslib/b
+          cd $HTSLIB
+          git submodule update --init --recursive
+          (autoreconf && mkdir b && ./configure --prefix=$HTSDIR && make && make install)
+          
+    - name: make and configure
+      run: |
+          export HTSDIR=${{ github.workspace }}/htslib/b
+          autoreconf -i
+          ./configure --with-htslib=$HTSDIR
+          make 
+          make check 
+          #make distcheck #TODO
+
+    # running ./Build dist and exporting files
+    - name: run ./Build dist  
+      run: |
+          make dist
+          export DIST_FILE=$(ls bambi-*.tar.gz)
+          export SHA256_FILE=$DIST_FILE.sha256
+          shasum -a 256 $DIST_FILE > $SHA256_FILE

Makefile.am

@@ -38,7 +38,8 @@ src_bambi_SOURCES = src/bambi.c \
                     src/parse_bam.c \
                     src/parse_bam.h \
                     src/crc.c \
-                    src/crc.h
+                    src/crc.h \
+                    src/substitution_analysis.c
 
 src_check_bcl_SOURCES = src/check_bcl.c src/bclfile.c src/bambi_utils.c src/array.c
 
@@ -60,7 +61,8 @@ TESTS = test/t_array \
         test/t_sf \
         test/t_seqchksum \
         test/t_adapters \
-        test/t_update
+        test/t_update \
+        test/t_sa
 
 dist_doc_DATA = README.md LICENSE
 
@@ -77,7 +79,8 @@ check_PROGRAMS = test/t_read2tags \
                  test/t_sf \
                  test/t_seqchksum \
                  test/t_adapters \
-                 test/t_update
+                 test/t_update \
+                 test/t_sa
 
 TEST_CFLAGS = -I$(top_srcdir)/src $(XML_CFLAGS) -DDATA_DIR=$(top_srcdir)/test/data
 TEST_LDADD = $(HTSLIB_LIBS) -lm
@@ -128,6 +131,10 @@ test_t_sf_SOURCES = test/t_sf.c src/bamit.c src/array.c src/bambi_utils.c
 test_t_sf_CFLAGS = $(TEST_CFLAGS)
 test_t_sf_LDADD = $(TEST_LDADD)
 
+test_t_sa_SOURCES = test/t_sa.c src/bamit.c src/bambi_utils.c
+test_t_sa_CFLAGS = $(TEST_CFLAGS)
+test_t_sa_LDADD = $(TEST_LDADD)
+
 test_t_seqchksum_SOURCES = test/t_seqchksum.c
 test_t_seqchksum_CFLAGS = $(TEST_CFLAGS)
 #test_t_seqchksum_LDADD = $(TEST_LDADD)

src/bambi.c

@@ -39,6 +39,7 @@ int main_spatial_filter(int argc, char *argv[]);
 int main_seqchksum(int argc, char *argv[]);
 int main_adapters(int argc, char *argv[]);
 int main_update(int argc, char *argv[]);
+int main_substitution_analysis(int argc, char *argv[]);
 
 const char *bambi_version()
 {
@@ -64,6 +65,7 @@ static void usage(FILE *fp)
 "     seqchksum      calculate checksums for a bam file\n"
 "     adapters       find and remove adapters\n"
 "     update         update an existing BAM/SAM/CRAM file\n"
+"     substitution_analysis   produce a substitution analysis table\n"
 "\n"
 "bambi <command> for help on a particular command\n"
 "\n");
@@ -93,6 +95,7 @@ int main(int argc, char *argv[])
     else if (strcmp(argv[1], "seqchksum") == 0) ret = main_seqchksum(argc-1, argv+1);
     else if (strcmp(argv[1], "adapters") == 0) ret = main_adapters(argc-1, argv+1);
     else if (strcmp(argv[1], "update") == 0) ret = main_update(argc-1, argv+1);
+    else if (strcmp(argv[1], "substitution_analysis") == 0) ret = main_substitution_analysis(argc-1, argv+1);
     else if (strcmp(argv[1], "--version") == 0) {
         printf( "bambi %s\n"
                 "Using htslib %s\n"

src/check_bcl.c

@@ -117,32 +117,6 @@ static char *uncompressBlock(char* abSrc, int nLenSrc, char* abDst, int nLenDst
     return msg;
 }
 
-#if 0
-// libdeflate version
-#include <libdeflate.h>
-static char *uncompressBlock(char* abSrc, int nLenSrc, char* abDst, int nLenDst )
-{
-    char *msg = NULL;
-    size_t actualOut;
-
-    if (nLenDst == 0) return 0;
-
-    int nErr;
-    struct libdeflate_decompressor *compressor = libdeflate_alloc_decompressor();
-    //LIBDEFLATEAPI *compressor = libdeflate_alloc_decompressor();
-
-        nErr = libdeflate_gzip_decompress(compressor, abSrc, nLenSrc, abDst, nLenDst, &actualOut);
-        if (nErr != LIBDEFLATE_SUCCESS) {
-            char buff[64];
-            sprintf(buff, "%02x %02x %02x %02x %02x", (uint8_t)abSrc[0], (uint8_t)abSrc[1], (uint8_t)abSrc[2], (uint8_t)abSrc[3], (uint8_t)abSrc[4]);
-            store_msg(&msg, "inflate() returned %d for data %s", nErr, buff);
-        }
-
-    libdeflate_free_decompressor(compressor);
-    return msg;
-}
-#endif
-
 static off_t find_tile_offset(bclfile_t *bcl, int tile, tilerec_t **ti_out)
 {
     off_t offset = bcl->header_size;
@@ -208,7 +182,7 @@ static int checkBclFile(char *fname, int verbose)
     bclfile_t *bcl;
     bcl = bclfile_open(fname, mt, tile);
     if (bcl->errmsg) {
-        display("File: %s\t%s\n", fname, bcl->errmsg);
+        display("Can't open File: %s\t%s\n", fname, bcl->errmsg);
         ret = 1;
     }
 
@@ -234,20 +208,17 @@ static int checkBclFile(char *fname, int verbose)
                 tilerec_t *tile = bcl->tiles->entries[n];
                 if (verbose) display("  %3d %6d %d\t%d\t%d\t", n, tile->tilenum, tile->nclusters, tile->uncompressed_blocksize, tile->compressed_blocksize);
                 char *msg = checkTile(bcl,tile->tilenum);
-                if (msg) ret = 1;
-                if (verbose) {
-                    if (msg) display("***FAIL***  %s\n", msg);
-                    else     display("Ok\n");
+                if (msg) {
+                    display("Failed Tile %6d for file %s: %s\n", tile->tilenum, fname, msg);
+                    ret = 1;
+                } else {
+                    if (verbose) display("Ok\n");
                 }
                 free(msg);
             }
         }
     }
 
-    if (!verbose && ret != 0) {
-        display("Failed %s check: %s\n", (ret==1 ? "Header" : "Tile"), fname);
-    }
-
     bclfile_close(bcl);
     return ret;
 }
@@ -343,7 +314,7 @@ int main(int argc, char *argv[])
     if (optind >= argc) Usage(stderr);
 
     if (isDirectory(argv[optind])) r = checkRunFolder(argv[optind]);
-    else                           r = checkBclFile(argv[optind], 1);
+    else                           r = checkBclFile(argv[optind], check_opts_verbose);
 
     return r;
 }

src/decode.c

@@ -633,7 +633,7 @@ int writeMetrics(va_t *barcodeArray, HashTable *tagHopHash, decode_opts_t *opts)
      * Now write tag hop metrics file - if there are any
      */
 
-    if (opts->idx2_len) {
+    if (opts->idx1_len && opts->idx2_len) {
         char *metrics_hops_name = malloc(strlen(opts->metrics_name)+6);
         strcpy(metrics_hops_name, opts->metrics_name);
         strcat(metrics_hops_name, ".hops");
@@ -968,7 +968,7 @@ char *findBarcodeName(char *barcode, va_t *barcodeArray, HashTable *barcodeHash,
     } else {
         bcd = findBestMatch(barcode, barcodeArray, barcodeHash, opts);
         if (isUpdateMetrics) updateMetrics(bcd, barcode, isPf);
-        if ((bcd == barcodeArray->entries[0]) && opts->idx2_len) {
+        if ((bcd == barcodeArray->entries[0]) && opts->idx1_len && opts->idx2_len) {
             bc_details_t *tag_hop = check_tag_hopping(barcode, barcodeArray, tagHopHash, opts);
             if (isUpdateMetrics && tag_hop) updateMetrics(tag_hop, barcode, isPf);
         }
@@ -1584,14 +1584,16 @@ size_t find_longest_barcode_name(va_t *barcodeArray)
 
 int get_barcode_metadata(va_t *barcodeArray, int idx,
                          const char **name_out, const char **lib_out,
-                         const char **sample_out, const char **desc_out) {
+                         const char **sample_out, const char **desc_out,
+                         const char **seq_out) {
     bc_details_t *barcode;
     if (!barcodeArray || idx >= barcodeArray->end) return -1;
     barcode = barcodeArray->entries[idx];
     if (name_out)   *name_out   = barcode->name;
     if (lib_out)    *lib_out    = barcode->lib;
     if (sample_out) *sample_out = barcode->sample;
     if (desc_out)   *desc_out   = barcode->desc;
+    if (seq_out)    *seq_out    = barcode->seq;
     return 0;
 }
 

src/decode.h

@@ -51,7 +51,8 @@ size_t find_longest_barcode_name(va_t *barcodeArray);
 // Get barcode metadata.  Returns -1 if idx is off the end of barcodeArray, else 0
 int get_barcode_metadata(va_t *barcodeArray, int idx,
                          const char **name_out, const char **lib_out,
-                         const char **sample_out, const char **desc_out);
+                         const char **sample_out, const char **desc_out,
+                         const char **seq_out);
 
 // Make a copy of a barcode array
 va_t *copy_barcode_array(va_t *barcode_array);

src/i2b.c

@@ -898,8 +898,8 @@ static int addHeader(samFile *output_file, bam_hdr_t *output_header, opts_t *opt
         char *pu = malloc(strlen(opts->platform_unit) + longest_name + 2);
         if (!id || !pu) die("Out of memory");
         for (int idx = 0; ; idx++) {
-            const char *name = NULL, *lib = NULL, *sample = NULL, *desc = NULL;
-            if (get_barcode_metadata(opts->barcodeArray, idx, &name, &lib, &sample, &desc) < 0) break;
+          const char *name = NULL, *lib = NULL, *sample = NULL, *desc = NULL, *seq = NULL;
+          if (get_barcode_metadata(opts->barcodeArray, idx, &name, &lib, &sample, &desc, &seq) < 0) break;
             if (idx == 0) {
                 lib    = opts->library_name;
                 sample = opts->sample_alias;
@@ -917,6 +917,7 @@ static int addHeader(samFile *output_file, bam_hdr_t *output_header, opts_t *opt
                         "CN", opts->sequencing_centre,
                         "PL", opts->platform,
                         (desc ? "DS" : NULL), (desc ? desc : NULL),
+                        (seq  ? "BC" : NULL), (seq ? seq : NULL),
                         NULL);
         }
         free(id);
@@ -2515,7 +2516,7 @@ static int createBAM(samFile *output_file, bam_hdr_t *output_header, hts_tpool *
         if (!tag_hops) die("Out of memory");
         barcodeHash = make_barcode_hash(opts->barcodeArray);
         longest_barcode_name = find_longest_barcode_name(opts->barcodeArray);
-        if (get_barcode_metadata(opts->barcodeArray, 0, &opts->unmatched_barcode_name, NULL, NULL, NULL) < 0) {
+        if (get_barcode_metadata(opts->barcodeArray, 0, &opts->unmatched_barcode_name, NULL, NULL, NULL, NULL) < 0) {
             opts->unmatched_barcode_name = "0";
         }
     }

src/parse_bam.c

@@ -327,11 +327,11 @@ bam1_t *parse_bam_readinfo( BAMit_t *fp,
         }
     }
 
-    *bam_lane = lane;
-    *bam_tile = tile;
-    *bam_x = x;
-    *bam_y = y;
-    *bam_read = read;
+    if (bam_lane) *bam_lane = lane;
+    if (bam_tile) *bam_tile = tile;
+    if (bam_x) *bam_x = x;
+    if (bam_y) *bam_y = y;
+    if (bam_read) *bam_read = read;
     if (bam_offset) *bam_offset = offset;
     return bam;
 }

src/spatial_filter.c

@@ -1146,7 +1146,7 @@ static void writeFilter(opts_t *s, RegionTable_t *rtsArray)
 	if (!fp) die("Can't open filter file %s: %s\n", s->filters->entries[0], strerror(errno));
 
 	strncpy(Fheader.region_magic, REGION_MAGIC, sizeof(Fheader.region_magic));
-	strncpy(Fheader.cmdLine, s->argv_list, sizeof(Fheader.cmdLine));
+	strncpy(Fheader.cmdLine, s->argv_list, sizeof(Fheader.cmdLine)-1);
     if (hwrite(fp, &Fheader, sizeof(Fheader)) < 0) die("writeFheader() failed\n");;
 
     for (int lane=1; lane < SF_MAX_LANES; lane++) {