Replace remaining LSB protocol implementations with ordinary functions

bench/cljam/io/sam_bench.clj

@@ -40,9 +40,10 @@
   (are [f decode-opts?]
        (c/quick-bench
         (with-open [r (sam/reader f)]
-          (doseq [a (sam/read-alignments r)
-                  opts (when decode-opts? (:options a))]
-            opts)))
+          (run! (fn [aln]
+                  (when decode-opts?
+                    (dorun (:options aln))))
+                (sam/read-alignments r))))
     tcommon/test-bam-file false
     tcommon/test-bam-file true
     tcommon/medium-bam-file false
@@ -53,7 +54,8 @@
   (are [f]
        (c/quick-bench
         (with-open [r (sam/reader f)]
-          (doseq [a (vec (take 10000000 (sam/read-alignments r)))
-                  opts (:options a)]
-            opts)))
+          (transduce (take 10000000)
+                     (completing #(dorun (:options %2)))
+                     nil
+                     (sam/read-alignments r))))
     tcommon/large-bam-file))

bench/cljam/io/vcf_bench.clj

@@ -1,17 +1,45 @@
 (ns cljam.io.vcf-bench
   (:require [cljam.io.vcf :as vcf]
             [cljam.test-common :as tcommon]
+            [cljam.util :as util]
+            [clojure.java.io :as cio]
             [libra.bench :refer [are defbench]]
             [libra.criterium :as c]))
 
-(defbench decode-small-bcf-bench
+(defbench encode-variant-small-bcf-bench
+  (are [f]
+       (util/with-temp-dir [d "encode-variant-small-bench"]
+         (with-open [r (vcf/reader f)
+                     w (vcf/writer (cio/file d "out.bcf")
+                                   (vcf/meta-info r)
+                                   (vcf/header r))]
+           (let [vs (vec (vcf/read-variants r))]
+             (c/quick-bench
+              (vcf/write-variants w vs)))))
+    tcommon/test-bcf-complex-file))
+
+(defbench encode-variant-large-bcf-bench
+  (tcommon/prepare-cavia!)
+  (are [f]
+       (util/with-temp-dir [d "encode-variant-large-bench"]
+         (with-open [r (vcf/reader f)
+                     w (vcf/writer (cio/file d "out.bcf")
+                                   (vcf/meta-info r)
+                                   (vcf/header r))]
+           (let [vs (vec (vcf/read-variants-randomly r {:chr "chr1" :end 30000000} {}))]
+             (c/quick-bench
+              (vcf/write-variants w vs)))))
+    tcommon/test-large-bcf-file))
+
+(defbench decode-variant-small-bcf-bench
   (are [f]
        (c/quick-bench
         (with-open [r (vcf/reader f)]
           (run! (constantly nil) (vcf/read-variants r))))
     tcommon/test-bcf-complex-file))
 
-(defbench decode-large-bcf-bench
+(defbench decode-variant-large-bcf-bench
+  (tcommon/prepare-cavia!)
   (are [f]
        (c/quick-bench
         (with-open [r (vcf/reader f)]

src/cljam/algo/convert.clj

@@ -1,19 +1,19 @@
 (ns cljam.algo.convert
   "Converters between equivalent formats: SAM/BAM and FASTA/TwoBit."
-  (:require [clojure.tools.logging :as logging]
-            [clojure.string :as cstr]
-            [cljam.common :refer [*n-threads* get-exec-n-threads]]
-            [cljam.io.sam :as sam]
+  (:require [cljam.common :refer [*n-threads* get-exec-n-threads]]
             [cljam.io.bam.encoder :as encoder]
+            [cljam.io.fastq :as fq]
+            [cljam.io.sam :as sam]
             [cljam.io.sam.util.flag :as flag]
             [cljam.io.sam.util.refs :as refs]
-            [cljam.util.sequence :as util-seq]
             [cljam.io.sequence :as cseq]
-            [cljam.io.fastq :as fq]
             [cljam.io.util :as io-util]
+            [cljam.util.sequence :as util-seq]
+            [clojure.string :as cstr]
+            [clojure.tools.logging :as logging]
             [com.climate.claypoole :as cp])
-  (:import [java.nio ByteBuffer]
-           [cljam.io.fastq FASTQRead]))
+  (:import [cljam.io.fastq FASTQRead]
+           [java.io ByteArrayOutputStream]))
 
 ;;; SAM <-> BAM
 
@@ -30,9 +30,9 @@
         n-threads (get-exec-n-threads)]
     (doseq [blocks (cp/pmap (if (= n-threads 1) :serial (dec n-threads))
                             (fn [chunk']
-                              (mapv #(let [bb (ByteBuffer/allocate (encoder/get-block-size %))]
-                                       (encoder/encode-alignment bb % refs)
-                                       {:data (.array bb)})
+                              (mapv #(let [baos (ByteArrayOutputStream. (encoder/get-block-size %))]
+                                       (encoder/encode-alignment baos % refs)
+                                       {:data (.toByteArray baos)})
                                     chunk'))
                             (partition-all num-block (sam/read-alignments rdr {})))]
       (sam/write-blocks wtr blocks))))

src/cljam/io/bam/core.clj

@@ -6,7 +6,7 @@
              [writer :as writer]]
             [cljam.io.bam-index :as bai]
             [cljam.io.util.bgzf :as bgzf]
-            [cljam.io.util.lsb :as lsb]
+            [cljam.io.util.lsb.data-io :as lsb]
             [cljam.util :as util])
   (:import java.util.Arrays
            [java.io DataInputStream DataOutputStream IOException FileNotFoundException]

src/cljam/io/bam/encoder.clj

@@ -7,7 +7,7 @@
             [cljam.io.sam.util.quality :as qual]
             [cljam.io.sam.util.cigar :as cigar]
             [cljam.io.sam.util.sequence :as seq]
-            [cljam.io.util.lsb :as lsb]
+            [cljam.io.util.lsb.io-stream :as lsb]
             [cljam.io.bam.common :as common]))
 
 (def ^:private ^:const fixed-tag-size 3)

src/cljam/io/bam/reader.clj

@@ -5,7 +5,7 @@
             [cljam.io.sam.util.header :as header]
             [cljam.io.bam-index.core :as bai]
             [cljam.io.bam.decoder :as decoder]
-            [cljam.io.util.lsb :as lsb])
+            [cljam.io.util.lsb.data-io :as lsb])
   (:import [java.io Closeable FileNotFoundException]
            [cljam.io.bam.decoder BAMRawBlock]
            [bgzf4j BGZFInputStream]))

src/cljam/io/bam/writer.clj

@@ -4,7 +4,7 @@
             [cljam.io.protocols :as protocols]
             [cljam.io.sam.util.refs :as refs]
             [cljam.io.sam.util.header :as header]
-            [cljam.io.util.lsb :as lsb]
+            [cljam.io.util.lsb.io-stream :as lsb]
             [cljam.io.bam.common :as common]
             [cljam.io.bam.encoder :as encoder]
             [cljam.io.bam.decoder :as bam-decoder]

src/cljam/io/bam_index/reader.clj

@@ -1,6 +1,6 @@
 (ns cljam.io.bam-index.reader
   (:require [clojure.java.io :as cio]
-            [cljam.io.util.lsb :as lsb]
+            [cljam.io.util.lsb.io-stream :as lsb]
             [cljam.io.bam-index.common :refer [bai-magic]]
             [cljam.io.util.chunk :as chunk]
             [cljam.util :as util])

src/cljam/io/bam_index/writer.clj

@@ -2,7 +2,7 @@
   (:require [com.climate.claypoole :as cp]
             [cljam.common :refer [get-exec-n-threads]]
             [cljam.io.util.bgzf :as bgzf]
-            [cljam.io.util.lsb :as lsb]
+            [cljam.io.util.lsb.io-stream :as lsb]
             [cljam.io.util.bin :as util-bin]
             [cljam.io.bam-index.common :refer [linear-index-shift
                                                linear-index-depth

src/cljam/io/bcf/reader.clj

@@ -4,7 +4,7 @@
             [cljam.io.protocols :as protocols]
             [cljam.io.util.bgzf :as bgzf]
             [cljam.io.util.byte-buffer :as bb]
-            [cljam.io.util.lsb :as lsb]
+            [cljam.io.util.lsb.io-stream :as lsb]
             [cljam.io.vcf.reader :as vcf-reader]
             [cljam.io.vcf.util :as vcf-util]
             [cljam.util :as util]

src/cljam/io/bcf/writer.clj

@@ -2,7 +2,7 @@
   (:require [clojure.string :as cstr]
             [cljam.io.protocols :as protocols]
             [cljam.io.util.bgzf :as bgzf]
-            [cljam.io.util.lsb :as lsb]
+            [cljam.io.util.lsb.io-stream :as lsb]
             [cljam.io.vcf.writer :as vw]
             [cljam.io.vcf.util :as vcf-util]
             [cljam.util :as util])

src/cljam/io/bigwig.clj

@@ -4,7 +4,7 @@
   specifications."
   (:require [clojure.java.io :as cio]
             [cljam.io.protocols :as protocols]
-            [cljam.io.util.lsb :as lsb]
+            [cljam.io.util.lsb.data-io :as lsb]
             [cljam.util :as util])
   (:import [java.net URL]
            [java.io Closeable IOException RandomAccessFile]

src/cljam/io/csi.clj

@@ -1,13 +1,14 @@
 (ns cljam.io.csi
   "Basic I/O of CSI:Coordinate Sorted Index files."
-  (:require [clojure.string :as cstr]
-            [cljam.io.util.bgzf :as bgzf]
-            [cljam.io.util.lsb :as lsb]
+  (:require [cljam.io.util.bgzf :as bgzf]
+            [cljam.io.util.bin :as util-bin]
             [cljam.io.util.chunk :as chunk]
-            [cljam.io.util.bin :as util-bin])
-  (:import java.util.Arrays
-           [java.io DataInputStream DataOutputStream IOException]
-           [java.nio ByteBuffer ByteOrder]))
+            [cljam.io.util.lsb.data-io :as lsb.data]
+            [cljam.io.util.lsb.io-stream :as lsb.stream]
+            [clojure.string :as cstr])
+  (:import [java.io DataInputStream DataOutputStream IOException]
+           [java.nio ByteBuffer ByteOrder]
+           java.util.Arrays))
 
 (def ^:const ^:private csi-magic "CSI\1")
 
@@ -69,32 +70,32 @@
 
 (defn- read-chunks!
   [rdr]
-  (let [n-chunk (lsb/read-int rdr)]
-    (->> #(let [beg (lsb/read-long rdr) end (lsb/read-long rdr)]
+  (let [n-chunk (lsb.data/read-int rdr)]
+    (->> #(let [beg (lsb.data/read-long rdr) end (lsb.data/read-long rdr)]
             (chunk/->Chunk beg end))
          (repeatedly n-chunk)
          vec)))
 
 (defn- read-bin-index
   [rdr]
-  (let [n-ref (lsb/read-int rdr)]
-    (->> #(let [bin (lsb/read-int rdr)
-                loffset (lsb/read-long rdr)
+  (let [n-ref (lsb.data/read-int rdr)]
+    (->> #(let [bin (lsb.data/read-int rdr)
+                loffset (lsb.data/read-long rdr)
                 chunks (read-chunks! rdr)]
             {:bin (long bin), :loffset loffset, :chunks chunks})
          (repeatedly n-ref)
          vec)))
 
 (defn- read-index*
   ^CSI [^DataInputStream rdr]
-  (when-not (Arrays/equals ^bytes (lsb/read-bytes rdr 4) (.getBytes csi-magic))
+  (when-not (Arrays/equals ^bytes (lsb.data/read-bytes rdr 4) (.getBytes csi-magic))
     (throw (IOException. "Invalid CSI file")))
-  (let [min-shift (lsb/read-int rdr)
-        depth (lsb/read-int rdr)
-        l-aux (lsb/read-int rdr)
-        aux (lsb/read-bytes rdr l-aux)
+  (let [min-shift (lsb.data/read-int rdr)
+        depth (lsb.data/read-int rdr)
+        l-aux (lsb.data/read-int rdr)
+        aux (lsb.data/read-bytes rdr l-aux)
         tabix-aux (try (parse-tabix-aux aux) (catch Throwable _ nil))
-        n-ref (lsb/read-int rdr)
+        n-ref (lsb.data/read-int rdr)
         bins (vec (repeatedly n-ref #(read-bin-index rdr)))
         max-bin (util-bin/max-bin depth)
         bidx (mapv #(into {} (map (juxt :bin :chunks)) %) bins)
@@ -211,24 +212,24 @@
   [f ^CSI csi]
   (let [max-bin (util-bin/max-bin (.depth csi))]
     (with-open [w (DataOutputStream. (bgzf/bgzf-output-stream f))]
-      (lsb/write-bytes w (.getBytes ^String csi-magic))
-      (lsb/write-int w (.min-shift csi))
-      (lsb/write-int w (.depth csi))
+      (lsb.stream/write-bytes w (.getBytes ^String csi-magic))
+      (lsb.stream/write-int w (.min-shift csi))
+      (lsb.stream/write-int w (.depth csi))
       (let [tabix-aux (some-> (.aux csi) create-tabix-aux)]
-        (lsb/write-int w (count tabix-aux))
+        (lsb.stream/write-int w (count tabix-aux))
         (when tabix-aux
-          (lsb/write-bytes w tabix-aux)))
-      (lsb/write-int w (count (.bidx csi)))
+          (lsb.stream/write-bytes w tabix-aux)))
+      (lsb.stream/write-int w (count (.bidx csi)))
       (doseq [[offsets loffset] (map vector (.bidx csi) (.loffset csi))]
-        (lsb/write-int w (count offsets))
+        (lsb.stream/write-int w (count offsets))
         (doseq [[bin chunks] offsets]
-          (lsb/write-int w bin)
-          (lsb/write-long
+          (lsb.stream/write-int w bin)
+          (lsb.stream/write-long
            w
            (if (<= (long bin) max-bin)
              (get loffset (util-bin/bin-beg bin (.min-shift csi) (.depth csi)))
              0))
-          (lsb/write-int w (count chunks))
+          (lsb.stream/write-int w (count chunks))
           (doseq [chunk' chunks]
-            (lsb/write-long w (:beg chunk'))
-            (lsb/write-long w (:end chunk'))))))))
+            (lsb.stream/write-long w (:beg chunk'))
+            (lsb.stream/write-long w (:end chunk'))))))))

src/cljam/io/tabix.clj

@@ -2,7 +2,7 @@
   "Alpha - subject to change.
   Reader of a TABIX format file."
   (:require [cljam.io.util.bgzf :as bgzf]
-            [cljam.io.util.lsb :as lsb]
+            [cljam.io.util.lsb.data-io :as lsb]
             [cljam.io.util.bin :as util-bin]
             [clojure.string :as cstr])
   (:import java.util.Arrays

src/cljam/io/util/lsb.clj

@@ -1,5 +1,6 @@
 (ns cljam.io.util.lsb
   "Reading/writing functions of stream and buffer for little-endian data."
+  {:deprecated "0.8.5"}
   (:refer-clojure :exclude [read-string])
   (:require [cljam.util :refer [string->bytes]])
   (:import [java.io DataInput InputStream DataOutputStream EOFException ByteArrayOutputStream]