Support conversion to sparse data format

- added a script to rename the dependency file after assembly

Author: Leonard Poon

rename-deps.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+BASE=$(dirname $0)
+TARGET="$BASE/target/scala-2.11/HLTA-assembly-1.1-deps.jar"
+if [[ -e $TARGET ]]; then
+  mv $TARGET $BASE/target/scala-2.11/HLTA-deps.jar
+else
+  echo "$TARGET does not exist."
+  exit -1
+fi

src/main/scala/tm/text/Convert.scala

@@ -16,13 +16,12 @@ import tm.util.ParMapReduce._
 import org.rogach.scallop._
 import tm.util.Arguments
 
-
 object Convert {
   class Conf(args: Seq[String]) extends Arguments(args) {
     banner("Usage: tm.text.Convert [OPTION]... name max-words max-n source")
     val name = trailArg[String](descr = "Name of data")
     val maxWords = trailArg[Int](descr = "Maximum number of words")
-    val concatenations = 
+    val concatenations =
       trailArg[Int](descr = "Number of concatentations for building n-grams")
     val source = trailArg[String](descr = "Source directory")
 
@@ -79,6 +78,10 @@ object Convert {
   def readFiles(name: String, source: Path): GenSeq[Document] = {
     logger.info("Finding files under {}", source)
     val paths = getFiles(source)
+    if (paths.isEmpty) {
+      logger.error("No text files found under {}", source)
+      throw new IllegalArgumentException("No text files found files under " + source)
+    }
     readFiles(paths)
   }
 

src/main/scala/tm/text/DataConverter.scala

@@ -50,6 +50,8 @@ object DataConverter {
     logger.info("Saving in HLCM format (binary data)")
     saveAsBinaryHlcm(name, s"${name}.txt",
       dictionary.words, countsByDocuments.seq, bowConverter)
+    logger.info("Saving in sparse data format (binary data)")
+    saveAsSparseData(s"${name}.sparse.txt", countsByDocuments.seq, dictionary.map)
 
     logger.info("done")
   }
@@ -233,11 +235,25 @@ object DataConverter {
     writer.close
   }
 
+  def saveAsSparseData(filename: String,
+    countsByDocuments: Seq[TokenCounts], indices: Map[NGram, Int]) = {
+    val writer = new PrintWriter(filename)
+
+    countsByDocuments.zipWithIndex.foreach { p =>
+      val rowId = p._2 + 1 // since the indices from zipWithIndex start with zero
+      // filter out any words not contained in the indices or those with zero counts 
+      p._1.filter(tc => indices.contains(tc._1) && tc._2 > 0)
+        .foreach { tc => writer.println(s"${rowId},${tc._1}") }
+    }
+
+    writer.close
+  }
+
   /**
    * Given a sequence of tokens, build the n-grams based on the tokens.  The
    * n-grams are built from two consecutive tokens.  Besides,
    * the constituent tokens must be contained in the given {@code base} dictionary.
-   * It is possible that after c concatenations n-grams, where n=2^c, may appear.  
+   * It is possible that after c concatenations n-grams, where n=2^c, may appear.
    */
   def buildNextNGrams(tokens: Seq[NGram], base: Dictionary): Iterator[NGram] =
     tokens.sliding(2)