Merge pull request #78 from jaa127/txn-filters

txn-filters: prototype implementation of transaction filters. PR is for review of design and for comments.

.gitignore

@@ -14,5 +14,5 @@ project/target/
 /gui/bin/
 /gui/.cache-main
 /gui/.cache-tests
-/testCases/*/out.*
-*.idea/
+/testCases/**/out.*
+/**/.idea/

base/src/main/scala/co/uproot/abandon/Ast.scala

@@ -118,6 +118,134 @@ case class Date(year: Int, month: Int, day: Int) {
   }
 }
 
+/**
+ * Trait for stackable Transaction filters
+ * These filters operate on "raw" AST level,
+ * so all filtering decisions are based on unprocessed
+ * information.
+ *
+ * TxnFilterStack is used to glue these filters together.
+ */
+sealed trait TransactionFilter {
+  def filter(txn: Transaction): Boolean
+  def description(): String
+  def xmlDescription(): xml.Node
+}
+
+/*
+ * Txn:Time filters
+ * To create filter for time span, stack "onOrAfter" and "before" filters.
+ */
+case class BeforeDateTxnFilter(before: Date) extends TransactionFilter {
+  override def filter(txn: Transaction) = { txn.date.toInt < before.toInt }
+  override def description() = { "before: Transaction date is before: " + before.formatISO8601Ext }
+  override def xmlDescription() = { <filter type="before" date={ before.formatISO8601Ext } />}
+}
+case class OnOrAfterDateTxnFilter(onOrAfter: Date) extends TransactionFilter {
+  override def filter(txn: Transaction) = { onOrAfter.toInt <= txn.date.toInt }
+  override def description() = { "onOrAfter: Transaction date is on or after: " + onOrAfter.formatISO8601Ext }
+  override def xmlDescription() = { <filter type="onOrAfter" date={ onOrAfter.formatISO8601Ext } />}
+}
+
+case class PayeeTxnFilter(regex: String) extends TransactionFilter {
+  val pattern = java.util.regex.Pattern.compile(regex)
+
+  override def filter(txn: Transaction) = {
+    pattern.matcher(txn.payeeOpt match {
+      case Some(payee) => payee
+      case None => ""
+    }).matches
+  }
+  override def description() = { "payee: Payee must match \"" + pattern.toString + "\""}
+  override def xmlDescription() = { <filter type="payee" pattern={ pattern.toString } /> }
+}
+
+/**
+ * Annotation Txn filter
+ *  - returns all transactions with matching annotation
+ */
+case class AnnotationTxnFilter(regex: String) extends TransactionFilter {
+  val pattern = java.util.regex.Pattern.compile(regex)
+
+  override def filter(txn: Transaction) = {
+    pattern.matcher(txn.annotationOpt match {
+      case Some(ann) => ann
+      case None => ""
+    }).matches
+  }
+  override def description() = { "annotation: Annotation must match \"" + pattern.toString + "\"" }
+  override def xmlDescription() = { <filter type="annotation" pattern={ pattern.toString }/> }
+}
+
+/**
+ * Account Name Txn filter
+ * Returns all transactions which have at least one matching account name
+ */
+case class AccountNameTxnFilter(regex: String) extends TransactionFilter {
+  val pattern = java.util.regex.Pattern.compile(regex)
+
+  override def filter(txn: Transaction) = {
+    txn.posts.exists { post =>
+      pattern.matcher(post.accName.toString).matches
+    }
+  }
+  override def description() = { "account: At least one of transaction's accounts must match \"" + pattern.toString + "\"" }
+  override def xmlDescription() = { <filter type="account" pattern={ pattern.toString }/> }
+}
+
+// TODO Txn comment filter
+// TODO Txn:Post comment filter
+
+object FilterStackHelper {
+  def getFilterWarnings(txnFilters: Option[TxnFilterStack], indent: String): List[String] = {
+    txnFilters match {
+      case Some(txnFilters) => {
+        indent + txnFilters.description() ::
+        txnFilters.filterDescriptions().map { desc =>
+          indent * 2 + desc
+        }.toList
+      }
+      case None => Nil
+    }
+  }
+}
+
+/**
+ * Trait for Transaction filter stacks
+ * Filter stack defines relationship between filters
+ * (e.g. f1 && f2,  f1 || f2 or some other, specialized logic)
+ */
+sealed trait TxnFilterStack {
+  def filter(txn: Transaction): Boolean
+  def description(): String
+  def filterDescriptions(): Seq[String]
+  def xmlDescription(): xml.Node
+}
+
+/**
+ * AND- TxnFilterStack (e.g. f1 && f2 && ..)
+ */
+case class ANDTxnFilterStack(filterStack: Seq[TransactionFilter]) extends TxnFilterStack {
+  override def filter(txn: Transaction): Boolean = {
+    filterStack.forall { f => f.filter(txn) }
+  }
+  override def description() = {
+    assert(!filterStack.isEmpty)
+    "All following conditions must be true:"
+  }
+  override def filterDescriptions() = {
+    assert(!filterStack.isEmpty)
+    filterStack.map({case f => f.description})
+  }
+  override def xmlDescription(): xml.Node = {
+    assert(!filterStack.isEmpty)
+    <filters type="every">
+    {
+      filterStack.map({ filt => filt.xmlDescription })
+    }
+    </filters>
+  }
+}
 
 object DateOrdering extends Ordering[Date] {
   def compare(x: Date, y: Date) = {

base/src/main/scala/co/uproot/abandon/Config.scala

@@ -16,6 +16,7 @@ class AbandonCLIConf(arguments: Seq[String]) extends ScallopConf(arguments) {
   val inputs = opt[List[String]]("input", short = 'i')
   val reports = opt[List[String]]("report", short = 'r')
   val config = opt[String]("config", short = 'c')
+  val filters = propsLong[String]("filter", descr="Transaction filters", keyName=" name")
   val unversioned = opt[Boolean]("unversioned", short = 'X')
   val quiet = opt[Boolean]("quiet", short = 'q')
   // val trail = trailArg[String]()
@@ -33,19 +34,57 @@ object SettingsHelper {
     }
   }
 
+  def createTxnFilter(key: String, value: String): TransactionFilter = {
+    def makeDate(date: String) = {
+      val jDate = java.time.LocalDate.parse(date,
+          java.time.format.DateTimeFormatter.ISO_DATE)
+      Date(jDate.getYear, jDate.getMonthValue, jDate.getDayOfMonth)
+    }
+
+    (key, value) match {
+      case (key, value) if (key == "onOrAfter") => {
+        OnOrAfterDateTxnFilter(makeDate(value))
+      }
+      case (key, value) if (key == "before") => {
+        BeforeDateTxnFilter(makeDate(value))
+      }
+      case (key, value) if (key == "payee") => {
+        PayeeTxnFilter(value)
+      }
+      case (key, value) if (key == "account") => {
+        AccountNameTxnFilter(value)
+      }
+      case (key, value) if (key == "annotation") => {
+        AnnotationTxnFilter(value)
+      }
+      case _ => {
+        throw new RuntimeException("Unknown filter: " + key)
+      }
+    }
+  }
+
   def getCompleteSettings(args: Seq[String]): Either[String, Settings] = {
     val cliConf = new AbandonCLIConf(args)
     cliConf.verify()
     val configOpt = cliConf.config.toOption
     val withoutVersion = cliConf.unversioned.getOrElse(false)
     val quiet = cliConf.quiet.getOrElse(false)
+    val txnFilters =
+      if (cliConf.filters.isEmpty) {
+        None
+      }else {
+        val txnfs: Seq[TransactionFilter] =
+          cliConf.filters.map({case (k,v) => createTxnFilter(k, v)}).toSeq
+        Option(ANDTxnFilterStack(txnfs))
+      }
+
     configOpt match {
       case Some(configFileName) =>
-        makeSettings(configFileName, withoutVersion, quiet)
+        makeSettings(configFileName, withoutVersion, quiet, txnFilters)
       case _ =>
         val inputs = cliConf.inputs.toOption.getOrElse(Nil)
         val allReport = BalanceReportSettings("All Balances", None, Nil, true)
-        Right(Settings(inputs, Nil, Nil, Seq(allReport), ReportOptions(Nil), Nil, None, quiet))
+        Right(Settings(inputs, Nil, Nil, Seq(allReport), ReportOptions(Nil), Nil, None, quiet, txnFilters))
     }
   }
 
@@ -61,7 +100,7 @@ object SettingsHelper {
     }
   }
 
-  def makeSettings(configFileName: String, withoutVersion: Boolean, quiet: Boolean) = {
+  def makeSettings(configFileName: String, withoutVersion: Boolean, quiet: Boolean, txnFiltersCLI: Option[TxnFilterStack]) = {
     def handleInput(input: String, confPath: String): List[String] = {
       val parentPath = Processor.mkParentDirPath(confPath)
       if (input.startsWith("glob:")) {
@@ -86,8 +125,26 @@ object SettingsHelper {
         val accountConfigs = config.optConfigList("accounts").getOrElse(Nil)
         val accounts = accountConfigs.map(makeAccountSettings)
         val eodConstraints = config.optConfigList("eodConstraints").getOrElse(Nil).map(makeEodConstraints(_))
+
+       /*
+        * filters, precedence
+        *  - conf none, cli none => None
+        *  - conf none, cli some => cli
+        *  - conf some, cli some => cli
+        */
+        val txnFilters = txnFiltersCLI match {
+          case Some(txnfs) => Option(txnfs)
+          case None =>
+            try {
+              val txnfs = config.getStringList("filters").asScala.map(s => s.split("=", 2)).
+                  map({ case Array(k, v) => createTxnFilter(k, v) })
+              Option(ANDTxnFilterStack(txnfs))
+            } catch {
+              case e: ConfigException.Missing => None
+            }
+        }
         val dateConstraints = config.optConfigList("dateConstraints").getOrElse(Nil).map(makeDateRangeConstraint(_))
-        Right(Settings(inputs, eodConstraints ++ dateConstraints, accounts, reports, ReportOptions(isRight), exports, Some(file), quiet))
+        Right(Settings(inputs, eodConstraints ++ dateConstraints, accounts, reports, ReportOptions(isRight), exports, Some(file), quiet, txnFilters))
       } catch {
         case e: ConfigException => Left(e.getMessage)
       }
@@ -277,7 +334,8 @@ case class Settings(
   reportOptions: ReportOptions,
   exports: Seq[ExportSettings],
   configFileOpt: Option[java.io.File],
-  quiet: Boolean) {
+  quiet: Boolean,
+  txnFilters: Option[TxnFilterStack]) {
   def getConfigRelativePath(path: String) = {
     configFileOpt.map(configFile => Processor.mkRelativeFileName(path, configFile.getAbsolutePath)).getOrElse(path)
   }

base/src/main/scala/co/uproot/abandon/Process.scala

@@ -201,9 +201,14 @@ object Processor {
     (new java.io.File(path)).getCanonicalPath
   }
 
-  def process(scope: Scope, accountSettings: Seq[AccountSettings]) = {
+  def process(scope: Scope, accountSettings: Seq[AccountSettings], txnFilters: Option[TxnFilterStack]) = {
     scope.checkDupes()
-    val transactions = scope.allTransactions
+    val transactions = (filterByType[ScopedTxn](scope.allTransactions)).filter { scopeTxn =>
+      txnFilters match {
+        case Some(filterStack) => filterStack.filter(scopeTxn.txn)
+        case None => { true }
+      }
+    }
     val sortedTxns = transactions.sortBy(_.txn.date)(DateOrdering)
     val accState = new AccountState()
     val aliasMap = accountSettings.collect{ case AccountSettings(name, Some(alias)) => alias -> name }.toMap

base/src/main/scala/co/uproot/abandon/Report.scala

@@ -252,9 +252,19 @@ object Reports {
     }
   }
 
-  def xmlBalanceExport(state: AppState, exportSettings: XmlExportSettings): xml.Node = {
+  def xmlBalanceExport(state: AppState, exportSettings: XmlExportSettings, filterXML: Option[xml.Node]): xml.Node = {
     val balance: Elem =
       <abandon>
+        {
+        filterXML match {
+          case Some(xml) => {
+            <info>
+              { xml }
+            </info>
+          }
+          case None => ;
+        }
+        }
         <balance>
           {state.accState.mkTree(exportSettings.isAccountMatching).toXML}
         </balance>
@@ -267,9 +277,19 @@ object Reports {
     }
   }
 
-  def xmlJournalExport(state: AppState, exportSettings: XmlExportSettings): xml.Node = {
+  def xmlJournalExport(state: AppState, exportSettings: XmlExportSettings, filterXML: Option[xml.Node]): xml.Node = {
     val journal: Elem =
       <abandon>
+        {
+        filterXML match {
+          case Some(xml) => {
+            <info>
+              { xml }
+            </info>
+          }
+          case None => ;
+        }
+        }
         <journal>
           <transactions>{
             val sortedGroups = state.accState.postGroups.sortBy(_.date.toInt)
@@ -299,10 +319,15 @@ object Reports {
 
   def addAttribute(n: Elem, k: String, v: String) = n % new xml.UnprefixedAttribute(k, v, xml.Null)
 
-  def xmlExport(state: AppState, exportSettings: XmlExportSettings): xml.Node = {
+  def xmlExport(state: AppState, exportSettings: XmlExportSettings, txnFilters: Option[TxnFilterStack]): xml.Node = {
+    val filterXML: Option[xml.Node] = txnFilters match {
+      case Some(filters) => Option(filters.xmlDescription)
+      case None => None
+    }
+
     exportSettings.exportType match {
-      case JournalType => xmlJournalExport(state, exportSettings)
-      case BalanceType => xmlBalanceExport(state, exportSettings)
+      case JournalType => xmlJournalExport(state, exportSettings, filterXML)
+      case BalanceType => xmlBalanceExport(state, exportSettings, filterXML)
     }
   }
 }

base/src/test/scala/co/uproot/abandon/ComplexProcessTest.scala

@@ -20,13 +20,13 @@ class ComplexProcessTest extends FlatSpec with Matchers with Inside {
 
 	  val xmlBalSettings = XmlExportSettings(BalanceType, None, Seq("not-used.xml"), true)
 	  val xmlTxnSettings = XmlExportSettings(JournalType, None, Seq("not-used.xml"), true)
-	  val settings = Settings(Nil, Nil, Nil, Nil, ReportOptions(Nil), Seq(xmlBalSettings), None, quiet)
+	  val settings = Settings(Nil, Nil, Nil, Nil, ReportOptions(Nil), Seq(xmlBalSettings), None, quiet, None)
 
-	  val appState = Processor.process(scope, settings.accounts)
+	  val appState = Processor.process(scope,settings.accounts, None)
 	  //TODO: Processor.checkConstaints(appState, settings.eodConstraints)
 
-	  val xmlBalance = Reports.xmlExport(appState, xmlBalSettings)
-	  val xmlJournal = Reports.xmlExport(appState, xmlTxnSettings)
+	  val xmlBalance = Reports.xmlExport(appState, xmlBalSettings, settings.txnFilters)
+	  val xmlJournal = Reports.xmlExport(appState, xmlTxnSettings, settings.txnFilters)
 
 	  val refXMLBalance = scala.xml.XML.loadFile("testCases/refSmallBalance.xml")
 	  val refXMLJournal = scala.xml.XML.loadFile("testCases/refSmallJournal.xml")