Release 0.16.0 (#175)

README.md

@@ -72,3 +72,4 @@ See the [documentation](doc/Developing.md) for full details.
 ## Support
 
 wdlTools is _not_ an official product of DNAnexus. Please do not contact DNAnexus (or any employees thereof) for support. To report a bug or feature request, please open an issue in the [issue tracker](https://github.com/dnanexus-rnd/wdlTools/issues).
+

RELEASE_NOTES.md

@@ -1,5 +1,9 @@
 # Change log
 
+## 0.16.0 (2021-09-09)
+
+* Adds parameters for runtime and hint overrides to `Eval.Runtime` and `Eval.Meta` classes
+
 ## 0.15.0 (2021-08-16)
 
 * No longer unwraps optional type for an input with a default value

build.sbt

@@ -20,7 +20,8 @@ ThisBuild / organization := "com.dnanexus"
 ThisBuild / scalaVersion := "2.13.2"
 ThisBuild / developers := List(
     Developer("orodeh", "orodeh", "orodeh@dnanexus.com", url("https://github.com/dnanexus-rnd")),
-    Developer("jdidion", "jdidion", "jdidion@dnanexus.com", url("https://github.com/dnanexus-rnd"))
+    Developer("jdidion", "jdidion", "jdidion@dnanexus.com", url("https://github.com/dnanexus-rnd")),
+    Developer("r-i-v-a", "Riva Nathans", "rnathans-cf@dnanexus.com", url("https://github.com/dnanexus-rnd"))
 )
 ThisBuild / homepage := Some(url("https://github.com/dnanexus-rnd/wdlTools"))
 ThisBuild / scmInfo := Some(
@@ -40,17 +41,17 @@ lazy val wdlTools = root.settings(
 )
 
 lazy val dependencies = {
-  val dxCommonVersion = "0.7.0"
-  val antlr4Version = "4.9"
+  val dxCommonVersion = "0.8.0"
+  val antlr4Version = "4.9.2"
   val scallopVersion = "3.4.0"
-  val typesafeVersion = "1.3.3"
-  val scalateVersion = "1.9.6"
+  val typesafeVersion = "1.4.1"
+  val scalateVersion = "1.9.7"
   //val beardVersion = "0.3.1"
-  val sprayVersion = "1.3.5"
+  val sprayVersion = "1.3.6"
   val katanVersion = "0.6.1"
-  val re2jVersion = "1.5"
+  val re2jVersion = "1.6"
   val graphVersion = "1.13.2"
-  val scalatestVersion = "3.1.1"
+  val scalatestVersion = "3.2.9"
 
   Seq(
       "com.dnanexus" % "dxcommon" % dxCommonVersion,

project/plugins.sbt

@@ -1,7 +1,7 @@
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
 addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.8")
 addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1")
-addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.2.1")
+addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.3")
 addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1")
 // sbt-sonatype plugin used to publish artifact to maven central via sonatype nexus
 // sbt-pgp plugin used to sign the artifcat with pgp keys

src/main/resources/application.conf

@@ -1,5 +1,5 @@
 wdlTools {
-    version = "0.15.0"
+    version = "0.16.0"
 }
 
 #

src/main/scala/wdlTools/eval/Eval.scala

@@ -549,6 +549,24 @@ case class Eval(paths: EvalPaths,
     }
   }
 
+  def applyMap(exprs: Map[String, (WdlTypes.T, TAT.Expr)],
+               ctx: Bindings[String, V]): Bindings[String, V] = {
+    val init: Bindings[String, V] = WdlValueBindings.empty
+    exprs.foldLeft(init) {
+      case (bindings, (name, (wdlType, expr))) =>
+        val value = ctx.get(name).getOrElse {
+          // if the output parameter is optional, then it is okay if the
+          // expression fails to evaluate - we set the value to None
+          try {
+            applyExprAndCoerce(expr, wdlType, ctx.update(bindings.toMap))
+          } catch {
+            case _: EvalException if TypeUtils.isOptional(wdlType) => WdlValues.V_Null
+          }
+        }
+        bindings.add(name, value)
+    }
+  }
+
   /**
     * Given a multi-line string, determine the largest w such that each line
     * begins with at least w whitespace characters. Trailing whitespace is

src/main/scala/wdlTools/eval/Meta.scala

@@ -47,22 +47,32 @@ object MetaMap {
   * @param kvs mapping of meta key to value
   * @param userDefaultValues default values supplied by the users - these override any built-in defaults
   */
-class Meta(kvs: Map[String, TAT.MetaValue], userDefaultValues: Option[VBindings] = None)
+class Meta(kvs: Map[String, TAT.MetaValue],
+           overrideValues: Option[VBindings] = None,
+           userDefaultValues: Option[VBindings] = None)
     extends MetaMap(kvs) {
   val defaults: Map[String, WdlValues.V] = Map.empty
 
-  def contains(id: String): Boolean = kvs.contains(id)
+  def contains(id: String): Boolean = {
+    overrideValues.exists(_.contains(id)) ||
+    kvs.contains(id) ||
+    userDefaultValues.exists(_.contains(id)) ||
+    defaults.contains(id)
+  }
 
   override def get(id: String, wdlTypes: Vector[WdlTypes.T] = Vector.empty): Option[WdlValues.V] = {
+    overrideValues.flatMap(_.get(id, wdlTypes))
     super
       .get(id, wdlTypes)
       .orElse(userDefaultValues.flatMap(_.get(id, wdlTypes)))
       .orElse(defaults.get(id))
   }
 }
 
-case class Draft2Meta(meta: Map[String, TAT.MetaValue], userDefaultValues: Option[VBindings] = None)
-    extends Meta(meta, userDefaultValues) {
+case class Draft2Meta(meta: Map[String, TAT.MetaValue],
+                      overrideValues: Option[VBindings] = None,
+                      userDefaultValues: Option[VBindings] = None)
+    extends Meta(meta, overrideValues, userDefaultValues) {
   override protected def applyKv(id: String,
                                  value: TAT.MetaValue,
                                  wdlTypes: Vector[WdlTypes.T] = Vector.empty): WdlValues.V = {
@@ -75,25 +85,30 @@ case class Draft2Meta(meta: Map[String, TAT.MetaValue], userDefaultValues: Optio
   }
 }
 
-case class V1Meta(meta: Map[String, TAT.MetaValue], userDefaultValues: Option[VBindings] = None)
-    extends Meta(meta, userDefaultValues)
+case class V1Meta(meta: Map[String, TAT.MetaValue],
+                  overrideValues: Option[VBindings] = None,
+                  userDefaultValues: Option[VBindings] = None)
+    extends Meta(meta, overrideValues, userDefaultValues)
 
 object Meta {
   def create(
       version: WdlVersion,
       meta: Option[TAT.MetaSection],
+      overrideValues: Option[VBindings] = None,
       userDefaultValues: Option[VBindings] = None
   ): Meta = {
     val kvs = meta.map(_.kvs.toMap).getOrElse(Map.empty)
     version match {
-      case WdlVersion.Draft_2 => Draft2Meta(kvs, userDefaultValues)
-      case _                  => V1Meta(kvs, userDefaultValues)
+      case WdlVersion.Draft_2 => Draft2Meta(kvs, overrideValues, userDefaultValues)
+      case _                  => V1Meta(kvs, overrideValues, userDefaultValues)
     }
   }
 }
 
-case class Hints(hints: Option[TAT.MetaSection], userDefaultValues: Option[VBindings] = None)
-    extends Meta(hints.map(_.kvs).getOrElse(Map.empty), userDefaultValues) {
+case class Hints(hints: Option[TAT.MetaSection],
+                 overrideValues: Option[VBindings] = None,
+                 userDefaultValues: Option[VBindings] = None)
+    extends Meta(hints.map(_.kvs).getOrElse(Map.empty), overrideValues, userDefaultValues) {
   override val defaults: Map[String, WdlValues.V] = Map(
       Hints.ShortTaskKey -> V_Boolean(false),
       Hints.LocalizationOptionalKey -> V_Boolean(false),
@@ -144,7 +159,8 @@ object Hints {
 
   def create(
       hints: Option[TAT.MetaSection],
+      overrideValues: Option[VBindings] = None,
       userDefaultValues: Option[VBindings] = None
   ): Hints =
-    Hints(hints, userDefaultValues)
+    Hints(hints, overrideValues, userDefaultValues)
 }

src/main/scala/wdlTools/eval/Runtime.scala

@@ -10,7 +10,13 @@ case class DiskRequest(size: Long,
                        mountPoint: Option[String] = None,
                        diskType: Option[String] = None)
 
+/**
+  * Evaluation of runtime section. For any given runtime attribute, the value is
+  * resolved in the following order:
+  * overrides > runtime section > user defaults > spec defaults
+  */
 abstract class Runtime(runtime: Map[String, TAT.Expr],
+                       overrideValues: Option[VBindings],
                        userDefaultValues: Option[VBindings],
                        runtimeLocation: SourceLocation) {
   private var cache: Map[String, Option[V]] = Map.empty
@@ -24,6 +30,7 @@ abstract class Runtime(runtime: Map[String, TAT.Expr],
   def contains(id: String): Boolean = {
     allows(id) && (
         cache.contains(id) ||
+        overrideValues.exists(_.contains(id)) ||
         runtime.contains(id) ||
         (aliases.contains(id) && contains(aliases.get(id))) ||
         userDefaultValues.exists(_.contains(id)) ||
@@ -34,19 +41,23 @@ abstract class Runtime(runtime: Map[String, TAT.Expr],
   protected def applyKv(id: String, expr: TAT.Expr, wdlType: Vector[T] = Vector.empty): V
 
   def get(id: String, wdlTypes: Vector[T] = Vector.empty): Option[V] = {
-    if (!cache.contains(id)) {
-      val value = runtime.get(id) match {
-        case Some(expr) =>
-          Some(applyKv(id, expr, wdlTypes))
-        case None if aliases.contains(id) =>
-          get(aliases.get(id), wdlTypes)
-        case None =>
-          // TODO: check type
-          userDefaultValues.flatMap(_.get(id, wdlTypes)).orElse(defaults.get(id))
-      }
-      cache += (id -> value)
-    }
-    cache(id)
+    cache.getOrElse(
+        id, {
+          // TODO: check type for overrides and user defaults
+          val value = overrideValues.flatMap(_.get(id)).orElse {
+            runtime.get(id) match {
+              case Some(expr) =>
+                Some(applyKv(id, expr, wdlTypes))
+              case None if aliases.contains(id) =>
+                get(aliases.get(id), wdlTypes)
+              case None =>
+                userDefaultValues.flatMap(_.get(id, wdlTypes)).orElse(defaults.get(id))
+            }
+          }
+          cache += (id -> value)
+          value
+        }
+    )
   }
 
   def getAll: Map[String, WdlValues.V] = {
@@ -103,9 +114,13 @@ case class DefaultRuntime(runtime: Option[TAT.RuntimeSection],
                           evaluator: Eval,
                           wdlVersion: WdlVersion,
                           ctx: Option[Bindings[String, V]],
+                          overrideValues: Option[VBindings] = None,
                           userDefaultValues: Option[VBindings],
                           runtimeLocation: SourceLocation)
-    extends Runtime(runtime.map(_.kvs).getOrElse(Map.empty), userDefaultValues, runtimeLocation) {
+    extends Runtime(runtime.map(_.kvs).getOrElse(Map.empty),
+                    overrideValues,
+                    userDefaultValues,
+                    runtimeLocation) {
   assert(wdlVersion < WdlVersion.V2, "DefaultRuntime is only for WDL versions < 2")
   val defaults: Map[String, V] = Map.empty
 
@@ -191,9 +206,13 @@ case class V2Runtime(runtime: Option[TAT.RuntimeSection],
                      evaluator: Eval,
                      wdlVersion: WdlVersion,
                      ctx: Option[Bindings[String, V]],
+                     overrideValues: Option[VBindings] = None,
                      userDefaultValues: Option[VBindings],
                      runtimeLocation: SourceLocation)
-    extends Runtime(runtime.map(_.kvs).getOrElse(Map.empty), userDefaultValues, runtimeLocation) {
+    extends Runtime(runtime.map(_.kvs).getOrElse(Map.empty),
+                    overrideValues,
+                    userDefaultValues,
+                    runtimeLocation) {
   assert(wdlVersion >= WdlVersion.V2, "V2Runtime only supports WDL versions >= 2")
 
   val defaults: Map[String, V] = Map(
@@ -365,11 +384,13 @@ object Runtime {
       task: TAT.Task,
       evaluator: Eval,
       ctx: Option[Bindings[String, V]] = None,
+      overrideValues: Option[VBindings] = None,
       defaultValues: Option[VBindings] = None
   ): Runtime = {
     create(task.runtime,
            evaluator,
            ctx,
+           overrideValues,
            defaultValues,
            Some(task.runtime.map(_.loc).getOrElse(task.loc)))
   }
@@ -378,6 +399,7 @@ object Runtime {
       runtime: Option[TAT.RuntimeSection],
       evaluator: Eval,
       ctx: Option[Bindings[String, V]] = None,
+      overrideValues: Option[VBindings] = None,
       defaultValues: Option[VBindings] = None,
       runtimeLocation: Option[SourceLocation] = None
   ): Runtime = {
@@ -390,9 +412,9 @@ object Runtime {
     evaluator.wdlVersion match {
       case None => throw new RuntimeException("no WdlVersion")
       case Some(wdlVersion: WdlVersion) if wdlVersion >= WdlVersion.V2 =>
-        V2Runtime(runtime, evaluator, wdlVersion, ctx, defaultValues, loc)
+        V2Runtime(runtime, evaluator, wdlVersion, ctx, overrideValues, defaultValues, loc)
       case Some(wdlVersion) =>
-        DefaultRuntime(runtime, evaluator, wdlVersion, ctx, defaultValues, loc)
+        DefaultRuntime(runtime, evaluator, wdlVersion, ctx, overrideValues, defaultValues, loc)
     }
   }