Merge pull request #139 from wri/develop

Develop to master
wri · Feb 10, 2022 · 06805db · 06805db
2 parents d264efc + c3c2960
commit 06805db
Show file tree

Hide file tree

Showing 195 changed files with 4,809 additions and 3,531 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -0,0 +1,71 @@
+name: Run tests and apply terraform changes for current branch
+
+on: [ push ]
+
+permissions:
+  id-token: write
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    container:
+        image: quay.io/azavea/openjdk-gdal:3.1-jdk8-slim
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: coursier/cache-action@v6
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-region: us-east-1
+          role-to-assume: arn:aws:iam::617001639586:role/gfw-data-lake-read
+
+      - name: run tests
+        run: ./sbt ++$SCALA_VERSION coverage test coverageReport
+
+      - name: Run codacy-coverage-reporter
+        uses: codacy/codacy-coverage-reporter-action@master
+        with:
+          project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
+          coverage-reports: target/scala-2.12/coverage-report/cobertura.xml
+
+      - name: Run CodeCOV action
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: target/scala-2.12/coverage-report/cobertura.xml
+          flags: unittests
+          name: codecov-umbrella
+          fail_ci_if_error: false
+          verbose: false
+
+  publish:
+    name: Publish Artifacts
+    needs: [build]
+    if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v'))
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: coursier/cache-action@v6
+
+      - uses: actions/setup-java@v2
+        with:
+          distribution: temurin
+          java-version: 11
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-region: us-east-1
+          role-to-assume: arn:aws:iam::617001639586:role/gfw-geotrellis-jars-write
+
+      - name: Build Assembly
+        run: sbt assembly
+
+      - name: Publish Assembly
+        run: aws s3 cp target/scala-2.12/*.jar s3://gfwpro-geotrellis-jars/release/
diff --git a/.github/workflows/terraform_build.yaml b/.github/workflows/terraform_build.yaml
diff --git a/.scalafmt.conf b/.scalafmt.conf
@@ -0,0 +1,16 @@
+version = "3.0.0-RC3"
+maxColumn = 138
+continuationIndent.defnSite = 2
+binPack.parentConstructors = true
+binPack.literalArgumentLists = false
+newlines.penalizeSingleSelectMultiArgList = false
+newlines.sometimesBeforeColonInMethodReturnType = false
+align.openParenCallSite = false
+align.openParenDefnSite = false
+rewriteTokens {
+  "⇒" = "=>"
+  "←" = "<-"
+}
+optIn.selfAnnotationNewline = false
+optIn.breakChainOnFirstMethodDot = true
+importSelectors = BinPack
diff --git a/README.md b/README.md
@@ -8,7 +8,7 @@ This project performs a polygonal summary on tree cover loss and intersecting la
 
 Currently the following analysis are implemented
 
-*   Tree Cover Loss
+*   Tree Cover Loss (for ArcPy client)
 *   Annual Update
 *   Annual Update minimal
 *   Carbon Flux Full Standard Model
@@ -19,18 +19,20 @@ Currently the following analysis are implemented
 
 ### Tree Cover Loss
 
-A simple analysis which only looks at Tree Cover Loss, Tree Cover Density (2000 or 2010) and optionally Primary Forest.
+A simple analysis which only looks at tree cover loss, tree cover density (2000 or 2010), aboveground biomass,
+gross GHG emissions, gross carbon removals, and net GHG flux. Aboveground carbon, belowground carbon, and soil carbon can be optionally analyzed.
 Users can select one or many tree cover thresholds. Output will be a flat file, with one row per input feature and tree cover density threshold.
+Optional contextual analysis layers include plantations and humid tropical primary forests. 
 The emissions outputs (annual and total) are from the forest carbon flux model (Harris et al. 2021 Nature Climate Change) [forest carbon flux model](https://github.com/wri/carbon-budget).
-Outputs also include carbon removals and carbon net flux (total only). 
+Outputs also include carbon removals and carbon net flux (total only) (Harris et al. 2021). 
 Emissions, removals, and net flux are reported for (> tree cover density 2000 threshold OR Hansen gain) because the model results
 include not just pixels above a certain tree cover density threshold, but also Hansen gain pixels.
-Other outputs from this analysis (loss, gain, biomass, etc.) use the simple tree cover density threshold. 
+Other outputs from this analysis (loss, gain, biomass, carbon pools, etc.) use the simple tree cover density threshold. 
 
 This type of analysis only supports simple features as input. Best used together with the [ArcPY Client](https://github.com/wri/gfw_forest_loss_geotrellis_arcpy_client).
 
 ```sbt
-sparkSubmitMain org.globalforestwatch.summarystats.SummaryMain --analysis treecoverloss --feature_type feature --features s3://bucket/prefix/file.tsv --output s3://bucket/prefix
+sparkSubmitMain org.globalforestwatch.summarystats.SummaryMain treecoverloss --feature_type feature --features s3://bucket/prefix/file.tsv --output s3://bucket/prefix --tcd 2000 --threshold 0 --threshold 30 --contextual layer is__gfw_plantations --carbon_pools
 ```
 
 ### Annual Update
@@ -69,7 +71,7 @@ sparkSubmitMain org.globalforestwatch.summarystats.SummaryMain --analysis annual
 
 Carbon Flux (full standard model) analysis is used to produce detailed statistics for carbon flux research, with additional model-specific contextual layers.
 It uses the same approach as the annual update analysis, but with all stock and flux outputs from the [forest carbon flux model](https://github.com/wri/carbon-budget). 
-It also analyzes several contextual layers that are unique to the carbon flux model and does not include many contextual layers used in the Tree Cover Loss analyses. 
+It also analyzes several contextual layers that are unique to the carbon flux model and does not include many contextual layers used in the annual update minimal analysis. 
 It currently only works with GADM features.
 
 ```sbt
@@ -165,9 +167,10 @@ The following options are supported:
 |admin2           |string|`gadm` features          |Filter by country Admin2 code                                                                                                    |
 |id_start         |int   |`feature` analysis       |Filter by IDs larger than or equal to given value                                                                                |
 |wdpa_status      |string|`wdpa` features          |Filter by WDPA Status                                                                                                            |
-|tcd              |int   |`treecover` analysis     |Select tree cover density year                                                                                                   |
-|threshold        |int   |`treecover` analysis     |Treecover threshold to apply (multiple)                                                                                          |
-|contextual_layer |string|`treecover` analysis     |Include (multiple) selected contextual layers: `is__umd_regional_primary_forest_2001`, `is__gfw_plantations`                     |
+|tcd              |int   |`treecoverloss` analysis |Select tree cover density year                                                                                                   |
+|threshold        |int   |`treecoverloss` analysis |Treecover threshold to apply (multiple)                                                                                          |
+|contextual_layer |string|`treecoverloss` analysis |Include (multiple) selected contextual layers: `is__umd_regional_primary_forest_2001`, `is__gfw_plantations`                     |
+|carbon_pools     |flag  |`treecoverloss` analysis |Optionally calculate stock sums for multiple carbon pools in 2000 (aboveground, belowground, soil)                                    |
 |tcl              |flag  |all                      |Filter input feature by TCL tile extent, requires boolean `tcl` field in input feature class                                     |
 |glad             |flag  |all                      |Filter input feature by GLAD tile extent, requires boolean `glad` field in input feature class                                   |
 |change_only      |flag  |all except `treecover`   |Process change only                                                                                                              |

diff --git a/build.sbt b/build.sbt
@@ -7,7 +7,7 @@ licenses := Seq(
 )
 
 scalaVersion := Version.scala
-scalaVersion in ThisBuild := Version.scala
+ThisBuild  / scalaVersion := Version.scala
 
 scalacOptions ++= Seq(
   "-deprecation",
@@ -19,10 +19,12 @@ scalacOptions ++= Seq(
   "-language:postfixOps",
   "-language:existentials",
   "-language:experimental.macros",
-  "-Ypartial-unification" // Required by Cats
+  "-Ypartial-unification", // Required by Cats
+  "-Ywarn-unused-import",
+  "-Yrangepos"
 )
 publishMavenStyle := true
-publishArtifact in Test := false
+Test / publishArtifact := false
 pomIncludeRepository := { _ =>
   false
 }
@@ -55,6 +57,7 @@ libraryDependencies ++= Seq(
   sparkCore,
   sparkSQL,
   sparkHive,
+  frameless,
   hadoopAws,
   hadoopCommon,
   hadoopMapReduceClientCore,
@@ -65,30 +68,33 @@ libraryDependencies ++= Seq(
   scalactic % Test,
   geotrellisSpark,
   geotrellisSparkTestKit % Test,
+  sparkFastTests % Test,
   geotrellisS3,
-  geotrellisShapefile,
-  geotrellisGeotools,
-  geotrellisVectorTile,
   geotrellisGdal,
-  logging,
-  decline,
   sedonaCore,
   sedonaSQL,
-  //  jtsCore,
-  //  jts2geojson,
-  geoToolsOGRBridj,
-  bridj,
   breeze,
   breezeNatives,
   breezeViz,
   sparkDaria,
+  "org.datasyslab" % "geotools-wrapper" % "geotools-24.1",
+  "org.wololo" % "jts2geojson" % "0.14.3",
+  jts
 )
 
 dependencyOverrides += "com.google.guava" % "guava" % "20.0"
 
+assembly / assemblyShadeRules := {
+  val shadePackage = "org.globalforestwatch.shaded"
+  Seq(
+    ShadeRule.rename("shapeless.**" -> s"$shadePackage.shapeless.@1").inAll,
+    ShadeRule.rename("cats.kernel.**" -> s"$shadePackage.cats.kernel.@1").inAll
+  )
+}
+
 // auto imports for local SBT console
 // can be used with `test:console` command
-initialCommands in console :=
+console / initialCommands :=
   """
 import java.net._
 //import geotrellis.raster._
@@ -119,17 +125,18 @@ import org.globalforestwatch.util._
 """
 
 // settings for local testing
-console / fork := true
+Compile / run := Defaults.runTask(Compile / fullClasspath, Compile / run / mainClass, Compile / run / runner).evaluated
+Compile / runMain := Defaults.runMainTask(Compile / fullClasspath , Compile / runMain / runner)
+
 Test / fork := true
 Test / parallelExecution := false
 Test / testOptions += Tests.Argument("-oD")
-Test / javaOptions ++= Seq("-Xms1024m", "-Xmx8144m")
+Test / javaOptions ++= Seq("-Xms1024m", "-Xmx8144m", "-Djts.overlay=ng")
 Test / envVars := Map("AWS_REQUEST_PAYER" -> "requester")
 
 // Settings for sbt-assembly plugin which builds fat jars for use by spark jobs
-test in assembly := {}
-assemblyOption in assembly := (assemblyOption in assembly).value.copy(appendContentHash = true)
-assemblyMergeStrategy in assembly := {
+assembly / test := {}
+assembly / assemblyMergeStrategy  := {
   case "reference.conf" => MergeStrategy.concat
   case "application.conf" => MergeStrategy.concat
   // both GeoSpark and Geotrellis bring in this library, need to use GeoSpark version

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -20,21 +20,23 @@ object Version {
   val breeze = "0.13.2"
   val decline = "1.3.0"
   val scala = "2.12.12"
-  val geotrellis = "3.5.2"
+  val geotrellis = "3.7.0-SNAPSHOT"
   val hadoop = "3.2.1"
   val jackson = "2.10.0"
-  val spark = "3.0.0"
+  val spark = "3.1.1"
   val sparkCompatible = "3.0"
-  val geotools = "23.1"
-  val sedona = "1.0.0-incubating"
+  val geotools = "24.1"
+  val sedona = "1.1.1-incubating"
+  val frameless = "0.9.0"
+  val jts = "1.18.2"
 }
 
 object Dependencies {
 
   private val dependencyScope = "provided"
 
   val sparkJts = "org.locationtech.geomesa" %% "geomesa-spark-jts" % "2.3.1"
-
+  val jts = "org.locationtech.jts" % "jts-core" % Version.jts
   val geotrellisSpark = "org.locationtech.geotrellis" %% "geotrellis-spark" % Version.geotrellis
   val geotrellisS3 = "org.locationtech.geotrellis" %% "geotrellis-s3" % Version.geotrellis
   val geotrellisRaster = "org.locationtech.geotrellis" %% "geotrellis-raster" % Version.geotrellis
@@ -60,6 +62,7 @@ object Dependencies {
   val sparkCore = "org.apache.spark" %% "spark-core" % Version.spark % dependencyScope exclude("org.apache.hadoop", "*")
   val sparkSQL = "org.apache.spark" %% "spark-sql" % Version.spark % dependencyScope exclude("org.apache.hadoop", "*")
   val sparkHive = "org.apache.spark" %% "spark-hive" % Version.spark % dependencyScope exclude("org.apache.hadoop", "*")
+  val frameless = "org.typelevel" %% "frameless-dataset" % Version.frameless
 
   val hadoopClient = "org.apache.hadoop" % "hadoop-client" % Version.hadoop % dependencyScope
   val hadoopMapReduceClientCore = "org.apache.hadoop" % "hadoop-mapreduce-client-core" % Version.hadoop % dependencyScope
@@ -72,8 +75,10 @@ object Dependencies {
   //  val sedonaSQL = "org.apache.sedona" %% "sedona-sql-".concat(
   //    Version.sparkCompatible
   //  ) % Version.sedona
-  val sedonaCore = "org.datasyslab" % "geospark" % "1.3.2-SNAPSHOT"
-  val sedonaSQL = "org.datasyslab" % "geospark-sql_3.0" % "1.3.2-SNAPSHOT"
+
+  val sedonaCore = "org.apache.sedona" %% "sedona-core-3.0" % Version.sedona
+  val sedonaSQL = "org.apache.sedona" %% "sedona-sql-3.0" % Version.sedona
+
   //  val jts2geojson = "org.wololo" % "jts2geojson" % "0.14.3" % "compile"
   //  val jtsCore = "org.locationtech.jts" % "jts-core" % "1.16.1" % "compile"  // 1.18.0 ?
   val geoToolsOGRBridj = "org.geotools" % "gt-ogr-bridj" % Version.geotools exclude("com.nativelibs4java", "bridj")
@@ -82,4 +87,5 @@ object Dependencies {
   val breezeNatives = "org.scalanlp" %% "breeze-natives" % Version.breeze
   val breezeViz = "org.scalanlp" %% "breeze-viz" % Version.breeze
   val sparkDaria = "com.github.mrpowers" % "spark-daria_2.12" % "0.38.2"
+  val sparkFastTests = "com.github.mrpowers" %% "spark-fast-tests" % "1.0.0"
 }
diff --git a/project/build.properties b/project/build.properties
@@ -1 +1 @@
-sbt.version=1.4.3
+sbt.version=1.5.8
diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -1,5 +1,7 @@
+addDependencyTreePlugin
 addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.5.4")
-addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.0")
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0")
 addSbtPlugin("net.pishen" % "sbt-lighter" % "1.2.0")
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.8.1")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.8.2")
+addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1")
+addCompilerPlugin("org.scalameta" % "semanticdb-scalac" % "4.4.30" cross CrossVersion.full)