diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..24858812 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,71 @@ +name: Run tests and apply terraform changes for current branch + +on: [ push ] + +permissions: + id-token: write + +jobs: + build: + + runs-on: ubuntu-latest + container: + image: quay.io/azavea/openjdk-gdal:3.1-jdk8-slim + + steps: + - uses: actions/checkout@v2 + + - uses: coursier/cache-action@v6 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: us-east-1 + role-to-assume: arn:aws:iam::617001639586:role/gfw-data-lake-read + + - name: run tests + run: ./sbt ++$SCALA_VERSION coverage test coverageReport + + - name: Run codacy-coverage-reporter + uses: codacy/codacy-coverage-reporter-action@master + with: + project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} + coverage-reports: target/scala-2.12/coverage-report/cobertura.xml + + - name: Run CodeCOV action + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: target/scala-2.12/coverage-report/cobertura.xml + flags: unittests + name: codecov-umbrella + fail_ci_if_error: false + verbose: false + + publish: + name: Publish Artifacts + needs: [build] + if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v')) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: coursier/cache-action@v6 + + - uses: actions/setup-java@v2 + with: + distribution: temurin + java-version: 11 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: us-east-1 + role-to-assume: arn:aws:iam::617001639586:role/gfw-geotrellis-jars-write + + - name: Build Assembly + run: sbt assembly + + - name: Publish Assembly + run: aws s3 cp target/scala-2.12/*.jar s3://gfwpro-geotrellis-jars/release/ \ No newline at end of file diff --git a/.github/workflows/terraform_build.yaml b/.github/workflows/terraform_build.yaml deleted file mode 100644 index 88ac614c..00000000 --- a/.github/workflows/terraform_build.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Run tests and apply terraform changes for current branch - -on: [ push ] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Test with pytest - run: | - ./scripts/test - - - name: Run codacy-coverage-reporter - uses: codacy/codacy-coverage-reporter-action@master - with: - project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} - coverage-reports: target/scala-2.12/coverage-report/cobertura.xml - - - name: Run CodeCOV action - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: target/scala-2.12/coverage-report/cobertura.xml - flags: unittests - name: codecov-umbrella - fail_ci_if_error: false - verbose: false \ No newline at end of file diff --git a/.scalafmt.conf b/.scalafmt.conf new file mode 100644 index 00000000..54f91cd6 --- /dev/null +++ b/.scalafmt.conf @@ -0,0 +1,16 @@ +version = "3.0.0-RC3" +maxColumn = 138 +continuationIndent.defnSite = 2 +binPack.parentConstructors = true +binPack.literalArgumentLists = false +newlines.penalizeSingleSelectMultiArgList = false +newlines.sometimesBeforeColonInMethodReturnType = false +align.openParenCallSite = false +align.openParenDefnSite = false +rewriteTokens { + "⇒" = "=>" + "←" = "<-" +} +optIn.selfAnnotationNewline = false +optIn.breakChainOnFirstMethodDot = true +importSelectors = BinPack \ No newline at end of file diff --git a/README.md b/README.md index 4dec942d..965962f9 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This project performs a polygonal summary on tree cover loss and intersecting la Currently the following analysis are implemented -* Tree Cover Loss +* Tree Cover Loss (for ArcPy client) * Annual Update * Annual Update minimal * Carbon Flux Full Standard Model @@ -19,18 +19,20 @@ Currently the following analysis are implemented ### Tree Cover Loss -A simple analysis which only looks at Tree Cover Loss, Tree Cover Density (2000 or 2010) and optionally Primary Forest. +A simple analysis which only looks at tree cover loss, tree cover density (2000 or 2010), aboveground biomass, +gross GHG emissions, gross carbon removals, and net GHG flux. Aboveground carbon, belowground carbon, and soil carbon can be optionally analyzed. Users can select one or many tree cover thresholds. Output will be a flat file, with one row per input feature and tree cover density threshold. +Optional contextual analysis layers include plantations and humid tropical primary forests. The emissions outputs (annual and total) are from the forest carbon flux model (Harris et al. 2021 Nature Climate Change) [forest carbon flux model](https://github.com/wri/carbon-budget). -Outputs also include carbon removals and carbon net flux (total only). +Outputs also include carbon removals and carbon net flux (total only) (Harris et al. 2021). Emissions, removals, and net flux are reported for (> tree cover density 2000 threshold OR Hansen gain) because the model results include not just pixels above a certain tree cover density threshold, but also Hansen gain pixels. -Other outputs from this analysis (loss, gain, biomass, etc.) use the simple tree cover density threshold. +Other outputs from this analysis (loss, gain, biomass, carbon pools, etc.) use the simple tree cover density threshold. This type of analysis only supports simple features as input. Best used together with the [ArcPY Client](https://github.com/wri/gfw_forest_loss_geotrellis_arcpy_client). ```sbt -sparkSubmitMain org.globalforestwatch.summarystats.SummaryMain --analysis treecoverloss --feature_type feature --features s3://bucket/prefix/file.tsv --output s3://bucket/prefix +sparkSubmitMain org.globalforestwatch.summarystats.SummaryMain treecoverloss --feature_type feature --features s3://bucket/prefix/file.tsv --output s3://bucket/prefix --tcd 2000 --threshold 0 --threshold 30 --contextual layer is__gfw_plantations --carbon_pools ``` ### Annual Update @@ -69,7 +71,7 @@ sparkSubmitMain org.globalforestwatch.summarystats.SummaryMain --analysis annual Carbon Flux (full standard model) analysis is used to produce detailed statistics for carbon flux research, with additional model-specific contextual layers. It uses the same approach as the annual update analysis, but with all stock and flux outputs from the [forest carbon flux model](https://github.com/wri/carbon-budget). -It also analyzes several contextual layers that are unique to the carbon flux model and does not include many contextual layers used in the Tree Cover Loss analyses. +It also analyzes several contextual layers that are unique to the carbon flux model and does not include many contextual layers used in the annual update minimal analysis. It currently only works with GADM features. ```sbt @@ -165,9 +167,10 @@ The following options are supported: |admin2 |string|`gadm` features |Filter by country Admin2 code | |id_start |int |`feature` analysis |Filter by IDs larger than or equal to given value | |wdpa_status |string|`wdpa` features |Filter by WDPA Status | -|tcd |int |`treecover` analysis |Select tree cover density year | -|threshold |int |`treecover` analysis |Treecover threshold to apply (multiple) | -|contextual_layer |string|`treecover` analysis |Include (multiple) selected contextual layers: `is__umd_regional_primary_forest_2001`, `is__gfw_plantations` | +|tcd |int |`treecoverloss` analysis |Select tree cover density year | +|threshold |int |`treecoverloss` analysis |Treecover threshold to apply (multiple) | +|contextual_layer |string|`treecoverloss` analysis |Include (multiple) selected contextual layers: `is__umd_regional_primary_forest_2001`, `is__gfw_plantations` | +|carbon_pools |flag |`treecoverloss` analysis |Optionally calculate stock sums for multiple carbon pools in 2000 (aboveground, belowground, soil) | |tcl |flag |all |Filter input feature by TCL tile extent, requires boolean `tcl` field in input feature class | |glad |flag |all |Filter input feature by GLAD tile extent, requires boolean `glad` field in input feature class | |change_only |flag |all except `treecover` |Process change only | diff --git a/build.sbt b/build.sbt index 5d4b150d..d9e08c43 100644 --- a/build.sbt +++ b/build.sbt @@ -7,7 +7,7 @@ licenses := Seq( ) scalaVersion := Version.scala -scalaVersion in ThisBuild := Version.scala +ThisBuild / scalaVersion := Version.scala scalacOptions ++= Seq( "-deprecation", @@ -19,10 +19,12 @@ scalacOptions ++= Seq( "-language:postfixOps", "-language:existentials", "-language:experimental.macros", - "-Ypartial-unification" // Required by Cats + "-Ypartial-unification", // Required by Cats + "-Ywarn-unused-import", + "-Yrangepos" ) publishMavenStyle := true -publishArtifact in Test := false +Test / publishArtifact := false pomIncludeRepository := { _ => false } @@ -55,6 +57,7 @@ libraryDependencies ++= Seq( sparkCore, sparkSQL, sparkHive, + frameless, hadoopAws, hadoopCommon, hadoopMapReduceClientCore, @@ -65,30 +68,33 @@ libraryDependencies ++= Seq( scalactic % Test, geotrellisSpark, geotrellisSparkTestKit % Test, + sparkFastTests % Test, geotrellisS3, - geotrellisShapefile, - geotrellisGeotools, - geotrellisVectorTile, geotrellisGdal, - logging, - decline, sedonaCore, sedonaSQL, - // jtsCore, - // jts2geojson, - geoToolsOGRBridj, - bridj, breeze, breezeNatives, breezeViz, sparkDaria, + "org.datasyslab" % "geotools-wrapper" % "geotools-24.1", + "org.wololo" % "jts2geojson" % "0.14.3", + jts ) dependencyOverrides += "com.google.guava" % "guava" % "20.0" +assembly / assemblyShadeRules := { + val shadePackage = "org.globalforestwatch.shaded" + Seq( + ShadeRule.rename("shapeless.**" -> s"$shadePackage.shapeless.@1").inAll, + ShadeRule.rename("cats.kernel.**" -> s"$shadePackage.cats.kernel.@1").inAll + ) +} + // auto imports for local SBT console // can be used with `test:console` command -initialCommands in console := +console / initialCommands := """ import java.net._ //import geotrellis.raster._ @@ -119,17 +125,18 @@ import org.globalforestwatch.util._ """ // settings for local testing -console / fork := true +Compile / run := Defaults.runTask(Compile / fullClasspath, Compile / run / mainClass, Compile / run / runner).evaluated +Compile / runMain := Defaults.runMainTask(Compile / fullClasspath , Compile / runMain / runner) + Test / fork := true Test / parallelExecution := false Test / testOptions += Tests.Argument("-oD") -Test / javaOptions ++= Seq("-Xms1024m", "-Xmx8144m") +Test / javaOptions ++= Seq("-Xms1024m", "-Xmx8144m", "-Djts.overlay=ng") Test / envVars := Map("AWS_REQUEST_PAYER" -> "requester") // Settings for sbt-assembly plugin which builds fat jars for use by spark jobs -test in assembly := {} -assemblyOption in assembly := (assemblyOption in assembly).value.copy(appendContentHash = true) -assemblyMergeStrategy in assembly := { +assembly / test := {} +assembly / assemblyMergeStrategy := { case "reference.conf" => MergeStrategy.concat case "application.conf" => MergeStrategy.concat // both GeoSpark and Geotrellis bring in this library, need to use GeoSpark version diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5e587ce5..99064ffb 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -20,13 +20,15 @@ object Version { val breeze = "0.13.2" val decline = "1.3.0" val scala = "2.12.12" - val geotrellis = "3.5.2" + val geotrellis = "3.7.0-SNAPSHOT" val hadoop = "3.2.1" val jackson = "2.10.0" - val spark = "3.0.0" + val spark = "3.1.1" val sparkCompatible = "3.0" - val geotools = "23.1" - val sedona = "1.0.0-incubating" + val geotools = "24.1" + val sedona = "1.1.1-incubating" + val frameless = "0.9.0" + val jts = "1.18.2" } object Dependencies { @@ -34,7 +36,7 @@ object Dependencies { private val dependencyScope = "provided" val sparkJts = "org.locationtech.geomesa" %% "geomesa-spark-jts" % "2.3.1" - + val jts = "org.locationtech.jts" % "jts-core" % Version.jts val geotrellisSpark = "org.locationtech.geotrellis" %% "geotrellis-spark" % Version.geotrellis val geotrellisS3 = "org.locationtech.geotrellis" %% "geotrellis-s3" % Version.geotrellis val geotrellisRaster = "org.locationtech.geotrellis" %% "geotrellis-raster" % Version.geotrellis @@ -60,6 +62,7 @@ object Dependencies { val sparkCore = "org.apache.spark" %% "spark-core" % Version.spark % dependencyScope exclude("org.apache.hadoop", "*") val sparkSQL = "org.apache.spark" %% "spark-sql" % Version.spark % dependencyScope exclude("org.apache.hadoop", "*") val sparkHive = "org.apache.spark" %% "spark-hive" % Version.spark % dependencyScope exclude("org.apache.hadoop", "*") + val frameless = "org.typelevel" %% "frameless-dataset" % Version.frameless val hadoopClient = "org.apache.hadoop" % "hadoop-client" % Version.hadoop % dependencyScope val hadoopMapReduceClientCore = "org.apache.hadoop" % "hadoop-mapreduce-client-core" % Version.hadoop % dependencyScope @@ -72,8 +75,10 @@ object Dependencies { // val sedonaSQL = "org.apache.sedona" %% "sedona-sql-".concat( // Version.sparkCompatible // ) % Version.sedona - val sedonaCore = "org.datasyslab" % "geospark" % "1.3.2-SNAPSHOT" - val sedonaSQL = "org.datasyslab" % "geospark-sql_3.0" % "1.3.2-SNAPSHOT" + + val sedonaCore = "org.apache.sedona" %% "sedona-core-3.0" % Version.sedona + val sedonaSQL = "org.apache.sedona" %% "sedona-sql-3.0" % Version.sedona + // val jts2geojson = "org.wololo" % "jts2geojson" % "0.14.3" % "compile" // val jtsCore = "org.locationtech.jts" % "jts-core" % "1.16.1" % "compile" // 1.18.0 ? val geoToolsOGRBridj = "org.geotools" % "gt-ogr-bridj" % Version.geotools exclude("com.nativelibs4java", "bridj") @@ -82,4 +87,5 @@ object Dependencies { val breezeNatives = "org.scalanlp" %% "breeze-natives" % Version.breeze val breezeViz = "org.scalanlp" %% "breeze-viz" % Version.breeze val sparkDaria = "com.github.mrpowers" % "spark-daria_2.12" % "0.38.2" + val sparkFastTests = "com.github.mrpowers" %% "spark-fast-tests" % "1.0.0" } diff --git a/project/build.properties b/project/build.properties index 947bdd30..e64c208f 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.4.3 +sbt.version=1.5.8 diff --git a/project/plugins.sbt b/project/plugins.sbt index 4e7b622a..855d38fe 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,5 +1,7 @@ +addDependencyTreePlugin addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.5.4") -addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.0") -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0") addSbtPlugin("net.pishen" % "sbt-lighter" % "1.2.0") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.8.1") \ No newline at end of file +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.8.2") +addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1") +addCompilerPlugin("org.scalameta" % "semanticdb-scalac" % "4.4.30" cross CrossVersion.full) \ No newline at end of file diff --git a/sbt b/sbt index bc1d9540..3c79c5f9 100755 --- a/sbt +++ b/sbt @@ -3,14 +3,42 @@ # A more capable sbt runner, coincidentally also called sbt. # Author: Paul Phillips # https://github.com/paulp/sbt-extras +# +# Generated from http://www.opensource.org/licenses/bsd-license.php +# Copyright (c) 2011, Paul Phillips. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set -o pipefail -declare -r sbt_release_version="0.13.16" -declare -r sbt_unreleased_version="0.13.16" +declare -r sbt_release_version="1.5.3" +declare -r sbt_unreleased_version="1.5.3" -declare -r latest_213="2.13.0-M4" -declare -r latest_212="2.12.6" +declare -r latest_213="2.13.6" +declare -r latest_212="2.12.14" declare -r latest_211="2.11.12" declare -r latest_210="2.10.7" declare -r latest_29="2.9.3" @@ -18,18 +46,17 @@ declare -r latest_28="2.8.2" declare -r buildProps="project/build.properties" -declare -r sbt_launch_ivy_release_repo="http://repo.typesafe.com/typesafe/ivy-releases" +declare -r sbt_launch_ivy_release_repo="https://repo.typesafe.com/typesafe/ivy-releases" declare -r sbt_launch_ivy_snapshot_repo="https://repo.scala-sbt.org/scalasbt/ivy-snapshots" -declare -r sbt_launch_mvn_release_repo="http://repo.scala-sbt.org/scalasbt/maven-releases" -declare -r sbt_launch_mvn_snapshot_repo="http://repo.scala-sbt.org/scalasbt/maven-snapshots" +declare -r sbt_launch_mvn_release_repo="https://repo1.maven.org/maven2" +declare -r sbt_launch_mvn_snapshot_repo="https://repo.scala-sbt.org/scalasbt/maven-snapshots" -declare -r default_jvm_opts_common="-Xms512m -Xmx1536m -Xss2m" -declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" +declare -r default_jvm_opts_common="-Xms512m -Xss2m -XX:MaxInlineLevel=18" +declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy -Dsbt.coursier.home=project/.coursier" declare sbt_jar sbt_dir sbt_create sbt_version sbt_script sbt_new declare sbt_explicit_version declare verbose noshare batch trace_level -declare debugUs declare java_cmd="java" declare sbt_launch_dir="$HOME/.sbt/launchers" @@ -41,13 +68,17 @@ declare -a java_args scalac_args sbt_commands residual_args # args to jvm/sbt via files or environment variables declare -a extra_jvm_opts extra_sbt_opts -echoerr () { echo >&2 "$@"; } -vlog () { [[ -n "$verbose" ]] && echoerr "$@"; } -die () { echo "Aborting: $@" ; exit 1; } +echoerr() { echo >&2 "$@"; } +vlog() { [[ -n "$verbose" ]] && echoerr "$@"; } +die() { + echo "Aborting: $*" + exit 1 +} -setTrapExit () { +setTrapExit() { # save stty and trap exit, to ensure echo is re-enabled if we are interrupted. - export SBT_STTY="$(stty -g 2>/dev/null)" + SBT_STTY="$(stty -g 2>/dev/null)" + export SBT_STTY # restore stty settings (echo in particular) onSbtRunnerExit() { @@ -63,11 +94,14 @@ setTrapExit () { # this seems to cover the bases on OSX, and someone will # have to tell me about the others. -get_script_path () { +get_script_path() { local path="$1" - [[ -L "$path" ]] || { echo "$path" ; return; } + [[ -L "$path" ]] || { + echo "$path" + return + } - local target="$(readlink "$path")" + local -r target="$(readlink "$path")" if [[ "${target:0:1}" == "/" ]]; then echo "$target" else @@ -75,10 +109,12 @@ get_script_path () { fi } -declare -r script_path="$(get_script_path "$BASH_SOURCE")" -declare -r script_name="${script_path##*/}" +script_path="$(get_script_path "${BASH_SOURCE[0]}")" +declare -r script_path +script_name="${script_path##*/}" +declare -r script_name -init_default_option_file () { +init_default_option_file() { local overriding_var="${!1}" local default_file="$2" if [[ ! -r "$default_file" && "$overriding_var" =~ ^@(.*)$ ]]; then @@ -90,67 +126,82 @@ init_default_option_file () { echo "$default_file" } -declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" -declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" +sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" +sbtx_opts_file="$(init_default_option_file SBTX_OPTS .sbtxopts)" +jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" -build_props_sbt () { - [[ -r "$buildProps" ]] && \ +build_props_sbt() { + [[ -r "$buildProps" ]] && grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }' } -set_sbt_version () { +set_sbt_version() { sbt_version="${sbt_explicit_version:-$(build_props_sbt)}" [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version export sbt_version } -url_base () { +url_base() { local version="$1" case "$version" in - 0.7.*) echo "http://simple-build-tool.googlecode.com" ;; - 0.10.* ) echo "$sbt_launch_ivy_release_repo" ;; + 0.7.*) echo "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/simple-build-tool" ;; + 0.10.*) echo "$sbt_launch_ivy_release_repo" ;; 0.11.[12]) echo "$sbt_launch_ivy_release_repo" ;; 0.*-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" - echo "$sbt_launch_ivy_snapshot_repo" ;; - 0.*) echo "$sbt_launch_ivy_release_repo" ;; - *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" - echo "$sbt_launch_mvn_snapshot_repo" ;; - *) echo "$sbt_launch_mvn_release_repo" ;; + echo "$sbt_launch_ivy_snapshot_repo" ;; + 0.*) echo "$sbt_launch_ivy_release_repo" ;; + *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmddThhMMss" + echo "$sbt_launch_mvn_snapshot_repo" ;; + *) echo "$sbt_launch_mvn_release_repo" ;; esac } -make_url () { +make_url() { local version="$1" local base="${sbt_launch_repo:-$(url_base "$version")}" case "$version" in - 0.7.*) echo "$base/files/sbt-launch-0.7.7.jar" ;; - 0.10.* ) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; + 0.7.*) echo "$base/sbt-launch-0.7.7.jar" ;; + 0.10.*) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; 0.11.[12]) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; - 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; - *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; + 0.*) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; + *) echo "$base/org/scala-sbt/sbt-launch/$version/sbt-launch-${version}.jar" ;; esac } -addJava () { vlog "[addJava] arg = '$1'" ; java_args+=("$1"); } -addSbt () { vlog "[addSbt] arg = '$1'" ; sbt_commands+=("$1"); } -addScalac () { vlog "[addScalac] arg = '$1'" ; scalac_args+=("$1"); } -addResidual () { vlog "[residual] arg = '$1'" ; residual_args+=("$1"); } +addJava() { + vlog "[addJava] arg = '$1'" + java_args+=("$1") +} +addSbt() { + vlog "[addSbt] arg = '$1'" + sbt_commands+=("$1") +} +addScalac() { + vlog "[addScalac] arg = '$1'" + scalac_args+=("$1") +} +addResidual() { + vlog "[residual] arg = '$1'" + residual_args+=("$1") +} -addResolver () { addSbt "set resolvers += $1"; } -addDebugger () { addJava "-Xdebug" ; addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; } -setThisBuild () { - vlog "[addBuild] args = '$@'" +addResolver() { addSbt "set resolvers += $1"; } + +addDebugger() { addJava "-Xdebug" && addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"; } + +setThisBuild() { + vlog "[addBuild] args = '$*'" local key="$1" && shift - addSbt "set $key in ThisBuild := $@" + addSbt "set $key in ThisBuild := $*" } -setScalaVersion () { +setScalaVersion() { [[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")' addSbt "++ $1" } -setJavaHome () { +setJavaHome() { java_cmd="$1/bin/java" setThisBuild javaHome "_root_.scala.Some(file(\"$1\"))" export JAVA_HOME="$1" @@ -159,13 +210,13 @@ setJavaHome () { } getJavaVersion() { - local str=$("$1" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d '"') + local -r str=$("$1" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d '"') # java -version on java8 says 1.8.x # but on 9 and 10 it's 9.x.y and 10.x.y. - if [[ "$str" =~ ^1\.([0-9]+)\..*$ ]]; then + if [[ "$str" =~ ^1\.([0-9]+)(\..*)?$ ]]; then echo "${BASH_REMATCH[1]}" - elif [[ "$str" =~ ^([0-9]+)\..*$ ]]; then + elif [[ "$str" =~ ^([0-9]+)(\..*)?$ ]]; then echo "${BASH_REMATCH[1]}" elif [[ -n "$str" ]]; then echoerr "Can't parse java version from: $str" @@ -175,8 +226,8 @@ getJavaVersion() { checkJava() { # Warn if there is a Java version mismatch between PATH and JAVA_HOME/JDK_HOME - [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java" - [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java" + [[ -n "$JAVA_HOME" && -e "$JAVA_HOME/bin/java" ]] && java="$JAVA_HOME/bin/java" + [[ -n "$JDK_HOME" && -e "$JDK_HOME/lib/tools.jar" ]] && java="$JDK_HOME/bin/java" if [[ -n "$java" ]]; then pathJavaVersion=$(getJavaVersion java) @@ -190,31 +241,32 @@ checkJava() { fi } -java_version () { - local version=$(getJavaVersion "$java_cmd") +java_version() { + local -r version=$(getJavaVersion "$java_cmd") vlog "Detected Java version: $version" echo "$version" } +is_apple_silicon() { [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; } + # MaxPermSize critical on pre-8 JVMs but incurs noisy warning on 8+ -default_jvm_opts () { - local v="$(java_version)" - if [[ $v -ge 8 ]]; then +default_jvm_opts() { + local -r v="$(java_version)" + if [[ $v -ge 10 ]]; then + if is_apple_silicon; then + # As of Dec 2020, JVM for Apple Silicon (M1) doesn't support JVMCI + echo "$default_jvm_opts_common" + else + echo "$default_jvm_opts_common -XX:+UnlockExperimentalVMOptions -XX:+UseJVMCICompiler" + fi + elif [[ $v -ge 8 ]]; then echo "$default_jvm_opts_common" else echo "-XX:MaxPermSize=384m $default_jvm_opts_common" fi } -build_props_scala () { - if [[ -r "$buildProps" ]]; then - versionLine="$(grep '^build.scala.versions' "$buildProps")" - versionString="${versionLine##build.scala.versions=}" - echo "${versionString%% .*}" - fi -} - -execRunner () { +execRunner() { # print the arguments one to a line, quoting any containing spaces vlog "# Executing command line:" && { for arg; do @@ -232,40 +284,36 @@ execRunner () { setTrapExit if [[ -n "$batch" ]]; then - "$@" < /dev/null + "$@" /dev/null 2>&1; then + if command -v curl >/dev/null 2>&1; then curl --fail --silent --location "$url" --output "$jar" - elif command -v wget > /dev/null 2>&1; then + elif command -v wget >/dev/null 2>&1; then wget -q -O "$jar" "$url" fi } && [[ -r "$jar" ]] } -acquire_sbt_jar () { +acquire_sbt_jar() { { sbt_jar="$(jar_file "$sbt_version")" [[ -r "$sbt_jar" ]] @@ -274,11 +322,66 @@ acquire_sbt_jar () { [[ -r "$sbt_jar" ]] } || { sbt_jar="$(jar_file "$sbt_version")" - download_url "$(make_url "$sbt_version")" "$sbt_jar" + jar_url="$(make_url "$sbt_version")" + + echoerr "Downloading sbt launcher for ${sbt_version}:" + echoerr " From ${jar_url}" + echoerr " To ${sbt_jar}" + + download_url "${jar_url}" "${sbt_jar}" + + case "${sbt_version}" in + 0.*) + vlog "SBT versions < 1.0 do not have published MD5 checksums, skipping check" + echo "" + ;; + *) verify_sbt_jar "${sbt_jar}" ;; + esac } } -usage () { +verify_sbt_jar() { + local jar="${1}" + local md5="${jar}.md5" + md5url="$(make_url "${sbt_version}").md5" + + echoerr "Downloading sbt launcher ${sbt_version} md5 hash:" + echoerr " From ${md5url}" + echoerr " To ${md5}" + + download_url "${md5url}" "${md5}" >/dev/null 2>&1 + + if command -v md5sum >/dev/null 2>&1; then + if echo "$(cat "${md5}") ${jar}" | md5sum -c -; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v md5 >/dev/null 2>&1; then + if [ "$(md5 -q "${jar}")" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + elif command -v openssl >/dev/null 2>&1; then + if [ "$(openssl md5 -r "${jar}" | awk '{print $1}')" == "$(cat "${md5}")" ]; then + rm -rf "${md5}" + return 0 + else + echoerr "Checksum does not match" + return 1 + fi + else + echoerr "Could not find an MD5 command" + return 1 + fi +} + +usage() { set_sbt_version cat < Run the specified file as a scala script # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version) - -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version - -sbt-version use the specified version of sbt (default: $sbt_release_version) - -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version - -sbt-jar use the specified jar as the sbt launcher - -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir) - -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version")) + -sbt-version use the specified version of sbt (default: $sbt_release_version) + -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version + -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version + -sbt-jar use the specified jar as the sbt launcher + -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir) + -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version")) # scala version (default: as chosen by sbt) - -28 use $latest_28 - -29 use $latest_29 - -210 use $latest_210 - -211 use $latest_211 - -212 use $latest_212 - -213 use $latest_213 - -scala-home use the scala build at the specified directory - -scala-version use the specified version of scala - -binary-version use the specified scala version when searching for dependencies + -28 use $latest_28 + -29 use $latest_29 + -210 use $latest_210 + -211 use $latest_211 + -212 use $latest_212 + -213 use $latest_213 + -scala-home use the scala build at the specified directory + -scala-version use the specified version of scala + -binary-version use the specified scala version when searching for dependencies # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) - -java-home alternate JAVA_HOME + -java-home alternate JAVA_HOME # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found - $(default_jvm_opts) - JVM_OPTS environment variable holding either the jvm args directly, or - the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') - Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. - -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) - -Dkey=val pass -Dkey=val directly to the jvm - -J-X pass option -X directly to the jvm (-J is stripped) + $(default_jvm_opts) + JVM_OPTS environment variable holding either the jvm args directly, or + the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') + Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. + -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) + -Dkey=val pass -Dkey=val directly to the jvm + -J-X pass option -X directly to the jvm (-J is stripped) # passing options to sbt, OR to this runner - SBT_OPTS environment variable holding either the sbt args directly, or - the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') - Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. - -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) - -S-X add -X to sbt's scalacOptions (-S is stripped) + SBT_OPTS environment variable holding either the sbt args directly, or + the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') + Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. + -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) + -S-X add -X to sbt's scalacOptions (-S is stripped) + + # passing options exclusively to this runner + SBTX_OPTS environment variable holding either the sbt-extras args directly, or + the reference to a file containing sbt-extras args if given path is prepended by '@' (e.g. '@/etc/sbtxopts') + Note: "@"-file is overridden by local '.sbtxopts' or '-sbtx-opts' argument. + -sbtx-opts file containing sbt-extras args (if not given, .sbtxopts in project root is used if present) EOM + exit 0 } -process_args () { - require_arg () { +process_args() { + require_arg() { local type="$1" local opt="$2" local arg="$3" @@ -365,50 +469,56 @@ process_args () { } while [[ $# -gt 0 ]]; do case "$1" in - -h|-help) usage; exit 0 ;; - -v) verbose=true && shift ;; - -d) addSbt "--debug" && shift ;; - -w) addSbt "--warn" && shift ;; - -q) addSbt "--error" && shift ;; - -x) debugUs=true && shift ;; - -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; - -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; - -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; - -no-share) noshare=true && shift ;; - -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; - -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; - -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; - -offline) addSbt "set offline in Global := true" && shift ;; - -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; - -batch) batch=true && shift ;; - -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; - -script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;; - - -sbt-create) sbt_create=true && shift ;; - -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; + -h | -help) usage ;; + -v) verbose=true && shift ;; + -d) addSbt "--debug" && shift ;; + -w) addSbt "--warn" && shift ;; + -q) addSbt "--error" && shift ;; + -x) shift ;; # currently unused + -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; + -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; + + -no-colors) addJava "-Dsbt.log.noformat=true" && addJava "-Dsbt.color=false" && shift ;; + -sbt-create) sbt_create=true && shift ;; + -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; + -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; + -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; + -no-share) noshare=true && shift ;; + -offline) addSbt "set offline in Global := true" && shift ;; + -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; + -batch) batch=true && shift ;; + -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; + -script) require_arg file "$1" "$2" && sbt_script="$2" && addJava "-Dsbt.main.class=sbt.ScriptMain" && shift 2 ;; + -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;; - -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; - -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; - -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; - -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; - -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; - -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; - -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;; - -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; - -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; - -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; - - -D*) addJava "$1" && shift ;; - -J*) addJava "${1:2}" && shift ;; - -S*) addScalac "${1:2}" && shift ;; - -28) setScalaVersion "$latest_28" && shift ;; - -29) setScalaVersion "$latest_29" && shift ;; - -210) setScalaVersion "$latest_210" && shift ;; - -211) setScalaVersion "$latest_211" && shift ;; - -212) setScalaVersion "$latest_212" && shift ;; - -213) setScalaVersion "$latest_213" && shift ;; - new) sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;; - *) addResidual "$1" && shift ;; + -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; + -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; + -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; + -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; + -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; + + -28) setScalaVersion "$latest_28" && shift ;; + -29) setScalaVersion "$latest_29" && shift ;; + -210) setScalaVersion "$latest_210" && shift ;; + -211) setScalaVersion "$latest_211" && shift ;; + -212) setScalaVersion "$latest_212" && shift ;; + -213) setScalaVersion "$latest_213" && shift ;; + + -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; + -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; + -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "_root_.scala.Some(file(\"$2\"))" && shift 2 ;; + -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; + -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; + -sbtx-opts) require_arg path "$1" "$2" && sbtx_opts_file="$2" && shift 2 ;; + -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; + + -D*) addJava "$1" && shift ;; + -J*) addJava "${1:2}" && shift ;; + -S*) addScalac "${1:2}" && shift ;; + + new) sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual "$1" && shift ;; + + *) addResidual "$1" && shift ;; esac done } @@ -420,19 +530,31 @@ process_args "$@" readConfigFile() { local end=false until $end; do - read || end=true + read -r || end=true [[ $REPLY =~ ^# ]] || [[ -z $REPLY ]] || echo "$REPLY" - done < "$1" + done <"$1" } # if there are file/environment sbt_opts, process again so we # can supply args to this runner if [[ -r "$sbt_opts_file" ]]; then vlog "Using sbt options defined in file $sbt_opts_file" - while read opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file") + while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file") elif [[ -n "$SBT_OPTS" && ! ("$SBT_OPTS" =~ ^@.*) ]]; then vlog "Using sbt options defined in variable \$SBT_OPTS" - extra_sbt_opts=( $SBT_OPTS ) + IFS=" " read -r -a extra_sbt_opts <<<"$SBT_OPTS" +else + vlog "No extra sbt options have been defined" +fi + +# if there are file/environment sbtx_opts, process again so we +# can supply args to this runner +if [[ -r "$sbtx_opts_file" ]]; then + vlog "Using sbt options defined in file $sbtx_opts_file" + while read -r opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbtx_opts_file") +elif [[ -n "$SBTX_OPTS" && ! ("$SBTX_OPTS" =~ ^@.*) ]]; then + vlog "Using sbt options defined in variable \$SBTX_OPTS" + IFS=" " read -r -a extra_sbt_opts <<<"$SBTX_OPTS" else vlog "No extra sbt options have been defined" fi @@ -451,24 +573,24 @@ checkJava # only exists in 0.12+ setTraceLevel() { case "$sbt_version" in - "0.7."* | "0.10."* | "0.11."* ) echoerr "Cannot set trace level in sbt version $sbt_version" ;; - *) setThisBuild traceLevel $trace_level ;; + "0.7."* | "0.10."* | "0.11."*) echoerr "Cannot set trace level in sbt version $sbt_version" ;; + *) setThisBuild traceLevel "$trace_level" ;; esac } # set scalacOptions if we were given any -S opts -[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[@]}\"" +[[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[*]}\"" [[ -n "$sbt_explicit_version" && -z "$sbt_new" ]] && addJava "-Dsbt.version=$sbt_explicit_version" vlog "Detected sbt version $sbt_version" if [[ -n "$sbt_script" ]]; then - residual_args=( $sbt_script ${residual_args[@]} ) + residual_args=("$sbt_script" "${residual_args[@]}") else # no args - alert them there's stuff in here - (( argumentCount > 0 )) || { + ((argumentCount > 0)) || { vlog "Starting $script_name: invoke with -help for other options" - residual_args=( shell ) + residual_args=(shell) } fi @@ -484,6 +606,7 @@ EOM } # pick up completion if present; todo +# shellcheck disable=SC1091 [[ -r .sbt_completion.sh ]] && source .sbt_completion.sh # directory to store sbt launchers @@ -493,7 +616,7 @@ EOM # no jar? download it. [[ -r "$sbt_jar" ]] || acquire_sbt_jar || { # still no jar? uh-oh. - echo "Download failed. Obtain the jar manually and place it at $sbt_jar" + echo "Could not download and verify the launcher. Obtain the jar manually and place it at $sbt_jar" exit 1 } @@ -503,12 +626,12 @@ if [[ -n "$noshare" ]]; then done else case "$sbt_version" in - "0.7."* | "0.10."* | "0.11."* | "0.12."* ) + "0.7."* | "0.10."* | "0.11."* | "0.12."*) [[ -n "$sbt_dir" ]] || { sbt_dir="$HOME/.sbt/$sbt_version" vlog "Using $sbt_dir as sbt dir, -sbt-dir to override." } - ;; + ;; esac if [[ -n "$sbt_dir" ]]; then @@ -518,58 +641,21 @@ fi if [[ -r "$jvm_opts_file" ]]; then vlog "Using jvm options defined in file $jvm_opts_file" - while read opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file") + while read -r opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file") elif [[ -n "$JVM_OPTS" && ! ("$JVM_OPTS" =~ ^@.*) ]]; then vlog "Using jvm options defined in \$JVM_OPTS variable" - extra_jvm_opts=( $JVM_OPTS ) + IFS=" " read -r -a extra_jvm_opts <<<"$JVM_OPTS" else vlog "Using default jvm options" - extra_jvm_opts=( $(default_jvm_opts) ) + IFS=" " read -r -a extra_jvm_opts <<<"$( default_jvm_opts)" fi # traceLevel is 0.12+ [[ -n "$trace_level" ]] && setTraceLevel -main () { - execRunner "$java_cmd" \ - "${extra_jvm_opts[@]}" \ - "${java_args[@]}" \ - -jar "$sbt_jar" \ - "${sbt_commands[@]}" \ - "${residual_args[@]}" -} - -# sbt inserts this string on certain lines when formatting is enabled: -# val OverwriteLine = "\r\u001BM\u001B[2K" -# ...in order not to spam the console with a million "Resolving" lines. -# Unfortunately that makes it that much harder to work with when -# we're not going to print those lines anyway. We strip that bit of -# line noise, but leave the other codes to preserve color. -mainFiltered () { - local ansiOverwrite='\r\x1BM\x1B[2K' - local excludeRegex=$(egrep -v '^#|^$' ~/.sbtignore | paste -sd'|' -) - - echoLine () { - local line="$1" - local line1="$(echo "$line" | sed 's/\r\x1BM\x1B\[2K//g')" # This strips the OverwriteLine code. - local line2="$(echo "$line1" | sed 's/\x1B\[[0-9;]*[JKmsu]//g')" # This strips all codes - we test regexes against this. - - if [[ $line2 =~ $excludeRegex ]]; then - [[ -n $debugUs ]] && echo "[X] $line1" - else - [[ -n $debugUs ]] && echo " $line1" || echo "$line1" - fi - } - - echoLine "Starting sbt with output filtering enabled." - main | while read -r line; do echoLine "$line"; done -} - -# Only filter if there's a filter file and we don't see a known interactive command. -# Obviously this is super ad hoc but I don't know how to improve on it. Testing whether -# stdin is a terminal is useless because most of my use cases for this filtering are -# exactly when I'm at a terminal, running sbt non-interactively. -shouldFilter () { [[ -f ~/.sbtignore ]] && ! egrep -q '\b(shell|console|consoleProject)\b' <<<"${residual_args[@]}"; } - -# run sbt -if shouldFilter; then mainFiltered; else main; fi +execRunner "$java_cmd" \ + "${extra_jvm_opts[@]}" \ + "${java_args[@]}" \ + -jar "$sbt_jar" \ + "${sbt_commands[@]}" \ + "${residual_args[@]}" diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 9a86f687..6bed85d1 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -14,7 +14,7 @@ geotrellis.jts = { # Use double precision floating point values for JTS geometry coordinates - precision.type = floating +; precision.type = floating # For fixed precision grid for JTS coordinates; the following gives 11 decimal places of precision precision.type = fixed @@ -23,4 +23,17 @@ geotrellis.jts = { # In operations requiring simplification, the following dictates precision of simplified coordinates # Here, use 12 decimal places of precision # simplification.scale = 1e12 +} + +geotrellis.raster.gdal.options { + # GDAL_NUM_THREADS = "ALL_CPUS" + # GDAL_DISABLE_READDIR_ON_OPEN = "YES" + # GDAL_MAX_DATASET_POOL_SIZE = "512" + # GDAL_CACHEMAX = "1000" + # CPL_VSIL_GZIP_WRITE_PROPERTIES = "NO" + # CPL_VSIL_CURL_CHUNK_SIZE = "64000" + # VSIS3_CHUNK_SIZE = "256000" + # VRT_SHARED_SOURCE = "0" + # CPL_DEBUG = "ON" + # GDAL_HTTP_MAX_RETRY = "10" } \ No newline at end of file diff --git a/src/main/resources/feature-flag-flagship.conf b/src/main/resources/feature-flag-flagship.conf new file mode 100644 index 00000000..509faa02 --- /dev/null +++ b/src/main/resources/feature-flag-flagship.conf @@ -0,0 +1,10 @@ +raster-layers { + BrazilBiomes: "s3://gfw-data-lake/ibge_bra_biomes/v2004/raster/epsg-4326///name/gdal-geotiff/.tif" + IndonesiaForestArea: "s3://gfw-data-lake/idn_forest_area/v201709/raster/epsg-4326///class/gdal-geotiff/.tif" + IntactForestLandscapes: "s3://gfw-data-lake/ifl_intact_forest_landscapes/v2018/raster/epsg-4326///year/gdal-geotiff/.tif" + Plantations: "s3://gfw-data-lake/gfw_plantations/v2014/raster/epsg-4326///type/gdal-geotiff/.tif" + ProtectedAreas: "s3://gfw-data-lake/wdpa_protected_areas/v202106/raster/epsg-4326///iucn_cat/gdal-geotiff/.tif" + SEAsiaLandCover: "s3://gfw-data-lake/rspo_southeast_asia_land_cover_2010/v2013/raster/epsg-4326///class/gdal-geotiff/.tif" + TreeCoverDensity2000: "s3://gfw-data-lake/umd_tree_cover_density_2000/v1.6/raster/epsg-4326///percent/gdal-geotiff/.tif" + TreeCoverDensity2010: "s3://gfw-data-lake/umd_tree_cover_density_2010/v1.6/raster/epsg-4326///percent/gdal-geotiff/.tif" +} \ No newline at end of file diff --git a/src/main/resources/feature-flag-pro.conf b/src/main/resources/feature-flag-pro.conf new file mode 100644 index 00000000..db523c1c --- /dev/null +++ b/src/main/resources/feature-flag-pro.conf @@ -0,0 +1,9 @@ +raster-layers { + BrazilBiomes: "s3://gfw-data-lake/bra_biomes/v20150601/raster/epsg-4326///name/gdal-geotiff/.tif" + IndonesiaForestArea: "s3://gfw-data-lake/idn_forest_area/v201709/raster/epsg-4326///class_compressed/gdal-geotiff/.tif" + IntactForestLandscapes: "s3://gfw-data-lake/ifl_intact_forest_landscapes/v20180628/raster/epsg-4326///year/gdal-geotiff/.tif" + Plantations: "s3://gfw-data-lake/gfw_plantations/v1.3/raster/epsg-4326///type/gdal-geotiff/.tif" + ProtectedAreas: "s3://gfw-data-lake/wdpa_protected_areas/v202010/raster/epsg-4326///iucn_cat/gdal-geotiff/.tif" + SEAsiaLandCover: "s3://gfw-data-lake/rspo_southeast_asia_land_cover_2010/v2013/raster/epsg-4326///land_cover_class/gdal-geotiff/.tif" + TreeCoverDensity2000: "s3://gfw-data-lake/umd_tree_cover_density_2000/v1.8/raster/epsg-4326///percent/gdal-geotiff/.tif" +} \ No newline at end of file diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index 68e9581a..0b0dc833 100644 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -11,6 +11,7 @@ log4j.logger.geotrellis.spark=INFO log4j.logger.org.datasyslab=WARN log4j.category.org.spark_project.jetty=WARN log4j.logger.org.apache.spark=WARN +log4j.logger.org.apache.spark.storage=ERROR log4j.logger.org.apache.hadoop=WARN log4j.logger.org.apache.spark.scheduler.DAGScheduler=INFO log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=WARN diff --git a/src/main/scala/org/globalforestwatch/ValidatedWorkflow.scala b/src/main/scala/org/globalforestwatch/ValidatedWorkflow.scala new file mode 100644 index 00000000..a1e903cc --- /dev/null +++ b/src/main/scala/org/globalforestwatch/ValidatedWorkflow.scala @@ -0,0 +1,50 @@ +package org.globalforestwatch + +import cats.data.Validated +import scala.reflect.ClassTag +import org.apache.spark.rdd.RDD +import org.apache.spark.storage.StorageLevel + +/** + * Working with RDDs in context of Validated, where per-row operations can fail but errors must be preserved. + * Once a row has errored it can not be processed further, but it must be carried through to final output. + * This is done in carrying errors in a seprate RDD, which is unioned with additional errors in later steps. + */ +final case class ValidatedWorkflow[E, A](invalid: RDD[Validated.Invalid[E]], valid: RDD[A]){ + /** Transformation over Valid RDD that is not expected to fail */ + def mapValid[B: ClassTag](f: RDD[A] => RDD[B]): ValidatedWorkflow[E, B] = { + ValidatedWorkflow(invalid, f(valid)) + } + + /** Transformation over Valid RDD that may fail per row, failures are reified into Validated */ + def mapValidToValidated[B: ClassTag](f: RDD[A] => RDD[Validated[E, B]]): ValidatedWorkflow[E, B] = { + val verifiedOutput = f(valid) + val ValidatedWorkflow(nextInvalid, nextValid) = ValidatedWorkflow(verifiedOutput) + ValidatedWorkflow(invalid.union(nextInvalid), nextValid) + } + + /** Unify the Valid and Invalid branches into a single RDD of Validated values */ + def unify: RDD[Validated[E, A]] = { + val validSide = valid.map(Validated.valid[E, A]) + val invalidSide = invalid.asInstanceOf[RDD[Validated[E, A]]] + validSide.union(invalidSide) + } + + def flatMap[B](f: RDD[A] => ValidatedWorkflow[E, B]): ValidatedWorkflow[E, B] = { + val nextInstance = f(valid) + ValidatedWorkflow(invalid.union(nextInstance.invalid), nextInstance.valid) + } +} + +object ValidatedWorkflow { + def apply[E, A: ClassTag](rdd: RDD[Validated[E, A]]): ValidatedWorkflow[E, A] = { + rdd.persist(StorageLevel.MEMORY_AND_DISK) + val valid = rdd.collect { + case Validated.Valid(a) => a + } + val invalid = rdd.collect{ + case row: Validated.Invalid[E] => row + }.repartition(math.max(1, math.log10(rdd.getNumPartitions).toInt)) + ValidatedWorkflow(invalid, valid) + } +} \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/config/GfwConfig.scala b/src/main/scala/org/globalforestwatch/config/GfwConfig.scala new file mode 100644 index 00000000..c5ada324 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/config/GfwConfig.scala @@ -0,0 +1,27 @@ +package org.globalforestwatch.config + +import pureconfig.generic.auto._ +import pureconfig.ConfigSource +import com.typesafe.scalalogging.LazyLogging + +case class GfwConfig( + rasterLayers: Map[String, String] +) + +object GfwConfig extends LazyLogging { + private val featureFlag: Option[String] = { + val flag = scala.util.Properties.envOrNone("GFW_FEATURE_FLAG").map(_.toLowerCase()) + logger.info(s"GFW_FEATURE_FLAG=$flag") + flag + } + + def isGfwPro: Boolean = featureFlag == Some("pro") + + lazy val get: GfwConfig = read(featureFlag.getOrElse("pro")) + + def read(flag: String): GfwConfig = { + val confFile = s"feature-flag-$flag.conf" + logger.info(s"Reading $confFile") + ConfigSource.resources(confFile).loadOrThrow[GfwConfig] + } +} \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/features/BurnedAreasFeature.scala b/src/main/scala/org/globalforestwatch/features/BurnedAreasFeature.scala index 09056f7b..03ab5367 100644 --- a/src/main/scala/org/globalforestwatch/features/BurnedAreasFeature.scala +++ b/src/main/scala/org/globalforestwatch/features/BurnedAreasFeature.scala @@ -1,26 +1,11 @@ package org.globalforestwatch.features -import geotrellis.vector.Geometry -import geotrellis.vector.io.wkb.WKB -import org.apache.spark.sql.Row -import org.globalforestwatch.util.GeometryReducer - object BurnedAreasFeature extends Feature { override val geomPos: Int = 1 val featureIdExpr = "alert__date as alertDate" val featureCount = 1 - def get(i: Row): geotrellis.vector.Feature[Geometry, FeatureId] = { - val featureId = getFeatureId(i) - val geom: Geometry = - GeometryReducer.reduce(GeometryReducer.gpr)( - WKB.read(i.getString(geomPos)) - ) - - geotrellis.vector.Feature(geom, featureId) - } - def getFeatureId(i: Array[String], parsed: Boolean = false): FeatureId = { val alertDate = i(0) BurnedAreasFeatureId(alertDate) diff --git a/src/main/scala/org/globalforestwatch/features/CombinedFeatureId.scala b/src/main/scala/org/globalforestwatch/features/CombinedFeatureId.scala index 3dafb874..afc03e44 100644 --- a/src/main/scala/org/globalforestwatch/features/CombinedFeatureId.scala +++ b/src/main/scala/org/globalforestwatch/features/CombinedFeatureId.scala @@ -1,5 +1,5 @@ package org.globalforestwatch.features case class CombinedFeatureId(featureId1: FeatureId, featureId2: FeatureId) extends FeatureId { - override def toString: String = s"$featureId1" + s"$featureId2" + override def toString: String = s"$featureId1\t$featureId2" } \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/features/Feature.scala b/src/main/scala/org/globalforestwatch/features/Feature.scala index 34140ffb..ee5531ee 100644 --- a/src/main/scala/org/globalforestwatch/features/Feature.scala +++ b/src/main/scala/org/globalforestwatch/features/Feature.scala @@ -1,15 +1,21 @@ package org.globalforestwatch.features import geotrellis.vector.Geometry -import org.apache.spark.sql.{DataFrame, Row, SparkSession} -import org.globalforestwatch.util.GeometryReducer -import org.globalforestwatch.util.Util.getAnyMapValue +import org.apache.spark.sql.{DataFrame, Row} +import org.globalforestwatch.util.GeotrellisGeometryValidator +import org.globalforestwatch.util.GeotrellisGeometryValidator.makeValidGeom +/** This trait defiens how to read a Feature from DataFrame, from what columns to parse its FeatureId and how to read its geometry */ trait Feature extends java.io.Serializable { val geomPos: Int val featureIdExpr: String - def get(i: Row): geotrellis.vector.Feature[Geometry, FeatureId] + def get(i: Row): geotrellis.vector.Feature[Geometry, FeatureId] = { + val featureId = getFeatureId(i) + val geom: Geometry = makeValidGeom(i.getString(geomPos)) + + geotrellis.vector.Feature(geom, featureId) + } def getFeatureId(i: Row): FeatureId = { getFeatureId(i.toSeq.map(_.asInstanceOf[String]).toArray) @@ -17,32 +23,32 @@ trait Feature extends java.io.Serializable { def getFeatureId(i: Array[String], parsed: Boolean = false): FeatureId - def isValidGeom(i: Row): Boolean = { - GeometryReducer.isValidGeom(i.getString(geomPos)) + def isNonEmptyGeom(i: Row): Boolean = { + GeotrellisGeometryValidator.isNonEmptyGeom(i.getString(geomPos)) } - def filter(filters: Map[String, Any])(df: DataFrame): DataFrame = { - - val spark: SparkSession = df.sparkSession - import spark.implicits._ - - val trueValues: List[String] = - List("t", "T", "true", "True", "TRUE", "1", "Yes", "yes", "YES") - val limit: Option[Int] = getAnyMapValue[Option[Int]](filters, "limit") - val tcl: Boolean = getAnyMapValue[Boolean](filters, "tcl") - val glad: Boolean = getAnyMapValue[Boolean](filters, "glad") - - val customFilterDF = df.transform(custom_filter(filters)) - - val gladDF = if (glad) customFilterDF.filter($"glad".isin(trueValues: _*)) else customFilterDF - - val tclDF = if (tcl) gladDF.filter($"tcl".isin(trueValues: _*)) else gladDF - - limit.foldLeft(tclDF)(_.limit(_)) - + def filter(filters: FeatureFilter)(df: DataFrame): DataFrame = { + val conditions = filters.filterConditions() + if (conditions.isEmpty) df else { + val condition = conditions.reduce(_ and _) + df.filter(condition) + } } +} - def custom_filter(filters: Map[String, Any])(df: DataFrame): DataFrame = { - df +object Feature { + def apply(name: String): Feature = name match { + case "gadm" => GadmFeature + case "feature" => SimpleFeature + case "wdpa" => WdpaFeature + case "geostore" => GeostoreFeature + case "viirs" => FireAlertViirsFeature + case "modis" => FireAlertModisFeature + case "burned_areas" => BurnedAreasFeature + case "gfwpro" => GfwProFeature + case value => + throw new IllegalArgumentException( + s"FeatureType must be one of 'gadm', 'wdpa', 'geostore', 'gfwpro', 'feature', 'viirs', 'modis', or 'burned_areas'. Got $value." + ) } -} +} \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/features/FeatureDF.scala b/src/main/scala/org/globalforestwatch/features/FeatureDF.scala index 9ed9ed7e..6da0c802 100644 --- a/src/main/scala/org/globalforestwatch/features/FeatureDF.scala +++ b/src/main/scala/org/globalforestwatch/features/FeatureDF.scala @@ -7,7 +7,7 @@ import org.apache.spark.sql.{DataFrame, SparkSession} object FeatureDF { def apply(input: NonEmptyList[String], featureObj: Feature, - filters: Map[String, Any], + filters: FeatureFilter, spark: SparkSession, delimiter: String = "\t"): DataFrame = spark.read diff --git a/src/main/scala/org/globalforestwatch/features/FeatureFactory.scala b/src/main/scala/org/globalforestwatch/features/FeatureFactory.scala deleted file mode 100644 index e3cf0a03..00000000 --- a/src/main/scala/org/globalforestwatch/features/FeatureFactory.scala +++ /dev/null @@ -1,21 +0,0 @@ -package org.globalforestwatch.features - -case class FeatureFactory(featureName: String, fireAlertType: Option[String] = None) { - val featureObj: Feature = featureName match { - case "gadm" => GadmFeature - case "feature" => SimpleFeature - case "wdpa" => WdpaFeature - case "geostore" => GeostoreFeature - case "firealerts" => fireAlertType match { - case Some("viirs") => FireAlertsViirsFeature - case Some("modis") => FireAlertsModisFeature - case Some("burned_areas") => BurnedAreasFeature - case _ => throw new IllegalArgumentException("Cannot provide fire alert feature type without selecting alert" + - "type 'modis' or 'viirs'") - } - case _ => - throw new IllegalArgumentException( - "Feature type must be one of 'gadm', 'wdpa' 'geostore' and 'feature'" - ) - } -} diff --git a/src/main/scala/org/globalforestwatch/features/FeatureFilter.scala b/src/main/scala/org/globalforestwatch/features/FeatureFilter.scala new file mode 100644 index 00000000..d7a955be --- /dev/null +++ b/src/main/scala/org/globalforestwatch/features/FeatureFilter.scala @@ -0,0 +1,34 @@ +package org.globalforestwatch.features + +import org.globalforestwatch.summarystats.SummaryCommand +import com.typesafe.scalalogging.LazyLogging +import org.apache.spark.sql.Column + +trait FeatureFilter { + def filterConditions(): List[Column] +} + +object FeatureFilter extends LazyLogging { + def empty: FeatureFilter = new FeatureFilter { + def filterConditions() = Nil + } + + def fromOptions(featureType: String, options: SummaryCommand.AllFilterOptions): FeatureFilter = { + val featureFilter = featureType match { + case "gadm" => GadmFeature.Filter(options.base, options.gadm) + case "feature" => SimpleFeature.Filter(options.base, options.featureId) + case "wdpa" => WdpaFeature.Filter(options.base, options.gadm, options.wdpa) + case "geostore" => FeatureFilter.empty + case "viirs" => FeatureFilter.empty + case "modis" => FeatureFilter.empty + case "burned_areas" => FeatureFilter.empty + case "gfwpro" => GfwProFeature.Filter(options.base, options.featureId) + case value => + throw new IllegalArgumentException( + s"FeatureType must be one of 'gadm', 'wdpa', 'geostore', 'gfwpro', 'feature', 'viirs', 'modis', or 'burned_areas'. Got $value." + ) + } + logger.info(s"Filter `$featureType` features with $featureFilter") + featureFilter + } +} diff --git a/src/main/scala/org/globalforestwatch/features/FeatureId.scala b/src/main/scala/org/globalforestwatch/features/FeatureId.scala index 9d03b262..b29a81f2 100644 --- a/src/main/scala/org/globalforestwatch/features/FeatureId.scala +++ b/src/main/scala/org/globalforestwatch/features/FeatureId.scala @@ -1,3 +1,24 @@ package org.globalforestwatch.features -abstract class FeatureId +trait FeatureId + +object FeatureId { + def fromUserData(featureType: String, value: String, delimiter: String = "\t"): FeatureId = { + val values = value.filterNot("[]".toSet).split(delimiter).map(_.trim) + + featureType match { + case "gadm" => GadmFeature.getFeatureId(values, true) + case "feature" => SimpleFeatureId(values(0).toInt) + case "wdpa" => WdpaFeature.getFeatureId(values, true) + case "geostore" => GeostoreFeature.getFeatureId(values, true) + case "gfwpro" => GfwProFeature.getFeatureId(values, true) + case "burned_areas" => BurnedAreasFeature.getFeatureId(values, true) + case "viirs" => FireAlertViirsFeature.getFeatureId(values, true) + case "modis" => FireAlertModisFeature.getFeatureId(values, true) + case value => + throw new IllegalArgumentException( + s"FeatureType must be one of 'gadm', 'wdpa', 'geostore', 'feature', 'gfwpro', 'viirs', 'modis', or 'burned_areas'. Got $value." + ) + } + } +} diff --git a/src/main/scala/org/globalforestwatch/features/FeatureIdFactory.scala b/src/main/scala/org/globalforestwatch/features/FeatureIdFactory.scala deleted file mode 100644 index 96baa2df..00000000 --- a/src/main/scala/org/globalforestwatch/features/FeatureIdFactory.scala +++ /dev/null @@ -1,26 +0,0 @@ -package org.globalforestwatch.features - -case class FeatureIdFactory(featureName: String) { - def featureId(id: Any): FeatureId = - (featureName, id) match { - case ("gadm", (iso: String, adm1: Integer, adm2: Integer)) => - GadmFeatureId(iso, adm1, adm2) - case ("feature", simple_id: Int) => SimpleFeatureId(simple_id) - case ( - "wdpa", - ( - wdpaId: Int, - name: String, - iucnCat: String, - iso: String, - status: String - ) - ) => - WdpaFeatureId(wdpaId, name, iucnCat, iso, status) - case ("geostore", geostore: String) => GeostoreFeatureId(geostore) - case _ => - throw new IllegalArgumentException( - "Feature type must be one of 'gadm', 'wdpa', 'geostore' or 'feature'" - ) - } -} diff --git a/src/main/scala/org/globalforestwatch/features/FeatureRDD.scala b/src/main/scala/org/globalforestwatch/features/FeatureRDD.scala index 738bcae0..b8ebda03 100644 --- a/src/main/scala/org/globalforestwatch/features/FeatureRDD.scala +++ b/src/main/scala/org/globalforestwatch/features/FeatureRDD.scala @@ -1,151 +1,214 @@ package org.globalforestwatch.features import cats.data.NonEmptyList +import org.locationtech.jts.geom._ import org.apache.log4j.Logger -import com.vividsolutions.jts.geom.{Geometry => GeoSparkGeometry, Point => GeoSparkPoint, Polygonal => GeoSparkPolygonal } -import com.vividsolutions.jts.io.WKTWriter +import geotrellis.store.index.zcurve.Z2 import geotrellis.vector -import geotrellis.vector.{Geometry, Polygon} +import geotrellis.vector.{Geometry, MultiPolygon} +import org.apache.spark.HashPartitioner +import org.apache.spark.api.java.JavaPairRDD import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, Row, SparkSession} -import org.datasyslab.geospark.spatialRDD.SpatialRDD -import org.globalforestwatch.util.GeometryReducer -import org.globalforestwatch.util.IntersectGeometry.{getIntersecting1x1Grid, intersectGeometries} -import org.globalforestwatch.util.Util.getAnyMapValue -import org.locationtech.jts.io.WKTReader - -import scala.annotation.tailrec +import org.apache.sedona.core.spatialRDD.SpatialRDD +import org.apache.sedona.sql.utils.Adapter +import org.globalforestwatch.util.{GridRDD, SpatialJoinRDD} +import org.globalforestwatch.util.IntersectGeometry.{ + extractPolygons, + intersectGeometries +} object FeatureRDD { val logger = Logger.getLogger("FeatureRDD") - def apply( - input: NonEmptyList[String], - featureObj: Feature, - kwargs: Map[String, Any], - spark: SparkSession, + def apply(input: NonEmptyList[String], + featureType: String, + filters: FeatureFilter, + splitFeatures: Boolean, + spark: SparkSession, ): RDD[geotrellis.vector.Feature[Geometry, FeatureId]] = { - val featuresDF: DataFrame = - FeatureDF(input, featureObj, kwargs, spark) - - val splitFeatures = getAnyMapValue[Boolean](kwargs, "splitFeatures") - - val featureRdd = featuresDF.rdd - .mapPartitions({ iter: Iterator[Row] => - for { - i <- iter - if splitFeatures || featureObj.isValidGeom(i) - } yield { - featureObj.get(i) - } - }, preservesPartitioning = true) - if (splitFeatures) featureRdd.flatMap { feature => - splitGeometry(feature) - } else featureRdd + if (splitFeatures) + splitGeometries(input, featureType, filters, spark) + else { + val featureObj: Feature = Feature(featureType) + val featuresDF: DataFrame = + FeatureDF(input, featureObj, filters, spark) + + featuresDF.rdd + .mapPartitions({ iter: Iterator[Row] => + for { + i <- iter + if featureObj.isNonEmptyGeom(i) + } yield { + featureObj.get(i) + } + }, preservesPartitioning = true) + } } /* Convert point-in-polygon join to feature RDD - */ - def apply( - featureObj: Feature, - spatialRDD: SpatialRDD[GeoSparkGeometry], - kwargs: Map[String, Any], - ): RDD[geotrellis.vector.Feature[Geometry, FeatureId]] = { - + */ + def pointInPolygonJoinAsFeature( + featureType: String, + spatialRDD: SpatialRDD[Geometry] + ): RDD[geotrellis.vector.Feature[Geometry, FeatureId]] = { val scalaRDD = org.apache.spark.api.java.JavaRDD.toRDD(spatialRDD.spatialPartitionedRDD) scalaRDD .map { - case pt: GeoSparkPoint => - val pointFeatureData = pt.getUserData.asInstanceOf[String].split('\t') + case pt: Point => + val pointFeatureData = pt.getUserData.asInstanceOf[String] //.split('\t') - val geom = GeometryReducer.reduce(GeometryReducer.gpr)( - vector.Point(pt.getX, pt.getY) - ) + // use implicit geometry converter + val geom: vector.Point = pt val pointFeatureId: FeatureId = - featureObj.getFeatureId(pointFeatureData) + FeatureId.fromUserData(featureType, pointFeatureData) + vector.Feature(geom, pointFeatureId) case _ => - throw new IllegalArgumentException("Point-in-polygon intersection must be points.") + throw new IllegalArgumentException( + "Point-in-polygon intersection must be points." + ) } - // In case we implement this method for other geometry types we will have to split geometries - // .flatMap { feature => - // splitGeometry(feature) - // } - } - /* Convert polygon-polygon intersection join to feature RDD */ def apply( - feature1Obj: Feature, - feature2Obj: Feature, - spatialRDD: SpatialRDD[GeoSparkGeometry], - kwargs: Map[String, Any], + feature1Type: String, + feature1Uris: NonEmptyList[String], + feature1Delimiter: String, + feature2Type: String, + feature2Uris: NonEmptyList[String], + feature2Delimiter: String, + filters: FeatureFilter, + spark: SparkSession ): RDD[geotrellis.vector.Feature[Geometry, FeatureId]] = { - val scalaRDD = - org.apache.spark.api.java.JavaRDD.toRDD(spatialRDD.spatialPartitionedRDD) - scalaRDD - .flatMap { - case shp: GeoSparkPolygonal => - val featureData = shp.getUserData.asInstanceOf[String].split('\t') - val writer: WKTWriter = new WKTWriter() - val wkt = writer.write(shp) - - val reader: WKTReader = new WKTReader() - val geom = GeometryReducer.reduce(GeometryReducer.gpr)( - reader.read(wkt) - ) - - val feature1Data = featureData.head.drop(1).dropRight(1).split(',') - val feature1Id: FeatureId = - feature1Obj.getFeatureId(feature1Data, parsed = true) - - val feature2Data = featureData.tail.head.drop(1).dropRight(1).split(',') - val feature2Id: FeatureId = - feature2Obj.getFeatureId(feature2Data, parsed = true) + val spatialDF: DataFrame = PolygonIntersectionDF( + feature1Uris, + feature1Type, + feature2Uris, + feature2Type, + spark, + filters, + feature1Delimiter, + feature2Delimiter, + ) + + val pairedRDD = spatialDF.rdd.map { row: Row => + val featureId1: FeatureId = FeatureId.fromUserData(feature1Type, row.getAs[Row](0).toString, ",") + val featureId2: FeatureId = FeatureId.fromUserData(feature2Type, row.getAs[Row](1).toString, ",") + val geom = row.getAs[Geometry](2) + + (CombinedFeatureId(featureId1, featureId2), geom) + } - Some(vector.Feature(geom, CombinedFeatureId(feature1Id, feature2Id))) - case _ => - // Polygon-polygon intersections can generate points or lines, which we just want to ignore - logger.warn( - "Cannot process geometry type" - ) - None - } + pairedRDD.flatMap { + case (id, geom) => + geom match { + case geomCol: GeometryCollection => + val maybePoly = extractPolygons(geomCol) + maybePoly match { + case Some(poly) => List(vector.Feature(poly, id)) + case _ => List() + } + case poly: Polygonal => + List(vector.Feature(poly, id)) + case _ => + // Polygon-polygon intersections can generate points or lines, which we just want to ignore + logger.warn("Cannot process geometry type") + List() + } + } } - private def splitGeometry( - feature: geotrellis.vector.Feature[Geometry, FeatureId] - ): List[geotrellis.vector.Feature[Geometry, FeatureId]] = { + private def splitGeometries( + input: NonEmptyList[String], + featureType: String, + filters: FeatureFilter, + spark: SparkSession + ): RDD[geotrellis.vector.Feature[Geometry, FeatureId]] = { - @tailrec def loop(geom: Geometry, - gridGeoms: IndexedSeq[Polygon], - acc: List[Geometry]): List[Geometry] = { - if (gridGeoms.isEmpty) acc - else { + val featureDF: DataFrame = + SpatialFeatureDF(input, featureType, filters, "geom", spark) - val gridGeom = gridGeoms.head; + val spatialRDD: SpatialRDD[Geometry] = + Adapter.toSpatialRdd(featureDF, "polyshape") + spatialRDD.analyze() - val intersections: List[Geometry] = - intersectGeometries(feature.geom, gridGeom) - loop(geom, gridGeoms.tail, acc ::: intersections) - } + spatialRDD.rawSpatialRDD = spatialRDD.rawSpatialRDD.rdd.map { geom: Geometry => + val featureId = FeatureId.fromUserData(featureType, geom.getUserData.asInstanceOf[String], delimiter = ",") + geom.setUserData(featureId) + geom } - val gridGeoms = getIntersecting1x1Grid(feature.geom) - - loop(feature.geom, gridGeoms, List()).map(vector.Feature(_, feature.data)) + splitGeometries(spatialRDD, spark) + } + def splitGeometries( + spatialFeatureRDD: SpatialRDD[Geometry], + spark: SparkSession + ): RDD[geotrellis.vector.Feature[Geometry, FeatureId]] = { + + val envelope: Envelope = spatialFeatureRDD.boundaryEnvelope + + val spatialGridRDD = GridRDD(envelope, spark, clip = true) + val flatJoin: JavaPairRDD[Polygon, Geometry] = + SpatialJoinRDD.flatSpatialJoin( + spatialFeatureRDD, + spatialGridRDD, + considerBoundaryIntersection = true + ) + + /* + partitions will come back very skewed and we will need to even them out for any downstream analysis + For the summary analysis we will eventually use a range partitioner. + However, the range partitioner uses sampling to come up with the break points for the different partitions. + If the input RDD is already heavily skewed, sampling will be off and the range partitioner won't do a good job. + */ + val hashPartitioner = new HashPartitioner(flatJoin.getNumPartitions) + + flatJoin.rdd + .keyBy({ pair: (Polygon, Geometry) => + Z2( + (pair._1.getCentroid.getX * 100).toInt, + (pair._1.getCentroid.getY * 100).toInt + ).z + }) + .partitionBy(hashPartitioner) + .flatMap { + case (_, (gridCell, geom)) => + val geometries = intersectGeometries(geom, gridCell) + geometries + } + .flatMap { intersection => + // use implicit converter to covert to Geotrellis Geometry + val geotrellisGeom: MultiPolygon = intersection + + if (!geotrellisGeom.isEmpty) { + + val userData = intersection.getUserData + val featureId = userData match { + case fid: FeatureId => fid + } + + Seq( + geotrellis.vector.Feature( + geotrellisGeom, + featureId + ) + ) + } + else Seq() + } } } diff --git a/src/main/scala/org/globalforestwatch/features/FeatureRDDFactory.scala b/src/main/scala/org/globalforestwatch/features/FeatureRDDFactory.scala deleted file mode 100644 index fab578e7..00000000 --- a/src/main/scala/org/globalforestwatch/features/FeatureRDDFactory.scala +++ /dev/null @@ -1,63 +0,0 @@ -package org.globalforestwatch.features - -import cats.data.NonEmptyList -import geotrellis.vector -import org.datasyslab.geospark.enums.GridType -import com.vividsolutions.jts.geom.Geometry -import org.datasyslab.geospark.spatialRDD.SpatialRDD -//import org.apache.sedona.core.enums.GridType -//import org.apache.sedona.sql.utils.Adapter -//import org.locationtech.jts.geom.Point -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.SparkSession -import org.globalforestwatch.util.Util._ - - -object FeatureRDDFactory { - def apply(analysis: String, - featureType: String, - featureUris: NonEmptyList[String], - kwargs: Map[String, Any], - spark: SparkSession): RDD[vector.Feature[vector.Geometry, FeatureId]] = { - val featureObj = FeatureFactory(featureType).featureObj - - analysis match { - case "firealerts" => - val fireAlertType = getAnyMapValue[String](kwargs, "fireAlertType") - val fireAlertObj = - FeatureFactory("firealerts", Some(fireAlertType)).featureObj - - fireAlertType match { - case "viirs" | "modis" => - val fireRDD: SpatialRDD[Geometry] = FireAlertRDD(spark, kwargs) - fireRDD.spatialPartitioning(GridType.QUADTREE) - - FeatureRDD(fireAlertObj, fireRDD, kwargs) - case "burned_areas" => - val burnedAreasUris: NonEmptyList[String] = getAnyMapValue[NonEmptyList[String]]( - kwargs, - "fireAlertSource" - ) - - val spatialRDD: SpatialRDD[Geometry] = PolygonIntersectionRDD( - featureUris, - featureObj, - featureType, - burnedAreasUris, - fireAlertObj, - fireAlertType, - spark, - kwargs, - feature2Delimiter = "," - ) - - spatialRDD.analyze() - spatialRDD.spatialPartitioning(GridType.QUADTREE) - - FeatureRDD(featureObj, fireAlertObj, spatialRDD, kwargs) - } - case _ => - FeatureRDD(featureUris, featureObj, kwargs, spark) - } - } -} diff --git a/src/main/scala/org/globalforestwatch/features/FireAlertsModisFeature.scala b/src/main/scala/org/globalforestwatch/features/FireAlertModisFeature.scala similarity index 74% rename from src/main/scala/org/globalforestwatch/features/FireAlertsModisFeature.scala rename to src/main/scala/org/globalforestwatch/features/FireAlertModisFeature.scala index 05a9896e..93d78fc6 100644 --- a/src/main/scala/org/globalforestwatch/features/FireAlertsModisFeature.scala +++ b/src/main/scala/org/globalforestwatch/features/FireAlertModisFeature.scala @@ -3,17 +3,17 @@ package org.globalforestwatch.features import geotrellis.vector import geotrellis.vector.Geometry import org.apache.spark.sql.Row -import org.globalforestwatch.util.GeometryReducer +import org.globalforestwatch.util.GeotrellisGeometryValidator.makeValidGeom -object FireAlertsModisFeature extends Feature { +object FireAlertModisFeature extends Feature { override val geomPos: Int = 0 val featureCount = 8 - val featureIdExpr = + val featureIdExpr: String = "latitude as lat, longitude as lon, acq_date as acqDate, acq_time as acqTime, confidence, " + "bright_t31 as brightT31, brightness, frp" - override def isValidGeom(i: Row): Boolean = { + override def isNonEmptyGeom(i: Row): Boolean = { val lon = i.getString(geomPos + 1).toDouble val lat = i.getString(geomPos).toDouble @@ -37,14 +37,13 @@ object FireAlertsModisFeature extends Feature { lat - 0.00001 else lat - val geom = GeometryReducer.reduce(GeometryReducer.gpr)( - vector.Point(adjustedLon, adjustedLat) - ) + val geom = makeValidGeom(vector.Point(adjustedLon, adjustedLat)) geotrellis.vector.Feature(geom, featureId) } - override def getFeatureId(i: Array[String], parsed: Boolean = false): FeatureId = { + override def getFeatureId(i: Array[String], + parsed: Boolean = false): FeatureId = { val lat: Double = i(0).toDouble val lon: Double = i(1).toDouble val acqDate: String = i(2) @@ -59,6 +58,16 @@ object FireAlertsModisFeature extends Feature { val brightT31: Float = i(6).toFloat val frp: Float = i(7).toFloat - FireAlertModisFeatureId(lon, lat, acqDate, acqTime, confidencePerc, confidenceCat, brightness, brightT31, frp) + FireAlertModisFeatureId( + lon, + lat, + acqDate, + acqTime, + confidencePerc, + confidenceCat, + brightness, + brightT31, + frp + ) } } diff --git a/src/main/scala/org/globalforestwatch/features/FireAlertRDD.scala b/src/main/scala/org/globalforestwatch/features/FireAlertRDD.scala index 173dd509..f60c8ebf 100644 --- a/src/main/scala/org/globalforestwatch/features/FireAlertRDD.scala +++ b/src/main/scala/org/globalforestwatch/features/FireAlertRDD.scala @@ -1,28 +1,24 @@ package org.globalforestwatch.features import cats.data.NonEmptyList -import com.vividsolutions.jts.geom.Geometry +import org.locationtech.jts.geom.Geometry import org.apache.spark.sql.SparkSession -import org.datasyslab.geospark.spatialRDD.SpatialRDD -import org.datasyslab.geosparksql.utils.Adapter -import org.globalforestwatch.util.Util.getAnyMapValue +import org.apache.sedona.core.spatialRDD.SpatialRDD +import org.apache.sedona.sql.utils.Adapter -object FireAlertRDD { - def apply(spark: SparkSession, - kwargs: Map[String, Any]): SpatialRDD[Geometry] = { - val fireAlertType = getAnyMapValue[String](kwargs, "fireAlertType") - val fireAlertUris - : NonEmptyList[String] = getAnyMapValue[NonEmptyList[String]]( - kwargs, - "fireAlertSource" - ) - val fireAlertObj = - FeatureFactory("firealerts", Some(fireAlertType)).featureObj +object FireAlertRDD { + def apply( + spark: SparkSession, + fireAlertType: String, + fireAlertSource: NonEmptyList[String], + filters: FeatureFilter + ): SpatialRDD[Geometry] = { + val fireAlertObj = Feature(fireAlertType) val fireAlertPointDF = SpatialFeatureDF( - fireAlertUris, + fireAlertSource, fireAlertObj, - kwargs, + filters, spark, "longitude", "latitude" diff --git a/src/main/scala/org/globalforestwatch/features/FireAlertsViirsFeature.scala b/src/main/scala/org/globalforestwatch/features/FireAlertViirsFeature.scala similarity index 86% rename from src/main/scala/org/globalforestwatch/features/FireAlertsViirsFeature.scala rename to src/main/scala/org/globalforestwatch/features/FireAlertViirsFeature.scala index 5565fd6d..5c9aac34 100644 --- a/src/main/scala/org/globalforestwatch/features/FireAlertsViirsFeature.scala +++ b/src/main/scala/org/globalforestwatch/features/FireAlertViirsFeature.scala @@ -3,9 +3,9 @@ package org.globalforestwatch.features import geotrellis.vector import geotrellis.vector.Geometry import org.apache.spark.sql.Row -import org.globalforestwatch.util.GeometryReducer +import org.globalforestwatch.util.GeotrellisGeometryValidator.makeValidGeom -object FireAlertsViirsFeature extends Feature { +object FireAlertViirsFeature extends Feature { override val geomPos: Int = 0 val featureCount = 8 @@ -13,7 +13,7 @@ object FireAlertsViirsFeature extends Feature { "latitude as lat, longitude as lon, acq_date as acqDate, acq_time as acqTime, confidence, " + "bright_ti4 as brightTi4, bright_ti5 as brightTi5, frp" - override def isValidGeom(i: Row): Boolean = { + override def isNonEmptyGeom(i: Row): Boolean = { val lon = i.getString(geomPos + 1).toDouble val lat = i.getString(geomPos).toDouble @@ -37,9 +37,7 @@ object FireAlertsViirsFeature extends Feature { lat - 0.00001 else lat - val geom = GeometryReducer.reduce(GeometryReducer.gpr)( - vector.Point(adjustedLon, adjustedLat) - ) + val geom = makeValidGeom(vector.Point(adjustedLon, adjustedLat)) geotrellis.vector.Feature(geom, featureId) } diff --git a/src/main/scala/org/globalforestwatch/features/GadmFeature.scala b/src/main/scala/org/globalforestwatch/features/GadmFeature.scala index ee1a8b0d..10aeb2c7 100644 --- a/src/main/scala/org/globalforestwatch/features/GadmFeature.scala +++ b/src/main/scala/org/globalforestwatch/features/GadmFeature.scala @@ -1,11 +1,7 @@ package org.globalforestwatch.features -import geotrellis.vector.Geometry -import geotrellis.vector.io.wkb.WKB -import org.apache.spark.sql.functions.substring -import org.apache.spark.sql.{DataFrame, Row, SparkSession} -import org.globalforestwatch.util.GeometryReducer -import org.globalforestwatch.util.Util._ +import org.apache.spark.sql.Column +import org.globalforestwatch.summarystats.SummaryCommand object GadmFeature extends Feature { val countryPos = 1 @@ -17,16 +13,6 @@ object GadmFeature extends Feature { "gid_0 as iso, split(split(gid_1, '\\\\.')[1], '_')[0] as adm1, split(split(gid_2, '\\\\.')[2], '_')[0] as adm2" - def get(i: Row): geotrellis.vector.Feature[Geometry, FeatureId] = { - val featureId = getFeatureId(i) - val geom: Geometry = - GeometryReducer.reduce(GeometryReducer.gpr)( - WKB.read(i.getString(geomPos)) - ) - - geotrellis.vector.Feature(geom, featureId) - } - def getFeatureId(i: Array[String], parsed: Boolean = false): FeatureId = { if (parsed) { val countryCode = i(0) @@ -61,37 +47,14 @@ object GadmFeature extends Feature { } } - override def custom_filter( - filters: Map[String, Any] - )(df: DataFrame): DataFrame = { - - val spark: SparkSession = df.sparkSession - import spark.implicits._ - - val isoFirst: Option[String] = - getAnyMapValue[Option[String]](filters, "isoFirst") - val isoStart: Option[String] = - getAnyMapValue[Option[String]](filters, "isoStart") - val isoEnd: Option[String] = - getAnyMapValue[Option[String]](filters, "isoEnd") - val iso: Option[String] = getAnyMapValue[Option[String]](filters, "iso") - val admin1: Option[String] = - getAnyMapValue[Option[String]](filters, "admin1") - val admin2: Option[String] = - getAnyMapValue[Option[String]](filters, "admin2") - - val isoFirstDF: DataFrame = isoFirst.foldLeft(df)( - (acc, i) => acc.filter(substring($"gid_0", 0, 1) === i(0)) - ) - val isoStartDF: DataFrame = - isoStart.foldLeft(isoFirstDF)((acc, i) => acc.filter($"gid_0" >= i)) - val isoEndDF: DataFrame = - isoEnd.foldLeft(isoStartDF)((acc, i) => acc.filter($"gid_0" < i)) - val isoDF: DataFrame = - iso.foldLeft(isoEndDF)((acc, i) => acc.filter($"gid_0" === i)) - val admin1DF: DataFrame = - admin1.foldRight(isoDF)((i, acc) => acc.filter($"gid_1" === i)) - admin2.foldRight(admin1DF)((i, acc) => acc.filter($"gid_2" === i)) - + case class Filter( + base: Option[SummaryCommand.BaseFilter], + gadm: Option[SummaryCommand.GadmFilter] + ) extends FeatureFilter { + def filterConditions(): List[Column]= { + import org.apache.spark.sql.functions.col + base.toList.flatMap(_.filters()) ++ + gadm.toList.flatMap(_.filters(isoColumn = col("gid_0"), admin1Column = col("gid_1"), admin2Column = col("gid_2"))).toList + } } } diff --git a/src/main/scala/org/globalforestwatch/features/GadmFeatureId.scala b/src/main/scala/org/globalforestwatch/features/GadmFeatureId.scala index 3d52a44d..a4ec8f22 100644 --- a/src/main/scala/org/globalforestwatch/features/GadmFeatureId.scala +++ b/src/main/scala/org/globalforestwatch/features/GadmFeatureId.scala @@ -1,5 +1,5 @@ package org.globalforestwatch.features case class GadmFeatureId(iso: String, adm1: Integer, adm2: Integer) extends FeatureId { - override def toString: String = s"$iso - $adm1 - $adm2" + override def toString: String = s"$iso.$adm1.$adm2" } diff --git a/src/main/scala/org/globalforestwatch/features/GeostoreFeature.scala b/src/main/scala/org/globalforestwatch/features/GeostoreFeature.scala index 9b863ea2..64d435dd 100644 --- a/src/main/scala/org/globalforestwatch/features/GeostoreFeature.scala +++ b/src/main/scala/org/globalforestwatch/features/GeostoreFeature.scala @@ -1,10 +1,5 @@ package org.globalforestwatch.features -import geotrellis.vector.Geometry -import geotrellis.vector.io.wkb.WKB -import org.apache.spark.sql.Row -import org.globalforestwatch.util.GeometryReducer - object GeostoreFeature extends Feature { val idPos = 0 @@ -13,17 +8,6 @@ object GeostoreFeature extends Feature { val featureIdExpr = "geostore_id as geostoreId" - def get( - i: Row - ): geotrellis.vector.Feature[Geometry, FeatureId] = { - val featureId = getFeatureId(i) - val geom: Geometry = - GeometryReducer.reduce(GeometryReducer.gpr)( - WKB.read(i.getString(geomPos)) - ) - geotrellis.vector.Feature(geom, featureId) - } - def getFeatureId(i: Array[String], parsed: Boolean = false): FeatureId = { val geostoreId: String = i(idPos) GeostoreFeatureId(geostoreId) diff --git a/src/main/scala/org/globalforestwatch/features/GfwProFeature.scala b/src/main/scala/org/globalforestwatch/features/GfwProFeature.scala new file mode 100644 index 00000000..d7fbde28 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/features/GfwProFeature.scala @@ -0,0 +1,37 @@ +package org.globalforestwatch.features + +import org.apache.spark.sql.functions.col +import org.globalforestwatch.summarystats.SummaryCommand +import org.apache.spark.sql.Column + +object GfwProFeature extends Feature { + + val listIdPos = 0 + val locationIdPos = 1 + val geomPos = 2 + + val featureIdExpr = "list_id as listId, cast(location_id as int) as locationId, ST_X(ST_Centroid(ST_GeomFromWKB(geom))) as x, ST_Y(ST_Centroid(ST_GeomFromWKB(geom))) as y" + + + def getFeatureId(i: Array[String], parsed: Boolean = false): FeatureId = { + + val listId: String = i(0) + val locationId: Int = i(1).toInt + val x: Double = i(2).toDouble + val y: Double = i(3).toDouble + + GfwProFeatureId(listId, locationId, x, y) + } + + case class Filter( + base: Option[SummaryCommand.BaseFilter], + id: Option[SummaryCommand.FeatureIdFilter] + ) extends FeatureFilter { + def filterConditions: List[Column] = { + // TODO: is "iso" column same as "iso3"? gadm.isoFirst was applied to "iso" before + // TODO: add wdpaID filter + base.toList.flatMap(_.filters()) ++ + id.toList.flatMap(_.filters(idColumn=col("location_id"))) + } + } +} diff --git a/src/main/scala/org/globalforestwatch/features/GfwProFeatureId.scala b/src/main/scala/org/globalforestwatch/features/GfwProFeatureId.scala new file mode 100644 index 00000000..ffe58582 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/features/GfwProFeatureId.scala @@ -0,0 +1,5 @@ +package org.globalforestwatch.features + +case class GfwProFeatureId(listId: String, locationId: Int, x: Double, y: Double) extends FeatureId { + override def toString: String = s"$listId, $locationId, $x, $y" +} diff --git a/src/main/scala/org/globalforestwatch/features/GridFeatureId.scala b/src/main/scala/org/globalforestwatch/features/GridFeatureId.scala deleted file mode 100644 index 36e10d76..00000000 --- a/src/main/scala/org/globalforestwatch/features/GridFeatureId.scala +++ /dev/null @@ -1,5 +0,0 @@ -package org.globalforestwatch.features - -case class GridFeatureId(featureId: FeatureId, gridId: String) extends FeatureId { - override def toString: String = s"$featureId - $gridId" -} diff --git a/src/main/scala/org/globalforestwatch/features/GridId.scala b/src/main/scala/org/globalforestwatch/features/GridId.scala new file mode 100644 index 00000000..8c0c7a08 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/features/GridId.scala @@ -0,0 +1,5 @@ +package org.globalforestwatch.features + +case class GridId(gridId: String) extends FeatureId { + override def toString: String = gridId +} diff --git a/src/main/scala/org/globalforestwatch/features/JoinedRDD.scala b/src/main/scala/org/globalforestwatch/features/JoinedRDD.scala deleted file mode 100644 index 534258fc..00000000 --- a/src/main/scala/org/globalforestwatch/features/JoinedRDD.scala +++ /dev/null @@ -1,38 +0,0 @@ -package org.globalforestwatch.features - -import com.vividsolutions.jts.geom.Geometry -import org.apache.spark.api.java.JavaPairRDD -import org.datasyslab.geospark.enums.{GridType, IndexType} -import org.datasyslab.geospark.spatialOperator.JoinQuery -import org.datasyslab.geospark.spatialRDD.SpatialRDD - -import java.util - -object JoinedRDD { - - def apply( - fireAlertSpatialRDD: SpatialRDD[Geometry], - featureSpatialRDD: SpatialRDD[Geometry] - ): JavaPairRDD[Geometry, util.HashSet[Geometry]] = { - - // Join Fire and Feature RDD - val buildOnSpatialPartitionedRDD = true // Set to TRUE only if run join query - val considerBoundaryIntersection = false // Only return gemeotries fully covered by each query window in queryWindowRDD - val usingIndex = false - - fireAlertSpatialRDD.spatialPartitioning(GridType.QUADTREE) - - featureSpatialRDD.spatialPartitioning(fireAlertSpatialRDD.getPartitioner) - featureSpatialRDD.buildIndex( - IndexType.QUADTREE, - buildOnSpatialPartitionedRDD - ) - - JoinQuery.SpatialJoinQuery( - fireAlertSpatialRDD, - featureSpatialRDD, - usingIndex, - considerBoundaryIntersection - ) - } -} diff --git a/src/main/scala/org/globalforestwatch/features/PolygonIntersectionDF.scala b/src/main/scala/org/globalforestwatch/features/PolygonIntersectionDF.scala new file mode 100644 index 00000000..8cece657 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/features/PolygonIntersectionDF.scala @@ -0,0 +1,44 @@ +package org.globalforestwatch.features + +import cats.data.NonEmptyList +import org.apache.spark.sql.{DataFrame, SparkSession} + +object PolygonIntersectionDF { + /* + Applies a spatial join between two polygonal datasets using GeoSpark, returning the + intersecting polygons with combined attributes. + + NOTE: The spatial join will partition/index on feature 1, so typically feature 1 should + be the larger dataset. + */ + def apply(feature1Uris: NonEmptyList[String], + feature1Type: String, + feature2Uris: NonEmptyList[String], + feature2Type: String, + spark: SparkSession, + filters: FeatureFilter, + feature1Delimiter: String = "\t", + feature2Delimiter: String = "\t"): DataFrame = { + + val feature1DF: DataFrame = + SpatialFeatureDF(feature1Uris, feature1Type, filters, "geom", spark, feature1Delimiter) + + val feature2DF: DataFrame = + SpatialFeatureDF(feature2Uris, feature2Type, filters, "geom", spark, feature2Delimiter) + + PolygonIntersectionDF(feature1DF, feature2DF, spark) + } + + + def apply(feature1DF: DataFrame, feature2DF: DataFrame, spark: SparkSession): DataFrame = { + feature1DF.createOrReplaceTempView("left") + feature2DF.createOrReplaceTempView("right") + spark.sql( + "SELECT " + + "left.featureId as featureId1, " + + "right.featureId as featureId2, " + + "ST_Intersection(left.polyshape, right.polyshape) as intersectedshape " + + "FROM left, right " + + "WHERE ST_Intersects(left.polyshape, right.polyshape)") + } +} diff --git a/src/main/scala/org/globalforestwatch/features/PolygonIntersectionRDD.scala b/src/main/scala/org/globalforestwatch/features/PolygonIntersectionRDD.scala deleted file mode 100644 index a56848ce..00000000 --- a/src/main/scala/org/globalforestwatch/features/PolygonIntersectionRDD.scala +++ /dev/null @@ -1,37 +0,0 @@ -package org.globalforestwatch.features - -import cats.data.NonEmptyList -import com.vividsolutions.jts.geom.Geometry -import org.apache.spark.sql.{DataFrame, SparkSession} -import org.datasyslab.geospark.spatialRDD.SpatialRDD -import org.datasyslab.geosparksql.utils.Adapter -import org.globalforestwatch.util.Util.getAnyMapValue -import org.apache.spark.sql.functions.{col, expr} - -object PolygonIntersectionRDD { - def apply(feature1Uris: NonEmptyList[String], - feature1Obj: Feature, - feature1Type: String, - feature2Uris: NonEmptyList[String], - feature2Obj: Feature, - feature2Type: String, - spark: SparkSession, - kwargs: Map[String, Any], - feature1Delimiter: String = "\t", - feature2Delimiter: String = "\t"): SpatialRDD[Geometry] = { - - val features1DF: DataFrame = - SpatialFeatureDF(feature1Uris, feature1Obj, kwargs, "geom", spark, feature1Delimiter).as(feature1Type) - - val features2DF: DataFrame = - SpatialFeatureDF(feature2Uris, feature2Obj, kwargs, "geom", spark, feature2Delimiter).as(feature2Type) - - val joinedDF = features1DF - .join(features2DF) - .where(s"ST_Intersects(${feature1Type}.polyshape, ${feature2Type}.polyshape)") - .withColumn("intersectedshape", expr(s"ST_Intersection(${feature1Type}.polyshape, ${feature2Type}.polyshape)")) - .select(col(s"${feature1Type}.featureId") as "featureId1", col(s"${feature2Type}.featureId") as "featureId2", col("intersectedshape")) - - Adapter.toSpatialRdd(joinedDF, "intersectedshape") - } -} diff --git a/src/main/scala/org/globalforestwatch/features/SimpleFeature.scala b/src/main/scala/org/globalforestwatch/features/SimpleFeature.scala index 58f2805c..45063001 100644 --- a/src/main/scala/org/globalforestwatch/features/SimpleFeature.scala +++ b/src/main/scala/org/globalforestwatch/features/SimpleFeature.scala @@ -1,10 +1,8 @@ package org.globalforestwatch.features -import geotrellis.vector.Geometry -import geotrellis.vector.io.wkb.WKB -import org.apache.spark.sql.{DataFrame, Row, SparkSession} -import org.globalforestwatch.util.GeometryReducer -import org.globalforestwatch.util.Util._ +import org.apache.spark.sql.functions.col +import org.globalforestwatch.summarystats.SummaryCommand +import org.apache.spark.sql.Column object SimpleFeature extends Feature { @@ -14,34 +12,19 @@ object SimpleFeature extends Feature { val featureIdExpr = "cast(fid as int) as featureId" - def get(i: Row): geotrellis.vector.Feature[Geometry, FeatureId] = { - val featureId = getFeatureId(i) - val geom: Geometry = - GeometryReducer.reduce(GeometryReducer.gpr)( - WKB.read(i.getString(geomPos)) - ) - geotrellis.vector.Feature(geom, featureId) - } def getFeatureId(i: Array[String], parsed: Boolean = false): FeatureId = { val feature_id: Int = i(idPos).toInt SimpleFeatureId(feature_id) } - override def custom_filter( - filters: Map[String, Any] - )(df: DataFrame): DataFrame = { - - val spark: SparkSession = df.sparkSession - import spark.implicits._ - - val idStart: Option[Int] = getAnyMapValue[Option[Int]](filters, "idStart") - // val idEnd: Option[Int] = getAnyMapValue[Option[Int]](filters, "idEnd") - - // val idStartDF: DataFrame = - idStart.foldLeft(df)((acc, i) => acc.filter($"fid" >= i)) - - // idEnd.foldLeft(idStartDF)((acc, i) => acc.filter($"fid" < i)) - + case class Filter( + base: Option[SummaryCommand.BaseFilter], + id: Option[SummaryCommand.FeatureIdFilter] + ) extends FeatureFilter { + def filterConditions: List[Column]= { + base.toList.flatMap(_.filters()) ++ + id.toList.flatMap(_.filters(idColumn=col("fid"))) + } } } diff --git a/src/main/scala/org/globalforestwatch/features/SpatialFeatureDF.scala b/src/main/scala/org/globalforestwatch/features/SpatialFeatureDF.scala index 8fdfb51f..56d92ef3 100644 --- a/src/main/scala/org/globalforestwatch/features/SpatialFeatureDF.scala +++ b/src/main/scala/org/globalforestwatch/features/SpatialFeatureDF.scala @@ -1,18 +1,19 @@ package org.globalforestwatch.features import cats.data.NonEmptyList +import org.locationtech.jts import org.apache.spark.sql.{DataFrame, SparkSession} - +import org.apache.spark.sql.functions.{col, isnull, udf} +import scala.util.Try object SpatialFeatureDF { - def apply(input: NonEmptyList[String], featureObj: Feature, - filters: Map[String, Any], + filters: FeatureFilter, spark: SparkSession, lonField: String, latField: String): DataFrame = { - val df: DataFrame = FeatureDF.apply(input, featureObj, filters, spark) + val df: DataFrame = FeatureDF(input, featureObj, filters, spark) val viewName = featureObj.getClass.getSimpleName.dropRight(1).toLowerCase df.createOrReplaceTempView(viewName) @@ -29,19 +30,81 @@ object SpatialFeatureDF { * Use GeoSpark to directly generate a DataFrame with a geometry column */ def apply(input: NonEmptyList[String], - featureObj: Feature, - filters: Map[String, Any], + featureType: String, + filters: FeatureFilter, wkbField: String, spark: SparkSession, delimiter: String = "\t"): DataFrame = { - val featureDF: DataFrame = FeatureDF.apply(input, featureObj, filters, spark, delimiter) + val featureObj: Feature = Feature(featureType) + SpatialFeatureDF(input, featureObj, filters, wkbField, spark, delimiter) + } + + def apply(input: NonEmptyList[String], + featureObj: Feature, + filters: FeatureFilter, + wkbField: String, + spark: SparkSession, + delimiter: String): DataFrame = { + + val featureDF: DataFrame = + FeatureDF(input, featureObj, filters, spark, delimiter) val emptyPolygonWKB = "0106000020E610000000000000" + // ST_PrecisionReduce may create invalid geometry if it contains a "sliver" that is below the precision threshold + // ST_Buffer(0) fixes these invalid geometries + featureDF + .selectExpr( + s"ST_Buffer(ST_PrecisionReduce(ST_GeomFromWKB(${wkbField}), 11), 0) AS polyshape", + s"struct(${featureObj.featureIdExpr}) as featureId" + ) + .where(s"${wkbField} != '${emptyPolygonWKB}'") + } + + /* + * Use GeoSpark to directly generate a DataFrame with a geometry column + * Any geometry that fails to be parsed as WKB will be dropped here + */ + def applyValidated( + input: NonEmptyList[String], + featureObj: Feature, + filters: FeatureFilter, + wkbField: String, + spark: SparkSession, + delimiter: String = "\t" + ): DataFrame = { + import spark.implicits._ + + val featureDF: DataFrame = FeatureDF(input, featureObj, filters, spark, delimiter) + val emptyPolygonWKB = "0106000020E610000000000000" + val readOptionWkbUDF = udf{ s: String => readOption(s) } featureDF - .selectExpr( - s"ST_PrecisionReduce(ST_GeomFromWKB(${wkbField}), 13) AS polyshape", - s"struct(${featureObj.featureIdExpr}) as featureId") .where(s"${wkbField} != '${emptyPolygonWKB}'") + .selectExpr( + s"${wkbField} AS wkb", + s"struct(${featureObj.featureIdExpr}) as featureId" + ) + .select( + readOptionWkbUDF (col("wkb")).as("polyshape"), + col("featureId") + ) + .where(!isnull('polyshape)) + } + + private val threadLocalWkbReader = new ThreadLocal[jts.io.WKBReader] + + def readOption(wkbHexString: String): Option[jts.geom.Geometry] = { + if (threadLocalWkbReader.get() == null) { + val precisionModel = new jts.geom.PrecisionModel(1e11) + val factory = new jts.geom.GeometryFactory(precisionModel) + val wkbReader = new jts.io.WKBReader(factory) + threadLocalWkbReader.set(wkbReader) + } + val wkbReader = threadLocalWkbReader.get() + + Try{ + val binWkb = javax.xml.bind.DatatypeConverter.parseHexBinary(wkbHexString) + wkbReader.read(binWkb) + }.toOption } } diff --git a/src/main/scala/org/globalforestwatch/features/ValidatedFeatureRDD.scala b/src/main/scala/org/globalforestwatch/features/ValidatedFeatureRDD.scala new file mode 100644 index 00000000..f771bdd2 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/features/ValidatedFeatureRDD.scala @@ -0,0 +1,108 @@ +package org.globalforestwatch.features + +import cats.data.{NonEmptyList, Validated} +import org.apache.log4j.Logger +import geotrellis.store.index.zcurve.Z2 +import geotrellis.vector.{Geometry, Feature => GTFeature} +import org.apache.spark.HashPartitioner +import org.apache.spark.api.java.JavaPairRDD +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{DataFrame, Row, SparkSession} +import org.apache.sedona.core.spatialRDD.SpatialRDD +import org.apache.sedona.sql.utils.Adapter +import org.globalforestwatch.summarystats.{Location, ValidatedLocation} +import org.globalforestwatch.util.{GridRDD, SpatialJoinRDD} +import org.globalforestwatch.util.IntersectGeometry.validatedIntersection +import org.locationtech.jts.geom._ + +object ValidatedFeatureRDD { + val logger = Logger.getLogger("FeatureRDD") + + /** + * Reads features from source and optionally splits them by 1x1 degree grid. + * - If the feature WKB is invalid, the feature will be dropped + * - If there is a problem with intersection logic, the erroring feature id will propagate to output + */ + def apply( + input: NonEmptyList[String], + featureType: String, + filters: FeatureFilter, + splitFeatures: Boolean + )(implicit spark: SparkSession): RDD[ValidatedLocation[Geometry]] = { + + if (splitFeatures) { + val featureObj: Feature = Feature(featureType) + val featureDF: DataFrame = SpatialFeatureDF.applyValidated(input, featureObj, filters, "geom", spark) + splitGeometries(featureType, featureDF, spark) + } else { + val featureObj: Feature = Feature(featureType) + val featuresDF: DataFrame = FeatureDF(input, featureObj, filters, spark) + + featuresDF.rdd + .mapPartitions( + { iter: Iterator[Row] => + for { + i <- iter + if featureObj.isNonEmptyGeom(i) + } yield { + val GTFeature(geom, id) = featureObj.get(i) + Validated.Valid(Location(id, geom)) + } + }, + preservesPartitioning = true + ) + } + } + + private def splitGeometries( + featureType: String, + featureDF: DataFrame, + spark: SparkSession + ): RDD[ValidatedLocation[Geometry]] = { + + val spatialFeatureRDD: SpatialRDD[Geometry] = Adapter.toSpatialRdd(featureDF, "polyshape") + spatialFeatureRDD.analyze() + + spatialFeatureRDD.rawSpatialRDD = spatialFeatureRDD.rawSpatialRDD.rdd.map { geom: Geometry => + val featureId = FeatureId.fromUserData(featureType, geom.getUserData.asInstanceOf[String], delimiter = ",") + geom.setUserData(featureId) + geom + } + + val envelope: Envelope = spatialFeatureRDD.boundaryEnvelope + val spatialGridRDD = GridRDD(envelope, spark, clip = true) + val flatJoin: JavaPairRDD[Polygon, Geometry] = + SpatialJoinRDD.flatSpatialJoin( + spatialFeatureRDD, + spatialGridRDD, + considerBoundaryIntersection = true + ) + + /* + partitions will come back very skewed and we will need to even them out for any downstream analysis + For the summary analysis we will eventually use a range partitioner. + However, the range partitioner uses sampling to come up with the break points for the different partitions. + If the input RDD is already heavily skewed, sampling will be off and the range partitioner won't do a good job. + */ + val hashPartitioner = new HashPartitioner(flatJoin.getNumPartitions) + + flatJoin.rdd + .keyBy({ pair: (Polygon, Geometry) => + Z2( + (pair._1.getCentroid.getX * 100).toInt, + (pair._1.getCentroid.getY * 100).toInt + ).z + }) + .partitionBy(hashPartitioner) + .flatMap { case (_, (gridCell, geom)) => + val fid = geom.getUserData.asInstanceOf[FeatureId] + validatedIntersection(geom, gridCell) + .leftMap { err => Location(fid, err) } + .map { geoms => geoms.map { geom => + // val gtGeom: Geometry = toGeotrellisGeometry(geom) + Location(fid, geom) + } } + .traverse(identity) + } + } +} diff --git a/src/main/scala/org/globalforestwatch/features/WdpaFeature.scala b/src/main/scala/org/globalforestwatch/features/WdpaFeature.scala index 48e7ca94..8990413e 100644 --- a/src/main/scala/org/globalforestwatch/features/WdpaFeature.scala +++ b/src/main/scala/org/globalforestwatch/features/WdpaFeature.scala @@ -1,11 +1,8 @@ package org.globalforestwatch.features -import geotrellis.vector.Geometry -import geotrellis.vector.io.wkb.WKB -import org.apache.spark.sql.functions.substring -import org.apache.spark.sql.{DataFrame, Row, SparkSession} -import org.globalforestwatch.util.GeometryReducer -import org.globalforestwatch.util.Util._ +import org.apache.spark.sql.functions.col +import org.globalforestwatch.summarystats.SummaryCommand +import org.apache.spark.sql.Column object WdpaFeature extends Feature { @@ -17,20 +14,11 @@ object WdpaFeature extends Feature { val geomPos = 7 val featureCount = 4 - val featureIdExpr = "cast(wdpaid as int) as wdpaId, name as name, iucn_cat as iucnCat, iso3 as iso, status" - - def get(i: Row): geotrellis.vector.Feature[Geometry, FeatureId] = { - val featureId = getFeatureId(i) - val geom: Geometry = - GeometryReducer.reduce(GeometryReducer.gpr)( - WKB.read(i.getString(geomPos)) - ) - - geotrellis.vector - .Feature(geom, featureId) - } + val featureIdExpr = + "cast(wdpaid as int) as wdpaId, name as name, iucn_cat as iucnCat, iso3 as iso, status" def getFeatureId(i: Array[String], parsed: Boolean = false): FeatureId = { + val wdpaId: Int = i(wdpaIdPos).toInt val name: String = i(namePos) val iucnCat: String = i(iucnCatPos) @@ -40,50 +28,16 @@ object WdpaFeature extends Feature { WdpaFeatureId(wdpaId, name, iucnCat, iso, status) } - override def custom_filter( - filters: Map[String, Any] - )(df: DataFrame): DataFrame = { - val spark: SparkSession = df.sparkSession - import spark.implicits._ - - val isoFirst: Option[String] = - getAnyMapValue[Option[String]](filters, "isoFirst") - val isoStart: Option[String] = - getAnyMapValue[Option[String]](filters, "isoStart") - val isoEnd: Option[String] = - getAnyMapValue[Option[String]](filters, "isoEnd") - val iso: Option[String] = getAnyMapValue[Option[String]](filters, "iso") - val wdpaIdStart: Option[Int] = - getAnyMapValue[Option[Int]](filters, "idStart") - //val wdpaIdEnd: Option[Int] = getAnyMapValue[Option[Int]](filters, "idEnd") - //val iucnCat: Option[String] = - // getAnyMapValue[Option[String]](filters, "iucnCat") - val wdpaStatus: Option[String] = - getAnyMapValue[Option[String]](filters, "wdpaStatus") - - val isoFirstDF: DataFrame = isoFirst.foldLeft(df)( - (acc, i) => acc.filter(substring($"iso", 0, 1) === i(0)) - ) - val isoStartDF: DataFrame = - isoStart.foldLeft(isoFirstDF)((acc, i) => acc.filter($"iso" >= i)) - val isoEndDF: DataFrame = - isoEnd.foldLeft(isoStartDF)((acc, i) => acc.filter($"iso" < i)) - val isoDF: DataFrame = - iso.foldLeft(isoEndDF)((acc, i) => acc.filter($"iso" === i)) - - val wdpaIdStartDF = - wdpaIdStart.foldLeft(isoDF)((acc, i) => acc.filter($"wdpaid" >= i)) - - /* - val wdpaIdEndtDF = - wdpaIdEnd.foldLeft(wdpaIdStartDF)((acc, i) => acc.filter($"wdpaid" < i)) - - val iucnCatDF = - wdpaIdEnd.foldLeft(wdpaIdEndtDF)( - (acc, i) => acc.filter($"iucn_cat" === i) - ) - */ - - wdpaStatus.foldLeft(isoDF)((acc, i) => acc.filter($"status" === i)) + case class Filter( + base: Option[SummaryCommand.BaseFilter], + gadm: Option[SummaryCommand.GadmFilter], + wdpa: Option[SummaryCommand.WdpaFilter] + ) extends FeatureFilter { + def filterConditions: List[Column]= { + // TODO: is "iso" column same as "iso3"? gadm.isoFirst was applied to "iso" before + base.toList.flatMap(_.filters()) ++ + wdpa.toList.flatMap(_.filters()) ++ + gadm.toList.flatMap(_.filters(isoColumn=col("iso3"), admin1Column=col("admin1"), admin2Column=col("admin2"))) + } } } diff --git a/src/main/scala/org/globalforestwatch/grids/Grid.scala b/src/main/scala/org/globalforestwatch/grids/Grid.scala index 9df7b822..552e3abd 100644 --- a/src/main/scala/org/globalforestwatch/grids/Grid.scala +++ b/src/main/scala/org/globalforestwatch/grids/Grid.scala @@ -2,7 +2,7 @@ package org.globalforestwatch.grids import geotrellis.raster.TileLayout import geotrellis.layer.{LayoutDefinition, SpatialKey} -import geotrellis.vector.{Extent, Point} +import geotrellis.vector.Extent import org.globalforestwatch.layers.{OptionalLayer, RequiredLayer} trait Grid[T <: GridSources] { @@ -16,8 +16,8 @@ trait Grid[T <: GridSources] { /** this represents the tile layout of 10x10 degrees */ lazy val rasterFileGrid: LayoutDefinition = { val tileLayout = TileLayout( - layoutCols = (gridExtent.xmin until gridExtent.xmax by gridSize).length, - layoutRows = (gridExtent.ymin until gridExtent.ymax by gridSize).length, + layoutCols = (gridExtent.xmin.toInt until gridExtent.xmax.toInt by gridSize).length, + layoutRows = (gridExtent.ymin.toInt until gridExtent.ymax.toInt by gridSize).length, tileCols = math.round(gridSize / pixelSize).toInt, tileRows = math.round(gridSize / pixelSize).toInt ) @@ -29,8 +29,8 @@ trait Grid[T <: GridSources] { */ lazy val stripedTileGrid: LayoutDefinition = { val tileLayout = TileLayout( - layoutCols = (gridExtent.xmin until gridExtent.xmax by gridSize).length, - layoutRows = (gridExtent.ymin until gridExtent.ymax by gridSize).length * (math + layoutCols = (gridExtent.xmin.toInt until gridExtent.xmax.toInt by gridSize).length, + layoutRows = (gridExtent.ymin.toInt until gridExtent.ymax.toInt by gridSize).length * (math .round(gridSize / pixelSize) .toInt / rowCount), tileCols = math.round(gridSize / pixelSize).toInt, @@ -44,10 +44,10 @@ trait Grid[T <: GridSources] { */ lazy val blockTileGrid: LayoutDefinition = { val tileLayout = TileLayout( - layoutCols = (gridExtent.xmin until gridExtent.xmax by gridSize).length * (math + layoutCols = (gridExtent.xmin.toInt until gridExtent.xmax.toInt by gridSize).length * (math .round(gridSize / pixelSize) .toInt / blockSize), - layoutRows = (gridExtent.ymin until gridExtent.ymax by gridSize).length * (math + layoutRows = (gridExtent.ymin.toInt until gridExtent.ymax.toInt by gridSize).length * (math .round(gridSize / pixelSize) .toInt / blockSize), tileCols = blockSize, @@ -61,7 +61,7 @@ trait Grid[T <: GridSources] { def checkSources(gridTile: GridTile, windowExtent: Extent, windowKey: SpatialKey, windowLayout: LayoutDefinition, kwargs: Map[String, Any]): T = { def ccToMap(cc: AnyRef): Map[String, Any] = - (Map[String, Any]() /: cc.getClass.getDeclaredFields) { (a, f) => + cc.getClass.getDeclaredFields.foldLeft(Map.empty[String, Any]) { (a, f) => f.setAccessible(true) a + (f.getName -> f.get(cc)) } diff --git a/src/main/scala/org/globalforestwatch/grids/GridId.scala b/src/main/scala/org/globalforestwatch/grids/GridId.scala index e4bbb523..8d52e9f1 100644 --- a/src/main/scala/org/globalforestwatch/grids/GridId.scala +++ b/src/main/scala/org/globalforestwatch/grids/GridId.scala @@ -7,6 +7,7 @@ object GridId { /** Translate from a point on a map to file grid ID of 10x10 grid * Top-Left corner, exclusive on south, east, inclusive on north and west */ + def pointGridId(point: Point, gridSize: Int): String = { val col = math.floor(point.getX / gridSize).toInt * gridSize val long: String = if (col >= 0) f"$col%03dE" else f"${-col}%03dW" @@ -17,6 +18,10 @@ object GridId { s"${lat}_$long" } + def pointGridId(x: Double, y: Double, gridSize: Int): String = { + pointGridId(Point(x, y), gridSize) + } + def toGladGridId(grid: String): String = { case class Corner(coord: Int, nsew: String) { override def toString: String = { diff --git a/src/main/scala/org/globalforestwatch/grids/GridSources.scala b/src/main/scala/org/globalforestwatch/grids/GridSources.scala index 73fd66e7..3a8f7245 100644 --- a/src/main/scala/org/globalforestwatch/grids/GridSources.scala +++ b/src/main/scala/org/globalforestwatch/grids/GridSources.scala @@ -3,10 +3,7 @@ package org.globalforestwatch.grids import com.typesafe.scalalogging.LazyLogging import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.{CellGrid, Raster} -import geotrellis.vector.Extent trait GridSources extends LazyLogging { - def readWindow(windowKey: SpatialKey, windowLayout: LayoutDefinition): Either[Throwable, Raster[CellGrid[Int]]] - } diff --git a/src/main/scala/org/globalforestwatch/layers/BrazilBiomes.scala b/src/main/scala/org/globalforestwatch/layers/BrazilBiomes.scala index 3429bdc9..201cbe58 100644 --- a/src/main/scala/org/globalforestwatch/layers/BrazilBiomes.scala +++ b/src/main/scala/org/globalforestwatch/layers/BrazilBiomes.scala @@ -1,11 +1,11 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile +import org.globalforestwatch.config.GfwConfig case class BrazilBiomes(gridTile: GridTile) extends StringLayer with OptionalILayer { - val uri: String = - s"$basePath/bra_biomes/v20150601/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/name/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers(getClass.getSimpleName()), gridTile) override val externalNoDataValue = "Not applicable" diff --git a/src/main/scala/org/globalforestwatch/layers/GFWProCoverage.scala b/src/main/scala/org/globalforestwatch/layers/GFWProCoverage.scala index cef93008..27cbac26 100644 --- a/src/main/scala/org/globalforestwatch/layers/GFWProCoverage.scala +++ b/src/main/scala/org/globalforestwatch/layers/GFWProCoverage.scala @@ -3,8 +3,8 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile case class GFWProCoverage(gridTile: GridTile) - extends MapLayer - with OptionalILayer { + extends MapILayer + with OptionalILayer { val uri: String = s"$basePath/gfwpro_forest_change_regions/v20210129/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/bit_encoding/gdal-geotiff/${gridTile.tileId}.tif" diff --git a/src/main/scala/org/globalforestwatch/layers/GFWProPeatlands.scala b/src/main/scala/org/globalforestwatch/layers/GFWProPeatlands.scala new file mode 100644 index 00000000..51c51887 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/layers/GFWProPeatlands.scala @@ -0,0 +1,10 @@ +package org.globalforestwatch.layers + +import org.globalforestwatch.grids.GridTile + +case class GFWProPeatlands(gridTile: GridTile) + extends BooleanLayer + with OptionalILayer { + val uri: String = + s"s3://gfw-data-lake/gfwpro_peatlands/v2019/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" +} diff --git a/src/main/scala/org/globalforestwatch/layers/GladAlerts.scala b/src/main/scala/org/globalforestwatch/layers/GladAlerts.scala index 8997f204..5325ac37 100644 --- a/src/main/scala/org/globalforestwatch/layers/GladAlerts.scala +++ b/src/main/scala/org/globalforestwatch/layers/GladAlerts.scala @@ -3,58 +3,24 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile import java.time.LocalDate -import java.time.format.DateTimeFormatter import org.globalforestwatch.grids.GridId.toGladGridId case class GladAlerts(gridTile: GridTile) extends DateConfLayer with OptionalILayer { - + val baseDate = LocalDate.of(2014,12,31) val gladGrid: String = toGladGridId(gridTile.tileId) val uri: String = s"s3://gfw2-data/forest_change/umd_landsat_alerts/prod/analysis/$gladGrid.tif" - override def lookup(value: Int): Option[(String, Boolean)] = { + override def lookup(value: Int): Option[(LocalDate, Boolean)] = { val confidence = value >= 30000 - val alertDate: Option[String] = { - - def isLeapYear(year: Int): Boolean = { - implicit def int2boolRev(i: Int): Boolean = i <= 0 - year % 4 - } - - def getDateString(days: Int, year: Int): String = { - val daysInYear = if (isLeapYear(year)) 366 else 365 - if (days <= daysInYear) s"$year" + "%03d".format(days) - else getDateString(days - daysInYear, year + 1) - } - - val julianDate = DateTimeFormatter.ofPattern("yyyyDDD") - val days: Int = if (confidence) value - 30000 else value - 20000 - - days match { - case d if d < 0 => None - case d if d == 0 => - Some( - LocalDate - .parse(getDateString(365, 2014), julianDate) - .format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) - ) - case _ => - Some( - LocalDate - .parse(getDateString(days, 2015), julianDate) - .format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) - ) - } - - } - - alertDate match { - case Some(d) => Some(d, confidence) - case None => None + val days: Int = if (confidence) value - 30000 else value - 20000 + if (days < 0) { + None + } else { + val date = baseDate.plusDays(days) + Some((date, confidence)) } - } - } diff --git a/src/main/scala/org/globalforestwatch/layers/IndonesiaForestArea.scala b/src/main/scala/org/globalforestwatch/layers/IndonesiaForestArea.scala index 301355b5..a4d3605f 100644 --- a/src/main/scala/org/globalforestwatch/layers/IndonesiaForestArea.scala +++ b/src/main/scala/org/globalforestwatch/layers/IndonesiaForestArea.scala @@ -1,13 +1,14 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile +import org.globalforestwatch.config.GfwConfig case class IndonesiaForestArea(gridTile: GridTile) extends StringLayer with OptionalILayer { - val uri: String = - s"$basePath/idn_forest_area/v201709/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/class_compressed/gdal-geotiff/${gridTile.tileId}.tif" + val datasetName = "idn_forest_area" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers(getClass.getSimpleName()), gridTile) override val externalNoDataValue: String = "" diff --git a/src/main/scala/org/globalforestwatch/layers/IndonesiaLandCover.scala b/src/main/scala/org/globalforestwatch/layers/IndonesiaLandCover.scala index 0f716e03..7f36a16a 100644 --- a/src/main/scala/org/globalforestwatch/layers/IndonesiaLandCover.scala +++ b/src/main/scala/org/globalforestwatch/layers/IndonesiaLandCover.scala @@ -1,15 +1,19 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile +import org.globalforestwatch.config.GfwConfig case class IndonesiaLandCover(gridTile: GridTile) extends StringLayer with OptionalILayer { val uri: String = s"$basePath/idn_land_cover_2017/v20180720/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/class/gdal-geotiff/${gridTile.tileId}.tif" override val externalNoDataValue: String = "" + private val fLookup = if (GfwConfig.isGfwPro) IndonesiaLandCover.proLabelTable else IndonesiaLandCover.flagshipLabelTable + def lookup(value: Int): String = fLookup(value) +} - def lookup(value: Int): String = value match { - +object IndonesiaLandCover { + val flagshipLabelTable: Int => String = { case 2001 => "Primary Dry Land Forest" case 2002 => "Secondary Dry Land Forest" case 2004 => "Primary Mangrove Forest" @@ -41,4 +45,37 @@ case class IndonesiaLandCover(gridTile: GridTile) extends StringLayer with Optio case 50011 => "Swamp" case _ => "" } + + val proLabelTable: Int => String = { + case 2001 => "Primary forest" + case 2002 => "Secondary forest" + case 2004 => "Primary forest" + case 2005 => "Primary forest" + case 2006 => "Timber plantation" + case 2007 => "Grassland/shrub" + case 2008 => "" + case 2010 => "Estate crop plantation" + case 2011 => "" + case 2012 => "Settlement" + case 2014 => "Bare land" + case 2020 => "" + case 2092 => "" + case 3000 => "Grassland/shrub" + case 20021 => "" + case 20041 => "Secondary forest" + case 20051 => "Secondary forest" + case 20071 => "Swamp" + case 20091 => "Agriculture" + case 20092 => "Agriculture" + case 20093 => "Agriculture" + case 20094 => "Fish pond" + case 20102 => "" + case 20121 => "Settlement" + case 20122 => "Settlement" + case 20141 => "Mining" + case 20191 => "" + case 5001 => "Body of water" + case 50011 => "Swamp" + case _ => "" + } } diff --git a/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala b/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala index 5052d6d3..2560002b 100644 --- a/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala +++ b/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala @@ -1,11 +1,12 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile +import org.globalforestwatch.config.GfwConfig case class IntactForestLandscapes(gridTile: GridTile) extends StringLayer with OptionalILayer { - val uri: String = s"$basePath/ifl_intact_forest_landscapes/v20180628/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/year/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers("IntactForestLandscapes"), gridTile) def lookup(value: Int): String = value match { case 0 => "" @@ -18,7 +19,7 @@ case class IntactForestLandscapes(gridTile: GridTile) case class IntactForestLandscapes2000(gridTile: GridTile) extends BooleanLayer with OptionalILayer { - val uri: String = s"$basePath/ifl_intact_forest_landscapes/v20180628/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/year/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers("IntactForestLandscapes"), gridTile) override def lookup(value: Int): Boolean = { value match { @@ -32,7 +33,7 @@ case class IntactForestLandscapes2000(gridTile: GridTile) case class IntactForestLandscapes2013(gridTile: GridTile) extends BooleanLayer with OptionalILayer { - val uri: String = s"$basePath/ifl_intact_forest_landscapes/v20180628/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/year/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers("IntactForestLandscapes"), gridTile) override def lookup(value: Int): Boolean = { value match { @@ -47,7 +48,7 @@ case class IntactForestLandscapes2013(gridTile: GridTile) case class IntactForestLandscapes2016(gridTile: GridTile) extends BooleanLayer with OptionalILayer { - val uri: String = s"$basePath/ifl_intact_forest_landscapes/v20180628/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/year/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers("IntactForestLandscapes"), gridTile) override def lookup(value: Int): Boolean = { value match { diff --git a/src/main/scala/org/globalforestwatch/layers/KeyBiodiversityAreas.scala b/src/main/scala/org/globalforestwatch/layers/KeyBiodiversityAreas.scala index 42de54b8..cc34b05c 100644 --- a/src/main/scala/org/globalforestwatch/layers/KeyBiodiversityAreas.scala +++ b/src/main/scala/org/globalforestwatch/layers/KeyBiodiversityAreas.scala @@ -5,5 +5,5 @@ import org.globalforestwatch.grids.GridTile case class KeyBiodiversityAreas(gridTile: GridTile) extends BooleanLayer with OptionalILayer { - val uri: String = s"$basePath/birdlife_key_biodiversity_areas/v20191211/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" + val uri: String = s"$basePath/birdlife_key_biodiversity_areas/v202106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/Landmark.scala b/src/main/scala/org/globalforestwatch/layers/Landmark.scala index f678d5ee..12b7e2e6 100644 --- a/src/main/scala/org/globalforestwatch/layers/Landmark.scala +++ b/src/main/scala/org/globalforestwatch/layers/Landmark.scala @@ -3,5 +3,5 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile case class Landmark(gridTile: GridTile) extends BooleanLayer with OptionalILayer { - val uri: String = s"$basePath/landmark_land_rights/v20191111/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" + val uri: String = s"$basePath/landmark_indigenous_and_community_lands/v20201215/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/Layer.scala b/src/main/scala/org/globalforestwatch/layers/Layer.scala index ea222ad9..4254810d 100644 --- a/src/main/scala/org/globalforestwatch/layers/Layer.scala +++ b/src/main/scala/org/globalforestwatch/layers/Layer.scala @@ -8,7 +8,6 @@ import geotrellis.layer.{LayoutDefinition, LayoutTileSource, SpatialKey} import geotrellis.raster.gdal.GDALRasterSource import geotrellis.raster.{ CellType, - IntCellType, IntCells, NoDataHandling, Tile, @@ -20,6 +19,8 @@ import software.amazon.awssdk.services.s3.model.{ NoSuchKeyException, RequestPayer } +import org.globalforestwatch.grids.GridTile +import java.time.LocalDate trait Layer { @@ -35,6 +36,13 @@ trait Layer { val externalNoDataValue: B val basePath: String = s"s3://gfw-data-lake" + + protected def uriForGrid(template: String, grid: GridTile): String = + template + .replace("", grid.gridSize.toString) + .replace("", grid.rowCount.toString) + .replace("", grid.tileId) + def lookup(a: A): B } @@ -382,11 +390,7 @@ trait IntegerLayer extends ILayer { trait DateConfLayer extends ILayer { - /** - * Layers which return an Integer type - * (use java.lang.Integer to be able to use null) - */ - type B = Option[(String, Boolean)] + type B = Option[(LocalDate, Boolean)] val internalNoDataValue: Int = 0 val externalNoDataValue: B = None @@ -483,7 +487,7 @@ trait StringLayer extends ILayer { } -trait MapLayer extends ILayer { +trait MapILayer extends ILayer { /** * Layers which return a String type @@ -494,3 +498,15 @@ trait MapLayer extends ILayer { val externalNoDataValue: Map[String, Boolean] = Map() } + +trait MapFLayer extends FLayer { + + /** + * Layers which return a String type + */ + type B = Map[String, Boolean] + + val internalNoDataValue: Float = 0 + val externalNoDataValue: Map[String, Boolean] = Map() + +} \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/layers/Logging.scala b/src/main/scala/org/globalforestwatch/layers/Logging.scala index eafe1e63..a7d96865 100644 --- a/src/main/scala/org/globalforestwatch/layers/Logging.scala +++ b/src/main/scala/org/globalforestwatch/layers/Logging.scala @@ -3,5 +3,5 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile case class Logging(gridTile: GridTile) extends BooleanLayer with OptionalILayer { - val uri: String = s"$basePath/gfw_managed_forests/v20190103/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" + val uri: String = s"$basePath/gfw_managed_forests/v202106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/Mangroves1996.scala b/src/main/scala/org/globalforestwatch/layers/Mangroves1996.scala index 287a5b89..5003bd35 100644 --- a/src/main/scala/org/globalforestwatch/layers/Mangroves1996.scala +++ b/src/main/scala/org/globalforestwatch/layers/Mangroves1996.scala @@ -4,5 +4,5 @@ import org.globalforestwatch.grids.GridTile case class Mangroves1996(gridTile: GridTile) extends BooleanLayer with OptionalILayer { val uri: String = - s"$basePath/gmw_mangroves_1996/v20180701/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" + s"$basePath/gmw_global_mangrove_extent_1996/v20180701/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/Mangroves2016.scala b/src/main/scala/org/globalforestwatch/layers/Mangroves2016.scala index 7739f6bb..a9e662cc 100644 --- a/src/main/scala/org/globalforestwatch/layers/Mangroves2016.scala +++ b/src/main/scala/org/globalforestwatch/layers/Mangroves2016.scala @@ -4,5 +4,5 @@ import org.globalforestwatch.grids.GridTile case class Mangroves2016(gridTile: GridTile) extends BooleanLayer with OptionalILayer { val uri: String = - s"$basePath/gmw_mangroves_2016/v20180701/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" + s"$basePath/gmw_global_mangrove_extent_2016/v20180701/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/Mining.scala b/src/main/scala/org/globalforestwatch/layers/Mining.scala index 506e9945..96b17f68 100644 --- a/src/main/scala/org/globalforestwatch/layers/Mining.scala +++ b/src/main/scala/org/globalforestwatch/layers/Mining.scala @@ -3,5 +3,5 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile case class Mining(gridTile: GridTile) extends BooleanLayer with OptionalILayer { - val uri: String = s"$basePath/gfw_mining/v20190205/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" + val uri: String = s"$basePath/gfw_mining_concessions/v202106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/Peatlands.scala b/src/main/scala/org/globalforestwatch/layers/Peatlands.scala index 353ba2fb..bb4af9b0 100644 --- a/src/main/scala/org/globalforestwatch/layers/Peatlands.scala +++ b/src/main/scala/org/globalforestwatch/layers/Peatlands.scala @@ -3,5 +3,5 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile case class Peatlands(gridTile: GridTile) extends BooleanLayer with OptionalILayer { - val uri: String = s"$basePath/gfw_peatlands/v20190103/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" + val uri: String = s"$basePath/gfw_peatlands/v20200807/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/PeruForestConcessions.scala b/src/main/scala/org/globalforestwatch/layers/PeruForestConcessions.scala index e6978dda..1cb08234 100644 --- a/src/main/scala/org/globalforestwatch/layers/PeruForestConcessions.scala +++ b/src/main/scala/org/globalforestwatch/layers/PeruForestConcessions.scala @@ -7,7 +7,7 @@ case class PeruForestConcessions(gridTile: GridTile) with OptionalILayer { val uri: String = - s"$basePath/per_forest_concessions/v20161001/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/type/gdal-geotiff/${gridTile.tileId}.tif" + s"$basePath/per_forest_concessions/v2016/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/type/gdal-geotiff/${gridTile.tileId}.tif" override val externalNoDataValue: String = "" diff --git a/src/main/scala/org/globalforestwatch/layers/PeruProductionForest.scala b/src/main/scala/org/globalforestwatch/layers/PeruProductionForest.scala index 1351bb72..898e25ac 100644 --- a/src/main/scala/org/globalforestwatch/layers/PeruProductionForest.scala +++ b/src/main/scala/org/globalforestwatch/layers/PeruProductionForest.scala @@ -5,6 +5,5 @@ import org.globalforestwatch.grids.GridTile case class PeruProductionForest(gridTile: GridTile) extends BooleanLayer with OptionalILayer { - val uri: String = - s"$basePath/per_permanent_production_forests/v20150901/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" + val uri: String = s"$basePath/osinfor_peru_permanent_production_forests/v2015/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/Plantations.scala b/src/main/scala/org/globalforestwatch/layers/Plantations.scala index 36c0c9c4..a5d6a52a 100644 --- a/src/main/scala/org/globalforestwatch/layers/Plantations.scala +++ b/src/main/scala/org/globalforestwatch/layers/Plantations.scala @@ -1,10 +1,11 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile +import org.globalforestwatch.config.GfwConfig case class Plantations(gridTile: GridTile) extends StringLayer with OptionalILayer { - val uri: String = s"$basePath/gfw_plantations/v1.3/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/type/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers("Plantations"), gridTile) def lookup(value: Int): String = value match { case 1 => "Fruit" @@ -24,6 +25,6 @@ case class Plantations(gridTile: GridTile) extends StringLayer with OptionalILay case class PlantationsBool(gridTile: GridTile) extends BooleanLayer with OptionalILayer { - val uri: String = s"$basePath/gfw_plantations/v1.3/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/type/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers("Plantations"), gridTile) -} \ No newline at end of file +} diff --git a/src/main/scala/org/globalforestwatch/layers/ProdesLossYear.scala b/src/main/scala/org/globalforestwatch/layers/ProdesLossYear.scala index c675aa2f..ec02098a 100644 --- a/src/main/scala/org/globalforestwatch/layers/ProdesLossYear.scala +++ b/src/main/scala/org/globalforestwatch/layers/ProdesLossYear.scala @@ -6,7 +6,7 @@ case class ProdesLossYear(gridTile: GridTile) extends IntegerLayer with OptionalILayer { val uri: String = - s"$basePath/inpe_amazonia_prodes/v20201201/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/year/gdal-geotiff/${gridTile.tileId}.tif" + s"$basePath/inpe_prodes/v202107/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/year/gdal-geotiff/${gridTile.tileId}.tif" override def lookup(value: Int): Integer = if (value == 0) null else value + 2000 } diff --git a/src/main/scala/org/globalforestwatch/layers/ProtectedAreas.scala b/src/main/scala/org/globalforestwatch/layers/ProtectedAreas.scala index 626ceb79..8c412348 100644 --- a/src/main/scala/org/globalforestwatch/layers/ProtectedAreas.scala +++ b/src/main/scala/org/globalforestwatch/layers/ProtectedAreas.scala @@ -1,10 +1,11 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile +import org.globalforestwatch.config.GfwConfig case class ProtectedAreas(gridTile: GridTile) extends StringLayer with OptionalILayer { - val uri: String = s"$basePath/wdpa_protected_areas/v202010/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/iucn_cat/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers(getClass.getSimpleName()), gridTile) def lookup(value: Int): String = value match { case 1 => "Category Ia/b or II" diff --git a/src/main/scala/org/globalforestwatch/layers/SEAsiaLandCover.scala b/src/main/scala/org/globalforestwatch/layers/SEAsiaLandCover.scala index 56403503..7ccb73fd 100644 --- a/src/main/scala/org/globalforestwatch/layers/SEAsiaLandCover.scala +++ b/src/main/scala/org/globalforestwatch/layers/SEAsiaLandCover.scala @@ -1,13 +1,13 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile +import org.globalforestwatch.config.GfwConfig case class SEAsiaLandCover(gridTile: GridTile) extends StringLayer with OptionalILayer { - val uri: String = - s"$basePath/rspo_southeast_asia_land_cover_2010/v2013/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/land_cover_class/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers(getClass.getSimpleName()), gridTile) override val externalNoDataValue = "Unknown" @@ -18,7 +18,7 @@ case class SEAsiaLandCover(gridTile: GridTile) case 8 => "Oil palm plantation" case 9 => "Timber plantation" case 10 => "Mixed tree crops" - case 11 | 15 => "Grassland/ shrub" + case 11 | 15 => "Grassland/shrub" case 12 | 16 => "Swamp" case 13 | 17 => "Agriculture" case 14 => "Settlements" diff --git a/src/main/scala/org/globalforestwatch/layers/SoyPlantedAreas.scala b/src/main/scala/org/globalforestwatch/layers/SoyPlantedAreas.scala index 0da952fd..815abad5 100644 --- a/src/main/scala/org/globalforestwatch/layers/SoyPlantedAreas.scala +++ b/src/main/scala/org/globalforestwatch/layers/SoyPlantedAreas.scala @@ -4,7 +4,42 @@ import org.globalforestwatch.grids.GridTile case class SoyPlantedAreas(gridTile: GridTile) extends BooleanLayer - with OptionalILayer { + with OptionalILayer { val uri: String = - s"$basePath/umd_soy_planted_area/v2019/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/soy_planted_area/geotiff/${gridTile.tileId}.tif" + s"$basePath/umd_soy_planted_area/v1/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is__year_2020/geotiff/${gridTile.tileId}.tif" + // + // def lookup(value: Float): Map[String, Boolean] = { + // // Geotrellis interprets Uint32 pixels as Float values + // // Somehow they also come back as very low values in `E` notation. + // // Here we convert them into the correct Long value ie 9.18355E-41 -> 918355 + // + // val valueString: String = value.toString + // val valueLong: Long = valueString.split("E")(0).replace(".", "").toLong + // + // if (valueLong > math.pow(2,20)) throw new RuntimeException(s"Value ${valueLong} out of Range") + // + // val bits = "0" * 19 + valueLong.toBinaryString takeRight 20 + // Map( + // "2001" -> (bits(19) == '1'), + // "2002" -> (bits(18) == '1'), + // "2003" -> (bits(17) == '1'), + // "2004" -> (bits(16) == '1'), + // "2005" -> (bits(15) == '1'), + // "2006" -> (bits(14) == '1'), + // "2007" -> (bits(13) == '1'), + // "2008" -> (bits(12) == '1'), + // "2009" -> (bits(11) == '1'), + // "2010" -> (bits(10) == '1'), + // "2011" -> (bits(9) == '1'), + // "2012" -> (bits(8) == '1'), + // "2013" -> (bits(7) == '1'), + // "2014" -> (bits(6) == '1'), + // "2015" -> (bits(5) == '1'), + // "2016" -> (bits(4) == '1'), + // "2017" -> (bits(3) == '1'), + // "2018" -> (bits(2) == '1'), + // "2019" -> (bits(1) == '1'), + // "2020" -> (bits(0) == '1') + // ) + // } } diff --git a/src/main/scala/org/globalforestwatch/layers/TreeCoverDensity.scala b/src/main/scala/org/globalforestwatch/layers/TreeCoverDensity.scala index 35ad80f9..e848c0b7 100644 --- a/src/main/scala/org/globalforestwatch/layers/TreeCoverDensity.scala +++ b/src/main/scala/org/globalforestwatch/layers/TreeCoverDensity.scala @@ -1,6 +1,7 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile +import org.globalforestwatch.config.GfwConfig trait TreeCoverDensityThreshold extends IntegerLayer with RequiredILayer { @@ -22,18 +23,18 @@ trait TreeCoverDensityThreshold extends IntegerLayer with RequiredILayer { case class TreeCoverDensityThreshold2000(gridTile: GridTile) extends TreeCoverDensityThreshold { - val uri: String = s"$basePath/umd_tree_cover_density_2000/v1.6/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/percent/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers("TreeCoverDensity2000"), gridTile) } case class TreeCoverDensityThreshold2010(gridTile: GridTile) extends TreeCoverDensityThreshold { - val uri: String = s"$basePath/umd_tree_cover_density_2010/v1.6/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/percent/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers("TreeCoverDensity2010"), gridTile) } case class TreeCoverDensity2010_60(gridTile: GridTile) extends BooleanLayer with RequiredILayer { - val uri: String = s"$basePath/umd_tree_cover_density_2010/v1.6/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/percent/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers("TreeCoverDensity2010"), gridTile) override def lookup(value: Int): Boolean = value > 60 @@ -43,12 +44,12 @@ case class TreeCoverDensityPercent2000(gridTile: GridTile) extends IntegerLayer with RequiredILayer { override val externalNoDataValue: Integer = 0 - val uri: String = s"$basePath/umd_tree_cover_density_2000/v1.6/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/percent/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers("TreeCoverDensity2000"), gridTile) } case class TreeCoverDensityPercent2010(gridTile: GridTile) extends IntegerLayer with RequiredILayer { override val externalNoDataValue: Integer = 0 - val uri: String = s"$basePath/umd_tree_cover_density_2010/v1.6/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/percent/gdal-geotiff/${gridTile.tileId}.tif" + val uri: String = uriForGrid(GfwConfig.get.rasterLayers("TreeCoverDensity2010"), gridTile) } diff --git a/src/main/scala/org/globalforestwatch/layers/TreeCoverLossDrivers.scala b/src/main/scala/org/globalforestwatch/layers/TreeCoverLossDrivers.scala index dce60465..548636fb 100644 --- a/src/main/scala/org/globalforestwatch/layers/TreeCoverLossDrivers.scala +++ b/src/main/scala/org/globalforestwatch/layers/TreeCoverLossDrivers.scala @@ -6,7 +6,7 @@ case class TreeCoverLossDrivers(gridTile: GridTile) extends StringLayer with OptionalILayer { val uri: String = - s"$basePath/umd_drivers/v2020/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/drivers/gdal-geotiff/${gridTile.tileId}.tif" + s"$basePath/tsc_tree_cover_loss_drivers/v2020/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/driver/gdal-geotiff/${gridTile.tileId}.tif" // s"s3://gfw-files/flux_1_2_1/drivers/${gridTile.tileId}.tif" override val internalNoDataValue = 0 diff --git a/src/main/scala/org/globalforestwatch/layers/WoodFiber.scala b/src/main/scala/org/globalforestwatch/layers/WoodFiber.scala index c385bb8f..0b8f07b7 100644 --- a/src/main/scala/org/globalforestwatch/layers/WoodFiber.scala +++ b/src/main/scala/org/globalforestwatch/layers/WoodFiber.scala @@ -3,5 +3,5 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile case class WoodFiber(gridTile: GridTile) extends BooleanLayer with OptionalILayer { - val uri: String = s"$basePath/gfw_wood_fiber/v20200725/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" + val uri: String = s"$basePath/gfw_wood_fiber/v202106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/is/geotiff/${gridTile.tileId}.tif" } diff --git a/src/main/scala/org/globalforestwatch/summarystats/ErrorSummaryRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/ErrorSummaryRDD.scala new file mode 100644 index 00000000..dcf7a2ba --- /dev/null +++ b/src/main/scala/org/globalforestwatch/summarystats/ErrorSummaryRDD.scala @@ -0,0 +1,167 @@ +package org.globalforestwatch.summarystats + +import cats.implicits._ +import com.typesafe.scalalogging.LazyLogging +import geotrellis.raster._ +import geotrellis.raster.rasterize.Rasterizer +import geotrellis.layer.{LayoutDefinition, SpatialKey} +import geotrellis.raster.summary.polygonal.{NoIntersection, PolygonalSummaryResult, Summary=>GTSummary} +import geotrellis.store.index.zcurve.Z2 +import geotrellis.vector._ +import org.apache.spark.rdd.RDD +import org.globalforestwatch.features.FeatureId +import org.globalforestwatch.grids.GridSources +import org.globalforestwatch.util.RepartitionSkewedRDD +import scala.reflect.ClassTag +import cats.kernel.Semigroup +import cats.data.Validated.{Valid, Invalid} + + +trait ErrorSummaryRDD extends LazyLogging with java.io.Serializable { + + type SOURCES <: GridSources + type SUMMARY <: Summary[SUMMARY] + type TILE <: CellGrid[Int] + + /** Produce RDD of tree cover loss from RDD of areas of interest* + * + * @param featureRDD areas of interest + * @param windowLayout window layout used for distribution of IO, subdivision of 10x10 degree grid + * @param partition flag of whether to partition RDD while processing + */ + def apply[FEATUREID <: FeatureId]( + featureRDD: RDD[Feature[Geometry, FEATUREID]], + windowLayout: LayoutDefinition, + kwargs: Map[String, Any], + partition: Boolean = true + )(implicit kt: ClassTag[SUMMARY], vt: ClassTag[FEATUREID]): RDD[ValidatedLocation[SUMMARY]] = { + + /* Intersect features with each tile from windowLayout grid and generate a record for each intersection. + * Each features will intersect one or more windows, possibly creating a duplicate record. + * Then create a key based off the Z curve value from the grid cell, to use for partitioning. + * Later we will calculate partial result for each intersection and merge them. + */ + + val keyedFeatureRDD: RDD[(Long, (SpatialKey, Feature[Geometry, FEATUREID]))] = featureRDD + .flatMap { feature: Feature[Geometry, FEATUREID] => + val keys: Set[SpatialKey] = + windowLayout.mapTransform.keysForGeometry(feature.geom) + keys.toSeq.map { key => + val z = Z2(key.col, key.row).z + (z, (key, feature)) + } + } + + /* + * Use a Range Partitioner based on the Z curve value to efficiently and evenly partition RDD for analysis, + * but still preserving locality which will both reduce the S3 reads per executor and make it more likely + * for features to be close together already during export. + */ + val partitionedFeatureRDD = if (partition) { + // if a single tile has more than 4096 features, split it up over partitions + RepartitionSkewedRDD.bySparseId(keyedFeatureRDD, 4096) + } else { + keyedFeatureRDD.values + } + + /* + * Here we're going to work with the features one partition at a time. + * We're going to use the tile key from windowLayout to read pixels from appropriate raster. + * Each record in this RDD may still represent only a partial result for that feature. + * + * The RDD is keyed by Id such that we can join and recombine partial results later. + */ + val featuresWithSummaries: RDD[(FEATUREID, ValidatedSummary[SUMMARY])] = + partitionedFeatureRDD.mapPartitions { + featurePartition: Iterator[(SpatialKey, Feature[Geometry, FEATUREID])] => + // Code inside .mapPartitions works in an Iterator of records + // Doing things this way allows us to reuse resources and perform other optimizations + // Grouping by spatial key allows us to minimize read thrashing from record to record + + val groupedByKey : Map[SpatialKey, Array[Feature[Geometry, FEATUREID]]] = + featurePartition.toArray.groupBy { + case (windowKey, _) => windowKey + }.mapValues(_.map{ case (_, feature) => feature }) + + groupedByKey.toIterator.flatMap { + case (windowKey, features) => + val maybeRasterSource: Either[JobError, SOURCES] = + getSources(windowKey, windowLayout, kwargs) + .left.map(ex => RasterReadError(ex.getMessage)) + + val maybeRaster: Either[JobError, Raster[TILE]] = + maybeRasterSource.flatMap { rs: SOURCES => + readWindow(rs, windowKey, windowLayout) + .left.map(ex => RasterReadError(s"Reading raster for $windowKey")) + } + + val partialSummaries: Array[(FEATUREID, ValidatedSummary[SUMMARY])] = + features.map { feature: Feature[Geometry, FEATUREID] => + val id: FEATUREID = feature.data + val summary: Either[JobError, PolygonalSummaryResult[SUMMARY]] = + maybeRaster.flatMap { raster => + Either.catchNonFatal { + runPolygonalSummary( + raster, + feature.geom, + ErrorSummaryRDD.rasterizeOptions, + kwargs) + }.left.map{ + // TODO: these should be moved into left side of PolygonalSummaryResult in GT + case ise: java.lang.IllegalStateException => + GeometryError(s"IllegalStateException") + case te: org.locationtech.jts.geom.TopologyException => + GeometryError(s"TopologyException") + case be: java.lang.ArrayIndexOutOfBoundsException => + GeometryError(s"ArrayIndexOutOfBoundsException") + case e: Throwable => + GeometryError(e.getMessage) + } + } + // Converting to Validated so errors across partial results can be accumulated + // @see https://typelevel.org/cats/datatypes/validated.html#validated-vs-either + (id, summary.toValidated) + } + + partialSummaries + } + } + + /* Group records by Id and combine their summaries + * The features may have intersected multiple grid blocks + */ + val featuresGroupedWithSummaries: RDD[ValidatedLocation[SUMMARY]] = + featuresWithSummaries + .reduceByKey(Semigroup.combine) + .map { case (fid, summary) => + summary match { + // If there was no intersection for any partial results, we consider this an invalid geometry + case Valid(NoIntersection) => + Invalid(Location(fid, NoIntersectionError)) + case Valid(GTSummary(result)) if result.isEmpty => + Invalid(Location(fid, NoIntersectionError)) + case Invalid(error) => + Invalid(Location(fid, error)) + case Valid(GTSummary(result)) => + Valid(Location(fid, result)) + } + } + + featuresGroupedWithSummaries + } + + def getSources(key: SpatialKey, windowLayout: LayoutDefinition, kwargs: Map[String, Any]): Either[Throwable, SOURCES] + + def readWindow(rs: SOURCES, windowKey: SpatialKey, windowLayout: LayoutDefinition): Either[Throwable, Raster[TILE]] + + def runPolygonalSummary(raster: Raster[TILE], + geometry: Geometry, + options: Rasterizer.Options, + kwargs: Map[String, Any]): PolygonalSummaryResult[SUMMARY] + +} + +object ErrorSummaryRDD { + val rasterizeOptions: Rasterizer.Options = + Rasterizer.Options(includePartial = false, sampleType = PixelIsPoint) +} diff --git a/src/main/scala/org/globalforestwatch/summarystats/JobError.scala b/src/main/scala/org/globalforestwatch/summarystats/JobError.scala new file mode 100644 index 00000000..150916eb --- /dev/null +++ b/src/main/scala/org/globalforestwatch/summarystats/JobError.scala @@ -0,0 +1,44 @@ +package org.globalforestwatch.summarystats + + +import cats.kernel.Semigroup +import cats.implicits._ +import io.circe.syntax._ +import io.circe.parser.decode + + +trait JobError + +case class RasterReadError(msg: String) extends JobError +case class GeometryError(msg: String) extends JobError +case object NoIntersectionError extends JobError +case class MultiError(errors: Set[String]) extends JobError { + def addError(err: JobError): MultiError = MultiError(errors + err.toString) + def addError(other: MultiError): MultiError = MultiError(errors ++ other.errors) +} + +object JobError { + implicit def jobErrorSemigroup[A, B]: Semigroup[JobError] = Semigroup.instance { + case (errs: MultiError, others: MultiError) => errs.addError(others) + case (errs: MultiError, err) => errs.addError(err) + case (err, errs: MultiError) => errs.addError(err) + case (err1, err2) => MultiError(Set(err1.toString, err2.toString)) + } + + + /** Convert from DataFrame error column encoding, will always produce MultiError instance */ + def fromErrorColumn(errors: String): Option[JobError] = { + decode[List[String]](errors) + .toOption + .map { errs => MultiError(errs.toSet) } + } + + /** Encode error as array of error strings */ + def toErrorColumn(jobError: JobError): String = { + val errors: List[String] = jobError match { + case MultiError(errs) => errs.toList.sorted + case err => List(err.toString) + } + errors.asJson.noSpaces + } +} diff --git a/src/main/scala/org/globalforestwatch/summarystats/LocationError.scala b/src/main/scala/org/globalforestwatch/summarystats/LocationError.scala new file mode 100644 index 00000000..7ae56c5c --- /dev/null +++ b/src/main/scala/org/globalforestwatch/summarystats/LocationError.scala @@ -0,0 +1,21 @@ +package org.globalforestwatch.summarystats + +import org.globalforestwatch.features.FeatureId +import cats.Functor + +object Location { + def apply[A](id: FeatureId, thing: A): Tuple2[FeatureId, A] = (id, thing) + + def unapply[A](location: Location[A]): Option[Tuple2[FeatureId, A]] = Some(location) + + implicit class methods[A](location: Location[A]) { + def id = location._1 + def thing = location._2 + } + + implicit val locationFunctor: Functor[Location] = new Functor[Location] { + def map[A, B](fa: Location[A])(f: A => B): Location[B] = Location(fa.id, f(fa.thing)) + } +} + +case class LocationError(id: FeatureId, error: JobError) diff --git a/src/main/scala/org/globalforestwatch/summarystats/Summary.scala b/src/main/scala/org/globalforestwatch/summarystats/Summary.scala index e264406f..8c9f861f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/Summary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/Summary.scala @@ -2,4 +2,5 @@ package org.globalforestwatch.summarystats trait Summary[Self <: Summary[Self]] { self: Self => def merge(other: Self): Self + def isEmpty: Boolean } diff --git a/src/main/scala/org/globalforestwatch/summarystats/SummaryAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/SummaryAnalysis.scala new file mode 100644 index 00000000..f90cd29b --- /dev/null +++ b/src/main/scala/org/globalforestwatch/summarystats/SummaryAnalysis.scala @@ -0,0 +1,25 @@ +package org.globalforestwatch.summarystats + +import org.apache.log4j.Logger +import org.globalforestwatch.util.Util.getAnyMapValue + +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +trait SummaryAnalysis { + + val name: String + val logger: Logger = Logger.getLogger("Summary Analysis") + + def getOutputUrl(kwargs: Map[String, Any], outputName: String = name): String = { + val noOutputPathSuffix: Boolean = getAnyMapValue[Boolean](kwargs, "noOutputPathSuffix") + val outputPath: String = getAnyMapValue[String](kwargs, "outputUrl") + + if (noOutputPathSuffix) outputPath + else s"${outputPath}/${outputName}_" + DateTimeFormatter + .ofPattern("yyyyMMdd_HHmm") + .format(LocalDateTime.now) + + } + +} diff --git a/src/main/scala/org/globalforestwatch/summarystats/SummaryAnalysisFactory.scala b/src/main/scala/org/globalforestwatch/summarystats/SummaryAnalysisFactory.scala deleted file mode 100644 index 80026927..00000000 --- a/src/main/scala/org/globalforestwatch/summarystats/SummaryAnalysisFactory.scala +++ /dev/null @@ -1,83 +0,0 @@ -package org.globalforestwatch.summarystats - -import geotrellis.vector.{Feature, Geometry} -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.SparkSession -import org.globalforestwatch.features.FeatureId -import org.globalforestwatch.summarystats.annualupdate_minimal.AnnualUpdateMinimalAnalysis -import org.globalforestwatch.summarystats.carbon_sensitivity.CarbonSensitivityAnalysis -import org.globalforestwatch.summarystats.carbonflux.CarbonFluxAnalysis -import org.globalforestwatch.summarystats.firealerts.FireAlertsAnalysis -import org.globalforestwatch.summarystats.forest_change_diagnostic.ForestChangeDiagnosticAnalysis -import org.globalforestwatch.summarystats.gfwpro_dashboard.GfwProDashboardAnalysis -import org.globalforestwatch.summarystats.gladalerts.GladAlertsAnalysis -import org.globalforestwatch.summarystats.treecoverloss.TreeLossAnalysis - -case class SummaryAnalysisFactory(analysis: String, - featureRDD: RDD[Feature[Geometry, FeatureId]], - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any]) { - - val runAnalysis: Unit = - analysis match { - case "annualupdate_minimal" => - AnnualUpdateMinimalAnalysis( - featureRDD: RDD[Feature[Geometry, FeatureId]], - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any] - ) - case "carbonflux" => - CarbonFluxAnalysis( - featureRDD: RDD[Feature[Geometry, FeatureId]], - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any] - ) - case "carbon_sensitivity" => - CarbonSensitivityAnalysis( - featureRDD: RDD[Feature[Geometry, FeatureId]], - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any] - ) - case "gladalerts" => - GladAlertsAnalysis( - featureRDD: RDD[Feature[Geometry, FeatureId]], - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any] - ) - case "treecoverloss" => - TreeLossAnalysis( - featureRDD: RDD[Feature[Geometry, FeatureId]], - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any] - ) - case "firealerts" => - FireAlertsAnalysis( - featureRDD: RDD[Feature[Geometry, FeatureId]], - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any] - ) - case "forest_change_diagnostic" => - ForestChangeDiagnosticAnalysis( - featureRDD: RDD[Feature[Geometry, FeatureId]], - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any] - ) - case "gfwpro_dashboard" => - GfwProDashboardAnalysis( - featureRDD: RDD[Feature[Geometry, FeatureId]], - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any] - ) - case _ => - throw new IllegalArgumentException("Not a valid analysis") - } -} diff --git a/src/main/scala/org/globalforestwatch/summarystats/SummaryCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/SummaryCommand.scala index 239cf427..d5a16a75 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/SummaryCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/SummaryCommand.scala @@ -1,14 +1,17 @@ package org.globalforestwatch.summarystats import cats.data.NonEmptyList -import com.monovore.decline.Opts import cats.implicits._ -import geotrellis.vector.{Feature, Geometry} -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.SparkSession -import org.globalforestwatch.features.{FeatureId, FeatureRDDFactory} +import com.monovore.decline.Opts +import org.apache.spark.sql.{SparkSession, Column} +import org.apache.spark.sql.functions.{substring, col} trait SummaryCommand { + import SummaryCommand._ + + val gfwPro: Opts[Boolean] = Opts + .flag("gfwpro", "Feature flag for PRO, changes landcover labels") + .orFalse val featuresOpt: Opts[NonEmptyList[String]] = Opts.options[String]("features", "URI of features in TSV format") @@ -16,10 +19,17 @@ trait SummaryCommand { val outputOpt: Opts[String] = Opts.option[String]("output", "URI of output dir for CSV files") + val overwriteOutputOpt: Opts[Boolean] = Opts + .flag( + "overwrite", + help = "Overwrite output location if already existing" + ) + .orFalse + val featureTypeOpt: Opts[String] = Opts .option[String]( "feature_type", - help = "Feature type: one of 'gadm', 'wdpa', 'geostore' or 'feature'" + help = "Feature type: one of 'gadm', 'wdpa', 'geostore', 'gfwpro' or 'feature'" ) .withDefault("feature") @@ -100,42 +110,111 @@ trait SummaryCommand { help = "URI of fire alerts in TSV format" ) - val defaultOptions: Opts[(String, NonEmptyList[String], String, Boolean)] = - (featureTypeOpt, featuresOpt, outputOpt, splitFeatures).tupled - val fireAlertOptions: Opts[(String, NonEmptyList[String])] = - (fireAlertTypeOpt, fireAlertSourceOpt).tupled - - val defaultFilterOptions: Opts[(Option[Int], Boolean, Boolean)] = - (limitOpt, tclOpt, gladOpt).tupled - val gdamFilterOptions: Opts[ - (Option[String], - Option[String], - Option[String], - Option[String], - Option[String], - Option[String]) - ] = (isoOpt, isoFirstOpt, isoStartOpt, isoEndOpt, admin1Opt, admin2Opt).tupled - val wdpaFilterOptions - : Opts[(Option[NonEmptyList[String]], Option[NonEmptyList[String]])] = - (wdpaStatusOpts, iucnCatOpts).tupled - val featureFilterOptions: Opts[(Option[Int], Option[Int])] = - (idStartOpt, idEndOpt).tupled - - def runAnalysis(analysis: String, - fType: String, - featureUris: NonEmptyList[String], - kwargs: Map[String, Any]): Unit = { - - val spark: SparkSession = - SummarySparkSession(s"${analysis} Session") - - /* Transition from DataFrame to RDD in order to work with GeoTrellis features */ - val featureRDD: RDD[Feature[Geometry, FeatureId]] = - FeatureRDDFactory(analysis, fType, featureUris, kwargs, spark) - - SummaryAnalysisFactory(analysis, featureRDD, fType, spark, kwargs).runAnalysis - - spark.stop + val noOutputPathSuffixOpt: Opts[Boolean] = Opts.flag("no_output_path_suffix", help = "Do not autogenerate output path suffix at runtime").orFalse + + val defaultOptions: Opts[BaseOptions] = + (featureTypeOpt, featuresOpt, outputOpt, overwriteOutputOpt, splitFeatures, noOutputPathSuffixOpt).mapN(BaseOptions) + + val fireAlertOptions: Opts[FireAlert] = + (fireAlertTypeOpt, fireAlertSourceOpt).mapN(FireAlert) + + val defaultFilterOptions: Opts[BaseFilter] = + (tclOpt, gladOpt).mapN(BaseFilter) + + val gdamFilterOptions: Opts[GadmFilter] = + (isoOpt, isoFirstOpt, isoStartOpt, isoEndOpt, admin1Opt, admin2Opt).mapN(GadmFilter) + + val wdpaFilterOptions : Opts[WdpaFilter] = + (wdpaStatusOpts, iucnCatOpts).mapN(WdpaFilter) + + val featureIdFilterOptions: Opts[FeatureIdFilter] = + (idStartOpt, idEndOpt).mapN(FeatureIdFilter) + + val featureFilterOptions: Opts[AllFilterOptions] = ( + defaultFilterOptions.orNone, + featureIdFilterOptions.orNone, + gdamFilterOptions.orNone, + wdpaFilterOptions.orNone + ).mapN(AllFilterOptions) + + def runAnalysis[A](analysis: SparkSession => A): A = { + val name = getClass().getSimpleName() + val spark = SummarySparkSession(name) + try { + analysis(spark) + } finally { + spark.stop() + } + } +} + +object SummaryCommand { + trait FilterOptions + + case class BaseOptions( + featureType: String, + featureUris: NonEmptyList[String], + outputUrl: String, + overwriteOutput: Boolean, + splitFeatures: Boolean, + noOutputPathSuffix: Boolean) + + case class BaseFilter(tcl: Boolean, glad: Boolean) extends FilterOptions { + def filters(): List[Column] = { + val trueValues: List[String] = List("t", "T", "true", "True", "TRUE", "1", "Yes", "yes", "YES") + List( + if (glad) Some(col("glad").isin(trueValues: _*)) else None, + if (tcl) Some(col("tcl").isin(trueValues: _*)) else None + ).flatten + } + } + + case class GadmFilter( + iso: Option[String], + isoFirst: Option[String], + isoStart: Option[String], + isoEnd: Option[String], + admin1: Option[String], + admin2: Option[String] + ) extends FilterOptions { + def filters(isoColumn: Column, admin1Column: Column, admin2Column: Column): List[Column] = { + List( + iso.map { code => isoColumn === code }, + isoStart.map { s => isoColumn >= s }, + isoEnd.map { s => isoColumn < s }, + isoFirst.map { s => substring(isoColumn, 0, 1) === s(0) }, + admin1.map { s => admin1Column === s }, + admin2.map { s => admin2Column === s } + ).flatten + } + } + + case class WdpaFilter( + wdpaStatus: Option[NonEmptyList[String]], + iucnCat: Option[NonEmptyList[String]] + ) extends FilterOptions { + def filters(): List[Column] = { + List( + wdpaStatus.map { s => col("status") === s }, + iucnCat.map { s => col("iucn_cat") === s } + ).flatten + } } + case class FeatureIdFilter(idStart: Option[Int], idEnd: Option[Int]) extends FilterOptions { + def filters(idColumn: Column): List[Column] = { + List( + idStart.map { id => idColumn >= id }, + idEnd.map { id => idColumn <= id } + ).flatten + } + } + + case class AllFilterOptions( + base: Option[BaseFilter], + featureId: Option[FeatureIdFilter], + gadm: Option[GadmFilter], + wdpa: Option[WdpaFilter]) + + case class FireAlert(alertType: String, alertSource: NonEmptyList[String]) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/SummaryDF.scala b/src/main/scala/org/globalforestwatch/summarystats/SummaryDF.scala new file mode 100644 index 00000000..673e14f1 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/summarystats/SummaryDF.scala @@ -0,0 +1,30 @@ +package org.globalforestwatch.summarystats + +import frameless.{TypedEncoder, TypedExpressionEncoder} +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder + +/** Extend this trait where Data classes are being convereted to Spark DataFrame or DataSet to produce a Summary DataFrame + * The implicit defined here will superceded the reflection based process of deriving encoders used in Spark. + * + * For instance, if these encoders are not used and you write a DataFrame containing ForestChangeDiagnosticDataLossYearly to csv + * you will see an error that says that csv format does nto support columns with map data type. + * This is because the Injection from ForestChangeDiagnosticDataLossYearly to JSON string was not found and used. + */ +trait SummaryDF { + /** High priority specific product encoder derivation. Without it, the default spark would be used. */ + implicit def productEncoder[T <: Product: TypedEncoder]: ExpressionEncoder[T] = + TypedExpressionEncoder[T].asInstanceOf[ExpressionEncoder[T]] +} + +object SummaryDF { + case class RowId(list_id: String, location_id: String) + + case class RowError(status_code: Int, location_error: String) + object RowError { + val empty: RowError = RowError(status_code = 2, location_error = null) + def fromJobError(err: JobError): RowError = RowError( + status_code = 3, + location_error = JobError.toErrorColumn(err) + ) + } +} \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/summarystats/SummaryExport.scala b/src/main/scala/org/globalforestwatch/summarystats/SummaryExport.scala index 3acffa8c..53a3b404 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/SummaryExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/SummaryExport.scala @@ -17,16 +17,18 @@ trait SummaryExport { def export(featureType: String, summaryDF: DataFrame, outputUrl: String, - kwargs: Map[String, Any]): Unit = { + kwargs: Map[String, Any] + ): Unit = { featureType match { case "gadm" => exportGadm(summaryDF, outputUrl, kwargs) case "feature" => exportFeature(summaryDF, outputUrl, kwargs) case "wdpa" => exportWdpa(summaryDF, outputUrl, kwargs) case "geostore" => exportGeostore(summaryDF, outputUrl, kwargs) + case "gfwpro" => exportGfwPro(summaryDF, outputUrl, kwargs) case _ => throw new IllegalArgumentException( - "Feature type must be one of 'gadm', 'wdpa', 'geostore' or 'feature'" + "Feature type must be one of 'gadm', 'wdpa', 'geostore' 'gfwpro' or 'feature'" ) } } @@ -46,4 +48,8 @@ trait SummaryExport { protected def exportGeostore(summaryDF: DataFrame, outputUrl: String, kwargs: Map[String, Any]): Unit = throw new NotImplementedError("Geostore feature analysis not implemented") + + protected def exportGfwPro(summaryDF: DataFrame, + outputUrl: String, + kwargs: Map[String, Any]): Unit = throw new NotImplementedError("GfwPro feature analysis not implemented") } diff --git a/src/main/scala/org/globalforestwatch/summarystats/SummaryMain.scala b/src/main/scala/org/globalforestwatch/summarystats/SummaryMain.scala index 8ceaa69a..84fdc5ae 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/SummaryMain.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/SummaryMain.scala @@ -1,6 +1,5 @@ package org.globalforestwatch.summarystats -import com.monovore.decline.CommandApp import org.globalforestwatch.summarystats.annualupdate_minimal.AnnualUpdateMinimalCommand.annualupdateMinimalCommand import org.globalforestwatch.summarystats.carbon_sensitivity.CarbonSensitivityCommand.carbonSensitivityCommand import org.globalforestwatch.summarystats.carbonflux.CarbonFluxCommand.carbonFluxCommand @@ -9,20 +8,29 @@ import org.globalforestwatch.summarystats.forest_change_diagnostic.ForestChangeD import org.globalforestwatch.summarystats.gfwpro_dashboard.GfwProDashboardCommand.gfwProDashboardCommand import org.globalforestwatch.summarystats.gladalerts.GladAlertsCommand.gladAlertsCommand import org.globalforestwatch.summarystats.treecoverloss.TreeCoverLossCommand.treeCoverLossCommand +import com.monovore.decline._ -object SummaryMain - extends CommandApp( - name = "geotrellis-summary-stats", - header = "Compute summary statistics for GFW data", - main = { - annualupdateMinimalCommand orElse - carbonSensitivityCommand orElse - carbonFluxCommand orElse - fireAlertsCommand orElse - forestChangeDiagnosticCommand orElse - gfwProDashboardCommand orElse - gladAlertsCommand orElse - treeCoverLossCommand +object SummaryMain { + val name = "geotrellis-summary-stats" + val header = "Compute summary statistics for GFW data" + val main = { + annualupdateMinimalCommand orElse + carbonSensitivityCommand orElse + carbonFluxCommand orElse + fireAlertsCommand orElse + forestChangeDiagnosticCommand orElse + gfwProDashboardCommand orElse + gladAlertsCommand orElse + treeCoverLossCommand + } + val command = Command(name, header, true)(main) + final def main(args: Array[String]): Unit = { + command.parse(args, sys.env) match { + case Left(help) => + System.err.println(help) + System.exit(2) + case Right(_) => () } - ) + } +} diff --git a/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala index 5b33a6c1..5a3e2a0d 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala @@ -7,15 +7,12 @@ import geotrellis.raster.rasterize.Rasterizer import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.summary.polygonal.{NoIntersection, PolygonalSummaryResult} import geotrellis.raster.summary.polygonal -import org.apache.commons.lang.NotImplementedException import geotrellis.store.index.zcurve.Z2 import geotrellis.vector._ import org.apache.spark.RangePartitioner import org.apache.spark.rdd.RDD import org.globalforestwatch.features.FeatureId import org.globalforestwatch.grids.GridSources - - import scala.reflect.ClassTag @@ -43,7 +40,6 @@ trait SummaryRDD extends LazyLogging with java.io.Serializable { * Later we will calculate partial result for each intersection and merge them. */ - val keyedFeatureRDD: RDD[(Long, (SpatialKey, Feature[Geometry, FEATUREID]))] = featureRDD .flatMap { feature: Feature[Geometry, FEATUREID] => val keys: Set[SpatialKey] = @@ -59,6 +55,7 @@ trait SummaryRDD extends LazyLogging with java.io.Serializable { * but still preserving locality which will both reduce the S3 reads per executor and make it more likely * for features to be close together already during export. */ + val partitionedFeatureRDD = if (partition) { val inputPartitionMultiplier = 64 val rangePartitioner = @@ -68,7 +65,10 @@ trait SummaryRDD extends LazyLogging with java.io.Serializable { keyedFeatureRDD } - /* Here we're going to work with the features one partition at a time. + // countRecordsPerPartition(partitionedFeatureRDD, SummarySparkSession("tmp")) + + /* + * Here we're going to work with the features one partition at a time. * We're going to use the tile key from windowLayout to read pixels from appropriate raster. * Each record in this RDD may still represent only a partial result for that feature. * @@ -81,7 +81,6 @@ trait SummaryRDD extends LazyLogging with java.io.Serializable { ] => // Code inside .mapPartitions works in an Iterator of records // Doing things this way allows us to reuse resources and perform other optimizations - // Grouping by spatial key allows us to minimize read thrashing from record to record val windowFeature = featurePartition.map { @@ -178,6 +177,7 @@ trait SummaryRDD extends LazyLogging with java.io.Serializable { def getSources(key: SpatialKey, windowLayout: LayoutDefinition, kwargs: Map[String, Any]): Either[Throwable, SOURCES] def readWindow(rs: SOURCES, windowKey: SpatialKey, windowLayout: LayoutDefinition): Either[Throwable, Raster[TILE]] + def runPolygonalSummary(raster: Raster[TILE], geometry: Geometry, options: Rasterizer.Options, diff --git a/src/main/scala/org/globalforestwatch/summarystats/SummarySparkSession.scala b/src/main/scala/org/globalforestwatch/summarystats/SummarySparkSession.scala index 73a002e0..119a21fb 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/SummarySparkSession.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/SummarySparkSession.scala @@ -3,8 +3,8 @@ package org.globalforestwatch.summarystats //import org.apache.sedona.core.serde.SedonaKryoRegistrator //import org.apache.sedona.sql.utils.SedonaSQLRegistrator -import org.datasyslab.geospark.serde.GeoSparkKryoRegistrator -import org.datasyslab.geosparksql.utils.GeoSparkSQLRegistrator +import org.apache.sedona.core.serde.SedonaKryoRegistrator +import org.apache.sedona.sql.utils.SedonaSQLRegistrator import org.apache.spark.SparkConf import org.apache.spark.sql.SparkSession @@ -16,9 +16,9 @@ object SummarySparkSession { .setAppName(name) .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryo.registrator", "geotrellis.spark.io.kryo.KryoRegistrator") - .set("spark.kryo.registrator", classOf[GeoSparkKryoRegistrator].getName) + .set("spark.kryo.registrator", classOf[SedonaKryoRegistrator].getName) .set("spark.debug.maxToStringFields", "255") - .set("geospark.join.gridtype", "kdbtree") + .set("sedona.join.gridtype", "kdbtree") // .set("spark.sql.crossJoin.enabled", "true") val localConf: SparkConf = conf @@ -33,8 +33,17 @@ object SummarySparkSession { case e: Throwable => throw e } - GeoSparkSQLRegistrator.registerAll(spark) + SedonaSQLRegistrator.registerAll(spark) spark } + + def run(name: String)(job: SparkSession => Unit): Unit = { + val spark = apply(name) + try { + job(spark) + } finally { + spark.stop() + } + } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalAnalysis.scala index f94afe31..285552c8 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalAnalysis.scala @@ -1,15 +1,14 @@ package org.globalforestwatch.summarystats.annualupdate_minimal -import java.time.LocalDateTime -import java.time.format.DateTimeFormatter - import geotrellis.vector.{Feature, Geometry} import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.globalforestwatch.features.FeatureId -import org.globalforestwatch.util.Util.getAnyMapValue +import org.globalforestwatch.summarystats.SummaryAnalysis + +object AnnualUpdateMinimalAnalysis extends SummaryAnalysis { + val name = "annualupdate_minimal" -object AnnualUpdateMinimalAnalysis { def apply(featureRDD: RDD[Feature[Geometry, FeatureId]], featureType: String, spark: SparkSession, @@ -33,10 +32,7 @@ object AnnualUpdateMinimalAnalysis { summaryDF.repartition($"id", $"data_group") - val runOutputUrl: String = getAnyMapValue[String](kwargs, "outputUrl") + - "/annualupdate_minimal_" + DateTimeFormatter - .ofPattern("yyyyMMdd_HHmm") - .format(LocalDateTime.now) + val runOutputUrl: String = getOutputUrl(kwargs) AnnualUpdateMinimalExport.export( featureType, diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalCommand.scala index 9fd6b8cc..74d0455e 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalCommand.scala @@ -3,40 +3,29 @@ package org.globalforestwatch.summarystats.annualupdate_minimal import com.monovore.decline.Opts import org.globalforestwatch.summarystats.SummaryCommand import cats.implicits._ +import org.globalforestwatch.features._ object AnnualUpdateMinimalCommand extends SummaryCommand { + val changeOnlyOpt: Opts[Boolean] = + Opts.flag("change_only", "Process change only").orFalse val annualupdateMinimalCommand: Opts[Unit] = Opts.subcommand( - name = "annualupdate_minimal", + name = AnnualUpdateMinimalAnalysis.name, help = "Compute summary statistics for GFW dashboards." ) { - ( - defaultOptions, - defaultFilterOptions, - gdamFilterOptions, - wdpaFilterOptions, - featureFilterOptions, - ).mapN { (default, defaultFilter, gadmFilter, wdpaFilter, featureFilter) => + (defaultOptions, featureFilterOptions, changeOnlyOpt).mapN { (default, filterOptions, changeOnly) => val kwargs = Map( - "outputUrl" -> default._3, - "splitFeatures" -> default._4, - "iso" -> gadmFilter._1, - "isoFirst" -> gadmFilter._2, - "isoStart" -> gadmFilter._3, - "isoEnd" -> gadmFilter._4, - "admin1" -> gadmFilter._5, - "admin2" -> gadmFilter._6, - "idStart" -> featureFilter._1, - "idEnd" -> featureFilter._2, - "wdpaStatus" -> wdpaFilter._1, - "iucnCat" -> wdpaFilter._2, - "limit" -> defaultFilter._1, - "tcl" -> defaultFilter._2, - "glad" -> defaultFilter._3 + "outputUrl" -> default.outputUrl, + "noOutputPathSuffix" -> default.noOutputPathSuffix, + "changeOnly" -> changeOnly ) - runAnalysis("annualupdate_minimum", default._1, default._2, kwargs) + val featureFilter = FeatureFilter.fromOptions(default.featureType, filterOptions) + runAnalysis { spark => + val featureRDD = FeatureRDD(default.featureUris, default.featureType, featureFilter, default.splitFeatures, spark) + AnnualUpdateMinimalAnalysis(featureRDD, default.featureType, spark, kwargs) + } } } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala index 4b3ad02c..738059c9 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala @@ -10,24 +10,24 @@ object AnnualUpdateMinimalDF { "umd_tree_cover_density__threshold", "tsc_tree_cover_loss_drivers__type", "esa_land_cover_2015__class", - "is__birdlife_alliance_for_zero_extinction_site", - "gfw_plantation__type", - "is__gmw_mangroves_1996", - "is__gmw_mangroves_2016", - "ifl_intact_forest_landscape__year", + "is__birdlife_alliance_for_zero_extinction_sites", + "gfw_plantations__type", + "is__gmw_global_mangrove_extent_1996", + "is__gmw_global_mangrove_extent_2016", + "ifl_intact_forest_landscapes__year", "is__umd_regional_primary_forest_2001", - "is__gfw_tiger_landscape", - "is__landmark_land_right", - "is__gfw_land_right", - "is__birdlife_key_biodiversity_area", + "is__gfw_tiger_landscapes", + "is__landmark_indigenous_and_community_lands", + "is__gfw_land_rights", + "is__birdlife_key_biodiversity_areas", "is__gfw_mining", - "is__peatland", + "is__gfw_peatlands", "is__gfw_oil_palm", "is__idn_forest_moratorium", "is__gfw_wood_fiber", - "is__gfw_resource_right", - "is__gfw_managed_forest", - "is__umd_tree_cover_gain_2000-2012" + "is__gfw_resource_rights", + "is__gfw_managed_forests", + "is__umd_tree_cover_gain" ) def unpackValues(cols: List[Column], @@ -43,45 +43,45 @@ object AnnualUpdateMinimalDF { $"data_group.drivers" as "tsc_tree_cover_loss_drivers__type", $"data_group.globalLandCover" as "esa_land_cover_2015__class", $"data_group.primaryForest" as "is__umd_regional_primary_forest_2001", - $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_site", - $"data_group.plantations" as "gfw_plantation__type", - $"data_group.mangroves1996" as "is__gmw_mangroves_1996", - $"data_group.mangroves2016" as "is__gmw_mangroves_2016", - $"data_group.intactForestLandscapes" as "ifl_intact_forest_landscape__year", - $"data_group.tigerLandscapes" as "is__gfw_tiger_landscape", - $"data_group.landmark" as "is__landmark_land_right", - $"data_group.landRights" as "is__gfw_land_right", - $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_area", + $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_sites", + $"data_group.plantations" as "gfw_plantations__type", + $"data_group.mangroves1996" as "is__gmw_global_mangrove_extent_1996", + $"data_group.mangroves2016" as "is__gmw_global_mangrove_extent_2016", + $"data_group.intactForestLandscapes" as "ifl_intact_forest_landscapes__year", + $"data_group.tigerLandscapes" as "is__gfw_tiger_landscapes", + $"data_group.landmark" as "is__landmark_indigenous_and_community_lands", + $"data_group.landRights" as "is__gfw_land_rights", + $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas", $"data_group.mining" as "is__gfw_mining", - $"data_group.peatlands" as "is__peatland", + $"data_group.peatlands" as "is__gfw_peatlands", $"data_group.oilPalm" as "is__gfw_oil_palm", $"data_group.idnForestMoratorium" as "is__idn_forest_moratorium", $"data_group.woodFiber" as "is__gfw_wood_fiber", - $"data_group.resourceRights" as "is__gfw_resource_right", - $"data_group.logging" as "is__gfw_managed_forest", - $"data_group.isGain" as "is__umd_tree_cover_gain_2000-2012", + $"data_group.resourceRights" as "is__gfw_resource_rights", + $"data_group.logging" as "is__gfw_managed_forests", + $"data_group.isGain" as "is__umd_tree_cover_gain", $"data.treecoverExtent2000" as "umd_tree_cover_extent_2000__ha", $"data.treecoverExtent2010" as "umd_tree_cover_extent_2010__ha", $"data.totalArea" as "area__ha", - $"data.totalGainArea" as "umd_tree_cover_gain_2000-2012__ha", + $"data.totalGainArea" as "umd_tree_cover_gain__ha", $"data.totalBiomass" as "whrc_aboveground_biomass_stock_2000__Mg", $"data.treecoverLoss" as "umd_tree_cover_loss__ha", $"data.biomassLoss" as "whrc_aboveground_biomass_loss__Mg", $"data.co2Emissions" as "whrc_aboveground_co2_emissions__Mg", $"data.totalCo2" as "whrc_aboveground_co2_stock_2000__Mg", - $"data.totalGrossCumulAbovegroundRemovalsCo2" as "gfw_gross_cumulative_aboveground_co2_removals__Mg", - $"data.totalGrossCumulBelowgroundRemovalsCo2" as "gfw_gross_cumulative_belowground_co2_removals__Mg", - $"data.totalGrossCumulAboveBelowgroundRemovalsCo2" as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", - $"data.totalNetFluxCo2" as "gfw_net_flux_co2e__Mg", - $"data.totalGrossEmissionsCo2eCo2Only" as "gfw_gross_emissions_co2e_co2_only__Mg", - $"data.totalGrossEmissionsCo2eNonCo2" as "gfw_gross_emissions_co2e_non_co2__Mg", - $"data.totalGrossEmissionsCo2e" as "gfw_gross_emissions_co2e_all_gases__Mg" + $"data.totalGrossCumulAbovegroundRemovalsCo2" as "gfw_forest_carbon_gross_removals_aboveground__Mg_CO2e", + $"data.totalGrossCumulBelowgroundRemovalsCo2" as "gfw_forest_carbon_gross_removals_belowground__Mg_CO2e", + $"data.totalGrossCumulAboveBelowgroundRemovalsCo2" as "gfw_forest_carbon_gross_removals__Mg_CO2e", + $"data.totalNetFluxCo2" as "gfw_forest_carbon_net_flux__Mg_CO2e", + $"data.totalGrossEmissionsCo2eCo2Only" as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", + $"data.totalGrossEmissionsCo2eNonCo2" as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", + $"data.totalGrossEmissionsCo2e" as "gfw_forest_carbon_gross_emissions__Mg_CO2e" ) val unpackCols = { if (!wdpa) { defaultUnpackCols ::: List( - $"data_group.wdpa" as "wdpa_protected_area__iucn_cat" + $"data_group.wdpa" as "wdpa_protected_areas__iucn_cat" ) } else defaultUnpackCols } @@ -97,7 +97,7 @@ object AnnualUpdateMinimalDF { val cols = if (!wdpa) groupByCols ::: contextualLayers ::: List( - "wdpa_protected_area__iucn_cat" + "wdpa_protected_areas__iucn_cat" ) else groupByCols ::: contextualLayers @@ -106,18 +106,18 @@ object AnnualUpdateMinimalDF { sum("umd_tree_cover_extent_2000__ha") as "umd_tree_cover_extent_2000__ha", sum("umd_tree_cover_extent_2010__ha") as "umd_tree_cover_extent_2010__ha", sum("area__ha") as "area__ha", - sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", + sum("umd_tree_cover_gain__ha") as "umd_tree_cover_gain__ha", sum("whrc_aboveground_biomass_stock_2000__Mg") as "whrc_aboveground_biomass_stock_2000__Mg", sum("whrc_aboveground_co2_stock_2000__Mg") as "whrc_aboveground_co2_stock_2000__Mg", sum("umd_tree_cover_loss__ha") as "umd_tree_cover_loss__ha", sum("whrc_aboveground_biomass_loss__Mg") as "whrc_aboveground_biomass_loss__Mg", - sum("gfw_gross_cumulative_aboveground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_co2_removals__Mg", - sum("gfw_gross_cumulative_belowground_co2_removals__Mg") as "gfw_gross_cumulative_belowground_co2_removals__Mg", - sum("gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", - sum("gfw_net_flux_co2e__Mg") as "gfw_net_flux_co2e__Mg", - sum("gfw_gross_emissions_co2e_co2_only__Mg") as "gfw_gross_emissions_co2e_co2_only__Mg", - sum("gfw_gross_emissions_co2e_non_co2__Mg") as "gfw_gross_emissions_co2e_non_co2__Mg", - sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg", + sum("gfw_forest_carbon_gross_removals_aboveground__Mg_CO2e") as "gfw_forest_carbon_gross_removals_aboveground__Mg_CO2e", + sum("gfw_forest_carbon_gross_removals_belowground__Mg_CO2e") as "gfw_forest_carbon_gross_removals_belowground__Mg_CO2e", + sum("gfw_forest_carbon_gross_removals__Mg_CO2e") as "gfw_forest_carbon_gross_removals__Mg_CO2e", + sum("gfw_forest_carbon_net_flux__Mg_CO2e") as "gfw_forest_carbon_net_flux__Mg_CO2e", + sum("gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", + sum("gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", + sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e", ) } @@ -127,7 +127,7 @@ object AnnualUpdateMinimalDF { val cols = if (!wdpa) groupByCols ::: contextualLayers ::: List( - "wdpa_protected_area__iucn_cat" + "wdpa_protected_areas__iucn_cat" ) else groupByCols ::: contextualLayers @@ -136,18 +136,18 @@ object AnnualUpdateMinimalDF { sum("umd_tree_cover_extent_2000__ha") as "umd_tree_cover_extent_2000__ha", sum("umd_tree_cover_extent_2010__ha") as "umd_tree_cover_extent_2010__ha", sum("area__ha") as "area__ha", - sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", + sum("umd_tree_cover_gain__ha") as "umd_tree_cover_gain__ha", sum("whrc_aboveground_biomass_stock_2000__Mg") as "whrc_aboveground_biomass_stock_2000__Mg", sum("whrc_aboveground_co2_stock_2000__Mg") as "whrc_aboveground_co2_stock_2000__Mg", sum("umd_tree_cover_loss__ha") as "umd_tree_cover_loss__ha", sum("whrc_aboveground_biomass_loss__Mg") as "whrc_aboveground_biomass_loss__Mg", - sum("gfw_gross_cumulative_aboveground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_co2_removals__Mg", - sum("gfw_gross_cumulative_belowground_co2_removals__Mg") as "gfw_gross_cumulative_belowground_co2_removals__Mg", - sum("gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", - sum("gfw_net_flux_co2e__Mg") as "gfw_net_flux_co2e__Mg", - sum("gfw_gross_emissions_co2e_co2_only__Mg") as "gfw_gross_emissions_co2e_co2_only__Mg", - sum("gfw_gross_emissions_co2e_non_co2__Mg") as "gfw_gross_emissions_co2e_non_co2__Mg", - sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg", + sum("gfw_forest_carbon_gross_removals_aboveground__Mg_CO2e") as "gfw_forest_carbon_gross_removals_aboveground__Mg_CO2e", + sum("gfw_forest_carbon_gross_removals_belowground__Mg_CO2e") as "gfw_forest_carbon_gross_removals_belowground__Mg_CO2e", + sum("gfw_forest_carbon_gross_removals__Mg_CO2e") as "gfw_forest_carbon_gross_removals__Mg_CO2e", + sum("gfw_forest_carbon_net_flux__Mg_CO2e") as "gfw_forest_carbon_net_flux__Mg_CO2e", + sum("gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", + sum("gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", + sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e", ) } @@ -157,7 +157,7 @@ object AnnualUpdateMinimalDF { val cols = if (!wdpa) groupByCols ::: List("umd_tree_cover_loss__year") ::: contextualLayers ::: List( - "wdpa_protected_area__iucn_cat" + "wdpa_protected_areas__iucn_cat" ) else groupByCols ::: List("umd_tree_cover_loss__year") ::: contextualLayers @@ -166,9 +166,9 @@ object AnnualUpdateMinimalDF { sum("umd_tree_cover_loss__ha") as "umd_tree_cover_loss__ha", sum("whrc_aboveground_biomass_loss__Mg") as "whrc_aboveground_biomass_loss__Mg", sum("whrc_aboveground_co2_emissions__Mg") as "whrc_aboveground_co2_emissions__Mg", - sum("gfw_gross_emissions_co2e_co2_only__Mg") as "gfw_gross_emissions_co2e_co2_only__Mg", - sum("gfw_gross_emissions_co2e_non_co2__Mg") as "gfw_gross_emissions_co2e_non_co2__Mg", - sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg" + sum("gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", + sum("gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", + sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e", ) } @@ -183,32 +183,32 @@ object AnnualUpdateMinimalDF { max(length($"esa_land_cover_2015__class")) .cast("boolean") as "esa_land_cover_2015__class", max($"is__umd_regional_primary_forest_2001") as "is__umd_regional_primary_forest_2001", - max($"is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", - max(length($"gfw_plantation__type")) - .cast("boolean") as "gfw_plantation__type", - max($"is__gmw_mangroves_1996") as "is__gmw_mangroves_1996", - max($"is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", - max(length($"ifl_intact_forest_landscape__year")) - .cast("boolean") as "ifl_intact_forest_landscape__year", - max($"is__gfw_tiger_landscape") as "is__gfw_tiger_landscape", - max($"is__landmark_land_right") as "is__landmark_land_right", - max($"is__gfw_land_right") as "is__gfw_land_right", - max($"is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", + max($"is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites", + max(length($"gfw_plantations__type")) + .cast("boolean") as "gfw_plantations__type", + max($"is__gmw_global_mangrove_extent_1996") as "is__gmw_global_mangrove_extent_1996", + max($"is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", + max(length($"ifl_intact_forest_landscapes__year")) + .cast("boolean") as "ifl_intact_forest_landscapes__year", + max($"is__gfw_tiger_landscapes") as "is__gfw_tiger_landscapes", + max($"is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", + max($"is__gfw_land_rights") as "is__gfw_land_rights", + max($"is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas", max($"is__gfw_mining") as "is__gfw_mining", - max($"is__peatland") as "is__peatland", + max($"is__gfw_peatlands") as "is__gfw_peatlands", max($"is__gfw_oil_palm") as "is__gfw_oil_palm", max($"is__idn_forest_moratorium") as "is__idn_forest_moratorium", max($"is__gfw_wood_fiber") as "is__gfw_wood_fiber", - max($"is__gfw_resource_right") as "is__gfw_resource_right", - max($"is__gfw_managed_forest") as "is__gfw_managed_forest", - max($"is__umd_tree_cover_gain_2000-2012") as "is__umd_tree_cover_gain_2000-2012" + max($"is__gfw_resource_rights") as "is__gfw_resource_rights", + max($"is__gfw_managed_forests") as "is__gfw_managed_forests", + max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain" ) val aggCols = if (!wdpa) defaultAggCols ::: List( - max(length($"wdpa_protected_area__iucn_cat")) - .cast("boolean") as "wdpa_protected_area__iucn_cat" + max(length($"wdpa_protected_areas__iucn_cat")) + .cast("boolean") as "wdpa_protected_areas__iucn_cat" ) else defaultAggCols @@ -227,28 +227,28 @@ object AnnualUpdateMinimalDF { max($"tsc_tree_cover_loss_drivers__type") as "tsc_tree_cover_loss_drivers__type", max($"esa_land_cover_2015__class") as "esa_land_cover_2015__class", max($"is__umd_regional_primary_forest_2001") as "is__umd_regional_primary_forest_2001", - max($"is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", - max($"gfw_plantation__type") as "gfw_plantation__type", - max($"is__gmw_mangroves_1996") as "is__gmw_mangroves_1996", - max($"is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", - max($"ifl_intact_forest_landscape__year") as "ifl_intact_forest_landscape__year", - max($"is__gfw_tiger_landscape") as "is__gfw_tiger_landscape", - max($"is__landmark_land_right") as "is__landmark_land_right", - max($"is__gfw_land_right") as "is__gfw_land_right", - max($"is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", + max($"is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites", + max($"gfw_plantations__type") as "gfw_plantations__type", + max($"is__gmw_global_mangrove_extent_1996") as "is__gmw_global_mangrove_extent_1996", + max($"is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", + max($"ifl_intact_forest_landscapes__year") as "ifl_intact_forest_landscapes__year", + max($"is__gfw_tiger_landscapes") as "is__gfw_tiger_landscapes", + max($"is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", + max($"is__gfw_land_rights") as "is__gfw_land_rights", + max($"is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas", max($"is__gfw_mining") as "is__gfw_mining", - max($"is__peatland") as "is__peatland", + max($"is__gfw_peatlands") as "is__gfw_peatlands", max($"is__gfw_oil_palm") as "is__gfw_oil_palm", max($"is__idn_forest_moratorium") as "is__idn_forest_moratorium", max($"is__gfw_wood_fiber") as "is__gfw_wood_fiber", - max($"is__gfw_resource_right") as "is__gfw_resource_right", - max($"is__gfw_managed_forest") as "is__gfw_managed_forest", - max($"is__umd_tree_cover_gain_2000-2012") as "is__umd_tree_cover_gain_2000-2012" + max($"is__gfw_resource_rights") as "is__gfw_resource_rights", + max($"is__gfw_managed_forests") as "is__gfw_managed_forests", + max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain" ) val aggCols = if (!wdpa) defaultAggCols ::: List( - max($"wdpa_protected_area__iucn_cat") as "wdpa_protected_area__iucn_cat" + max($"wdpa_protected_areas__iucn_cat") as "wdpa_protected_areas__iucn_cat" ) else defaultAggCols diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDownloadDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDownloadDF.scala index 034c4e46..1c0dc475 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDownloadDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDownloadDF.scala @@ -20,7 +20,7 @@ object AnnualUpdateMinimalDownloadDF { .pivot("umd_tree_cover_loss__year", yearRange) .agg( sum("umd_tree_cover_loss__ha") as "umd_tree_cover_loss__ha", - sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg" + sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e" ) .as("annual") .na.fill(0, Seq("adm1", "adm2")) @@ -31,14 +31,14 @@ object AnnualUpdateMinimalDownloadDF { sum("umd_tree_cover_extent_2000__ha") as "umd_tree_cover_extent_2000__ha", sum("umd_tree_cover_extent_2010__ha") as "umd_tree_cover_extent_2010__ha", sum("area__ha") as "area__ha", - sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", + sum("umd_tree_cover_gain__ha") as "umd_tree_cover_gain__ha", sum("whrc_aboveground_biomass_stock_2000__Mg") as "whrc_aboveground_biomass_stock_2000__Mg", sum("whrc_aboveground_biomass_stock_2000__Mg") / sum( "umd_tree_cover_extent_2000__ha" ) as "avg_whrc_aboveground_biomass_2000_Mg_ha-1", - sum($"gfw_gross_emissions_co2e_all_gases__Mg") / fluxModelTotalYears as "gfw_gross_emissions_co2e_all_gases__Mg_yr-1", - sum($"gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg") / fluxModelTotalYears as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg_yr-1", - sum($"gfw_net_flux_co2e__Mg") / fluxModelTotalYears as "gfw_net_flux_co2e__Mg_yr-1" + sum($"gfw_forest_carbon_gross_emissions__Mg_CO2e") / fluxModelTotalYears as "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1", + sum($"gfw_forest_carbon_gross_removals__Mg_CO2e") / fluxModelTotalYears as "gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1", + sum($"gfw_forest_carbon_net_flux__Mg_CO2e") / fluxModelTotalYears as "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" ) .as("total") .na.fill(0, Seq("adm1", "adm2")) @@ -65,7 +65,7 @@ object AnnualUpdateMinimalDownloadDF { val totalGrossEmissionsCo2eAllGasesCols = (for (i <- treecoverLossMinYear to treecoverLossMaxYear) yield { - sum($"${i}_gfw_gross_emissions_co2e_all_gases__Mg") as s"gfw_gross_emissions_co2e_all_gases_${i}__Mg" + sum($"${i}_gfw_forest_carbon_gross_emissions__Mg_CO2e") as s"gfw_forest_carbon_gross_emissions_${i}__Mg_CO2e" }).toList _sumDownload( @@ -88,7 +88,7 @@ object AnnualUpdateMinimalDownloadDF { val totalGrossEmissionsCo2eAllGasesCols = (for (i <- treecoverLossMinYear to treecoverLossMaxYear) yield { - sum($"gfw_gross_emissions_co2e_all_gases_${i}__Mg") as s"gfw_gross_emissions_co2e_all_gases_${i}__Mg" + sum($"gfw_forest_carbon_gross_emissions_${i}__Mg_CO2e") as s"gfw_forest_carbon_gross_emissions_${i}__Mg_CO2e" }).toList _sumDownload( @@ -111,14 +111,14 @@ object AnnualUpdateMinimalDownloadDF { sum($"umd_tree_cover_extent_2000__ha") as "umd_tree_cover_extent_2000__ha", sum($"umd_tree_cover_extent_2010__ha") as "umd_tree_cover_extent_2010__ha", sum($"area__ha") as "area__ha", - sum($"umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", + sum($"umd_tree_cover_gain__ha") as "umd_tree_cover_gain__ha", sum($"whrc_aboveground_biomass_stock_2000__Mg") as "whrc_aboveground_biomass_stock_2000__Mg", sum($"whrc_aboveground_biomass_stock_2000__Mg") / sum( $"umd_tree_cover_extent_2000__ha" ) as "avg_whrc_aboveground_biomass_2000_Mg_ha-1", - sum($"gfw_gross_emissions_co2e_all_gases__Mg_yr-1") as "gfw_gross_emissions_co2e_all_gases__Mg_yr-1", - sum($"gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg_yr-1") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg_yr-1", - sum($"gfw_net_flux_co2e__Mg_yr-1") as "gfw_net_flux_co2e__Mg_yr-1" + sum($"gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1") as "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1", + sum($"gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1") as "gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1", + sum($"gfw_forest_carbon_net_flux__Mg_CO2e_yr-1") as "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" ) ::: treecoverLossCols ::: totalGrossEmissionsCo2eAllGasesCols df.groupBy( @@ -141,7 +141,7 @@ object AnnualUpdateMinimalDownloadDF { val totalGrossEmissionsCo2eAllGasesCols = (for (i <- treecoverLossMinYear to treecoverLossMaxYear) yield { - round($"${i}_gfw_gross_emissions_co2e_all_gases__Mg") as s"gfw_gross_emissions_co2e_all_gases_${i}__Mg" + round($"${i}_gfw_forest_carbon_gross_emissions__Mg_CO2e") as s"gfw_forest_carbon_gross_emissions_${i}__Mg_CO2e" }).toList _roundDownload( @@ -164,7 +164,7 @@ object AnnualUpdateMinimalDownloadDF { val totalGrossEmissionsCo2eAllGasesCols = (for (i <- treecoverLossMinYear to treecoverLossMaxYear) yield { - round($"gfw_gross_emissions_co2e_all_gases_${i}__Mg") as s"gfw_gross_emissions_co2e_all_gases_${i}__Mg" + round($"gfw_forest_carbon_gross_emissions_${i}__Mg_CO2e") as s"gfw_forest_carbon_gross_emissions_${i}__Mg_CO2e" }).toList _roundDownload( @@ -189,12 +189,12 @@ object AnnualUpdateMinimalDownloadDF { round($"umd_tree_cover_extent_2000__ha") as "umd_tree_cover_extent_2000__ha", round($"umd_tree_cover_extent_2010__ha") as "umd_tree_cover_extent_2010__ha", round($"area__ha") as "area__ha", - round($"umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", + round($"umd_tree_cover_gain__ha") as "umd_tree_cover_gain__ha", round($"whrc_aboveground_biomass_stock_2000__Mg") as "whrc_aboveground_biomass_stock_2000__Mg", round($"avg_whrc_aboveground_biomass_2000_Mg_ha-1") as "avg_whrc_aboveground_biomass_2000_Mg_ha-1", - round($"gfw_gross_emissions_co2e_all_gases__Mg_yr-1") as "gfw_gross_emissions_co2e_all_gases__Mg_yr-1", - round($"gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg_yr-1") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg_yr-1", - round($"gfw_net_flux_co2e__Mg_yr-1") as "gfw_net_flux_co2e__Mg_yr-1" + round($"gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1") as "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1", + round($"gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1") as "gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1", + round($"gfw_forest_carbon_net_flux__Mg_CO2e_yr-1") as "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" ) df.select( @@ -213,10 +213,10 @@ object AnnualUpdateMinimalDownloadDF { val totalGrossEmissionsCo2eAllGasesCols = (for (i <- treecoverLossMinYear to treecoverLossMaxYear) yield { - s"${i}_gfw_gross_emissions_co2e_all_gases__Mg" + s"${i}_gfw_forest_carbon_gross_emissions__Mg_CO2e" }).toList - val cols = "avg_whrc_aboveground_biomass_2000_Mg_ha-1" :: "gfw_gross_emissions_co2e_all_gases__Mg_yr-1" :: "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg_yr-1" :: "gfw_net_flux_co2e__Mg_yr-1" :: treecoverLossCols ::: totalGrossEmissionsCo2eAllGasesCols + val cols = "avg_whrc_aboveground_biomass_2000_Mg_ha-1" :: "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1" :: "gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1" :: "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" :: treecoverLossCols ::: totalGrossEmissionsCo2eAllGasesCols val nullColumns = df .select(cols.head, cols.tail: _*) .columns @@ -233,10 +233,10 @@ object AnnualUpdateMinimalDownloadDF { val totalGrossEmissionsCo2eAllGasesCols = (for (i <- treecoverLossMinYear to treecoverLossMaxYear) yield { - s"${i}_gfw_gross_emissions_co2e_all_gases__Mg" + s"${i}_gfw_forest_carbon_gross_emissions__Mg_CO2e" }).toList - val cols = "gfw_gross_emissions_co2e_all_gases__Mg_yr-1" :: "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg_yr-1" :: "gfw_net_flux_co2e__Mg_yr-1" :: totalGrossEmissionsCo2eAllGasesCols + val cols = "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1" :: "gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1" :: "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" :: totalGrossEmissionsCo2eAllGasesCols val carbonColumns = df .select(cols.head, cols.tail: _*) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala index 7bb1a269..c0c68423 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala @@ -98,7 +98,7 @@ object AnnualUpdateMinimalExport extends SummaryExport { val adm2ApiDF = df .filter($"umd_tree_cover_loss__year".isNotNull && - ($"umd_tree_cover_loss__ha" > 0 || $"gfw_gross_emissions_co2e_all_gases__Mg" > 0)) + ($"umd_tree_cover_loss__ha" > 0 || $"gfw_forest_carbon_gross_emissions__Mg_CO2e" > 0)) .transform(AnnualUpdateMinimalDF.aggChange(List("iso", "adm1", "adm2"))) .coalesce(133) // this should result in an avg file size of 100MB @@ -194,11 +194,11 @@ object AnnualUpdateMinimalExport extends SummaryExport { import spark.implicits._ val idCols: List[String] = List( - "wdpa_protected_area__id", - "wdpa_protected_area__name", - "wdpa_protected_area__iucn_cat", - "wdpa_protected_area__iso", - "wdpa_protected_area__status" + "wdpa_protected_areas__id", + "wdpa_protected_areas__name", + "wdpa_protected_areas__iucn_cat", + "wdpa_protected_areas__iso", + "wdpa_protected_areas__status" ) val changeOnly: Boolean = @@ -208,11 +208,11 @@ object AnnualUpdateMinimalExport extends SummaryExport { .transform( AnnualUpdateMinimalDF.unpackValues( List( - $"id.wdpaId" as "wdpa_protected_area__id", - $"id.name" as "wdpa_protected_area__name", - $"id.iucnCat" as "wdpa_protected_area__iucn_cat", - $"id.iso" as "wdpa_protected_area__iso", - $"id.status" as "wdpa_protected_area__status" + $"id.wdpaId" as "wdpa_protected_areas__id", + $"id.name" as "wdpa_protected_areas__name", + $"id.iucnCat" as "wdpa_protected_areas__iucn_cat", + $"id.iso" as "wdpa_protected_areas__iso", + $"id.status" as "wdpa_protected_areas__status" ), wdpa = true ) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala index 4e4bf051..b05d08e7 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala @@ -3,7 +3,6 @@ package org.globalforestwatch.summarystats.annualupdate_minimal import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.Raster -import geotrellis.vector.Extent import org.globalforestwatch.grids.{GridSources, GridTile} import org.globalforestwatch.layers._ diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalRDD.scala index feabbe23..78ccfe16 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalRDD.scala @@ -3,7 +3,6 @@ package org.globalforestwatch.summarystats.annualupdate_minimal import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.summary.polygonal._ -import geotrellis.raster.summary.GridVisitor import geotrellis.raster._ import geotrellis.raster.rasterize.Rasterizer import geotrellis.vector._ diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala index ad51779f..7ee59610 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala @@ -19,6 +19,8 @@ case class AnnualUpdateMinimalSummary( // the years.combine method uses LossData.lossDataSemigroup instance to perform per value combine on the map AnnualUpdateMinimalSummary(stats.combine(other.stats)) } + + def isEmpty = stats.isEmpty } object AnnualUpdateMinimalSummary { diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityAnalysis.scala index 71b3682e..54e545e7 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityAnalysis.scala @@ -1,15 +1,15 @@ package org.globalforestwatch.summarystats.carbon_sensitivity -import java.time.LocalDateTime -import java.time.format.DateTimeFormatter - import geotrellis.vector.{Feature, Geometry} import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.globalforestwatch.features.FeatureId +import org.globalforestwatch.summarystats.SummaryAnalysis import org.globalforestwatch.util.Util.getAnyMapValue -object CarbonSensitivityAnalysis { +object CarbonSensitivityAnalysis extends SummaryAnalysis { + val name = "carbon_sensitivity" + def apply(featureRDD: RDD[Feature[Geometry, FeatureId]], featureType: String, spark: SparkSession, @@ -17,7 +17,7 @@ object CarbonSensitivityAnalysis { import spark.implicits._ - val model:String = getAnyMapValue[String](kwargs,"sensitivityType") + val model: String = getAnyMapValue[String](kwargs, "sensitivityType") val summaryRDD: RDD[(FeatureId, CarbonSensitivitySummary)] = CarbonSensitivityRDD(featureRDD, CarbonSensitivityGrid.blockTileGrid, kwargs) @@ -31,10 +31,7 @@ object CarbonSensitivityAnalysis { summaryDF.repartition($"id", $"dataGroup") - val runOutputUrl: String = getAnyMapValue[String](kwargs, "outputUrl") + - s"/carbon_sensitivity_${model}_" + DateTimeFormatter - .ofPattern("yyyyMMdd_HHmm") - .format(LocalDateTime.now) + val runOutputUrl: String = getOutputUrl(kwargs) CarbonSensitivityExport.export(featureType, summaryDF, runOutputUrl, kwargs) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityCommand.scala index 4f6c8fe7..048d6f35 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityCommand.scala @@ -3,6 +3,7 @@ package org.globalforestwatch.summarystats.carbon_sensitivity import com.monovore.decline.Opts import org.globalforestwatch.summarystats.SummaryCommand import cats.implicits._ +import org.globalforestwatch.features._ object CarbonSensitivityCommand extends SummaryCommand { @@ -14,33 +15,27 @@ object CarbonSensitivityCommand extends SummaryCommand { .withDefault("standard") val carbonSensitivityCommand: Opts[Unit] = Opts.subcommand( - name = "carbon_sensitivity", + name = CarbonSensitivityAnalysis.name, help = "Compute summary statistics for Carbon Sensitivity Models." ) { ( defaultOptions, sensitivityTypeOpt, - defaultFilterOptions, - gdamFilterOptions - ).mapN { (default, sensitivityType, defaultFilter, gadmFilter) => + featureFilterOptions + ).mapN { (default, sensitivityType, filterOptions) => val kwargs = Map( - "outputUrl" -> default._3, - "splitFeatures" -> default._4, - "sensitivityType" -> sensitivityType, - "iso" -> gadmFilter._1, - "isoFirst" -> gadmFilter._2, - "isoStart" -> gadmFilter._3, - "isoEnd" -> gadmFilter._4, - "admin1" -> gadmFilter._5, - "admin2" -> gadmFilter._6, - "limit" -> defaultFilter._1, - "tcl" -> defaultFilter._2, - "glad" -> defaultFilter._3 + "outputUrl" -> default.outputUrl, + "noOutputPathSuffix" -> default.noOutputPathSuffix, + "sensitivityType" -> sensitivityType ) - runAnalysis("carbon_sensitivity", default._1, default._2, kwargs) + val featureFilter = FeatureFilter.fromOptions(default.featureType, filterOptions) + runAnalysis { spark => + val featureRDD = FeatureRDD(default.featureUris, default.featureType, featureFilter, default.splitFeatures, spark) + CarbonSensitivityAnalysis(featureRDD, default.featureType, spark, kwargs) + } } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala index 6ff71757..e2b2724b 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala @@ -3,7 +3,6 @@ package org.globalforestwatch.summarystats.carbon_sensitivity import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.Raster -import geotrellis.vector.Extent import org.globalforestwatch.grids.{GridSources, GridTile} import org.globalforestwatch.layers._ import org.globalforestwatch.util.Util.getAnyMapValue diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityRDD.scala index fe1bb67c..8af4ff60 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityRDD.scala @@ -3,7 +3,6 @@ package org.globalforestwatch.summarystats.carbon_sensitivity import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.summary.polygonal._ -import geotrellis.raster.summary.GridVisitor import geotrellis.raster._ import geotrellis.raster.rasterize.Rasterizer import geotrellis.vector._ diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivitySummary.scala b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivitySummary.scala index df72519c..b142b218 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivitySummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivitySummary.scala @@ -5,7 +5,6 @@ import geotrellis.raster._ import geotrellis.raster.summary.GridVisitor import org.globalforestwatch.summarystats.Summary import org.globalforestwatch.util.Geodesy -import org.globalforestwatch.util.Implicits._ import org.globalforestwatch.util.Util.getAnyMapValue import scala.annotation.tailrec @@ -20,6 +19,7 @@ case class CarbonSensitivitySummary( // the years.combine method uses LossData.lossDataSemigroup instance to perform per value combine on the map CarbonSensitivitySummary(stats.combine(other.stats)) } + def isEmpty = stats.isEmpty } object CarbonSensitivitySummary { diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxAnalysis.scala index 6d63f072..8eed21c4 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxAnalysis.scala @@ -1,15 +1,14 @@ package org.globalforestwatch.summarystats.carbonflux -import java.time.LocalDateTime -import java.time.format.DateTimeFormatter - import geotrellis.vector.{Feature, Geometry} import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.globalforestwatch.features.FeatureId -import org.globalforestwatch.util.Util.getAnyMapValue +import org.globalforestwatch.summarystats.SummaryAnalysis + +object CarbonFluxAnalysis extends SummaryAnalysis { + val name = "carbonflux" -object CarbonFluxAnalysis { def apply(featureRDD: RDD[Feature[Geometry, FeatureId]], featureType: String, spark: SparkSession, @@ -29,10 +28,7 @@ object CarbonFluxAnalysis { summaryDF.repartition($"id", $"dataGroup") - val runOutputUrl: String = getAnyMapValue[String](kwargs, "outputUrl") + - "/carbonflux_" + DateTimeFormatter - .ofPattern("yyyyMMdd_HHmm") - .format(LocalDateTime.now) + val runOutputUrl: String = getOutputUrl(kwargs) CarbonFluxExport.export(featureType, summaryDF, runOutputUrl, kwargs) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxCommand.scala index 001d9c2b..e2d8d781 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxCommand.scala @@ -3,40 +3,29 @@ package org.globalforestwatch.summarystats.carbonflux import org.globalforestwatch.summarystats.SummaryCommand import cats.implicits._ import com.monovore.decline.Opts +import org.globalforestwatch.features._ object CarbonFluxCommand extends SummaryCommand { val carbonFluxCommand: Opts[Unit] = Opts.subcommand( - name = "carbonflux", + name = CarbonFluxAnalysis.name, help = "Compute summary statistics for GFW dashboards." ) { ( defaultOptions, - defaultFilterOptions, - gdamFilterOptions, - wdpaFilterOptions, featureFilterOptions - ).mapN { (default, defaultFilter, gadmFilter, wdpaFilter, featureFilter) => + ).mapN { (default, filterOptions) => val kwargs = Map( - "outputUrl" -> default._3, - "splitFeatures" -> default._4, - "iso" -> gadmFilter._1, - "isoFirst" -> gadmFilter._2, - "isoStart" -> gadmFilter._3, - "isoEnd" -> gadmFilter._4, - "admin1" -> gadmFilter._5, - "admin2" -> gadmFilter._6, - "idStart" -> featureFilter._1, - "idEnd" -> featureFilter._2, - "wdpaStatus" -> wdpaFilter._1, - "iucnCat" -> wdpaFilter._2, - "limit" -> defaultFilter._1, - "tcl" -> defaultFilter._2, - "glad" -> defaultFilter._3 + "outputUrl" -> default.outputUrl, + "noOutputPathSuffix" -> default.noOutputPathSuffix ) - runAnalysis("carbonflux", default._1, default._2, kwargs) + val featureFilter = FeatureFilter.fromOptions(default.featureType, filterOptions) + runAnalysis { spark => + val featureRDD = FeatureRDD(default.featureUris, default.featureType, featureFilter, default.splitFeatures, spark) + CarbonFluxAnalysis(featureRDD, default.featureType, spark, kwargs) + } } } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala index b18bfc4e..fe2a71e3 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala @@ -3,7 +3,6 @@ package org.globalforestwatch.summarystats.carbonflux import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.Raster -import geotrellis.vector.Extent import org.globalforestwatch.grids.{GridSources, GridTile} import org.globalforestwatch.layers._ diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxRDD.scala index 201053e9..1f925b8f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxRDD.scala @@ -4,7 +4,6 @@ import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster._ import geotrellis.raster.rasterize.Rasterizer -import geotrellis.raster.summary.GridVisitor import geotrellis.raster.summary.polygonal._ import geotrellis.vector._ import org.globalforestwatch.summarystats.SummaryRDD diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxSummary.scala index 88309471..e4a74613 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxSummary.scala @@ -18,6 +18,7 @@ case class CarbonFluxSummary( // the years.combine method uses LossData.lossDataSemigroup instance to perform per value combine on the map CarbonFluxSummary(stats.combine(other.stats)) } + def isEmpty = stats.isEmpty } object CarbonFluxSummary { diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala index 9288e47c..1a9be885 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala @@ -1,19 +1,22 @@ package org.globalforestwatch.summarystats.firealerts -import java.time.LocalDateTime -import java.time.format.DateTimeFormatter import geotrellis.vector.Feature import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, SparkSession} -import org.globalforestwatch.features.{FeatureDF, FeatureFactory, FeatureId, SpatialFeatureDF} +import org.globalforestwatch.features._ import org.globalforestwatch.util.Util._ import cats.data.NonEmptyList import geotrellis.vector +import org.globalforestwatch.summarystats.SummaryAnalysis -object FireAlertsAnalysis { +object FireAlertsAnalysis extends SummaryAnalysis { + + val name = "firealerts" + def apply(featureRDD: RDD[Feature[vector.Geometry, FeatureId]], featureType: String, + featureFilter: FeatureFilter, spark: SparkSession, kwargs: Map[String, Any]): Unit = { @@ -21,26 +24,29 @@ object FireAlertsAnalysis { val fireAlertType = getAnyMapValue[String](kwargs, "fireAlertType") val layoutDefinition = fireAlertType match { - case "viirs" => ViirsGrid.blockTileGrid - case "modis" | "burned_areas" => ModisGrid.blockTileGrid + case "viirs" | "burned_areas" => ViirsGrid.blockTileGrid + case "modis" => ModisGrid.blockTileGrid + } + + val partition = fireAlertType match { + case "modis" | "viirs" | "burned_areas" => false + case _ => true } val summaryRDD: RDD[(FeatureId, FireAlertsSummary)] = - FireAlertsRDD(featureRDD, layoutDefinition, kwargs, partition = false) + FireAlertsRDD(featureRDD, layoutDefinition, kwargs, partition = partition) val summaryDF = fireAlertType match { case "modis" | "viirs" => - joinWithFeatures(summaryRDD, featureType, spark, kwargs) + joinWithFeatures(summaryRDD, featureType, featureFilter, spark, kwargs) case "burned_areas" => FireAlertsDFFactory(featureType, summaryRDD, spark, kwargs).getDataFrame } summaryDF.repartition(partitionExprs = $"featureId") - val runOutputUrl: String = getAnyMapValue[String](kwargs, "outputUrl") + - s"/firealerts_${fireAlertType}_" + DateTimeFormatter - .ofPattern("yyyyMMdd_HHmm") - .format(LocalDateTime.now) + val runOutputUrl: String = getOutputUrl(kwargs, s"${name}_${fireAlertType}") + FireAlertsExport.export( featureType, @@ -52,6 +58,7 @@ object FireAlertsAnalysis { def joinWithFeatures(summaryRDD: RDD[(FeatureId, FireAlertsSummary)], featureType: String, + featureFilter: FeatureFilter, spark: SparkSession, kwargs: Map[String, Any]): DataFrame = { val fireDF = FireAlertsDFFactory(featureType, summaryRDD, spark, kwargs).getDataFrame @@ -59,10 +66,9 @@ object FireAlertsAnalysis { val firePointDF = fireDF .selectExpr("ST_Point(CAST(fireId.lon AS Decimal(24,10)),CAST(fireId.lat AS Decimal(24,10))) AS pointshape", "*") - val featureObj = FeatureFactory(featureType).featureObj val featureUris: NonEmptyList[String] = getAnyMapValue[NonEmptyList[String]](kwargs, "featureUris") - val featureDF = SpatialFeatureDF(featureUris, featureObj, kwargs, "geom", spark) + val featureDF = SpatialFeatureDF(featureUris, featureType, featureFilter, "geom", spark) firePointDF .join(featureDF) diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsCommand.scala index 31d3c8bb..42dc82dd 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsCommand.scala @@ -3,54 +3,62 @@ package org.globalforestwatch.summarystats.firealerts import org.globalforestwatch.summarystats.SummaryCommand import cats.implicits._ import com.monovore.decline.Opts +import org.globalforestwatch.features._ +import org.apache.sedona.core.enums.GridType object FireAlertsCommand extends SummaryCommand { val changeOnlyOpt: Opts[Boolean] = Opts.flag("change_only", "Process change only").orFalse val fireAlertsCommand: Opts[Unit] = Opts.subcommand( - name = "firealerts", + name = FireAlertsAnalysis.name, help = "Compute summary fire alert statistics for GFW dashboards." ) { ( defaultOptions, changeOnlyOpt, fireAlertOptions, - defaultFilterOptions, - gdamFilterOptions, - wdpaFilterOptions, - featureFilterOptions, - ).mapN { - (default, - changeOnly, - fireAlert, - defaultFilter, - gadmFilter, - wdpaFilter, - featureFilter) => - val kwargs = Map( - "featureUris" -> default._2, - "outputUrl" -> default._3, - "splitFeatures" -> default._4, - "changeOnly" -> changeOnly, - "fireAlertType" -> fireAlert._1, - "fireAlertSource" -> fireAlert._2, - "iso" -> gadmFilter._1, - "isoFirst" -> gadmFilter._2, - "isoStart" -> gadmFilter._3, - "isoEnd" -> gadmFilter._4, - "admin1" -> gadmFilter._5, - "admin2" -> gadmFilter._6, - "idStart" -> featureFilter._1, - "idEnd" -> featureFilter._2, - "wdpaStatus" -> wdpaFilter._1, - "iucnCat" -> wdpaFilter._2, - "limit" -> defaultFilter._1, - "tcl" -> defaultFilter._2, - "glad" -> defaultFilter._3 - ) + featureFilterOptions + ).mapN { (default, changeOnly, fireAlert, filterOptions) => + val kwargs = Map( + "featureUris" -> default.featureUris, + "outputUrl" -> default.outputUrl, + "splitFeatures" -> default.splitFeatures, + "noOutputPathSuffix" -> default.noOutputPathSuffix, + "changeOnly" -> changeOnly + ) + + val featureFilter = FeatureFilter.fromOptions(default.featureType, filterOptions) + + runAnalysis { spark => + val featureRDD = fireAlert.alertType match { + case "viirs" | "modis" => + val fireRDD = FireAlertRDD(spark, fireAlert.alertType, fireAlert.alertSource, FeatureFilter.empty) + fireRDD.spatialPartitioning(GridType.QUADTREE) + FeatureRDD.pointInPolygonJoinAsFeature(fireAlert.alertType, fireRDD) - runAnalysis("firealerts", default._1, default._2, kwargs) + case "burned_areas" => + val burnedAreasUris = fireAlert.alertSource + FeatureRDD( + fireAlert.alertType, + burnedAreasUris, + ",", + default.featureType, + default.featureUris, + "\t", + featureFilter, + spark + ) + } + + FireAlertsAnalysis( + featureRDD, + default.featureType, + FeatureFilter.empty, + spark, + kwargs + ) + } } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala index 1adc9ab5..196ef326 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala @@ -9,23 +9,23 @@ object FireAlertsDF { val contextualLayers: List[String] = List( "umd_tree_cover_density__threshold", "is__umd_regional_primary_forest_2001", - "is__birdlife_alliance_for_zero_extinction_site", - "is__birdlife_key_biodiversity_area", - "is__landmark_land_right", - "gfw_plantation__type", - "is__gfw_mining", - "is__gfw_managed_forest", + "is__birdlife_alliance_for_zero_extinction_sites", + "is__birdlife_key_biodiversity_areas", + "is__landmark_indigenous_and_community_lands", + "gfw_plantations__type", + "is__gfw_mining_concessions", + "is__gfw_managed_forests", "rspo_oil_palm__certification_status", "is__gfw_wood_fiber", - "is__peatland", + "is__gfw_peatlands", "is__idn_forest_moratorium", "is__gfw_oil_palm", - "idn_forest_area__type", - "per_forest_concession__type", + "idn_forest_area__class", + "per_forest_concessions__type", "is__gfw_oil_gas", - "is__gmw_mangroves_2016", - "is__ifl_intact_forest_landscape_2016", - "bra_biome__name" + "is__gmw_global_mangrove_extent_2016", + "is__ifl_intact_forest_landscapes_2016", + "ibge_bra_biomes__name" ) def unpackValues(unpackCols: List[Column], @@ -40,28 +40,28 @@ object FireAlertsDF { List( $"data_group.threshold" as "umd_tree_cover_density__threshold", $"data_group.primaryForest" as "is__umd_regional_primary_forest_2001", - $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_site", - $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_area", - $"data_group.landmark" as "is__landmark_land_right", - $"data_group.plantations" as "gfw_plantation__type", - $"data_group.mining" as "is__gfw_mining", - $"data_group.logging" as "is__gfw_managed_forest", + $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_sites", + $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas", + $"data_group.landmark" as "is__landmark_indigenous_and_community_lands", + $"data_group.plantations" as "gfw_plantations__type", + $"data_group.mining" as "is__gfw_mining_concessions", + $"data_group.logging" as "is__gfw_managed_forests", $"data_group.rspo" as "rspo_oil_palm__certification_status", $"data_group.woodFiber" as "is__gfw_wood_fiber", - $"data_group.peatlands" as "is__peatland", + $"data_group.peatlands" as "is__gfw_peatlands", $"data_group.indonesiaForestMoratorium" as "is__idn_forest_moratorium", $"data_group.oilPalm" as "is__gfw_oil_palm", - $"data_group.indonesiaForestArea" as "idn_forest_area__type", - $"data_group.peruForestConcessions" as "per_forest_concession__type", + $"data_group.indonesiaForestArea" as "idn_forest_area__class", + $"data_group.peruForestConcessions" as "per_forest_concessions__type", $"data_group.oilGas" as "is__gfw_oil_gas", - $"data_group.mangroves2016" as "is__gmw_mangroves_2016", - $"data_group.intactForestLandscapes2016" as "is__ifl_intact_forest_landscape_2016", - $"data_group.braBiomes" as "bra_biome__name", + $"data_group.mangroves2016" as "is__gmw_global_mangrove_extent_2016", + $"data_group.intactForestLandscapes2016" as "is__ifl_intact_forest_landscapes_2016", + $"data_group.braBiomes" as "ibge_bra_biomes__name", ) val cols = if (!wdpa) - unpackCols ::: ($"data_group.protectedAreas" as "wdpa_protected_area__iucn_cat") :: defaultCols + unpackCols ::: ($"data_group.protectedAreas" as "wdpa_protected_areas__iucn_cat") :: defaultCols else unpackCols ::: defaultCols df.select(cols: _*) @@ -79,7 +79,7 @@ object FireAlertsDF { val cols = if (!wdpa) - groupByCols ::: fireCols ::: "wdpa_protected_area__iucn_cat" :: contextualLayers + groupByCols ::: fireCols ::: "wdpa_protected_areas__iucn_cat" :: contextualLayers else groupByCols ::: fireCols ::: contextualLayers @@ -125,14 +125,12 @@ object FireAlertsDF { aggCol: String, wdpa: Boolean = false): DataFrame = { val spark = df.sparkSession - import spark.implicits._ - val confCols = if (!aggCol.equals("burned_area__ha")) List("confidence__cat") else List() val fireCols2 = List("alert__year", "alert__week") ::: confCols val aggCols = List(col(aggCol)) val contextLayers: List[String] = - if (!wdpa) "wdpa_protected_area__iucn_cat" :: contextualLayers + if (!wdpa) "wdpa_protected_areas__iucn_cat" :: contextualLayers else contextualLayers val selectCols: List[Column] = cols.foldRight(Nil: List[Column])( @@ -157,33 +155,33 @@ object FireAlertsDF { val defaultAggCols = List( max("is__umd_regional_primary_forest_2001") as "is__umd_regional_primary_forest_2001", - max("is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", - max("is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", - max("is__landmark_land_right") as "is__landmark_land_right", - max(length($"gfw_plantation__type")) - .cast("boolean") as "gfw_plantation__type", - max("is__gfw_mining") as "is__gfw_mining", - max("is__gfw_managed_forest") as "is__gfw_managed_forest", + max("is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites", + max("is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas", + max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", + max(length($"gfw_plantations__type")) + .cast("boolean") as "gfw_plantations__type", + max("is__gfw_mining_concessions") as "is__gfw_mining_concessions", + max("is__gfw_managed_forests") as "is__gfw_managed_forests", max(length($"rspo_oil_palm__certification_status")) .cast("boolean") as "rspo_oil_palm__certification_status", max("is__gfw_wood_fiber") as "is__gfw_wood_fiber", - max("is__peatland") as "is__peatland", + max("is__gfw_peatlands") as "is__gfw_peatlands", max("is__idn_forest_moratorium") as "is__idn_forest_moratorium", max("is__gfw_oil_palm") as "is__gfw_oil_palm", - max(length($"idn_forest_area__type")) - .cast("boolean") as "idn_forest_area__type", - max(length($"per_forest_concession__type")) - .cast("boolean") as "per_forest_concession__type", + max(length($"idn_forest_area__class")) + .cast("boolean") as "idn_forest_area__class", + max(length($"per_forest_concessions__type")) + .cast("boolean") as "per_forest_concessions__type", max("is__gfw_oil_gas") as "is__gfw_oil_gas", - max("is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", - max("is__ifl_intact_forest_landscape_2016") as "is__ifl_intact_forest_landscape_2016", - max(length($"bra_biome__name")).cast("boolean") as "bra_biome__name" + max("is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", + max("is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", + max(length($"ibge_bra_biomes__name")).cast("boolean") as "ibge_bra_biomes__name" ) val aggCols = if (!wdpa) - (max(length($"wdpa_protected_area__iucn_cat")) - .cast("boolean") as "wdpa_protected_area__iucn_cat") :: defaultAggCols + (max(length($"wdpa_protected_areas__iucn_cat")) + .cast("boolean") as "wdpa_protected_areas__iucn_cat") :: defaultAggCols else defaultAggCols df.groupBy(groupByCols.head, groupByCols.tail: _*) @@ -195,31 +193,31 @@ object FireAlertsDF { val defaultAggCols = List( max("is__umd_regional_primary_forest_2001") as "is__umd_regional_primary_forest_2001", - max("is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", - max("is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", - max("is__landmark_land_right") as "is__landmark_land_right", - max("gfw_plantation__type") as "gfw_plantation__type", - max("is__gfw_mining") as "is__gfw_mining", - max("is__gfw_managed_forest") as "is__gfw_managed_forest", + max("is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites", + max("is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas", + max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", + max("gfw_plantations__type") as "gfw_plantations__type", + max("is__gfw_mining_concessions") as "is__gfw_mining_concessions", + max("is__gfw_managed_forests") as "is__gfw_managed_forests", max("rspo_oil_palm__certification_status") as "rspo_oil_palm__certification_status", max("is__gfw_wood_fiber") as "is__gfw_wood_fiber", - max("is__peatland") as "is__peatland", + max("is__gfw_peatlands") as "is__gfw_peatlands", max("is__idn_forest_moratorium") as "is__idn_forest_moratorium", max("is__gfw_oil_palm") as "is__gfw_oil_palm", - max("idn_forest_area__type") as "idn_forest_area__type", - max("per_forest_concession__type") as "per_forest_concession__type", + max("idn_forest_area__class") as "idn_forest_area__class", + max("per_forest_concessions__type") as "per_forest_concessions__type", max("is__gfw_oil_gas") as "is__gfw_oil_gas", - max("is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", - max("is__ifl_intact_forest_landscape_2016") as "is__ifl_intact_forest_landscape_2016", - max("bra_biome__name") as "bra_biome__name" + max("is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", + max("is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", + max("ibge_bra_biomes__name") as "ibge_bra_biomes__name" ) val aggCols = if (!wdpa) - (max("wdpa_protected_area__iucn_cat") as "wdpa_protected_area__iucn_cat") :: defaultAggCols + (max("wdpa_protected_areas__iucn_cat") as "wdpa_protected_areas__iucn_cat") :: defaultAggCols else defaultAggCols df.groupBy(groupByCols.head, groupByCols.tail: _*) .agg(aggCols.head, aggCols.tail: _*) } -} \ No newline at end of file +} diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDFFactory.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDFFactory.scala index 27508471..5e5ab0c3 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDFFactory.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDFFactory.scala @@ -5,7 +5,6 @@ import org.apache.spark.sql.{DataFrame, SparkSession} import org.globalforestwatch.features._ import org.globalforestwatch.util.Util.getAnyMapValue -import scala.collection.immutable case class FireAlertsDFFactory( featureType: String, @@ -59,7 +58,7 @@ case class FireAlertsDFFactory( id match { case combinedId: CombinedFeatureId => combinedId match { - case CombinedFeatureId(gadmId: GadmFeatureId, burnedAreaId: BurnedAreasFeatureId) => + case CombinedFeatureId(burnedAreaId: BurnedAreasFeatureId, gadmId: GadmFeatureId) => BurnedAreasRowGadm(burnedAreaId, gadmId, dataGroup, data) case _ => throw new IllegalArgumentException("Not a valid GADM-Burned Areas ID") @@ -79,7 +78,7 @@ case class FireAlertsDFFactory( id match { case combinedId: CombinedFeatureId => combinedId match { - case CombinedFeatureId(wdpaId: WdpaFeatureId, burnedAreaId: BurnedAreasFeatureId) => + case CombinedFeatureId(burnedAreaId: BurnedAreasFeatureId, wdpaId: WdpaFeatureId) => BurnedAreasRowWdpa(burnedAreaId, wdpaId, dataGroup, data) case _ => throw new IllegalArgumentException("Not a valid WDPA-Burned Areas ID") @@ -99,7 +98,7 @@ case class FireAlertsDFFactory( id match { case combinedId: CombinedFeatureId => combinedId match { - case CombinedFeatureId(geostoreId: GeostoreFeatureId, burnedAreaId: BurnedAreasFeatureId) => + case CombinedFeatureId(burnedAreaId: BurnedAreasFeatureId, geostoreId: GeostoreFeatureId) => BurnedAreasRowGeostore(burnedAreaId, geostoreId, dataGroup, data) case _ => throw new IllegalArgumentException("Not a valid Geostore-Burned Areas ID") diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDataGroup.scala index 6a2b17ab..2cefced5 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDataGroup.scala @@ -1,6 +1,5 @@ package org.globalforestwatch.summarystats.firealerts -import org.globalforestwatch.util.Mercantile case class FireAlertsDataGroup(threshold: Integer, primaryForest: Boolean, diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsExport.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsExport.scala index 512358c5..fb327d1d 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsExport.scala @@ -131,18 +131,18 @@ object FireAlertsExport extends SummaryExport { import spark.implicits._ val groupByCols = List( - "wdpa_protected_area__id", - "wdpa_protected_area__name", - "wdpa_protected_area__iucn_cat", - "wdpa_protected_area__iso", - "wdpa_protected_area__status", + "wdpa_protected_areas__id", + "wdpa_protected_areas__name", + "wdpa_protected_areas__iucn_cat", + "wdpa_protected_areas__iso", + "wdpa_protected_areas__status", ) val unpackCols = List( - $"featureId.wdpaId" as "wdpa_protected_area__id", - $"featureId.name" as "wdpa_protected_area__name", - $"featureId.iucnCat" as "wdpa_protected_area__iucn_cat", - $"featureId.iso" as "wdpa_protected_area__iso", - $"featureId.status" as "wdpa_protected_area__status" + $"featureId.wdpaId" as "wdpa_protected_areas__id", + $"featureId.name" as "wdpa_protected_areas__name", + $"featureId.iucnCat" as "wdpa_protected_areas__iucn_cat", + $"featureId.iso" as "wdpa_protected_areas__iso", + $"featureId.status" as "wdpa_protected_areas__status" ) _export(summaryDF, outputUrl + "/wdpa", kwargs, groupByCols, unpackCols, wdpa = true) @@ -205,10 +205,10 @@ object FireAlertsExport extends SummaryExport { df.cache() // for now only export VIIRS GADM all -// df.coalesce(ceil(numPartitions / 40.0).toInt) -// .write -// .options(csvOptions) -// .csv(path = outputUrl + "/all") + // df.coalesce(ceil(numPartitions / 40.0).toInt) + // .write + // .options(csvOptions) + // .csv(path = outputUrl + "/all") if (!changeOnly) { df.transform(FireAlertsDF.whitelist(cols, wdpa = wdpa)) @@ -274,4 +274,4 @@ object FireAlertsExport extends SummaryExport { case "burned_areas" => "burned_area__ha" } } -} +} \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsGridSources.scala index 0440511c..7de283c4 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsGridSources.scala @@ -1,9 +1,7 @@ package org.globalforestwatch.summarystats.firealerts -import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.Raster -import geotrellis.vector.Extent import org.globalforestwatch.grids.{GridSources, GridTile} import org.globalforestwatch.layers._ diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsRDD.scala index a2235499..f95a4acd 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsRDD.scala @@ -3,7 +3,6 @@ package org.globalforestwatch.summarystats.firealerts import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.summary.polygonal._ -import geotrellis.raster.summary.GridVisitor import geotrellis.raster._ import geotrellis.raster.rasterize.Rasterizer import geotrellis.vector._ @@ -22,8 +21,8 @@ object FireAlertsRDD extends SummaryRDD { Either.catchNonFatal { fireAlertType match { - case "viirs" => ViirsGrid.getRasterSource(windowKey, windowLayout, kwargs) - case "modis" | "burned_areas" => ModisGrid.getRasterSource(windowKey, windowLayout, kwargs) + case "viirs" | "burned_areas" => ViirsGrid.getRasterSource(windowKey, windowLayout, kwargs) + case "modis" => ModisGrid.getRasterSource(windowKey, windowLayout, kwargs) } } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsSummary.scala index 780ee97e..6b01aa74 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsSummary.scala @@ -5,7 +5,7 @@ import geotrellis.raster._ import geotrellis.raster.summary.GridVisitor import org.globalforestwatch.summarystats.Summary import org.globalforestwatch.util.Util.getAnyMapValue -import org.globalforestwatch.util.{Geodesy, Mercantile} +import org.globalforestwatch.util.Geodesy /** LossData Summary by year */ case class FireAlertsSummary(stats: Map[FireAlertsDataGroup, FireAlertsData] = @@ -17,6 +17,7 @@ case class FireAlertsSummary(stats: Map[FireAlertsDataGroup, FireAlertsData] = // the years.combine method uses LossData.lossDataSemigroup instance to perform per value combine on the map FireAlertsSummary(stats.combine(other.stats)) } + def isEmpty = stats.isEmpty } object FireAlertsSummary { @@ -73,7 +74,7 @@ object FireAlertsSummary { else { val pKey = FireAlertsDataGroup( - tcd2000, + thresholds.head, primaryForest, protectedAreas, aze, diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/ModisGrid.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/ModisGrid.scala index 110f9762..cd79bd16 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/ModisGrid.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/ModisGrid.scala @@ -1,7 +1,7 @@ package org.globalforestwatch.summarystats.firealerts import geotrellis.vector.Extent -import org.globalforestwatch.grids.{GridTile, NinetyByNinety1kmGrid, TenByTen30mGrid} +import org.globalforestwatch.grids.{GridTile, NinetyByNinety1kmGrid} object ModisGrid extends NinetyByNinety1kmGrid[FireAlertsGridSources] { diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala index 46ac9d47..2f163ce3 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysis.scala @@ -1,835 +1,218 @@ package org.globalforestwatch.summarystats.forest_change_diagnostic -import cats.data.NonEmptyList - -import java.time.LocalDateTime -import java.time.format.DateTimeFormatter +import cats.data.Validated.{Invalid, Valid} +import cats.implicits._ +import scala.collection.JavaConverters._ +import scala.collection.immutable.SortedMap import geotrellis.vector.{Feature, Geometry} -import com.vividsolutions.jts.geom.{Geometry => GeoSparkGeometry} -import geotrellis.vector -import org.apache.log4j.Logger -import org.datasyslab.geosparksql.utils.Adapter -import org.globalforestwatch.features.{ - FeatureDF, - FeatureIdFactory, - FireAlertRDD, - GridFeatureId, - JoinedRDD, - SimpleFeature, - SpatialFeatureDF -} -import org.globalforestwatch.grids.GridId.pointGridId - -import java.util - -//import org.apache.sedona.core.enums.{FileDataSplitter, GridType, IndexType} -//import org.apache.sedona.core.spatialOperator.JoinQuery -//import org.apache.sedona.core.spatialRDD.PointRDD -//import org.apache.sedona.sql.utils.{Adapter, SedonaSQLRegistrator} +import org.locationtech.jts.geom.Geometry import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession -import org.globalforestwatch.features.{ - FeatureFactory, - FeatureId, - SimpleFeatureId -} -import org.globalforestwatch.util.Util.getAnyMapValue - -import scala.collection.JavaConverters._ -import scala.collection.immutable.SortedMap - -object ForestChangeDiagnosticAnalysis { - - val logger: Logger = Logger.getLogger("ForestChangeDiagnosticAnalysis") - - def apply(mainRDD: RDD[Feature[Geometry, FeatureId]], - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any]): Unit = { - - val intermediateListSource = getAnyMapValue[Option[NonEmptyList[String]]]( - kwargs, - "intermediateListSource" - ) - - mainRDD.cache() - - val gridFilter: List[String] = - mainRDD - .filter { - case f: Feature[Geometry, SimpleFeatureId] - if f.data.featureId == -2 => - true - case _ => false - } - .map(f => pointGridId(f.geom.getCentroid, 1)) - .collect - .toList - - val featureRDD: RDD[Feature[Geometry, FeatureId]] = - toFeatureRdd(mainRDD, gridFilter, intermediateListSource.isDefined) - - mainRDD.unpersist() - - val summaryRDD: RDD[(FeatureId, ForestChangeDiagnosticSummary)] = - ForestChangeDiagnosticRDD( - featureRDD, - ForestChangeDiagnosticGrid.blockTileGrid, - kwargs - ) - - val fireCount: RDD[(FeatureId, ForestChangeDiagnosticDataLossYearly)] = - ForestChangeDiagnosticAnalysis.fireStats(featureType, spark, kwargs) - - val dataRDD: RDD[(FeatureId, ForestChangeDiagnosticData)] = - reformatSummaryData(summaryRDD) - .reduceByKey(_ merge _) - .map { case (id, data) => updateCommodityRisk(id, data) } - .leftOuterJoin(fireCount) - .mapValues { - case (data, fire) => - data.update( - fireThreatIndicator = - fire.getOrElse(ForestChangeDiagnosticDataLossYearly.empty) - ) - } - - dataRDD.cache() - - val runOutputUrl: String = getAnyMapValue[String](kwargs, "outputUrl") + - "/forest_change_diagnostic_" + DateTimeFormatter - .ofPattern("yyyyMMdd_HHmm") - .format(LocalDateTime.now) - - val finalRDD = combineIntermediateList(dataRDD, gridFilter, runOutputUrl, spark, kwargs) +import org.apache.sedona.core.spatialRDD.SpatialRDD +import org.globalforestwatch.features.{CombinedFeatureId, FeatureId, GfwProFeatureId, GridId } +import org.globalforestwatch.grids.GridId.pointGridId +import org.globalforestwatch.summarystats.{SummaryAnalysis, ValidatedLocation, Location} +import org.globalforestwatch.util.SpatialJoinRDD +import org.apache.spark.storage.StorageLevel +import org.globalforestwatch.ValidatedWorkflow - val summaryDF = - ForestChangeDiagnosticDFFactory(featureType, finalRDD, spark, kwargs).getDataFrame +object ForestChangeDiagnosticAnalysis extends SummaryAnalysis { - ForestChangeDiagnosticExport.export( - featureType, - summaryDF, - runOutputUrl, - kwargs - ) - } + val name = "forest_change_diagnostic" - /** - * GFW Pro hand of a input features in a TSV file - * TSV file contains the individual list items, the merged list geometry and the geometric difference from the current merged list geometry and the former one. - * Individual list items have location IDs >= 0 - * Merged list geometry has location ID -1 - * Geometric difference to previous version has location ID -2 + /** GFW Pro hand of a input features in a TSV file TSV file contains the individual list items, the merged list geometry and the + * geometric difference from the current merged list geometry and the former one. + * - Individual list items have location IDs >= 0 + * - Merged list geometry has location ID -1 + * - Geometric difference to previous version has location ID -2 * - * Merged list and geometric difference may or may be not present. - * If geometric difference is present, we only need to process chunks of the merged list which fall into the same grid cells as the geometric difference. - * Later in the analysis we will then read cached values for the remaining chunks and use them to aggregate list level results. - * */ - private def toFeatureRdd( - mainRDD: RDD[Feature[Geometry, FeatureId]], - gridFilter: List[String], - useFilter: Boolean - ): RDD[Feature[Geometry, FeatureId]] = { - - val featureRDD: RDD[Feature[Geometry, FeatureId]] = mainRDD - .filter { - case f: Feature[Geometry, SimpleFeatureId] if f.data.featureId >= 0 => - true - case f: Feature[Geometry, SimpleFeatureId] if f.data.featureId == -1 => - // If no geometric difference or intermediate result table is present process entire merged list geometry - if (gridFilter.isEmpty || !useFilter) true - // Otherwise only process chunks which fall into the same grid cells as the geometric difference - else gridFilter.contains(pointGridId(f.geom.getCentroid, 1)) - case _ => false - } - .map { - case f: Feature[Geometry, SimpleFeatureId] if f.data.featureId >= 0 => - f - case f => - val grid = pointGridId(f.geom.getCentroid, 1) - // For merged list, update data to contain the GridFeatureId - vector.Feature(f.geom, GridFeatureId(f.data, grid)) + * Merged list and geometric difference may or may be not present. If geometric difference is present, we only need to process chunks + * of the merged list which fall into the same grid cells as the geometric difference. Later in the analysis we will then read cached + * values for the remaining chunks and use them to aggregate list level results. + * + * This function assumes that all features have already been split by 1x1 degree grid. This function will exclude diff geometry + * locations from output (id=-2). + */ + def apply( + features: RDD[ValidatedLocation[Geometry]], + intermediateResultsRDD: Option[RDD[ValidatedLocation[ForestChangeDiagnosticData]]], + fireAlerts: SpatialRDD[Geometry], + saveIntermidateResults: RDD[ValidatedLocation[ForestChangeDiagnosticData]] => Unit, + kwargs: Map[String, Any] + )(implicit spark: SparkSession): RDD[ValidatedLocation[ForestChangeDiagnosticData]] = { + features.persist(StorageLevel.MEMORY_AND_DISK) + + val diffGridIds: List[GridId] = + if (intermediateResultsRDD.nonEmpty) collectDiffGridIds(features) + else List.empty + + // These records are not covered by diff geometry, they're still valid and can be re-used + val cachedIntermidateResultsRDD = intermediateResultsRDD.map { rdd => + rdd.filter { + case Valid(Location(CombinedFeatureId(fid1, fid2), _)) => + !diffGridIds.contains(fid2) + case Invalid(Location(CombinedFeatureId(fid1, fid2), _)) => + !diffGridIds.contains(fid2) + case _ => + false } + } - featureRDD - - } + val partialResult: RDD[ValidatedLocation[ForestChangeDiagnosticData]] = + ValidatedWorkflow(features) + .flatMap { locationGeometries => + val diffLocations = filterDiffGridCells(locationGeometries, diffGridIds) - def combineIntermediateList( - dataRDD: RDD[(FeatureId, ForestChangeDiagnosticData)], - gridFilter: List[String], - outputUrl: String, - spark: SparkSession, - kwargs: Map[String, Any] - ): RDD[(FeatureId, ForestChangeDiagnosticData)] = { + val fireCount: RDD[Location[ForestChangeDiagnosticDataLossYearly]] = + fireStats(diffLocations, fireAlerts, spark) - val intermediateListSource = getAnyMapValue[Option[NonEmptyList[String]]]( - kwargs, - "intermediateListSource" - ) - - // Get merged list RDD - val listRDD: RDD[(FeatureId, ForestChangeDiagnosticData)] = { - dataRDD.filter({ - case a: (FeatureId, ForestChangeDiagnosticData) => - a._1 match { - case _: GridFeatureId => true - case _ => false + val locationSummaries: RDD[ValidatedLocation[ForestChangeDiagnosticSummary]] = { + val tmp = diffLocations.map { case Location(id, geom) => Feature(geom, id) } + ForestChangeDiagnosticRDD(tmp, ForestChangeDiagnosticGrid.blockTileGrid, kwargs) } - case _ => false - }) - } - - // Get row RDD - val rowRDD = dataRDD.filter({ - case a: (FeatureId, ForestChangeDiagnosticData) => - a._1 match { - case _: SimpleFeatureId => true - case _ => false - } - case _ => false - }) - - // combine filtered List with filtered intermediate results - val combinedListRDD = { - if (intermediateListSource.isDefined) { - val intermediateRDD: RDD[(FeatureId, ForestChangeDiagnosticData)] = - getIntermediateRDD(intermediateListSource.get, spark, kwargs) - listRDD ++ - intermediateRDD.filter({ - case (id, _) => - id match { - case gridId: GridFeatureId => - !gridFilter.contains(gridId.gridId) - case _ => false + ValidatedWorkflow(locationSummaries).mapValid { summaries => + summaries + .mapValues { _.toForestChangeDiagnosticData().withUpdatedCommodityRisk() } + .leftOuterJoin(fireCount) + .mapValues { case (data, fire) => + data.copy( + commodity_threat_fires = fire.getOrElse(ForestChangeDiagnosticDataLossYearly.empty), + tree_cover_loss_soy_yearly = data.tree_cover_loss_soy_yearly.limitToMaxYear(2019) + ) } - case _ => false - }) - - } else listRDD + } + } + .unify + .persist(StorageLevel.MEMORY_AND_DISK) + + cachedIntermidateResultsRDD match { + case Some(cachedResults) => + val mergedResults = partialResult.union(cachedResults) + saveIntermidateResults(mergedResults) + combineGridResults(mergedResults) + case None => + combineGridResults(partialResult) } + } - // EXPORT new intermediate results - val combinedListDF = ForestChangeDiagnosticDFFactory( - "grid", - combinedListRDD, - spark, - kwargs - ).getDataFrame + /** Filter only to those rows covered by gridFilter, these are areas where location geometries have changed If gridFilter is empty list, + * all locations except diff geom will be preserved + */ + def filterDiffGridCells( + rdd: RDD[Location[Geometry]], + gridFilter: List[GridId] + ): RDD[Location[Geometry]] = { + def keepLocationCell(locationId: Int, geom: Geometry): Boolean = + (locationId >= -1) && (gridFilter.isEmpty || gridFilter.contains(GridId(pointGridId(geom.getCentroid, 1)))) + + rdd.collect { + case Location(gfwFid @ GfwProFeatureId(_, lid, _, _), geom) if keepLocationCell(lid, geom) => + val grid = pointGridId(geom.getCentroid, 1) + val fid = CombinedFeatureId(gfwFid, GridId(grid)) + Location(fid, geom) + } + } - ForestChangeDiagnosticExport.exportIntermediateList( - combinedListDF, - outputUrl - ) + /** Collect lists of GridIds for which diff geometry is present (id=-2) */ + def collectDiffGridIds(rdd: RDD[ValidatedLocation[Geometry]]): List[GridId] = { + rdd + .collect { + case Valid(Location(GfwProFeatureId(_, locationId, _, _), geom)) if locationId == -2 => + GridId(pointGridId(geom.getCentroid, 1)) + } + .collect + .toList + } - // Reduce by feature ID and update commodity risk - val updatedListRDD = combinedListRDD + /** Combine per grid results named by CombinedFeatureId to per location results named by FeatureId Some of the per-grid results fo may + * be Invalid errors. Combining per-grid results will aggregate errors up to Location level. + */ + def combineGridResults( + rdd: RDD[ValidatedLocation[ForestChangeDiagnosticData]] + )(implicit spark: SparkSession): RDD[ValidatedLocation[ForestChangeDiagnosticData]] = { + rdd .map { - case (id, data) => - id match { - case gridFeatureId: GridFeatureId => - (gridFeatureId.featureId, data) - } + case Valid(Location(CombinedFeatureId(fid, _), data)) => + (fid, Valid(data)) + case Invalid(Location(CombinedFeatureId(fid, _), err)) => + (fid, Invalid(err)) + case Valid(Location(fid, data)) => + (fid, Valid(data)) + case Invalid(Location(fid, err)) => + (fid, Invalid(err)) + } + .reduceByKey(_ combine _) + .map { + case (fid, Valid(data)) => + Valid(Location(fid, data.withUpdatedCommodityRisk())) + case (fid, Invalid(err)) => + Invalid(Location(fid, err)) } - .reduceByKey(_ merge _) - .map { case (id, data) => updateCommodityRisk(id, data) } - - // Merge with row RDD - rowRDD ++ updatedListRDD - } def fireStats( - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any] - ): RDD[(FeatureId, ForestChangeDiagnosticDataLossYearly)] = { - - // FIRE RDD - val fireAlertSpatialRDD = FireAlertRDD(spark, kwargs) - - // Feature RDD - val featureObj = FeatureFactory(featureType).featureObj - val featureUris: NonEmptyList[String] = - getAnyMapValue[NonEmptyList[String]](kwargs, "featureUris") - val featurePolygonDF = - SpatialFeatureDF( - featureUris, - featureObj, - kwargs, - "geom", - spark, + featureRDD: RDD[Location[Geometry]], + fireAlertRDD: SpatialRDD[Geometry], + spark: SparkSession + ): RDD[Location[ForestChangeDiagnosticDataLossYearly]] = { + // Convert FeatureRDD to SpatialRDD + val polyRDD = featureRDD.map { case Location(fid, geom) => + geom.setUserData(fid) + geom + } + val spatialFeatureRDD = new SpatialRDD[Geometry] + spatialFeatureRDD.rawSpatialRDD = polyRDD.toJavaRDD() + spatialFeatureRDD.fieldNames = seqAsJavaList(List("FeatureId")) + spatialFeatureRDD.analyze() + + val joinedRDD = + SpatialJoinRDD.spatialjoin( + spatialFeatureRDD, + fireAlertRDD, + usingIndex = true ) - val featureSpatialRDD = Adapter.toSpatialRdd(featurePolygonDF, "polyshape") - - featureSpatialRDD.analyze() - - val joinedRDD = JoinedRDD(fireAlertSpatialRDD, featureSpatialRDD) joinedRDD.rdd - .map { - case (poly, points) => - toForestChangeDiagnosticFireData(featureType, poly, points) + .map { case (poly, points) => + val fid = poly.getUserData.asInstanceOf[FeatureId] + val fireCount = points.asScala.foldLeft(SortedMap.empty[Int, Double]) { (acc, point) => + // extract year from acq_date column + val year = point.getUserData + .asInstanceOf[String] + .split("\t")(2) + .substring(0, 4) + .toInt + val count = acc.getOrElse(year, 0.0) + 1.0 + acc.updated(year, count) + } + (fid, ForestChangeDiagnosticDataLossYearly(fireCount)) } .reduceByKey(_ merge _) .mapValues { fires => - aggregateFireData(fires) - } - } - - private def toForestChangeDiagnosticFireData( - featureType: String, - poly: GeoSparkGeometry, - points: util.HashSet[GeoSparkGeometry] - ): (FeatureId, ForestChangeDiagnosticDataLossYearly) = { - ( { - val id = { - poly.getUserData.asInstanceOf[String].filterNot("[]".toSet).toInt - + aggregateFireData(fires.merge(ForestChangeDiagnosticDataLossYearly.prefilled)).limitToMaxYear(2019) } - FeatureIdFactory(featureType).featureId(id) - - }, { - val fireCount = - points.asScala.toList.foldLeft(SortedMap[Int, Double]()) { - (z: SortedMap[Int, Double], point) => { - // extract year from acq_date column - val year = point.getUserData - .asInstanceOf[String] - .split("\t")(2) - .substring(0, 4) - .toInt - val count = z.getOrElse(year, 0.0) + 1.0 - z.updated(year, count) - } - } - - ForestChangeDiagnosticDataLossYearly.prefilled - .merge(ForestChangeDiagnosticDataLossYearly(fireCount)) - }) - } - - private def reformatSummaryData( - summaryRDD: RDD[(FeatureId, ForestChangeDiagnosticSummary)] - ): RDD[(FeatureId, ForestChangeDiagnosticData)] = { - - summaryRDD - .flatMap { - case (id, summary) => - // We need to convert the Map to a List in order to correctly flatmap the data - summary.stats.toList.map { - case (dataGroup, data) => - // id match { - // case featureId: SimpleFeatureId => - toForestChangeDiagnosticData(id, dataGroup, data) - // case _ => - // throw new IllegalArgumentException("Not a SimpleFeatureId") - // } - } - } - } - private def aggregateFireData( - fires: ForestChangeDiagnosticDataLossYearly - ): ForestChangeDiagnosticDataLossYearly = { + def aggregateFireData( + fires: ForestChangeDiagnosticDataLossYearly + ): ForestChangeDiagnosticDataLossYearly = { val minFireYear = fires.value.keysIterator.min val maxFireYear = fires.value.keysIterator.max val years: List[Int] = List.range(minFireYear + 1, maxFireYear + 1) ForestChangeDiagnosticDataLossYearly( SortedMap( - years.map( - year => - (year, { + years.map(year => + ( + year, { val thisYearFireCount: Double = fires.value.getOrElse(year, 0) val lastYearFireCount: Double = fires.value.getOrElse(year - 1, 0) (thisYearFireCount + lastYearFireCount) / 2 - }) + } + ) ): _* ) ) } - - private def toForestChangeDiagnosticData( - featureId: FeatureId, - dataGroup: ForestChangeDiagnosticRawDataGroup, - data: ForestChangeDiagnosticRawData - ): (FeatureId, ForestChangeDiagnosticData) = { - ( - featureId, - ForestChangeDiagnosticData( - treeCoverLossTcd30Yearly = ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.umdTreeCoverLossYear, - data.totalArea, - dataGroup.isUMDLoss - ), - treeCoverLossTcd90Yearly = ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.umdTreeCoverLossYear, - data.totalArea, - dataGroup.isUMDLoss && dataGroup.isTreeCoverExtent90 - ), - treeCoverLossPrimaryForestYearly = - ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.umdTreeCoverLossYear, - data.totalArea, - dataGroup.isPrimaryForest && dataGroup.isUMDLoss - ), - treeCoverLossPeatLandYearly = ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.umdTreeCoverLossYear, - data.totalArea, - dataGroup.isPeatlands && dataGroup.isUMDLoss - ), - treeCoverLossIntactForestYearly = - ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.umdTreeCoverLossYear, - data.totalArea, - dataGroup.isIntactForestLandscapes2000 && dataGroup.isUMDLoss - ), - treeCoverLossProtectedAreasYearly = - ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.umdTreeCoverLossYear, - data.totalArea, - dataGroup.isProtectedArea && dataGroup.isUMDLoss - ), - treeCoverLossSEAsiaLandCoverYearly = - ForestChangeDiagnosticDataLossYearlyCategory.fill( - dataGroup.seAsiaLandCover, - dataGroup.umdTreeCoverLossYear, - data.totalArea, - include = dataGroup.isUMDLoss - ), - treeCoverLossIDNLandCoverYearly = - ForestChangeDiagnosticDataLossYearlyCategory.fill( - dataGroup.idnLandCover, - dataGroup.umdTreeCoverLossYear, - data.totalArea, - include = dataGroup.isUMDLoss - ), - treeCoverLossSoyPlanedAreasYearly = - ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.umdTreeCoverLossYear, - data.totalArea, - dataGroup.isSoyPlantedAreas && dataGroup.isUMDLoss - ), - treeCoverLossIDNForestAreaYearly = - ForestChangeDiagnosticDataLossYearlyCategory.fill( - dataGroup.idnForestArea, - dataGroup.umdTreeCoverLossYear, - data.totalArea, - include = dataGroup.isUMDLoss - ), - treeCoverLossIDNForestMoratoriumYearly = - ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.umdTreeCoverLossYear, - data.totalArea, - dataGroup.isIdnForestMoratorium && dataGroup.isUMDLoss - ), - prodesLossYearly = ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.prodesLossYear, - data.totalArea, - dataGroup.isProdesLoss - ), - prodesLossProtectedAreasYearly = - ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.prodesLossYear, - data.totalArea, - dataGroup.isProdesLoss && dataGroup.isProtectedArea - ), - prodesLossProdesPrimaryForestYearly = - ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.prodesLossYear, - data.totalArea, - dataGroup.isProdesLoss && dataGroup.isPrimaryForest - ), - treeCoverLossBRABiomesYearly = - ForestChangeDiagnosticDataLossYearlyCategory.fill( - dataGroup.braBiomes, - dataGroup.umdTreeCoverLossYear, - data.totalArea, - include = dataGroup.isUMDLoss - ), - treeCoverExtent = ForestChangeDiagnosticDataDouble - .fill(data.totalArea, dataGroup.isTreeCoverExtent30), - treeCoverExtentPrimaryForest = ForestChangeDiagnosticDataDouble.fill( - data.totalArea, - dataGroup.isTreeCoverExtent30 && dataGroup.isPrimaryForest - ), - treeCoverExtentProtectedAreas = ForestChangeDiagnosticDataDouble.fill( - data.totalArea, - dataGroup.isTreeCoverExtent30 && dataGroup.isProtectedArea - ), - treeCoverExtentPeatlands = ForestChangeDiagnosticDataDouble.fill( - data.totalArea, - dataGroup.isTreeCoverExtent30 && dataGroup.isPeatlands - ), - treeCoverExtentIntactForests = ForestChangeDiagnosticDataDouble.fill( - data.totalArea, - dataGroup.isTreeCoverExtent30 && dataGroup.isIntactForestLandscapes2000 - ), - primaryForestArea = ForestChangeDiagnosticDataDouble - .fill(data.totalArea, dataGroup.isPrimaryForest), - intactForest2016Area = ForestChangeDiagnosticDataDouble - .fill(data.totalArea, dataGroup.isIntactForestLandscapes2000), - totalArea = ForestChangeDiagnosticDataDouble.fill(data.totalArea), - protectedAreasArea = ForestChangeDiagnosticDataDouble - .fill(data.totalArea, dataGroup.isProtectedArea), - peatlandsArea = ForestChangeDiagnosticDataDouble - .fill(data.totalArea, dataGroup.isPeatlands), - braBiomesArea = ForestChangeDiagnosticDataDoubleCategory - .fill(dataGroup.braBiomes, data.totalArea), - idnForestAreaArea = ForestChangeDiagnosticDataDoubleCategory - .fill(dataGroup.idnForestArea, data.totalArea), - seAsiaLandCoverArea = ForestChangeDiagnosticDataDoubleCategory - .fill(dataGroup.seAsiaLandCover, data.totalArea), - idnLandCoverArea = ForestChangeDiagnosticDataDoubleCategory - .fill(dataGroup.idnLandCover, data.totalArea), - idnForestMoratoriumArea = ForestChangeDiagnosticDataDouble - .fill(data.totalArea, dataGroup.isIdnForestMoratorium), - southAmericaPresence = ForestChangeDiagnosticDataBoolean - .fill(dataGroup.southAmericaPresence), - legalAmazonPresence = ForestChangeDiagnosticDataBoolean - .fill(dataGroup.legalAmazonPresence), - braBiomesPresence = ForestChangeDiagnosticDataBoolean - .fill(dataGroup.braBiomesPresence), - cerradoBiomesPresence = ForestChangeDiagnosticDataBoolean - .fill(dataGroup.cerradoBiomesPresence), - seAsiaPresence = - ForestChangeDiagnosticDataBoolean.fill(dataGroup.seAsiaPresence), - idnPresence = - ForestChangeDiagnosticDataBoolean.fill(dataGroup.idnPresence), - filteredTreeCoverExtent = ForestChangeDiagnosticDataDouble - .fill( - data.totalArea, - dataGroup.isTreeCoverExtent90 && !dataGroup.isPlantation - ), - filteredTreeCoverExtentYearly = - ForestChangeDiagnosticDataValueYearly.empty, - filteredTreeCoverLossYearly = ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.umdTreeCoverLossYear, - data.totalArea, - dataGroup.isUMDLoss && dataGroup.isTreeCoverExtent90 && !dataGroup.isPlantation - ), - filteredTreeCoverLossPeatYearly = - ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.umdTreeCoverLossYear, - data.totalArea, - dataGroup.isUMDLoss && dataGroup.isTreeCoverExtent90 && !dataGroup.isPlantation && dataGroup.isPeatlands - ), - filteredTreeCoverLossProtectedAreasYearly = - ForestChangeDiagnosticDataLossYearly.fill( - dataGroup.umdTreeCoverLossYear, - data.totalArea, - dataGroup.isUMDLoss && dataGroup.isTreeCoverExtent90 && !dataGroup.isPlantation && dataGroup.isProtectedArea - ), - plantationArea = ForestChangeDiagnosticDataDouble - .fill(data.totalArea, dataGroup.isPlantation), - plantationOnPeatArea = ForestChangeDiagnosticDataDouble - .fill( - data.totalArea, - dataGroup.isPlantation && dataGroup.isPeatlands - ), - plantationInProtectedAreasArea = ForestChangeDiagnosticDataDouble - .fill( - data.totalArea, - dataGroup.isPlantation && dataGroup.isProtectedArea - ), - forestValueIndicator = ForestChangeDiagnosticDataValueYearly.empty, - peatValueIndicator = ForestChangeDiagnosticDataValueYearly.empty, - protectedAreaValueIndicator = - ForestChangeDiagnosticDataValueYearly.empty, - deforestationThreatIndicator = - ForestChangeDiagnosticDataLossYearly.empty, - peatThreatIndicator = ForestChangeDiagnosticDataLossYearly.empty, - protectedAreaThreatIndicator = - ForestChangeDiagnosticDataLossYearly.empty, - fireThreatIndicator = ForestChangeDiagnosticDataLossYearly.empty - ) - ) - - } - - private def updateCommodityRisk( - featureId: FeatureId, - data: ForestChangeDiagnosticData - ): (FeatureId, ForestChangeDiagnosticData) = { - - val minLossYear = - ForestChangeDiagnosticDataLossYearly.prefilled.value.keysIterator.min - - val maxLossYear = - ForestChangeDiagnosticDataLossYearly.prefilled.value.keysIterator.max - - val years: List[Int] = List.range(minLossYear + 1, maxLossYear + 1) - - val forestValueIndicator: ForestChangeDiagnosticDataValueYearly = - ForestChangeDiagnosticDataValueYearly.fill( - data.filteredTreeCoverExtent.value, - data.filteredTreeCoverLossYearly.value, - 2 - ) - val peatValueIndicator: ForestChangeDiagnosticDataValueYearly = - ForestChangeDiagnosticDataValueYearly.fill(data.peatlandsArea.value) - val protectedAreaValueIndicator: ForestChangeDiagnosticDataValueYearly = - ForestChangeDiagnosticDataValueYearly.fill(data.protectedAreasArea.value) - val deforestationThreatIndicator: ForestChangeDiagnosticDataLossYearly = - ForestChangeDiagnosticDataLossYearly( - SortedMap( - years.map( - year => - (year, { - - // Somehow the compiler cannot infer the types correctly - // I hence declare them here explicitly to help him out. - val thisYearLoss: Double = - data.filteredTreeCoverLossYearly.value - .getOrElse(year, 0) - - val lastYearLoss: Double = - data.filteredTreeCoverLossYearly.value - .getOrElse(year - 1, 0) - - thisYearLoss + lastYearLoss - }) - ): _* - ) - ) - val peatThreatIndicator: ForestChangeDiagnosticDataLossYearly = - ForestChangeDiagnosticDataLossYearly( - SortedMap( - years.map( - year => - (year, { - // Somehow the compiler cannot infer the types correctly - // I hence declare them here explicitly to help him out. - val thisYearPeatLoss: Double = - data.filteredTreeCoverLossPeatYearly.value - .getOrElse(year, 0) - - val lastYearPeatLoss: Double = - data.filteredTreeCoverLossPeatYearly.value - .getOrElse(year - 1, 0) - - thisYearPeatLoss + lastYearPeatLoss + data.plantationOnPeatArea.value - - }) - ): _* - ) - ) - val protectedAreaThreatIndicator: ForestChangeDiagnosticDataLossYearly = - ForestChangeDiagnosticDataLossYearly( - SortedMap( - years.map( - year => - (year, { - // Somehow the compiler cannot infer the types correctly - // I hence declare them here explicitly to help him out. - val thisYearProtectedAreaLoss: Double = - data.filteredTreeCoverLossProtectedAreasYearly.value - .getOrElse(year, 0) - - val lastYearProtectedAreaLoss: Double = - data.filteredTreeCoverLossProtectedAreasYearly.value - .getOrElse(year - 1, 0) - - thisYearProtectedAreaLoss + lastYearProtectedAreaLoss + data.plantationInProtectedAreasArea.value - }) - ): _* - ) - ) - - val new_data = data.update( - forestValueIndicator = forestValueIndicator, - peatValueIndicator = peatValueIndicator, - protectedAreaValueIndicator = protectedAreaValueIndicator, - deforestationThreatIndicator = deforestationThreatIndicator, - peatThreatIndicator = peatThreatIndicator, - protectedAreaThreatIndicator = protectedAreaThreatIndicator - ) - (featureId, new_data) - } - - private def getIntermediateRDD( - intermediateListSource: NonEmptyList[String], - spark: SparkSession, - kwargs: Map[String, Any] - ): RDD[(FeatureId, ForestChangeDiagnosticData)] = { - val intermediateDF = { - FeatureDF(intermediateListSource, SimpleFeature, kwargs, spark) - } - - intermediateDF.rdd.map(row => { - val simpleFeatureId = SimpleFeatureId(row.getString(0).toInt) - val gridId = row.getString(1) - val treeCoverLossTcd30Yearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(2)) - val treeCoverLossPrimaryForestYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(3)) - val treeCoverLossPeatLandYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(4)) - val treeCoverLossIntactForestYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(5)) - val treeCoverLossProtectedAreasYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(6)) - val treeCoverLossSEAsiaLandCoverYearly = - ForestChangeDiagnosticDataLossYearlyCategory.fromString( - row.getString(7) - ) - val treeCoverLossIDNLandCoverYearly = - ForestChangeDiagnosticDataLossYearlyCategory.fromString( - row.getString(8) - ) - val treeCoverLossSoyPlanedAreasYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(9)) - val treeCoverLossIDNForestAreaYearly = - ForestChangeDiagnosticDataLossYearlyCategory.fromString( - row.getString(10) - ) - val treeCoverLossIDNForestMoratoriumYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(11)) - val prodesLossYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(12)) - val prodesLossProtectedAreasYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(13)) - val prodesLossProdesPrimaryForestYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(14)) - val treeCoverLossBRABiomesYearly = - ForestChangeDiagnosticDataLossYearlyCategory.fromString( - row.getString(15) - ) - val treeCoverExtent = - ForestChangeDiagnosticDataDouble(row.getString(16).toDouble) - val treeCoverExtentPrimaryForest = - ForestChangeDiagnosticDataDouble(row.getString(17).toDouble) - val treeCoverExtentProtectedAreas = - ForestChangeDiagnosticDataDouble(row.getString(18).toDouble) - val treeCoverExtentPeatlands = - ForestChangeDiagnosticDataDouble(row.getString(19).toDouble) - val treeCoverExtentIntactForests = - ForestChangeDiagnosticDataDouble(row.getString(20).toDouble) - val primaryForestArea = - ForestChangeDiagnosticDataDouble(row.getString(21).toDouble) - val intactForest2016Area = - ForestChangeDiagnosticDataDouble(row.getString(22).toDouble) - val totalArea = - ForestChangeDiagnosticDataDouble(row.getString(23).toDouble) - val protectedAreasArea = - ForestChangeDiagnosticDataDouble(row.getString(24).toDouble) - val peatlandsArea = - ForestChangeDiagnosticDataDouble(row.getString(25).toDouble) - val braBiomesArea = - ForestChangeDiagnosticDataDoubleCategory.fromString(row.getString(26)) - val idnForestAreaArea = - ForestChangeDiagnosticDataDoubleCategory.fromString(row.getString(27)) - val seAsiaLandCoverArea = - ForestChangeDiagnosticDataDoubleCategory.fromString(row.getString(28)) - val idnLandCoverArea = - ForestChangeDiagnosticDataDoubleCategory.fromString(row.getString(29)) - val idnForestMoratoriumArea = - ForestChangeDiagnosticDataDouble(row.getString(30).toDouble) - val southAmericaPresence = - ForestChangeDiagnosticDataBoolean(row.getString(31).toBoolean) - val legalAmazonPresence = - ForestChangeDiagnosticDataBoolean(row.getString(32).toBoolean) - val braBiomesPresence = - ForestChangeDiagnosticDataBoolean(row.getString(33).toBoolean) - val cerradoBiomesPresence = - ForestChangeDiagnosticDataBoolean(row.getString(34).toBoolean) - val seAsiaPresence = - ForestChangeDiagnosticDataBoolean(row.getString(35).toBoolean) - val idnPresence = - ForestChangeDiagnosticDataBoolean(row.getString(36).toBoolean) - val forestValueIndicator = - ForestChangeDiagnosticDataValueYearly.fromString(row.getString(37)) - val peatValueIndicator = - ForestChangeDiagnosticDataValueYearly.fromString(row.getString(38)) - val protectedAreaValueIndicator = - ForestChangeDiagnosticDataValueYearly.fromString(row.getString(39)) - val deforestationThreatIndicator = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(40)) - val peatThreatIndicator = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(41)) - val protectedAreaThreatIndicator = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(42)) - val fireThreatIndicator = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(43)) - - val treeCoverLossTcd90Yearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(44)) - val filteredTreeCoverExtent = - ForestChangeDiagnosticDataDouble(row.getString(45).toDouble) - val filteredTreeCoverExtentYearly = - ForestChangeDiagnosticDataValueYearly.fromString(row.getString(46)) - val filteredTreeCoverLossYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(47)) - val filteredTreeCoverLossPeatYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(48)) - val filteredTreeCoverLossProtectedAreasYearly = - ForestChangeDiagnosticDataLossYearly.fromString(row.getString(49)) - val plantationArea = - ForestChangeDiagnosticDataDouble(row.getString(50).toDouble) - val plantationOnPeatArea = - ForestChangeDiagnosticDataDouble(row.getString(51).toDouble) - val plantationInProtectedAreasArea = - ForestChangeDiagnosticDataDouble(row.getString(52).toDouble) - - ( - GridFeatureId(simpleFeatureId, gridId), - ForestChangeDiagnosticData( - treeCoverLossTcd30Yearly, - treeCoverLossTcd90Yearly, - treeCoverLossPrimaryForestYearly, - treeCoverLossPeatLandYearly, - treeCoverLossIntactForestYearly, - treeCoverLossProtectedAreasYearly, - treeCoverLossSEAsiaLandCoverYearly, - treeCoverLossIDNLandCoverYearly, - treeCoverLossSoyPlanedAreasYearly, - treeCoverLossIDNForestAreaYearly, - treeCoverLossIDNForestMoratoriumYearly, - prodesLossYearly, - prodesLossProtectedAreasYearly, - prodesLossProdesPrimaryForestYearly, - treeCoverLossBRABiomesYearly, - treeCoverExtent, - treeCoverExtentPrimaryForest, - treeCoverExtentProtectedAreas, - treeCoverExtentPeatlands, - treeCoverExtentIntactForests, - primaryForestArea, - intactForest2016Area, - totalArea, - protectedAreasArea, - peatlandsArea, - braBiomesArea, - idnForestAreaArea, - seAsiaLandCoverArea, - idnLandCoverArea, - idnForestMoratoriumArea, - southAmericaPresence, - legalAmazonPresence, - braBiomesPresence, - cerradoBiomesPresence, - seAsiaPresence, - idnPresence, - filteredTreeCoverExtent, - filteredTreeCoverExtentYearly, - filteredTreeCoverLossYearly, - filteredTreeCoverLossPeatYearly, - filteredTreeCoverLossProtectedAreasYearly, - plantationArea, - plantationOnPeatArea, - plantationInProtectedAreasArea, - forestValueIndicator, - peatValueIndicator, - protectedAreaValueIndicator, - deforestationThreatIndicator, - peatThreatIndicator, - protectedAreaThreatIndicator, - fireThreatIndicator - ) - ) - }) - } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala index c68e43e3..3f0f052c 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticCommand.scala @@ -4,43 +4,62 @@ import cats.data.NonEmptyList import org.globalforestwatch.summarystats.SummaryCommand import cats.implicits._ import com.monovore.decline.Opts +import org.globalforestwatch.features._ +import com.typesafe.scalalogging.LazyLogging +import org.globalforestwatch.summarystats.ValidatedLocation +import org.apache.spark.rdd.RDD -object ForestChangeDiagnosticCommand extends SummaryCommand { - +object ForestChangeDiagnosticCommand extends SummaryCommand with LazyLogging { val intermediateListSourceOpt: Opts[Option[NonEmptyList[String]]] = Opts .options[String]( "intermediate_list_source", help = "URI of intermediate list results in TSV format" - ).orNone + ) + .orNone val forestChangeDiagnosticCommand: Opts[Unit] = Opts.subcommand( - name = "forest_change_diagnostic", + name = ForestChangeDiagnosticAnalysis.name, help = "Compute summary statistics for GFW Pro Forest Change Diagnostic." ) { ( defaultOptions, intermediateListSourceOpt, fireAlertOptions, - defaultFilterOptions, - featureFilterOptions, - ).mapN { (default, intermediateListSource, fireAlert, defaultFilter, featureFilter) => + featureFilterOptions + ).mapN { (default, intermediateListSource, fireAlert, filterOptions) => val kwargs = Map( - "featureUris" -> default._2, - "outputUrl" -> default._3, - "splitFeatures" -> true, // force to split features - "intermediateListSource" -> intermediateListSource, - "fireAlertType" -> fireAlert._1, - "fireAlertSource" -> fireAlert._2, - "idStart" -> featureFilter._1, - "idEnd" -> featureFilter._2, - "limit" -> defaultFilter._1, - "tcl" -> defaultFilter._2, - "glad" -> defaultFilter._3 + "outputUrl" -> default.outputUrl, + "noOutputPathSuffix" -> default.noOutputPathSuffix, + "overwriteOutput" -> default.overwriteOutput ) - runAnalysis("forest_change_diagnostic", default._1, default._2, kwargs) + if (!default.splitFeatures) logger.warn("Forcing splitFeatures = true") + val featureFilter = FeatureFilter.fromOptions(default.featureType, filterOptions) + + runAnalysis { implicit spark => + val featureRDD = ValidatedFeatureRDD(default.featureUris, default.featureType, featureFilter, splitFeatures = true) + val fireAlertRDD = FireAlertRDD(spark, fireAlert.alertType, fireAlert.alertSource, FeatureFilter.empty) + + val intermediateResultsRDD = intermediateListSource.map { sources => + ForestChangeDiagnosticDF.readIntermidateRDD(sources, spark) + } + val saveIntermidateResults: RDD[ValidatedLocation[ForestChangeDiagnosticData]] => Unit = { rdd => + val df = ForestChangeDiagnosticDF.getGridFeatureDataFrame(rdd, spark) + ForestChangeDiagnosticExport.export("intermediate", df, default.outputUrl, kwargs) + } + + val fcdRDD = ForestChangeDiagnosticAnalysis( + featureRDD, + intermediateResultsRDD, + fireAlertRDD, + saveIntermidateResults, + kwargs + ) + val fcdDF = ForestChangeDiagnosticDF.getFeatureDataFrame(fcdRDD, spark) + ForestChangeDiagnosticExport.export(default.featureType, fcdDF, default.outputUrl, kwargs) + } } } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDF.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDF.scala new file mode 100644 index 00000000..3b7e5a2c --- /dev/null +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDF.scala @@ -0,0 +1,151 @@ +package org.globalforestwatch.summarystats.forest_change_diagnostic + +import cats.data.NonEmptyList +import cats.data.Validated.{Invalid, Valid} +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.globalforestwatch.features._ +import org.globalforestwatch.summarystats.{JobError, ValidatedLocation, Location} +import org.globalforestwatch.util.Util.{colsFor, fieldsFromCol} +import org.globalforestwatch.summarystats.SummaryDF +import org.globalforestwatch.summarystats.SummaryDF.{RowError, RowId} + +object ForestChangeDiagnosticDF extends SummaryDF { + + def getFeatureDataFrame( + dataRDD: RDD[ValidatedLocation[ForestChangeDiagnosticData]], + spark: SparkSession + ): DataFrame = { + import spark.implicits._ + + val rowId: FeatureId => RowId = { + case gfwproId: GfwProFeatureId => + RowId(gfwproId.listId, gfwproId.locationId.toString) + case gadmId: GadmFeatureId => + RowId("GADM 3.6", gadmId.toString) + case wdpaId: WdpaFeatureId => + RowId("WDPA", wdpaId.toString) + case id => + throw new IllegalArgumentException(s"Can't produce DataFrame for $id") + } + + dataRDD.map { + case Valid(Location(fid, data)) => + (rowId(fid), RowError.empty, data) + case Invalid(Location(fid, err)) => + (rowId(fid), RowError.fromJobError(err), ForestChangeDiagnosticData.empty) + } + .toDF("id", "error", "data") + .select($"id.*" :: $"error.*" :: fieldsFromCol($"data", featureFields): _*) + } + + def getGridFeatureDataFrame( + dataRDD: RDD[ValidatedLocation[ForestChangeDiagnosticData]], + spark: SparkSession + ): DataFrame = { + import spark.implicits._ + + val rowId: FeatureId => RowGridId = { + case CombinedFeatureId(gfwproId: GfwProFeatureId, gridId: GridId) => + RowGridId(gfwproId, gridId) + case _ => + throw new IllegalArgumentException("Not a CombinedFeatureId") + } + + dataRDD.map { + case Valid(Location(fid, data)) => + (rowId(fid), RowError.empty, data) + case Invalid(Location(fid, err)) => + (rowId(fid), RowError.fromJobError(err), ForestChangeDiagnosticData.empty) + } + .toDF("id", "error", "data") + .select($"id.*" :: $"error.*" :: fieldsFromCol($"data", featureFields) ::: fieldsFromCol($"data", gridFields): _*) + } + + def readIntermidateRDD( + sources: NonEmptyList[String], + spark: SparkSession, + ): RDD[ValidatedLocation[ForestChangeDiagnosticData]] = { + val df = FeatureDF(sources, GfwProFeature, FeatureFilter.empty, spark) + val ds = df.select( + colsFor[RowGridId].as[RowGridId], + colsFor[RowError].as[RowError], + colsFor[ForestChangeDiagnosticData].as[ForestChangeDiagnosticData]) + + ds.rdd.map { case (id, error, data) => + if (error.status_code == 2) Valid(Location(id.toFeatureID, data)) + else Invalid(Location(id.toFeatureID, JobError.fromErrorColumn(error.location_error).get)) + } + } + + case class RowGridId(list_id: String, location_id: Int, x: Double, y: Double, grid: String) { + def toFeatureID = CombinedFeatureId(GfwProFeatureId(list_id, location_id , x, y), GridId(grid)) + } + + object RowGridId { + def apply(gfwProId: GfwProFeatureId, gridId: GridId): RowGridId = RowGridId ( + list_id = gfwProId.listId, + location_id = gfwProId.locationId, + x = gfwProId.x, + y = gfwProId.y, + grid = gridId.gridId) + } + + val featureFields = List( + "tree_cover_loss_total_yearly", // treeCoverLossYearly + "tree_cover_loss_primary_forest_yearly", // treeCoverLossPrimaryForestYearly + "tree_cover_loss_peat_yearly", //treeCoverLossPeatLandYearly + "tree_cover_loss_intact_forest_yearly", // treeCoverLossIntactForestYearly + "tree_cover_loss_protected_areas_yearly", // treeCoverLossProtectedAreasYearly + "tree_cover_loss_sea_landcover_yearly", // treeCoverLossSEAsiaLandCoverYearly + "tree_cover_loss_idn_landcover_yearly", // treeCoverLossIDNLandCoverYearly + "tree_cover_loss_soy_yearly", // treeCoverLossSoyPlanedAreasYearly + "tree_cover_loss_idn_legal_yearly", // treeCoverLossIDNForestAreaYearly + "tree_cover_loss_idn_forest_moratorium_yearly", // treeCoverLossIDNForestMoratoriumYearly + "tree_cover_loss_prodes_yearly", // prodesLossYearly + "tree_cover_loss_prodes_wdpa_yearly", // prodesLossProtectedAreasYearly + "tree_cover_loss_prodes_primary_forest_yearly", // prodesLossProdesPrimaryForestYearly + "tree_cover_loss_brazil_biomes_yearly", // treeCoverLossBRABiomesYearly + "tree_cover_extent_total", // treeCoverExtent + "tree_cover_extent_primary_forest", // treeCoverExtentPrimaryForest + "tree_cover_extent_protected_areas", // treeCoverExtentProtectedAreas + "tree_cover_extent_peat", // treeCoverExtentPeatlands + "tree_cover_extent_intact_forest", // treeCoverExtentIntactForests + "natural_habitat_primary", // primaryForestArea + "natural_habitat_intact_forest", //intactForest2016Area + "total_area", // totalArea + "protected_areas_area", // protectedAreasArea + "peat_area", // peatlandsArea + "brazil_biomes", // braBiomesArea + "idn_legal_area", // idnForestAreaArea + "sea_landcover_area", // seAsiaLandCoverArea + "idn_landcover_area", // idnLandCoverArea + "idn_forest_moratorium_area", // idnForestMoratoriumArea + "south_america_presence", // southAmericaPresence, + "legal_amazon_presence", // legalAmazonPresence, + "brazil_biomes_presence", // braBiomesPresence, + "cerrado_biome_presence", // cerradoBiomesPresence, + "southeast_asia_presence", // seAsiaPresence, + "indonesia_presence", // idnPresence + "commodity_value_forest_extent", // forestValueIndicator + "commodity_value_peat", // peatValueIndicator + "commodity_value_protected_areas", // protectedAreaValueIndicator + "commodity_threat_deforestation", // deforestationThreatIndicator + "commodity_threat_peat", // peatThreatIndicator + "commodity_threat_protected_areas", // protectedAreaThreatIndicator + "commodity_threat_fires" // fireThreatIndicator + ) + + val gridFields = List( + "tree_cover_loss_tcd90_yearly", // treeCoverLossTcd90Yearly + "filtered_tree_cover_extent", // filteredTreeCoverExtent + "filtered_tree_cover_extent_yearly", //filteredTreeCoverExtentYearly + "filtered_tree_cover_loss_yearly", //filteredTreeCoverLossYearly + "filtered_tree_cover_loss_peat_yearly", //filteredTreeCoverLossPeatYearly + "filtered_tree_cover_loss_protected_areas_yearly", // filteredTreeCoverLossProtectedAreasYearly + "plantation_area", // plantationArea + "plantation_on_peat_area", // plantationOnPeatArea + "plantation_in_protected_areas_area" //plantationInProtectedAreasArea + ) + +} \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDFFactory.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDFFactory.scala deleted file mode 100644 index 09adba68..00000000 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDFFactory.scala +++ /dev/null @@ -1,197 +0,0 @@ -package org.globalforestwatch.summarystats.forest_change_diagnostic - -import io.circe.syntax._ -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.{DataFrame, SparkSession} -import org.globalforestwatch.features.{ - FeatureId, - GridFeatureId, - SimpleFeatureId -} -import org.globalforestwatch.util.CaseClassConstrutor.createCaseClassFromMap - -case class ForestChangeDiagnosticDFFactory( - featureType: String, - dataRDD: RDD[(FeatureId, ForestChangeDiagnosticData)], - spark: SparkSession, - kwargs: Map[String, Any] - ) { - - import spark.implicits._ - - def getDataFrame: DataFrame = { - featureType match { - case "feature" => getFeatureDataFrame - case "grid" => getGridFeatureDataFrame - case _ => - throw new IllegalArgumentException("Not a valid FeatureId") - } - } - - private def getFeatureDataFrame: DataFrame = { - - dataRDD - .map { - case (id, data) => - id match { - case simpleId: SimpleFeatureId => - createCaseClassFromMap[ForestChangeDiagnosticRowSimple]( - Map("id" -> simpleId.featureId.asJson.noSpaces) ++ - featureFieldMap(data) - ) - - case _ => - throw new IllegalArgumentException("Not a SimpleFeatureId") - } - } - .toDF("location_id" :: featureFieldNames: _*) - } - - private def getGridFeatureDataFrame: DataFrame = { - - dataRDD - .map { - case (id, data) => - id match { - case gridId: GridFeatureId => - val grid = gridId.gridId - gridId.featureId match { - case simpleId: SimpleFeatureId => - createCaseClassFromMap[ForestChangeDiagnosticRowGrid]( - Map( - "id" -> simpleId.featureId.asJson.noSpaces, - "grid" -> grid.asJson.noSpaces - ) ++ - featureFieldMap(data) - ++ - gridFieldMap(data) - ) - case _ => - throw new IllegalArgumentException("Not a SimpleFeatureId") - } - case _ => - throw new IllegalArgumentException("Not a SimpleFeatureId") - } - } - .toDF("id" :: "grid" :: featureFieldNames ++ gridFieldNames: _*) - } - - private def featureFieldMap(data: ForestChangeDiagnosticData) = { - Map( - "treeCoverLossTcd30Yearly" -> data.treeCoverLossTcd30Yearly.toJson, - "treeCoverLossPrimaryForestYearly" -> data.treeCoverLossPrimaryForestYearly.toJson, - "treeCoverLossPeatLandYearly" -> data.treeCoverLossPeatLandYearly.toJson, - "treeCoverLossIntactForestYearly" -> data.treeCoverLossIntactForestYearly.toJson, - "treeCoverLossProtectedAreasYearly" -> data.treeCoverLossProtectedAreasYearly.toJson, - "treeCoverLossSEAsiaLandCoverYearly" -> data.treeCoverLossSEAsiaLandCoverYearly.toJson, - "treeCoverLossIDNLandCoverYearly" -> data.treeCoverLossIDNLandCoverYearly.toJson, - "treeCoverLossSoyPlanedAreasYearly" -> data.treeCoverLossSoyPlanedAreasYearly.toJson, - "treeCoverLossIDNForestAreaYearly" -> data.treeCoverLossIDNForestAreaYearly.toJson, - "treeCoverLossIDNForestMoratoriumYearly" -> data.treeCoverLossIDNForestMoratoriumYearly.toJson, - "prodesLossYearly" -> data.prodesLossYearly.toJson, - "prodesLossProtectedAreasYearly" -> data.prodesLossProtectedAreasYearly.toJson, - "prodesLossProdesPrimaryForestYearly" -> data.prodesLossProdesPrimaryForestYearly.toJson, - "treeCoverLossBRABiomesYearly" -> data.treeCoverLossBRABiomesYearly.toJson, - "treeCoverExtent" -> data.treeCoverExtent.toJson, - "treeCoverExtentPrimaryForest" -> data.treeCoverExtentPrimaryForest.toJson, - "treeCoverExtentProtectedAreas" -> data.treeCoverExtentProtectedAreas.toJson, - "treeCoverExtentPeatlands" -> data.treeCoverExtentPeatlands.toJson, - "treeCoverExtentIntactForests" -> data.treeCoverExtentIntactForests.toJson, - "primaryForestArea" -> data.primaryForestArea.toJson, - "intactForest2016Area" -> data.intactForest2016Area.toJson, - "totalArea" -> data.totalArea.toJson, - "protectedAreasArea" -> data.protectedAreasArea.toJson, - "peatlandsArea" -> data.peatlandsArea.toJson, - "braBiomesArea" -> data.braBiomesArea.toJson, - "idnForestAreaArea" -> data.idnForestAreaArea.toJson, - "seAsiaLandCoverArea" -> data.seAsiaLandCoverArea.toJson, - "idnLandCoverArea" -> data.idnLandCoverArea.toJson, - "idnForestMoratoriumArea" -> data.idnForestMoratoriumArea.toJson, - "southAmericaPresence" -> data.southAmericaPresence.toJson, - "legalAmazonPresence" -> data.legalAmazonPresence.toJson, - "braBiomesPresence" -> data.braBiomesPresence.toJson, - "cerradoBiomesPresence" -> data.cerradoBiomesPresence.toJson, - "seAsiaPresence" -> data.seAsiaPresence.toJson, - "idnPresence" -> data.idnPresence.toJson, - "forestValueIndicator" -> data.forestValueIndicator.toJson, - "peatValueIndicator" -> data.peatValueIndicator.toJson, - "protectedAreaValueIndicator" -> data.protectedAreaValueIndicator.toJson, - "deforestationThreatIndicator" -> data.deforestationThreatIndicator.toJson, - "peatThreatIndicator" -> data.peatThreatIndicator.toJson, - "protectedAreaThreatIndicator" -> data.protectedAreaThreatIndicator.toJson, - "fireThreatIndicator" -> data.fireThreatIndicator.toJson - ) - } - - private def gridFieldMap(data: ForestChangeDiagnosticData) = { - Map( - "treeCoverLossTcd90Yearly" -> data.treeCoverLossTcd90Yearly.toJson, - "filteredTreeCoverExtent" -> data.filteredTreeCoverExtent.toJson, - "filteredTreeCoverExtentYearly" -> data.filteredTreeCoverExtentYearly.toJson, - "filteredTreeCoverLossYearly" -> data.filteredTreeCoverLossYearly.toJson, - "filteredTreeCoverLossPeatYearly" -> data.filteredTreeCoverLossPeatYearly.toJson, - "filteredTreeCoverLossProtectedAreasYearly" -> data.filteredTreeCoverLossProtectedAreasYearly.toJson, - "plantationArea" -> data.plantationArea.toJson, - "plantationOnPeatArea" -> data.plantationOnPeatArea.toJson, - "plantationInProtectedAreasArea" -> data.plantationInProtectedAreasArea.toJson - ) - } - - val featureFieldNames = List( - "tree_cover_loss_total_yearly", // treeCoverLossYearly - "tree_cover_loss_primary_forest_yearly", // treeCoverLossPrimaryForestYearly - "tree_cover_loss_peat_yearly", //treeCoverLossPeatLandYearly - "tree_cover_loss_intact_forest_yearly", // treeCoverLossIntactForestYearly - "tree_cover_loss_protected_areas_yearly", // treeCoverLossProtectedAreasYearly - "tree_cover_loss_sea_landcover_yearly", // treeCoverLossSEAsiaLandCoverYearly - "tree_cover_loss_idn_landcover_yearly", // treeCoverLossIDNLandCoverYearly - "tree_cover_loss_soy_yearly", // treeCoverLossSoyPlanedAreasYearly - "tree_cover_loss_idn_legal_yearly", // treeCoverLossIDNForestAreaYearly - "tree_cover_loss_idn_forest_moratorium_yearly", // treeCoverLossIDNForestMoratoriumYearly - "tree_cover_loss_prodes_yearly", // prodesLossYearly - "tree_cover_loss_prodes_wdpa_yearly", // prodesLossProtectedAreasYearly - "tree_cover_loss_prodes_primary_forest_yearly", // prodesLossProdesPrimaryForestYearly - "tree_cover_loss_brazil_biomes_yearly", // treeCoverLossBRABiomesYearly - "tree_cover_extent_total", // treeCoverExtent - "tree_cover_extent_primary_forest", // treeCoverExtentPrimaryForest - "tree_cover_extent_protected_areas", // treeCoverExtentProtectedAreas - "tree_cover_extent_peat", // treeCoverExtentPeatlands - "tree_cover_extent_intact_forest", // treeCoverExtentIntactForests - "natural_habitat_primary", // primaryForestArea - "natural_habitat_intact_forest", //intactForest2016Area - "total_area", // totalArea - "protected_areas_area", // protectedAreasArea - "peat_area", // peatlandsArea - "brazil_biomes", // braBiomesArea - "idn_legal_area", // idnForestAreaArea - "sea_landcover_area", // seAsiaLandCoverArea - "idn_landcover_area", // idnLandCoverArea - "idn_forest_moratorium_area", // idnForestMoratoriumArea - "south_america_presence", // southAmericaPresence, - "legal_amazon_presence", // legalAmazonPresence, - "brazil_biomes_presence", // braBiomesPresence, - "cerrado_biome_presence", // cerradoBiomesPresence, - "southeast_asia_presence", // seAsiaPresence, - "indonesia_presence", // idnPresence - "commodity_value_forest_extent", // forestValueIndicator - "commodity_value_peat", // peatValueIndicator - "commodity_value_protected_areas", // protectedAreaValueIndicator - "commodity_threat_deforestation", // deforestationThreatIndicator - "commodity_threat_peat", // peatThreatIndicator - "commodity_threat_protected_areas", // protectedAreaThreatIndicator - "commodity_threat_fires" // fireThreatIndicator - ) - - val gridFieldNames = List( - "tree_cover_Loss_tcd90_yearly", // treeCoverLossTcd90Yearly - "filtered_tree_cover_extent", // filteredTreeCoverExtent - "filtered_tree_cover_extent_yearly", //filteredTreeCoverExtentYearly - "filtered_tree_cover_loss_yearly", //filteredTreeCoverLossYearly - "filtered_tree_cover_loss_peat_yearly", //filteredTreeCoverLossPeatYearly - "filtered_tree_cover_loss_protected_areas_yearly", // filteredTreeCoverLossProtectedAreasYearly - "plantation_area", // plantationArea - "plantation_on_peat_area", // plantationOnPeatArea - "plantation_in_protected_areas_area" //plantationInProtectedAreasArea - ) - -} diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala index eaa33707..a04e53c1 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticData.scala @@ -2,296 +2,265 @@ package org.globalforestwatch.summarystats.forest_change_diagnostic import cats.Semigroup -/** Summary data per class +import scala.collection.immutable.SortedMap +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder + +/** Summary per class * * Note: This case class contains mutable values */ case class ForestChangeDiagnosticData( - treeCoverLossTcd30Yearly: ForestChangeDiagnosticDataLossYearly, - treeCoverLossTcd90Yearly: ForestChangeDiagnosticDataLossYearly, - treeCoverLossPrimaryForestYearly: ForestChangeDiagnosticDataLossYearly, - treeCoverLossPeatLandYearly: ForestChangeDiagnosticDataLossYearly, - treeCoverLossIntactForestYearly: ForestChangeDiagnosticDataLossYearly, - treeCoverLossProtectedAreasYearly: ForestChangeDiagnosticDataLossYearly, - treeCoverLossSEAsiaLandCoverYearly: ForestChangeDiagnosticDataLossYearlyCategory, - treeCoverLossIDNLandCoverYearly: ForestChangeDiagnosticDataLossYearlyCategory, - treeCoverLossSoyPlanedAreasYearly: ForestChangeDiagnosticDataLossYearly, - treeCoverLossIDNForestAreaYearly: ForestChangeDiagnosticDataLossYearlyCategory, - treeCoverLossIDNForestMoratoriumYearly: ForestChangeDiagnosticDataLossYearly, - prodesLossYearly: ForestChangeDiagnosticDataLossYearly, - prodesLossProtectedAreasYearly: ForestChangeDiagnosticDataLossYearly, - prodesLossProdesPrimaryForestYearly: ForestChangeDiagnosticDataLossYearly, - treeCoverLossBRABiomesYearly: ForestChangeDiagnosticDataLossYearlyCategory, - treeCoverExtent: ForestChangeDiagnosticDataDouble, - treeCoverExtentPrimaryForest: ForestChangeDiagnosticDataDouble, - treeCoverExtentProtectedAreas: ForestChangeDiagnosticDataDouble, - treeCoverExtentPeatlands: ForestChangeDiagnosticDataDouble, - treeCoverExtentIntactForests: ForestChangeDiagnosticDataDouble, - primaryForestArea: ForestChangeDiagnosticDataDouble, - intactForest2016Area: ForestChangeDiagnosticDataDouble, - totalArea: ForestChangeDiagnosticDataDouble, - protectedAreasArea: ForestChangeDiagnosticDataDouble, - peatlandsArea: ForestChangeDiagnosticDataDouble, - braBiomesArea: ForestChangeDiagnosticDataDoubleCategory, - idnForestAreaArea: ForestChangeDiagnosticDataDoubleCategory, - seAsiaLandCoverArea: ForestChangeDiagnosticDataDoubleCategory, - idnLandCoverArea: ForestChangeDiagnosticDataDoubleCategory, - idnForestMoratoriumArea: ForestChangeDiagnosticDataDouble, - southAmericaPresence: ForestChangeDiagnosticDataBoolean, - legalAmazonPresence: ForestChangeDiagnosticDataBoolean, - braBiomesPresence: ForestChangeDiagnosticDataBoolean, - cerradoBiomesPresence: ForestChangeDiagnosticDataBoolean, - seAsiaPresence: ForestChangeDiagnosticDataBoolean, - idnPresence: ForestChangeDiagnosticDataBoolean, - filteredTreeCoverExtent: ForestChangeDiagnosticDataDouble, - filteredTreeCoverExtentYearly: ForestChangeDiagnosticDataValueYearly, - filteredTreeCoverLossYearly: ForestChangeDiagnosticDataLossYearly, - filteredTreeCoverLossPeatYearly: ForestChangeDiagnosticDataLossYearly, - filteredTreeCoverLossProtectedAreasYearly: ForestChangeDiagnosticDataLossYearly, - plantationArea: ForestChangeDiagnosticDataDouble, - plantationOnPeatArea: ForestChangeDiagnosticDataDouble, - plantationInProtectedAreasArea: ForestChangeDiagnosticDataDouble, - forestValueIndicator: ForestChangeDiagnosticDataValueYearly, - peatValueIndicator: ForestChangeDiagnosticDataValueYearly, - protectedAreaValueIndicator: ForestChangeDiagnosticDataValueYearly, - deforestationThreatIndicator: ForestChangeDiagnosticDataLossYearly, - peatThreatIndicator: ForestChangeDiagnosticDataLossYearly, - protectedAreaThreatIndicator: ForestChangeDiagnosticDataLossYearly, - fireThreatIndicator: ForestChangeDiagnosticDataLossYearly - ) { + /** Tree Cover Loss TCD 30 */ + tree_cover_loss_total_yearly: ForestChangeDiagnosticDataLossYearly, + tree_cover_loss_tcd90_yearly: ForestChangeDiagnosticDataLossYearly, + tree_cover_loss_primary_forest_yearly: ForestChangeDiagnosticDataLossYearly, + tree_cover_loss_peat_yearly: ForestChangeDiagnosticDataLossYearly, + tree_cover_loss_intact_forest_yearly: ForestChangeDiagnosticDataLossYearly, + tree_cover_loss_protected_areas_yearly: ForestChangeDiagnosticDataLossYearly, + /** Tree cover loss in south east asia */ + tree_cover_loss_sea_landcover_yearly: ForestChangeDiagnosticDataLossYearlyCategory, + tree_cover_loss_idn_landcover_yearly: ForestChangeDiagnosticDataLossYearlyCategory, + /** treeCoverLossSoyPlanedAreasYearly */ + tree_cover_loss_soy_yearly: ForestChangeDiagnosticDataLossYearly, + /** treeCoverLossIDNForestAreaYearly */ + tree_cover_loss_idn_legal_yearly: ForestChangeDiagnosticDataLossYearlyCategory, + tree_cover_loss_idn_forest_moratorium_yearly: ForestChangeDiagnosticDataLossYearly, + tree_cover_loss_prodes_yearly: ForestChangeDiagnosticDataLossYearly, + /** prodesLossProtectedAreasYearly */ + tree_cover_loss_prodes_wdpa_yearly: ForestChangeDiagnosticDataLossYearly, + tree_cover_loss_prodes_primary_forest_yearly: ForestChangeDiagnosticDataLossYearly, + tree_cover_loss_brazil_biomes_yearly: ForestChangeDiagnosticDataLossYearlyCategory, + tree_cover_extent_total: ForestChangeDiagnosticDataDouble, + tree_cover_extent_primary_forest: ForestChangeDiagnosticDataDouble, + tree_cover_extent_protected_areas: ForestChangeDiagnosticDataDouble, + tree_cover_extent_peat: ForestChangeDiagnosticDataDouble, + tree_cover_extent_intact_forest: ForestChangeDiagnosticDataDouble, + /** Primary Forest Area */ + natural_habitat_primary: ForestChangeDiagnosticDataDouble, + /** Intact Forest 2016 Area */ + natural_habitat_intact_forest: ForestChangeDiagnosticDataDouble, + total_area: ForestChangeDiagnosticDataDouble, + protected_areas_area: ForestChangeDiagnosticDataDouble, + /** Peatland Area */ + peat_area: ForestChangeDiagnosticDataDouble, + brazil_biomes: ForestChangeDiagnosticDataDoubleCategory, + /** IDN Forest Area */ + idn_legal_area: ForestChangeDiagnosticDataDoubleCategory, + /** Southeast Asia land cover area */ + sea_landcover_area: ForestChangeDiagnosticDataDoubleCategory, + idn_landcover_area: ForestChangeDiagnosticDataDoubleCategory, + idn_forest_moratorium_area: ForestChangeDiagnosticDataDouble, + south_america_presence: ForestChangeDiagnosticDataBoolean, + legal_amazon_presence: ForestChangeDiagnosticDataBoolean, + brazil_biomes_presence: ForestChangeDiagnosticDataBoolean, + cerrado_biome_presence: ForestChangeDiagnosticDataBoolean, + southeast_asia_presence: ForestChangeDiagnosticDataBoolean, + indonesia_presence: ForestChangeDiagnosticDataBoolean, + filtered_tree_cover_extent: ForestChangeDiagnosticDataDouble, + filtered_tree_cover_extent_yearly: ForestChangeDiagnosticDataValueYearly, + filtered_tree_cover_loss_yearly: ForestChangeDiagnosticDataLossYearly, + filtered_tree_cover_loss_peat_yearly: ForestChangeDiagnosticDataLossYearly, + filtered_tree_cover_loss_protected_areas_yearly: ForestChangeDiagnosticDataLossYearly, + plantation_area: ForestChangeDiagnosticDataDouble, + plantation_on_peat_area: ForestChangeDiagnosticDataDouble, + plantation_in_protected_areas_area: ForestChangeDiagnosticDataDouble, + commodity_value_forest_extent: ForestChangeDiagnosticDataValueYearly, + /** Repeats the peat_area value for each year of diagnostic, ease of use for commodity threat calc */ + commodity_value_peat: ForestChangeDiagnosticDataValueYearly, + /** Repeats the protected_areas_area value for each year of diagnostic, ease of use for commodity threat calc */ + commodity_value_protected_areas: ForestChangeDiagnosticDataValueYearly, + /** Sum of moving two year window over filtered_tree_cover_loss_yearly */ + commodity_threat_deforestation: ForestChangeDiagnosticDataLossYearly, + /** Sum of moving two year window over filtered_tree_cover_loss_peat_yearly + plantation_peat_area */ + commodity_threat_peat: ForestChangeDiagnosticDataLossYearly, + /** Sum of moving to year window over filtered_tree_cover_loss_protected_areas_yearly + plantation_in_protected_areas_area */ + commodity_threat_protected_areas: ForestChangeDiagnosticDataLossYearly, + commodity_threat_fires: ForestChangeDiagnosticDataLossYearly +) { def merge(other: ForestChangeDiagnosticData): ForestChangeDiagnosticData = { ForestChangeDiagnosticData( - treeCoverLossTcd30Yearly.merge(other.treeCoverLossTcd30Yearly), - treeCoverLossTcd90Yearly.merge(other.treeCoverLossTcd90Yearly), - treeCoverLossPrimaryForestYearly.merge( - other.treeCoverLossPrimaryForestYearly + tree_cover_loss_total_yearly.merge(other.tree_cover_loss_total_yearly), + tree_cover_loss_tcd90_yearly.merge(other.tree_cover_loss_tcd90_yearly), + tree_cover_loss_primary_forest_yearly.merge( + other.tree_cover_loss_primary_forest_yearly ), - treeCoverLossPeatLandYearly.merge(other.treeCoverLossPeatLandYearly), - treeCoverLossIntactForestYearly.merge( - other.treeCoverLossIntactForestYearly + tree_cover_loss_peat_yearly.merge(other.tree_cover_loss_peat_yearly), + tree_cover_loss_intact_forest_yearly.merge( + other.tree_cover_loss_intact_forest_yearly ), - treeCoverLossProtectedAreasYearly.merge( - other.treeCoverLossProtectedAreasYearly + tree_cover_loss_protected_areas_yearly.merge( + other.tree_cover_loss_protected_areas_yearly ), - treeCoverLossSEAsiaLandCoverYearly.merge( - other.treeCoverLossSEAsiaLandCoverYearly + tree_cover_loss_sea_landcover_yearly.merge( + other.tree_cover_loss_sea_landcover_yearly ), - treeCoverLossIDNLandCoverYearly.merge( - other.treeCoverLossIDNLandCoverYearly + tree_cover_loss_idn_landcover_yearly.merge( + other.tree_cover_loss_idn_landcover_yearly ), - treeCoverLossSoyPlanedAreasYearly.merge( - other.treeCoverLossSoyPlanedAreasYearly + tree_cover_loss_soy_yearly.merge( + other.tree_cover_loss_soy_yearly ), - treeCoverLossIDNForestAreaYearly.merge( - other.treeCoverLossIDNForestAreaYearly + tree_cover_loss_idn_legal_yearly.merge( + other.tree_cover_loss_idn_legal_yearly ), - treeCoverLossIDNForestMoratoriumYearly.merge( - other.treeCoverLossIDNForestMoratoriumYearly + tree_cover_loss_idn_forest_moratorium_yearly.merge( + other.tree_cover_loss_idn_forest_moratorium_yearly ), - prodesLossYearly.merge(other.prodesLossYearly), - prodesLossProtectedAreasYearly.merge( - other.prodesLossProtectedAreasYearly + tree_cover_loss_prodes_yearly.merge(other.tree_cover_loss_prodes_yearly), + tree_cover_loss_prodes_wdpa_yearly.merge( + other.tree_cover_loss_prodes_wdpa_yearly ), - prodesLossProdesPrimaryForestYearly.merge( - other.prodesLossProdesPrimaryForestYearly + tree_cover_loss_prodes_primary_forest_yearly.merge( + other.tree_cover_loss_prodes_primary_forest_yearly ), - treeCoverLossBRABiomesYearly.merge(other.treeCoverLossBRABiomesYearly), - treeCoverExtent.merge(other.treeCoverExtent), - treeCoverExtentPrimaryForest.merge(other.treeCoverExtentPrimaryForest), - treeCoverExtentProtectedAreas.merge(other.treeCoverExtentProtectedAreas), - treeCoverExtentPeatlands.merge(other.treeCoverExtentPeatlands), - treeCoverExtentIntactForests.merge(other.treeCoverExtentIntactForests), - primaryForestArea.merge(other.primaryForestArea), - intactForest2016Area.merge(other.intactForest2016Area), - totalArea.merge(other.totalArea), - protectedAreasArea.merge(other.protectedAreasArea), - peatlandsArea.merge(other.peatlandsArea), - braBiomesArea.merge(other.braBiomesArea), - idnForestAreaArea.merge(other.idnForestAreaArea), - seAsiaLandCoverArea.merge(other.seAsiaLandCoverArea), - idnLandCoverArea.merge(other.idnLandCoverArea), - idnForestMoratoriumArea.merge(other.idnForestMoratoriumArea), - southAmericaPresence.merge(other.southAmericaPresence), - legalAmazonPresence.merge(other.legalAmazonPresence), - braBiomesPresence.merge(other.braBiomesPresence), - cerradoBiomesPresence.merge(other.cerradoBiomesPresence), - seAsiaPresence.merge(other.seAsiaPresence), - idnPresence.merge(other.idnPresence), - filteredTreeCoverExtent.merge(other.filteredTreeCoverExtent), - filteredTreeCoverExtentYearly.merge(other.filteredTreeCoverExtentYearly), - filteredTreeCoverLossYearly.merge(other.filteredTreeCoverLossYearly), - filteredTreeCoverLossPeatYearly.merge( - other.filteredTreeCoverLossPeatYearly + tree_cover_loss_brazil_biomes_yearly.merge(other.tree_cover_loss_brazil_biomes_yearly), + tree_cover_extent_total.merge(other.tree_cover_extent_total), + tree_cover_extent_primary_forest.merge(other.tree_cover_extent_primary_forest), + tree_cover_extent_protected_areas.merge(other.tree_cover_extent_protected_areas), + tree_cover_extent_peat.merge(other.tree_cover_extent_peat), + tree_cover_extent_intact_forest.merge(other.tree_cover_extent_intact_forest), + natural_habitat_primary.merge(other.natural_habitat_primary), + natural_habitat_intact_forest.merge(other.natural_habitat_intact_forest), + total_area.merge(other.total_area), + protected_areas_area.merge(other.protected_areas_area), + peat_area.merge(other.peat_area), + brazil_biomes.merge(other.brazil_biomes), + idn_legal_area.merge(other.idn_legal_area), + sea_landcover_area.merge(other.sea_landcover_area), + idn_landcover_area.merge(other.idn_landcover_area), + idn_forest_moratorium_area.merge(other.idn_forest_moratorium_area), + south_america_presence.merge(other.south_america_presence), + legal_amazon_presence.merge(other.legal_amazon_presence), + brazil_biomes_presence.merge(other.brazil_biomes_presence), + cerrado_biome_presence.merge(other.cerrado_biome_presence), + southeast_asia_presence.merge(other.southeast_asia_presence), + indonesia_presence.merge(other.indonesia_presence), + filtered_tree_cover_extent.merge(other.filtered_tree_cover_extent), + filtered_tree_cover_extent_yearly.merge(other.filtered_tree_cover_extent_yearly), + filtered_tree_cover_loss_yearly.merge(other.filtered_tree_cover_loss_yearly), + filtered_tree_cover_loss_peat_yearly.merge( + other.filtered_tree_cover_loss_peat_yearly ), - filteredTreeCoverLossProtectedAreasYearly.merge( - other.filteredTreeCoverLossProtectedAreasYearly + filtered_tree_cover_loss_protected_areas_yearly.merge( + other.filtered_tree_cover_loss_protected_areas_yearly ), - plantationArea.merge(other.plantationArea), - plantationOnPeatArea.merge(other.plantationOnPeatArea), - plantationInProtectedAreasArea.merge( - other.plantationInProtectedAreasArea + plantation_area.merge(other.plantation_area), + plantation_on_peat_area.merge(other.plantation_on_peat_area), + plantation_in_protected_areas_area.merge( + other.plantation_in_protected_areas_area ), - forestValueIndicator.merge(other.forestValueIndicator), - peatValueIndicator.merge(other.peatValueIndicator), - protectedAreaValueIndicator.merge(other.protectedAreaValueIndicator), - deforestationThreatIndicator.merge(other.deforestationThreatIndicator), - peatThreatIndicator.merge(other.peatThreatIndicator), - protectedAreaThreatIndicator.merge(other.protectedAreaThreatIndicator), - fireThreatIndicator.merge(other.fireThreatIndicator) + commodity_value_forest_extent.merge(other.commodity_value_forest_extent), + commodity_value_peat.merge(other.commodity_value_peat), + commodity_value_protected_areas.merge(other.commodity_value_protected_areas), + commodity_threat_deforestation.merge(other.commodity_threat_deforestation), + commodity_threat_peat.merge(other.commodity_threat_peat), + commodity_threat_protected_areas.merge(other.commodity_threat_protected_areas), + commodity_threat_fires.merge(other.commodity_threat_fires) ) } - def update( - treeCoverLossTcd30Yearly: ForestChangeDiagnosticDataLossYearly = - this.treeCoverLossTcd30Yearly, - treeCoverLossTcd90Yearly: ForestChangeDiagnosticDataLossYearly = - this.treeCoverLossTcd90Yearly, - treeCoverLossPrimaryForestYearly: ForestChangeDiagnosticDataLossYearly = - this.treeCoverLossPrimaryForestYearly, - treeCoverLossPeatLandYearly: ForestChangeDiagnosticDataLossYearly = - this.treeCoverLossPeatLandYearly, - treeCoverLossIntactForestYearly: ForestChangeDiagnosticDataLossYearly = - this.treeCoverLossProtectedAreasYearly, - treeCoverLossProtectedAreasYearly: ForestChangeDiagnosticDataLossYearly = - this.treeCoverLossProtectedAreasYearly, - treeCoverLossSEAsiaLandCoverYearly: ForestChangeDiagnosticDataLossYearlyCategory = - this.treeCoverLossSEAsiaLandCoverYearly, - treeCoverLossIDNLandCoverYearly: ForestChangeDiagnosticDataLossYearlyCategory = - this.treeCoverLossIDNLandCoverYearly, - treeCoverLossSoyPlanedAreasYearly: ForestChangeDiagnosticDataLossYearly = - this.treeCoverLossSoyPlanedAreasYearly, - treeCoverLossIDNForestAreaYearly: ForestChangeDiagnosticDataLossYearlyCategory = - this.treeCoverLossIDNForestAreaYearly, - treeCoverLossIDNForestMoratoriumYearly: ForestChangeDiagnosticDataLossYearly = - this.treeCoverLossIDNForestMoratoriumYearly, - prodesLossYearly: ForestChangeDiagnosticDataLossYearly = - this.prodesLossYearly, - prodesLossProtectedAreasYearly: ForestChangeDiagnosticDataLossYearly = - this.prodesLossProtectedAreasYearly, - prodesLossProdesPrimaryForestYearly: ForestChangeDiagnosticDataLossYearly = - this.prodesLossProdesPrimaryForestYearly, - treeCoverLossBRABiomesYearly: ForestChangeDiagnosticDataLossYearlyCategory = - this.treeCoverLossBRABiomesYearly, - treeCoverExtent: ForestChangeDiagnosticDataDouble = this.treeCoverExtent, - treeCoverExtentPrimaryForest: ForestChangeDiagnosticDataDouble = - this.treeCoverExtentPrimaryForest, - treeCoverExtentProtectedAreas: ForestChangeDiagnosticDataDouble = - this.treeCoverExtentProtectedAreas, - treeCoverExtentPeatlands: ForestChangeDiagnosticDataDouble = - this.treeCoverExtentPeatlands, - treeCoverExtentIntactForests: ForestChangeDiagnosticDataDouble = - this.treeCoverExtentIntactForests, - primaryForestArea: ForestChangeDiagnosticDataDouble = this.primaryForestArea, - intactForest2016Area: ForestChangeDiagnosticDataDouble = - this.intactForest2016Area, - totalArea: ForestChangeDiagnosticDataDouble = this.totalArea, - protectedAreasArea: ForestChangeDiagnosticDataDouble = - this.protectedAreasArea, - peatlandsArea: ForestChangeDiagnosticDataDouble = this.peatlandsArea, - braBiomesArea: ForestChangeDiagnosticDataDoubleCategory = this.braBiomesArea, - idnForestAreaArea: ForestChangeDiagnosticDataDoubleCategory = - this.idnForestAreaArea, - seAsiaLandCoverArea: ForestChangeDiagnosticDataDoubleCategory = - this.seAsiaLandCoverArea, - idnLandCoverArea: ForestChangeDiagnosticDataDoubleCategory = - this.idnLandCoverArea, - idnForestMoratoriumArea: ForestChangeDiagnosticDataDouble = - this.idnForestMoratoriumArea, - southAmericaPresence: ForestChangeDiagnosticDataBoolean = - this.southAmericaPresence, - legalAmazonPresence: ForestChangeDiagnosticDataBoolean = - this.legalAmazonPresence, - braBiomesPresence: ForestChangeDiagnosticDataBoolean = - this.braBiomesPresence, - cerradoBiomesPresence: ForestChangeDiagnosticDataBoolean = - this.cerradoBiomesPresence, - seAsiaPresence: ForestChangeDiagnosticDataBoolean = this.seAsiaPresence, - idnPresence: ForestChangeDiagnosticDataBoolean = this.idnPresence, - filteredTreeCoverExtent: ForestChangeDiagnosticDataDouble = this.filteredTreeCoverExtent, - filteredTreeCoverExtentYearly: ForestChangeDiagnosticDataValueYearly = - this.filteredTreeCoverExtentYearly, - filteredTreeCoverLossYearly: ForestChangeDiagnosticDataLossYearly = - this.filteredTreeCoverLossYearly, - filteredTreeCoverLossPeatYearly: ForestChangeDiagnosticDataLossYearly = - this.filteredTreeCoverLossPeatYearly, - filteredTreeCoverLossProtectedAreasYearly: ForestChangeDiagnosticDataLossYearly = - this.filteredTreeCoverLossProtectedAreasYearly, - plantationArea: ForestChangeDiagnosticDataDouble = this.plantationArea, - plantationOnPeatArea: ForestChangeDiagnosticDataDouble = - this.plantationOnPeatArea, - plantationInProtectedAreasArea: ForestChangeDiagnosticDataDouble = - this.plantationInProtectedAreasArea, - forestValueIndicator: ForestChangeDiagnosticDataValueYearly = - this.forestValueIndicator, - peatValueIndicator: ForestChangeDiagnosticDataValueYearly = - this.peatValueIndicator, - protectedAreaValueIndicator: ForestChangeDiagnosticDataValueYearly = - this.protectedAreaValueIndicator, - deforestationThreatIndicator: ForestChangeDiagnosticDataLossYearly = - this.deforestationThreatIndicator, - peatThreatIndicator: ForestChangeDiagnosticDataLossYearly = - this.peatThreatIndicator, - protectedAreaThreatIndicator: ForestChangeDiagnosticDataLossYearly = - this.protectedAreaThreatIndicator, - fireThreatIndicator: ForestChangeDiagnosticDataLossYearly = - this.fireThreatIndicator - ): ForestChangeDiagnosticData = { + /** + * @see https://docs.google.com/presentation/d/1nAq4mFNkv1q5vFvvXWReuLr4Znvr-1q-BDi6pl_5zTU/edit#slide=id.p + */ + def withUpdatedCommodityRisk(): ForestChangeDiagnosticData = { - ForestChangeDiagnosticData( - treeCoverLossTcd30Yearly, - treeCoverLossTcd90Yearly, - treeCoverLossPrimaryForestYearly, - treeCoverLossPeatLandYearly, - treeCoverLossIntactForestYearly, - treeCoverLossProtectedAreasYearly, - treeCoverLossSEAsiaLandCoverYearly, - treeCoverLossIDNLandCoverYearly, - treeCoverLossSoyPlanedAreasYearly, - treeCoverLossIDNForestAreaYearly, - treeCoverLossIDNForestMoratoriumYearly, - prodesLossYearly, - prodesLossProtectedAreasYearly, - prodesLossProdesPrimaryForestYearly, - treeCoverLossBRABiomesYearly, - treeCoverExtent, - treeCoverExtentPrimaryForest, - treeCoverExtentProtectedAreas, - treeCoverExtentPeatlands, - treeCoverExtentIntactForests, - primaryForestArea, - intactForest2016Area, - totalArea, - protectedAreasArea, - peatlandsArea, - braBiomesArea, - idnForestAreaArea, - seAsiaLandCoverArea, - idnLandCoverArea, - idnForestMoratoriumArea, - southAmericaPresence, - legalAmazonPresence, - braBiomesPresence, - cerradoBiomesPresence, - seAsiaPresence, - idnPresence, - filteredTreeCoverExtent, - filteredTreeCoverExtentYearly, - filteredTreeCoverLossYearly, - filteredTreeCoverLossPeatYearly, - filteredTreeCoverLossProtectedAreasYearly, - plantationArea, - plantationOnPeatArea, - plantationInProtectedAreasArea, - forestValueIndicator, - peatValueIndicator, - protectedAreaValueIndicator, - deforestationThreatIndicator, - peatThreatIndicator, - protectedAreaThreatIndicator, - fireThreatIndicator - ) + /* Exclude the last year, limit data to 2019 to sync with palm risk tool: + commodity_threat_deforestation, commodity_threat_peat, commodity_threat_protected_areas use year n and year n-1. + Including information from the current year would under-represent these values as it's in progress. + */ + val minLossYear = ForestChangeDiagnosticDataLossYearly.prefilled.value.keys.min + val maxLossYear = 2019 + val years: List[Int] = List.range(minLossYear + 1, maxLossYear + 1) + + val forestValueIndicator: ForestChangeDiagnosticDataValueYearly = + ForestChangeDiagnosticDataValueYearly.fill( + filtered_tree_cover_extent.value, + filtered_tree_cover_loss_yearly.value, + 2 + ).limitToMaxYear(maxLossYear) + + val peatValueIndicator: ForestChangeDiagnosticDataValueYearly = + ForestChangeDiagnosticDataValueYearly.fill(peat_area.value).limitToMaxYear(maxLossYear) + + val protectedAreaValueIndicator: ForestChangeDiagnosticDataValueYearly = + ForestChangeDiagnosticDataValueYearly.fill(protected_areas_area.value).limitToMaxYear(maxLossYear) + + val deforestationThreatIndicator: ForestChangeDiagnosticDataLossYearly = + ForestChangeDiagnosticDataLossYearly( + SortedMap( + years.map( + year => + (year, { + // Somehow the compiler cannot infer the types correctly + // I hence declare them here explicitly to help him out. + val thisYearLoss: Double = + filtered_tree_cover_loss_yearly.value + .getOrElse(year, 0) + + val lastYearLoss: Double = + filtered_tree_cover_loss_yearly.value + .getOrElse(year - 1, 0) + + thisYearLoss + lastYearLoss + }) + ): _* + ) + ).limitToMaxYear(maxLossYear) + + val peatThreatIndicator: ForestChangeDiagnosticDataLossYearly = + ForestChangeDiagnosticDataLossYearly( + SortedMap( + years.map( + year => + (year, { + // Somehow the compiler cannot infer the types correctly + // I hence declare them here explicitly to help him out. + val thisYearPeatLoss: Double = + filtered_tree_cover_loss_peat_yearly.value + .getOrElse(year, 0) + + val lastYearPeatLoss: Double = + filtered_tree_cover_loss_peat_yearly.value + .getOrElse(year - 1, 0) + + thisYearPeatLoss + lastYearPeatLoss + plantation_on_peat_area.value + + }) + ): _* + ) + ).limitToMaxYear(maxLossYear) + + val protectedAreaThreatIndicator: ForestChangeDiagnosticDataLossYearly = + ForestChangeDiagnosticDataLossYearly( + SortedMap( + years.map( + year => + (year, { + // Somehow the compiler cannot infer the types correctly + // I hence declare them here explicitly to help him out. + val thisYearProtectedAreaLoss: Double = + filtered_tree_cover_loss_protected_areas_yearly.value + .getOrElse(year, 0) + + val lastYearProtectedAreaLoss: Double = + filtered_tree_cover_loss_protected_areas_yearly.value + .getOrElse(year - 1, 0) + + thisYearProtectedAreaLoss + lastYearProtectedAreaLoss + plantation_in_protected_areas_area.value + }) + ): _* + ) + ).limitToMaxYear(maxLossYear) + + copy( + commodity_value_forest_extent = forestValueIndicator, + commodity_value_peat = peatValueIndicator, + commodity_value_protected_areas = protectedAreaValueIndicator, + commodity_threat_deforestation = deforestationThreatIndicator, + commodity_threat_peat = peatThreatIndicator, + commodity_threat_protected_areas = protectedAreaThreatIndicator) } } @@ -360,4 +329,7 @@ object ForestChangeDiagnosticData { x.merge(y) } + implicit def dataExpressionEncoder: ExpressionEncoder[ForestChangeDiagnosticData] = + frameless.TypedExpressionEncoder[ForestChangeDiagnosticData] + .asInstanceOf[ExpressionEncoder[ForestChangeDiagnosticData]] } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataBoolean.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataBoolean.scala index 70baff36..e5dc79ab 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataBoolean.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataBoolean.scala @@ -1,5 +1,6 @@ package org.globalforestwatch.summarystats.forest_change_diagnostic +import frameless.Injection import io.circe.syntax._ case class ForestChangeDiagnosticDataBoolean(value: Boolean) @@ -23,4 +24,6 @@ object ForestChangeDiagnosticDataBoolean { ForestChangeDiagnosticDataBoolean(value) } + implicit def injection: Injection[ForestChangeDiagnosticDataBoolean, String] = + Injection(_.toJson, s => ForestChangeDiagnosticDataBoolean(s.toBoolean)) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDouble.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDouble.scala index 01d3fd71..2c7e3369 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDouble.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDouble.scala @@ -1,4 +1,5 @@ package org.globalforestwatch.summarystats.forest_change_diagnostic +import frameless.Injection import org.globalforestwatch.util.Implicits._ import io.circe.syntax._ @@ -25,4 +26,7 @@ object ForestChangeDiagnosticDataDouble { ForestChangeDiagnosticDataDouble(value * include) } + implicit def injection: Injection[ForestChangeDiagnosticDataDouble, String] = + Injection(_.toJson, s => ForestChangeDiagnosticDataDouble(s.toDouble)) + } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleCategory.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleCategory.scala index bd3c55c6..3ae5cf5c 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleCategory.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataDoubleCategory.scala @@ -1,7 +1,7 @@ package org.globalforestwatch.summarystats.forest_change_diagnostic +import frameless.Injection import io.circe.syntax._ -import org.globalforestwatch.util.Implicits._ import io.circe.parser.decode case class ForestChangeDiagnosticDataDoubleCategory( @@ -56,4 +56,5 @@ object ForestChangeDiagnosticDataDoubleCategory { } + implicit def injection: Injection[ForestChangeDiagnosticDataDoubleCategory , String] = Injection(_.toJson, fromString) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearly.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearly.scala index 3dd561ba..7f016d24 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearly.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearly.scala @@ -1,23 +1,25 @@ package org.globalforestwatch.summarystats.forest_change_diagnostic +import frameless.Injection + import scala.collection.immutable.SortedMap import io.circe.syntax._ import io.circe.parser.decode +import cats.kernel.Semigroup -case class ForestChangeDiagnosticDataLossYearly(value: SortedMap[Int, Double]) extends ForestChangeDiagnosticDataParser[ForestChangeDiagnosticDataLossYearly] { - def merge( - other: ForestChangeDiagnosticDataLossYearly - ): ForestChangeDiagnosticDataLossYearly = { +case class ForestChangeDiagnosticDataLossYearly(value: SortedMap[Int, Double]) + extends ForestChangeDiagnosticDataParser[ForestChangeDiagnosticDataLossYearly] { - ForestChangeDiagnosticDataLossYearly(value ++ other.value.map { - case (key, otherValue) => - key -> - (value.getOrElse(key, 0.0) + otherValue) - }) + def merge(other: ForestChangeDiagnosticDataLossYearly): ForestChangeDiagnosticDataLossYearly = { + ForestChangeDiagnosticDataLossYearly(Semigroup[SortedMap[Int, Double]].combine(value, other.value)) } def round: SortedMap[Int, Double] = this.value.map { case (key, value) => key -> this.round(value) } + def limitToMaxYear(maxYear: Int): ForestChangeDiagnosticDataLossYearly = { + ForestChangeDiagnosticDataLossYearly(value.filterKeys{ year => year <= maxYear }) + } + def toJson: String = { this.round.asJson.noSpaces } @@ -50,7 +52,8 @@ object ForestChangeDiagnosticDataLossYearly { 2016 -> 0, 2017 -> 0, 2018 -> 0, - 2019 -> 0 + 2019 -> 0, + 2020 -> 0 ) ) @@ -79,4 +82,5 @@ object ForestChangeDiagnosticDataLossYearly { ForestChangeDiagnosticDataLossYearly(sortedMap.getOrElse(SortedMap())) } + implicit def injection: Injection[ForestChangeDiagnosticDataLossYearly, String] = Injection(_.toJson, fromString) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyCategory.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyCategory.scala index 8f3bac11..783ed9f8 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyCategory.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataLossYearlyCategory.scala @@ -1,8 +1,9 @@ package org.globalforestwatch.summarystats.forest_change_diagnostic +import frameless.Injection import io.circe.syntax._ import io.circe.parser.decode -import scala.reflect.internal.pickling.ByteCodecs + case class ForestChangeDiagnosticDataLossYearlyCategory( value: Map[String, ForestChangeDiagnosticDataLossYearly] @@ -69,4 +70,5 @@ object ForestChangeDiagnosticDataLossYearlyCategory { } + implicit def injection: Injection[ForestChangeDiagnosticDataLossYearlyCategory, String] = Injection(_.toJson, fromString) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataValueYearly.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataValueYearly.scala index 31d8f418..9920e967 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataValueYearly.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataValueYearly.scala @@ -1,22 +1,21 @@ package org.globalforestwatch.summarystats.forest_change_diagnostic +import frameless.Injection + import scala.collection.immutable.SortedMap import io.circe.syntax._ import io.circe.parser.decode +import cats.kernel.Semigroup case class ForestChangeDiagnosticDataValueYearly(value: SortedMap[Int, Double]) - extends ForestChangeDiagnosticDataParser[ - ForestChangeDiagnosticDataValueYearly - ] { - def merge( - other: ForestChangeDiagnosticDataValueYearly - ): ForestChangeDiagnosticDataValueYearly = { - - ForestChangeDiagnosticDataValueYearly(value ++ other.value.map { - case (key, otherValue) => - key -> - (value.getOrElse(key, 0.0) + otherValue) - }) + extends ForestChangeDiagnosticDataParser[ForestChangeDiagnosticDataValueYearly] { + + def merge(other: ForestChangeDiagnosticDataValueYearly): ForestChangeDiagnosticDataValueYearly = { + ForestChangeDiagnosticDataValueYearly(Semigroup[SortedMap[Int, Double]].combine(value, other.value)) + } + + def limitToMaxYear(maxYear: Int): ForestChangeDiagnosticDataValueYearly= { + ForestChangeDiagnosticDataValueYearly(value.filterKeys{ year => year <= maxYear }) } def toJson: String = { @@ -75,7 +74,8 @@ object ForestChangeDiagnosticDataValueYearly { 2016 -> 0, 2017 -> 0, 2018 -> 0, - 2019 -> 0 + 2019 -> 0, + 2020 -> 0 ) ) @@ -83,4 +83,6 @@ object ForestChangeDiagnosticDataValueYearly { val sortedMap = decode[SortedMap[Int, Double]](value) ForestChangeDiagnosticDataValueYearly(sortedMap.getOrElse(SortedMap())) } + + implicit def injection: Injection[ForestChangeDiagnosticDataValueYearly, String] = Injection(_.toJson, fromString) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticExport.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticExport.scala index 314452ea..d2a6d991 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticExport.scala @@ -1,7 +1,8 @@ package org.globalforestwatch.summarystats.forest_change_diagnostic -import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.{DataFrame, SaveMode} import org.globalforestwatch.summarystats.SummaryExport +import org.globalforestwatch.util.Util.getAnyMapValue object ForestChangeDiagnosticExport extends SummaryExport { @@ -14,25 +15,40 @@ object ForestChangeDiagnosticExport extends SummaryExport { "nullValue" -> null, "emptyValue" -> null ) - override protected def exportFeature(summaryDF: DataFrame, - outputUrl: String, - kwargs: Map[String, Any]): Unit = { - summaryDF - .coalesce(1) - .write - .options(csvOptions) - .csv(path = outputUrl + "/final") + override def export( + featureType: String, + summaryDF: DataFrame, + outputUrl: String, + kwargs: Map[String, Any] + ): Unit = { + val saveMode: SaveMode = + if (getAnyMapValue[Boolean](kwargs, "overwriteOutput")) + SaveMode.Overwrite + else + SaveMode.ErrorIfExists - } + featureType match { + case "gfwpro" | "wdpa" | "gadm" => + summaryDF + .repartition(1) + .write + .mode(saveMode) + .options(csvOptions) + .csv(path = outputUrl + "/final") - def exportIntermediateList(intermediateListDF: DataFrame, - outputUrl: String): Unit = { + case "intermediate" => + summaryDF + .repartition(1) + .write + .mode(saveMode) + .options(csvOptions) + .csv(path = outputUrl + "/intermediate") - intermediateListDF - .coalesce(1) - .write - .options(csvOptions) - .csv(path = outputUrl + "/intermediate") + case _ => + throw new IllegalArgumentException( + "Feature type must be one of 'gfwpro', 'intermediate', 'wdpa', or 'gadm'" + ) + } } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticGridSources.scala index 1fbd3660..7b44fb09 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticGridSources.scala @@ -15,7 +15,7 @@ case class ForestChangeDiagnosticGridSources(gridTile: GridTile) val treeCoverLoss = TreeCoverLoss(gridTile) val treeCoverDensity2000 = TreeCoverDensityPercent2000(gridTile) val isPrimaryForest = PrimaryForest(gridTile) - val isPeatlands = CiforPeatlands(gridTile) + val isPeatlands = GFWProPeatlands(gridTile) val isIntactForestLandscapes2000 = IntactForestLandscapes2000(gridTile) val protectedAreas = ProtectedAreas(gridTile) val seAsiaLandCover = SEAsiaLandCover(gridTile) diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRDD.scala index da4da1db..4f13be8c 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRDD.scala @@ -6,9 +6,9 @@ import geotrellis.raster._ import geotrellis.raster.rasterize.Rasterizer import geotrellis.raster.summary.polygonal._ import geotrellis.vector._ -import org.globalforestwatch.summarystats.SummaryRDD +import org.globalforestwatch.summarystats.ErrorSummaryRDD -object ForestChangeDiagnosticRDD extends SummaryRDD { +object ForestChangeDiagnosticRDD extends ErrorSummaryRDD { type SOURCES = ForestChangeDiagnosticGridSources type SUMMARY = ForestChangeDiagnosticSummary @@ -23,7 +23,7 @@ object ForestChangeDiagnosticRDD extends SummaryRDD { windowLayout, kwargs ) - } + }.left.map { ex => logger.error("Error in ErrorForestChangeDiagnosticRDD.getSources", ex); ex} } def readWindow( diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawData.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawData.scala index 330e6bad..5b35ed53 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawData.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawData.scala @@ -7,17 +7,9 @@ import cats.Semigroup * * Note: This case class contains mutable values */ -case class ForestChangeDiagnosticRawData( - var totalArea: Double, - - ) { +case class ForestChangeDiagnosticRawData(var totalArea: Double) { def merge(other: ForestChangeDiagnosticRawData): ForestChangeDiagnosticRawData = { - - ForestChangeDiagnosticRawData( - - totalArea + other.totalArea, - - ) + ForestChangeDiagnosticRawData(totalArea + other.totalArea) } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawDataGroup.scala index e9719373..22fec5aa 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRawDataGroup.scala @@ -22,6 +22,193 @@ case class ForestChangeDiagnosticRawDataGroup(umdTreeCoverLossYear: Int, braBiomesPresence: Boolean, cerradoBiomesPresence: Boolean, seAsiaPresence: Boolean, - idnPresence: Boolean) + idnPresence: Boolean) { + + /** Produce a partial ForestChangeDiagnosticData only for the loss year in this data group */ + def toForestChangeDiagnosticData(totalArea: Double): ForestChangeDiagnosticData = ForestChangeDiagnosticData( + tree_cover_loss_total_yearly = ForestChangeDiagnosticDataLossYearly.fill( + umdTreeCoverLossYear, + totalArea, + isUMDLoss + ), + tree_cover_loss_tcd90_yearly = ForestChangeDiagnosticDataLossYearly.fill( + umdTreeCoverLossYear, + totalArea, + isUMDLoss && isTreeCoverExtent90 + ), + tree_cover_loss_primary_forest_yearly = + ForestChangeDiagnosticDataLossYearly.fill( + umdTreeCoverLossYear, + totalArea, + isPrimaryForest && isUMDLoss + ), + tree_cover_loss_peat_yearly = ForestChangeDiagnosticDataLossYearly.fill( + umdTreeCoverLossYear, + totalArea, + isPeatlands && isUMDLoss + ), + tree_cover_loss_intact_forest_yearly = + ForestChangeDiagnosticDataLossYearly.fill( + umdTreeCoverLossYear, + totalArea, + isIntactForestLandscapes2000 && isUMDLoss + ), + tree_cover_loss_protected_areas_yearly = + ForestChangeDiagnosticDataLossYearly.fill( + umdTreeCoverLossYear, + totalArea, + isProtectedArea && isUMDLoss + ), + tree_cover_loss_sea_landcover_yearly = + ForestChangeDiagnosticDataLossYearlyCategory.fill( + seAsiaLandCover, + umdTreeCoverLossYear, + totalArea, + include = isUMDLoss + ), + tree_cover_loss_idn_landcover_yearly = + ForestChangeDiagnosticDataLossYearlyCategory.fill( + idnLandCover, + umdTreeCoverLossYear, + totalArea, + include = isUMDLoss + ), + tree_cover_loss_soy_yearly = + ForestChangeDiagnosticDataLossYearly.fill( + umdTreeCoverLossYear, + totalArea, + isSoyPlantedAreas && isUMDLoss + ), + tree_cover_loss_idn_legal_yearly = + ForestChangeDiagnosticDataLossYearlyCategory.fill( + idnForestArea, + umdTreeCoverLossYear, + totalArea, + include = isUMDLoss + ), + tree_cover_loss_idn_forest_moratorium_yearly = + ForestChangeDiagnosticDataLossYearly.fill( + umdTreeCoverLossYear, + totalArea, + isIdnForestMoratorium && isUMDLoss + ), + tree_cover_loss_prodes_yearly = ForestChangeDiagnosticDataLossYearly.fill( + prodesLossYear, + totalArea, + isProdesLoss + ), + tree_cover_loss_prodes_wdpa_yearly = + ForestChangeDiagnosticDataLossYearly.fill( + prodesLossYear, + totalArea, + isProdesLoss && isProtectedArea + ), + tree_cover_loss_prodes_primary_forest_yearly = + ForestChangeDiagnosticDataLossYearly.fill( + prodesLossYear, + totalArea, + isProdesLoss && isPrimaryForest + ), + tree_cover_loss_brazil_biomes_yearly = + ForestChangeDiagnosticDataLossYearlyCategory.fill( + braBiomes, + umdTreeCoverLossYear, + totalArea, + include = isUMDLoss + ), + tree_cover_extent_total = ForestChangeDiagnosticDataDouble + .fill(totalArea, isTreeCoverExtent30), + tree_cover_extent_primary_forest = ForestChangeDiagnosticDataDouble.fill( + totalArea, + isTreeCoverExtent30 && isPrimaryForest + ), + tree_cover_extent_protected_areas = ForestChangeDiagnosticDataDouble.fill( + totalArea, + isTreeCoverExtent30 && isProtectedArea + ), + tree_cover_extent_peat = ForestChangeDiagnosticDataDouble.fill( + totalArea, + isTreeCoverExtent30 && isPeatlands + ), + tree_cover_extent_intact_forest = ForestChangeDiagnosticDataDouble.fill( + totalArea, + isTreeCoverExtent30 && isIntactForestLandscapes2000 + ), + natural_habitat_primary = ForestChangeDiagnosticDataDouble + .fill(totalArea, isPrimaryForest), + natural_habitat_intact_forest = ForestChangeDiagnosticDataDouble + .fill(totalArea, isIntactForestLandscapes2000), + total_area = ForestChangeDiagnosticDataDouble.fill(totalArea), + protected_areas_area = ForestChangeDiagnosticDataDouble + .fill(totalArea, isProtectedArea), + peat_area = ForestChangeDiagnosticDataDouble + .fill(totalArea, isPeatlands), + brazil_biomes = ForestChangeDiagnosticDataDoubleCategory + .fill(braBiomes, totalArea), + idn_legal_area = ForestChangeDiagnosticDataDoubleCategory + .fill(idnForestArea, totalArea), + sea_landcover_area = ForestChangeDiagnosticDataDoubleCategory + .fill(seAsiaLandCover, totalArea), + idn_landcover_area = ForestChangeDiagnosticDataDoubleCategory + .fill(idnLandCover, totalArea), + idn_forest_moratorium_area = ForestChangeDiagnosticDataDouble + .fill(totalArea, isIdnForestMoratorium), + south_america_presence = ForestChangeDiagnosticDataBoolean + .fill(southAmericaPresence), + legal_amazon_presence = ForestChangeDiagnosticDataBoolean + .fill(legalAmazonPresence), + brazil_biomes_presence = ForestChangeDiagnosticDataBoolean + .fill(braBiomesPresence), + cerrado_biome_presence = ForestChangeDiagnosticDataBoolean + .fill(cerradoBiomesPresence), + southeast_asia_presence = + ForestChangeDiagnosticDataBoolean.fill(seAsiaPresence), + indonesia_presence = + ForestChangeDiagnosticDataBoolean.fill(idnPresence), + filtered_tree_cover_extent = ForestChangeDiagnosticDataDouble + .fill( + totalArea, + isTreeCoverExtent90 && !isPlantation + ), + filtered_tree_cover_extent_yearly = + ForestChangeDiagnosticDataValueYearly.empty, + filtered_tree_cover_loss_yearly = ForestChangeDiagnosticDataLossYearly.fill( + umdTreeCoverLossYear, + totalArea, + isUMDLoss && isTreeCoverExtent90 && !isPlantation + ), + filtered_tree_cover_loss_peat_yearly = + ForestChangeDiagnosticDataLossYearly.fill( + umdTreeCoverLossYear, + totalArea, + isUMDLoss && isTreeCoverExtent90 && !isPlantation && isPeatlands + ), + filtered_tree_cover_loss_protected_areas_yearly = + ForestChangeDiagnosticDataLossYearly.fill( + umdTreeCoverLossYear, + totalArea, + isUMDLoss && isTreeCoverExtent90 && !isPlantation && isProtectedArea + ), + plantation_area = ForestChangeDiagnosticDataDouble + .fill(totalArea, isPlantation), + plantation_on_peat_area = ForestChangeDiagnosticDataDouble + .fill( + totalArea, + isPlantation && isPeatlands + ), + plantation_in_protected_areas_area = ForestChangeDiagnosticDataDouble + .fill( + totalArea, + isPlantation && isProtectedArea + ), + commodity_value_forest_extent = ForestChangeDiagnosticDataValueYearly.empty, + commodity_value_peat = ForestChangeDiagnosticDataValueYearly.empty, + commodity_value_protected_areas = ForestChangeDiagnosticDataValueYearly.empty, + commodity_threat_deforestation = ForestChangeDiagnosticDataLossYearly.empty, + commodity_threat_peat = ForestChangeDiagnosticDataLossYearly.empty, + commodity_threat_protected_areas = ForestChangeDiagnosticDataLossYearly.empty, + commodity_threat_fires = ForestChangeDiagnosticDataLossYearly.empty + ) + } diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRowGrid.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRowGrid.scala deleted file mode 100644 index 9d2d0859..00000000 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRowGrid.scala +++ /dev/null @@ -1,58 +0,0 @@ -package org.globalforestwatch.summarystats.forest_change_diagnostic - -case class ForestChangeDiagnosticRowGrid( - id: String, - grid: String, - treeCoverLossTcd30Yearly: String, - treeCoverLossPrimaryForestYearly: String, - treeCoverLossPeatLandYearly: String, - treeCoverLossIntactForestYearly: String, - treeCoverLossProtectedAreasYearly: String, - treeCoverLossSEAsiaLandCoverYearly: String, - treeCoverLossIDNLandCoverYearly: String, - treeCoverLossSoyPlanedAreasYearly: String, - treeCoverLossIDNForestAreaYearly: String, - treeCoverLossIDNForestMoratoriumYearly: String, - prodesLossYearly: String, - prodesLossProtectedAreasYearly: String, - prodesLossProdesPrimaryForestYearly: String, - treeCoverLossBRABiomesYearly: String, - treeCoverExtent: String, - treeCoverExtentPrimaryForest: String, - treeCoverExtentProtectedAreas: String, - treeCoverExtentPeatlands: String, - treeCoverExtentIntactForests: String, - primaryForestArea: String, - intactForest2016Area: String, - totalArea: String, - protectedAreasArea: String, - peatlandsArea: String, - braBiomesArea: String, - idnForestAreaArea: String, - seAsiaLandCoverArea: String, - idnLandCoverArea: String, - idnForestMoratoriumArea: String, - southAmericaPresence: String, - legalAmazonPresence: String, - braBiomesPresence: String, - cerradoBiomesPresence: String, - seAsiaPresence: String, - idnPresence: String, - forestValueIndicator: String, - peatValueIndicator: String, - protectedAreaValueIndicator: String, - deforestationThreatIndicator: String, - peatThreatIndicator: String, - protectedAreaThreatIndicator: String, - fireThreatIndicator: String, - // extra columns - treeCoverLossTcd90Yearly: String, - filteredTreeCoverExtent: String, - filteredTreeCoverExtentYearly: String, - filteredTreeCoverLossYearly: String, - filteredTreeCoverLossPeatYearly: String, - filteredTreeCoverLossProtectedAreasYearly: String, - plantationArea: String, - plantationOnPeatArea: String, - plantationInProtectedAreasArea: String, - ) diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRowSimple.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRowSimple.scala deleted file mode 100644 index f633c3b3..00000000 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticRowSimple.scala +++ /dev/null @@ -1,46 +0,0 @@ -package org.globalforestwatch.summarystats.forest_change_diagnostic - -case class ForestChangeDiagnosticRowSimple(id: String, - treeCoverLossTcd30Yearly: String, - treeCoverLossPrimaryForestYearly: String, - treeCoverLossPeatLandYearly: String, - treeCoverLossIntactForestYearly: String, - treeCoverLossProtectedAreasYearly: String, - treeCoverLossSEAsiaLandCoverYearly: String, - treeCoverLossIDNLandCoverYearly: String, - treeCoverLossSoyPlanedAreasYearly: String, - treeCoverLossIDNForestAreaYearly: String, - treeCoverLossIDNForestMoratoriumYearly: String, - prodesLossYearly: String, - prodesLossProtectedAreasYearly: String, - prodesLossProdesPrimaryForestYearly: String, - treeCoverLossBRABiomesYearly: String, - treeCoverExtent: String, - treeCoverExtentPrimaryForest: String, - treeCoverExtentProtectedAreas: String, - treeCoverExtentPeatlands: String, - treeCoverExtentIntactForests: String, - primaryForestArea: String, - intactForest2016Area: String, - totalArea: String, - protectedAreasArea: String, - peatlandsArea: String, - braBiomesArea: String, - idnForestAreaArea: String, - seAsiaLandCoverArea: String, - idnLandCoverArea: String, - idnForestMoratoriumArea: String, - southAmericaPresence: String, - legalAmazonPresence: String, - braBiomesPresence: String, - cerradoBiomesPresence: String, - seAsiaPresence: String, - idnPresence: String, - forestValueIndicator: String, - peatValueIndicator: String, - protectedAreaValueIndicator: String, - deforestationThreatIndicator: String, - peatThreatIndicator: String, - protectedAreaThreatIndicator: String, - fireThreatIndicator: String - ) diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticSummary.scala index fb53a3c3..be235ae5 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticSummary.scala @@ -8,8 +8,9 @@ import org.globalforestwatch.util.Geodesy /** LossData Summary by year */ case class ForestChangeDiagnosticSummary( - stats: Map[ForestChangeDiagnosticRawDataGroup, ForestChangeDiagnosticRawData] = Map.empty -) extends Summary[ForestChangeDiagnosticSummary] { + stats: Map[ForestChangeDiagnosticRawDataGroup, + ForestChangeDiagnosticRawData] = Map.empty + ) extends Summary[ForestChangeDiagnosticSummary] { /** Combine two Maps and combine their LossData when a year is present in both */ def merge( @@ -18,6 +19,14 @@ case class ForestChangeDiagnosticSummary( // the years.combine method uses LossData.lossDataSemigroup instance to perform per value combine on the map ForestChangeDiagnosticSummary(stats.combine(other.stats)) } + + /** Pivot raw data to ForestChangeDiagnosticData and aggregate across years */ + def toForestChangeDiagnosticData(): ForestChangeDiagnosticData = { + stats + .map { case (group, data) => group.toForestChangeDiagnosticData(data.totalArea) } + .foldLeft(ForestChangeDiagnosticData.empty)( _ merge _) + } + def isEmpty = stats.isEmpty } object ForestChangeDiagnosticSummary { @@ -80,7 +89,8 @@ object ForestChangeDiagnosticSummary { raster.tile.isIDNForestMoratorium.getData(col, row) val braBiomes: String = raster.tile.braBiomes.getData(col, row) val isPlantation: Boolean = raster.tile.isPlantation.getData(col, row) - val gfwProCoverage: Map[String, Boolean] = raster.tile.gfwProCoverage.getData(col, row) + val gfwProCoverage: Map[String, Boolean] = + raster.tile.gfwProCoverage.getData(col, row) // compute Booleans val isTreeCoverExtent30: Boolean = tcd2000 > 30 @@ -89,10 +99,13 @@ object ForestChangeDiagnosticSummary { val isProtectedArea: Boolean = wdpa != "" val isProdesLoss: Boolean = prodesLossYear > 0 - val southAmericaPresence = gfwProCoverage.getOrElse("South America", false) - val legalAmazonPresence = gfwProCoverage.getOrElse("Legal Amazon", false) + val southAmericaPresence = + gfwProCoverage.getOrElse("South America", false) + val legalAmazonPresence = + gfwProCoverage.getOrElse("Legal Amazon", false) val braBiomesPresence = gfwProCoverage.getOrElse("Brazil Biomes", false) - val cerradoBiomesPresence = gfwProCoverage.getOrElse("Cerrado Biomes", false) + val cerradoBiomesPresence = + gfwProCoverage.getOrElse("Cerrado Biomes", false) val seAsiaPresence = gfwProCoverage.getOrElse("South East Asia", false) val idnPresence = gfwProCoverage.getOrElse("Indonesia", false) @@ -119,8 +132,8 @@ object ForestChangeDiagnosticSummary { braBiomesPresence, cerradoBiomesPresence, seAsiaPresence, - idnPresence) - + idnPresence + ) val summaryData: ForestChangeDiagnosticRawData = acc.stats.getOrElse( diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticTile.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticTile.scala index 36110d92..87cd425f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticTile.scala @@ -12,7 +12,7 @@ case class ForestChangeDiagnosticTile( loss: TreeCoverLoss#ITile, tcd2000: TreeCoverDensityPercent2000#ITile, isPrimaryForest: PrimaryForest#OptionalITile, - isPeatlands: CiforPeatlands#OptionalITile, + isPeatlands: GFWProPeatlands#OptionalITile, isIntactForestLandscapes2000: IntactForestLandscapes2000#OptionalITile, wdpaProtectedAreas: ProtectedAreas#OptionalITile, seAsiaLandCover: SEAsiaLandCover#OptionalITile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/package.scala b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/package.scala new file mode 100644 index 00000000..9fbc3bab --- /dev/null +++ b/src/main/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/package.scala @@ -0,0 +1,17 @@ +package org.globalforestwatch.summarystats + +import frameless.TypedEncoder + +package object forest_change_diagnostic { + implicit def dataDoubleTypedEncoder: TypedEncoder[ForestChangeDiagnosticDataDouble] = + TypedEncoder.usingInjection[ForestChangeDiagnosticDataDouble, String] + + implicit def dataBooleanTypedEncoder: TypedEncoder[ForestChangeDiagnosticDataBoolean] = + TypedEncoder.usingInjection[ForestChangeDiagnosticDataBoolean, String] + + implicit def dataDoubleCategoryTypedEncoder: TypedEncoder[ForestChangeDiagnosticDataDoubleCategory] = + TypedEncoder.usingInjection[ForestChangeDiagnosticDataDoubleCategory, String] + + implicit def dataLossYearlyCategoryTypedEncoder: TypedEncoder[ForestChangeDiagnosticDataLossYearlyCategory] = + TypedEncoder.usingInjection[ForestChangeDiagnosticDataLossYearlyCategory, String] +} diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardAnalysis.scala index 165b1836..c54cb9f7 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardAnalysis.scala @@ -1,163 +1,165 @@ package org.globalforestwatch.summarystats.gfwpro_dashboard -import cats.data.NonEmptyList +import cats.data.{NonEmptyList, Validated} import geotrellis.vector.{Feature, Geometry} -import com.vividsolutions.jts.geom.{Geometry => GeoSparkGeometry} -import org.apache.log4j.Logger -import org.datasyslab.geosparksql.utils.Adapter -import org.globalforestwatch.features.{FeatureIdFactory, FireAlertRDD, JoinedRDD, SpatialFeatureDF} - -import java.time.LocalDateTime -import java.time.format.DateTimeFormatter -import java.util -//import org.apache.sedona.core.enums.{FileDataSplitter, GridType, IndexType} -//import org.apache.sedona.core.spatialOperator.JoinQuery -//import org.apache.sedona.core.spatialRDD.PointRDD -//import org.apache.sedona.sql.utils.{Adapter, SedonaSQLRegistrator} +import geotrellis.store.index.zcurve.Z2 +import org.apache.spark.HashPartitioner +import org.globalforestwatch.features._ +import org.globalforestwatch.summarystats._ +import org.globalforestwatch.util.GeometryConstructor.createPoint +import org.globalforestwatch.util.{RDDAdapter, SpatialJoinRDD} +import org.globalforestwatch.util.RDDAdapter +import org.globalforestwatch.ValidatedWorkflow + import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession -import org.globalforestwatch.features.{FeatureFactory, FeatureId} -import org.globalforestwatch.util.Util.getAnyMapValue +import org.apache.spark.storage.StorageLevel +import org.globalforestwatch.features.FeatureId +import org.apache.sedona.sql.utils.Adapter +import org.apache.sedona.core.spatialRDD.SpatialRDD import scala.collection.JavaConverters._ +import java.time.LocalDate +import org.globalforestwatch.util.IntersectGeometry +import scala.reflect.ClassTag + +object GfwProDashboardAnalysis extends SummaryAnalysis { + + val name = "gfwpro_dashboard" + + def apply( + featureRDD: RDD[ValidatedLocation[Geometry]], + featureType: String, + contextualFeatureType: String, + contextualFeatureUrl: NonEmptyList[String], + fireAlertRDD: SpatialRDD[Geometry], + spark: SparkSession, + kwargs: Map[String, Any] + ): Unit = { + featureRDD.persist(StorageLevel.MEMORY_AND_DISK) + + val summaryRDD = ValidatedWorkflow(featureRDD).flatMap { rdd => + val spatialContextualDF = SpatialFeatureDF(contextualFeatureUrl, contextualFeatureType, FeatureFilter.empty, "geom", spark) + val spatialContextualRDD = Adapter.toSpatialRdd(spatialContextualDF, "polyshape") + val spatialFeatureRDD = RDDAdapter.toSpatialRDDfromLocationRdd(rdd) + + /* Enrich the feature RDD by intersecting it with contextual features + * The resulting FeatuerId carries combined identity of source fature and contextual geometry + */ + val enrichedRDD = + SpatialJoinRDD + .flatSpatialJoin(spatialContextualRDD, spatialFeatureRDD, considerBoundaryIntersection = true, usingIndex = true) + .rdd + .flatMap { case (feature, context) => + refineContextualIntersection(feature, context, contextualFeatureType) + } -object GfwProDashboardAnalysis { - - val logger = Logger.getLogger("GfwProDashboardAnalysis") - - def apply(featureRDD: RDD[Feature[Geometry, FeatureId]], - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any]): Unit = { - - val summaryRDD: RDD[(FeatureId, GfwProDashboardSummary)] = - GfwProDashboardRDD(featureRDD, GfwProDashboardGrid.blockTileGrid, kwargs) - - val fireCount: RDD[(FeatureId, GfwProDashboardDataDateCount)] = - GfwProDashboardAnalysis.fireStats(featureType, spark, kwargs) - - val dataRDD: RDD[(FeatureId, GfwProDashboardData)] = - formatGfwProDashboardData(summaryRDD) - .reduceByKey(_ merge _) - .leftOuterJoin(fireCount) - .mapValues { - case (data, fire) => - data.update( - viirsAlertsDaily = - fire.getOrElse(GfwProDashboardDataDateCount.empty) - ) + ValidatedWorkflow(enrichedRDD) + .mapValidToValidated { rdd => + rdd.map { case row@Location(fid, geom) => + if (geom.isEmpty()) + Validated.invalid[Location[JobError], Location[Geometry]](Location(fid, GeometryError(s"Empty Geometry"))) + else + Validated.valid[Location[JobError], Location[Geometry]](row) + } } + .flatMap { enrichedRDD => + val fireStatsRDD = fireStats(enrichedRDD, fireAlertRDD, spark) + val tmp = enrichedRDD.map { case Location(id, geom) => Feature(geom, id) } + val validatedSummaryStatsRdd = GfwProDashboardRDD(tmp, GfwProDashboardGrid.blockTileGrid, kwargs) + ValidatedWorkflow(validatedSummaryStatsRdd).mapValid { summaryStatsRDD => + // fold in fireStatsRDD after polygonal summary and accumulate the errors + summaryStatsRDD + .mapValues(_.toGfwProDashboardData()) + .leftOuterJoin(fireStatsRDD) + .mapValues { case (data, fire) => + data.copy(viirs_alerts_daily = fire.getOrElse(GfwProDashboardDataDateCount.empty)) + } + } + } + } - val summaryDF = - GfwProDashboardDFFactory(featureType, dataRDD, spark, kwargs).getDataFrame + val summaryDF = GfwProDashboardDF.getFeatureDataFrameFromVerifiedRdd(summaryRDD.unify, spark) + val runOutputUrl: String = getOutputUrl(kwargs) + GfwProDashboardExport.export(featureType, summaryDF, runOutputUrl, kwargs) - val runOutputUrl: String = getAnyMapValue[String](kwargs, "outputUrl") + - "/gfwpro_dashboard_" + DateTimeFormatter - .ofPattern("yyyyMMdd_HHmm") - .format(LocalDateTime.now) + } - GfwProDashboardExport.export(featureType, summaryDF, runOutputUrl, kwargs) + /** These geometries touch, apply application specific logic of how to treat that. + * - For intersection of location geometries only keep those where centroid of location is in the contextual geom (this ensures that + * any location is only assigned to a single contextual area even if it intersects more) + * - For dissolved geometry of list report all contextual areas it intersects + */ + private def refineContextualIntersection( + featureGeom: Geometry, + contextualGeom: Geometry, + contextualFeatureType: String + ): List[ValidatedLocation[Geometry]] = { + val featureId = featureGeom.getUserData.asInstanceOf[FeatureId] + val contextualId = FeatureId.fromUserData(contextualFeatureType, contextualGeom.getUserData.asInstanceOf[String], delimiter = ",") + + featureId match { + case gfwproId: GfwProFeatureId if gfwproId.locationId >= 0 => + val featureCentroid = createPoint(gfwproId.x, gfwproId.y) + if (contextualGeom.contains(featureCentroid)) { + val fid = CombinedFeatureId(gfwproId, contextualId) + // val gtGeom: Geometry = toGeotrellisGeometry(featureGeom) + List(Validated.Valid(Location(fid, featureGeom))) + } else Nil + + case gfwproId: GfwProFeatureId if gfwproId.locationId < 0 => + IntersectGeometry + .validatedIntersection(featureGeom, contextualGeom) + .leftMap { err => Location(featureId, err) } + .map { geometries => + geometries.map { geom => + // val gtGeom: Geometry = toGeotrellisGeometry(geom) + Location(CombinedFeatureId(gfwproId, contextualId), geom) + } + } + .traverse(identity) // turn validated list of geometries into list of validated geometries + } } - def fireStats( - featureType: String, - spark: SparkSession, - kwargs: Map[String, Any] - ): RDD[(FeatureId, GfwProDashboardDataDateCount)] = { - - // FIRE RDD - - val fireAlertSpatialRDD = FireAlertRDD(spark, kwargs) - - // Feature RDD - val featureObj = FeatureFactory(featureType).featureObj - val featureUris: NonEmptyList[String] = - getAnyMapValue[NonEmptyList[String]](kwargs, "featureUris") - val featurePolygonDF = - SpatialFeatureDF( - featureUris, - featureObj, - kwargs, - "geom", - spark, + private def partitionByZIndex[A: ClassTag](rdd: RDD[A])(getGeom: A => Geometry): RDD[A] = { + val hashPartitioner = new HashPartitioner(rdd.getNumPartitions) + + rdd + .keyBy({ row => + val geom = getGeom(row) + Z2( + (geom.getCentroid.getX * 100).toInt, + (geom.getCentroid.getY * 100).toInt + ).z + }) + .partitionBy(hashPartitioner) + .mapPartitions( + { iter: Iterator[(Long, A)] => + for (i <- iter) yield i._2 + }, + preservesPartitioning = true ) - val featureSpatialRDD = Adapter.toSpatialRdd(featurePolygonDF, "polyshape") - - featureSpatialRDD.analyze() - - val joinedRDD = JoinedRDD(fireAlertSpatialRDD, - featureSpatialRDD) - - joinedRDD.rdd - .map { - case (poly, points) => - toGfwProDashboardFireData(featureType, poly, points) - } - .reduceByKey(_ merge _) } - private def toGfwProDashboardFireData( - featureType: String, - poly: GeoSparkGeometry, - points: util.HashSet[GeoSparkGeometry] - ): (FeatureId, GfwProDashboardDataDateCount) = { - ( { - val id = - poly.getUserData.asInstanceOf[String].filterNot("[]".toSet).toInt - FeatureIdFactory(featureType).featureId(id) - }, { - - points.asScala.toList.foldLeft(GfwProDashboardDataDateCount.empty) { - (z, point) => { - // extract year from acq_date column - val alertDate = - point.getUserData.asInstanceOf[String].split("\t")(2) - z.merge( - GfwProDashboardDataDateCount - .fill(Some(alertDate), 1, viirs = true) - ) + private def fireStats( + featureRDD: RDD[Location[Geometry]], + fireAlertRDD: SpatialRDD[Geometry], + spark: SparkSession + ): RDD[Location[GfwProDashboardDataDateCount]] = { + val featureSpatialRDD = RDDAdapter.toSpatialRDDfromLocationRdd(featureRDD) + val joinedRDD = SpatialJoinRDD.spatialjoin(featureSpatialRDD, fireAlertRDD) + joinedRDD.rdd + .map { case (poly, points) => + val fid = poly.getUserData.asInstanceOf[FeatureId] + val data = points.asScala.foldLeft(GfwProDashboardDataDateCount.empty) { (z, point) => + // extract year from acq_date column is YYYY-MM-DD + val acqDate = point.getUserData.asInstanceOf[String].split("\t")(2) + val alertDate = LocalDate.parse(acqDate) + z.merge(GfwProDashboardDataDateCount.fillDaily(Some(alertDate), 1)) } + (fid, data) } - - }) - } - - private def formatGfwProDashboardData( - summaryRDD: RDD[(FeatureId, GfwProDashboardSummary)] - ): RDD[(FeatureId, GfwProDashboardData)] = { - - summaryRDD - .flatMap { - case (featureId, summary) => - // We need to convert the Map to a List in order to correctly flatmap the data - summary.stats.toList.map { - case (dataGroup, data) => - toGfwProDashboardData(featureId, dataGroup, data) - - } - } - } - - private def toGfwProDashboardData( - featureId: FeatureId, - dataGroup: GfwProDashboardRawDataGroup, - data: GfwProDashboardRawData - ): (FeatureId, GfwProDashboardData) = { - - ( - featureId, - GfwProDashboardData( - dataGroup.alertCoverage, - gladAlertsDaily = GfwProDashboardDataDateCount - .fill(dataGroup.alertDate, data.alertCount), - gladAlertsWeekly = GfwProDashboardDataDateCount - .fill(dataGroup.alertDate, data.alertCount, weekly = true), - gladAlertsMonthly = GfwProDashboardDataDateCount - .fill(dataGroup.alertDate, data.alertCount, monthly = true), - viirsAlertsDaily = GfwProDashboardDataDateCount.empty - ) - ) - + .reduceByKey(_ merge _) } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardCommand.scala index 3114959a..e28b76f2 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardCommand.scala @@ -1,36 +1,59 @@ package org.globalforestwatch.summarystats.gfwpro_dashboard +import cats.data.NonEmptyList import org.globalforestwatch.summarystats.SummaryCommand import cats.implicits._ import com.monovore.decline.Opts +import org.globalforestwatch.features._ object GfwProDashboardCommand extends SummaryCommand { + val contextualFeatureUrlOpt: Opts[NonEmptyList[String]] = Opts + .options[String]( + "contextual_feature_url", + help = "URI of contextual features in TSV format" + ) + + val contextualFeatureTypeOpt: Opts[String] = Opts + .option[String]( + "contextual_feature_type", + help = "Type of contextual features" + ) + val gfwProDashboardCommand: Opts[Unit] = Opts.subcommand( - name = "gfwpro_dashboard", + name = GfwProDashboardAnalysis.name, help = "Compute summary statistics for GFW Pro Dashboard." ) { ( defaultOptions, fireAlertOptions, - defaultFilterOptions, - featureFilterOptions - ).mapN { (default, fireAlert, defaultFilter, featureFilter) => + featureFilterOptions, + contextualFeatureUrlOpt, + contextualFeatureTypeOpt + ).mapN { (default, fireAlert, filterOptions, contextualFeatureUrl, contextualFeatureType) => val kwargs = Map( - "featureUris" -> default._2, - "outputUrl" -> default._3, - "splitFeatures" -> default._4, - "fireAlertType" -> fireAlert._1, - "fireAlertSource" -> fireAlert._2, - "idStart" -> featureFilter._1, - "idEnd" -> featureFilter._2, - "limit" -> defaultFilter._1, - "tcl" -> defaultFilter._2, - "glad" -> defaultFilter._3 + "outputUrl" -> default.outputUrl, + "noOutputPathSuffix" -> default.noOutputPathSuffix, + "overwriteOutput" -> default.overwriteOutput, ) + // TODO: move building the feature object into options + val featureFilter = FeatureFilter.fromOptions(default.featureType, filterOptions) + + runAnalysis { implicit spark => + val featureRDD = ValidatedFeatureRDD(default.featureUris, default.featureType, featureFilter, default.splitFeatures) - runAnalysis("gfwpro_dashboard", default._1, default._2, kwargs) + val fireAlertRDD = FireAlertRDD(spark, fireAlert.alertType, fireAlert.alertSource, FeatureFilter.empty) + GfwProDashboardAnalysis( + featureRDD, + default.featureType, + contextualFeatureType = contextualFeatureType, + contextualFeatureUrl = contextualFeatureUrl, + fireAlertRDD, + spark, + kwargs + ) + } } } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardDF.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardDF.scala new file mode 100644 index 00000000..3520fafa --- /dev/null +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardDF.scala @@ -0,0 +1,57 @@ +package org.globalforestwatch.summarystats.gfwpro_dashboard + +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.globalforestwatch.features.{CombinedFeatureId, FeatureId, GadmFeatureId, GfwProFeatureId} +import org.globalforestwatch.summarystats._ +import cats.data.Validated.{Valid, Invalid} + +object GfwProDashboardDF extends SummaryDF { + case class RowGadmId(list_id: String, location_id: String, gadm_id: String) + + def getFeatureDataFrameFromVerifiedRdd( + dataRDD: RDD[ValidatedLocation[GfwProDashboardData]], + spark: SparkSession + ): DataFrame = { + import spark.implicits._ + + val rowId: FeatureId => RowGadmId = { + case CombinedFeatureId(proId: GfwProFeatureId, gadmId: GadmFeatureId) => + RowGadmId(proId.listId, proId.locationId.toString, gadmId.toString()) + case proId: GfwProFeatureId => + RowGadmId(proId.listId, proId.locationId.toString, "none") + case _ => + throw new IllegalArgumentException("Not a CombinedFeatureId[GfwProFeatureId, GadmFeatureId]") + } + dataRDD.map { + case Valid(Location(id, data)) => + (rowId(id), SummaryDF.RowError.empty, data) + case Invalid(Location(id, err)) => + (rowId(id), SummaryDF.RowError.fromJobError(err), GfwProDashboardData.empty) + } + .toDF("id", "error", "data") + .select($"id.*", $"error.*", $"data.*") + } + + def getFeatureDataFrame( + dataRDD: RDD[(FeatureId, ValidatedRow[GfwProDashboardData])], + spark: SparkSession + ): DataFrame = { + import spark.implicits._ + + dataRDD.mapValues { + case Valid(data) => + (SummaryDF.RowError.empty, data) + case Invalid(err) => + (SummaryDF.RowError.fromJobError(err), GfwProDashboardData.empty) + }.map { + case (CombinedFeatureId(proId: GfwProFeatureId, gadmId: GadmFeatureId), (error, data)) => + val rowId = RowGadmId(proId.listId, proId.locationId.toString, gadmId.toString()) + (rowId, error, data) + case _ => + throw new IllegalArgumentException("Not a CombinedFeatureId[GfwProFeatureId, GadmFeatureId]") + } + .toDF("id", "error", "data") + .select($"id.*", $"error.*", $"data.*") + } +} diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardDFFactory.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardDFFactory.scala deleted file mode 100644 index 23e52cdd..00000000 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardDFFactory.scala +++ /dev/null @@ -1,54 +0,0 @@ -package org.globalforestwatch.summarystats.gfwpro_dashboard - -import io.circe.syntax._ -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.{DataFrame, SparkSession} -import org.globalforestwatch.features.{FeatureId, SimpleFeatureId} - - -case class GfwProDashboardDFFactory( - featureType: String, - dataRDD: RDD[(FeatureId, GfwProDashboardData)], - spark: SparkSession, - kwargs: Map[String, Any] - ) { - - import spark.implicits._ - - def getDataFrame: DataFrame = { - featureType match { - case "feature" => getFeatureDataFrame - case _ => - throw new IllegalArgumentException("Not a valid FeatureId") - } - } - - private def getFeatureDataFrame: DataFrame = { - dataRDD - .map { - case (id, data) => - id match { - case simpleId: SimpleFeatureId => - GfwProDashboardRowSimple( - simpleId.featureId.asJson.noSpaces, - data.gladAlertsCoverage.toString, - data.gladAlertsDaily.toJson, - data.gladAlertsWeekly.toJson, - data.gladAlertsMonthly.toJson, - data.viirsAlertsDaily.toJson - ) - case _ => - throw new IllegalArgumentException("Not a SimpleFeatureId") - } - } - .toDF( - "location_id", - "glad_alerts_coverage", // gladAlertsCoverage - "glad_alerts_daily", // gladAlertsDaily - "glad_alerts_weekly", // gladAlertsWeekly - "glad_alerts_monthly", // gladAlertsMonthly - "viirs_alerts_daily", // viirsAlertsDaily - - ) - } -} diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardData.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardData.scala index 97ac2f7e..e6f55d2f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardData.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardData.scala @@ -1,70 +1,59 @@ package org.globalforestwatch.summarystats.gfwpro_dashboard import cats.Semigroup +import org.globalforestwatch.summarystats.forest_change_diagnostic.ForestChangeDiagnosticDataDouble +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder /** Summary data per class * * Note: This case class contains mutable values */ case class GfwProDashboardData( - gladAlertsCoverage: Boolean, - gladAlertsDaily: GfwProDashboardDataDateCount, - gladAlertsWeekly: GfwProDashboardDataDateCount, - gladAlertsMonthly: GfwProDashboardDataDateCount, - viirsAlertsDaily: GfwProDashboardDataDateCount, - - - ) { + /** Location intersects GLAD Alert tiles, GLAD alerts are possible */ + glad_alerts_coverage: Boolean, + /** How many hacters of location geometry had tree cover extent > 30% in 2000 */ + tree_cover_extent_total: ForestChangeDiagnosticDataDouble, + /** GLAD alert count within location geometry grouped by day */ + glad_alerts_daily: GfwProDashboardDataDateCount, + /** GLAD alert count within location geometry grouped by ISO year-week */ + glad_alerts_weekly: GfwProDashboardDataDateCount, + /** GLAD alert count within location geometry grouped by year-month */ + glad_alerts_monthly: GfwProDashboardDataDateCount, + /** VIIRS alerts for location geometry grouped by day */ + viirs_alerts_daily: GfwProDashboardDataDateCount, +) { def merge(other: GfwProDashboardData): GfwProDashboardData = { - - GfwProDashboardData( - gladAlertsCoverage || other.gladAlertsCoverage, - gladAlertsDaily.merge(other.gladAlertsDaily), - gladAlertsWeekly.merge(other.gladAlertsWeekly), - gladAlertsMonthly.merge(other.gladAlertsMonthly), - viirsAlertsDaily.merge( - other.viirsAlertsDaily - ) - ) - } - - def update(gladAlertsCoverage: Boolean = this.gladAlertsCoverage, - gladAlertsDaily: GfwProDashboardDataDateCount = this.gladAlertsDaily, - gladAlertsWeekly: GfwProDashboardDataDateCount = this.gladAlertsWeekly, - gladAlertsMonthly: GfwProDashboardDataDateCount = this.gladAlertsMonthly, - viirsAlertsDaily: GfwProDashboardDataDateCount = this.viirsAlertsDaily, - ): GfwProDashboardData = { - GfwProDashboardData( - gladAlertsCoverage, - gladAlertsDaily, - gladAlertsWeekly, - gladAlertsMonthly, - viirsAlertsDaily + glad_alerts_coverage || other.glad_alerts_coverage, + tree_cover_extent_total.merge(other.tree_cover_extent_total), + glad_alerts_daily.merge(other.glad_alerts_daily), + glad_alerts_weekly.merge(other.glad_alerts_weekly), + glad_alerts_monthly.merge(other.glad_alerts_monthly), + viirs_alerts_daily.merge(other.viirs_alerts_daily) ) } - } - object GfwProDashboardData { def empty: GfwProDashboardData = GfwProDashboardData( - gladAlertsCoverage = false, - GfwProDashboardDataDateCount.empty, + glad_alerts_coverage = false, + tree_cover_extent_total = ForestChangeDiagnosticDataDouble.empty, GfwProDashboardDataDateCount.empty, GfwProDashboardDataDateCount.empty, GfwProDashboardDataDateCount.empty, + GfwProDashboardDataDateCount.empty ) implicit val gfwProDashboardDataSemigroup: Semigroup[GfwProDashboardData] = new Semigroup[GfwProDashboardData] { - def combine(x: GfwProDashboardData, - y: GfwProDashboardData): GfwProDashboardData = + def combine(x: GfwProDashboardData, y: GfwProDashboardData): GfwProDashboardData = x.merge(y) } + implicit def dataExpressionEncoder: ExpressionEncoder[GfwProDashboardData] = + frameless.TypedExpressionEncoder[GfwProDashboardData] + .asInstanceOf[ExpressionEncoder[GfwProDashboardData]] } - diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardDataDateCount.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardDataDateCount.scala index e55865db..d5478cb6 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardDataDateCount.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardDataDateCount.scala @@ -2,99 +2,72 @@ package org.globalforestwatch.summarystats.gfwpro_dashboard import io.circe.syntax._ -import java.util.Calendar import scala.collection.immutable.SortedMap +import frameless.Injection +import cats.implicits._ +import java.time.LocalDate +import java.time.format._ +import java.time.temporal._ case class GfwProDashboardDataDateCount(value: SortedMap[String, Int]) { - def merge( - other: GfwProDashboardDataDateCount - ): GfwProDashboardDataDateCount = { - - GfwProDashboardDataDateCount(value ++ other.value.map { - case (key, otherValue) => - key -> - (value.getOrElse(key, 0) + otherValue) - }) - } - def toJson: String = { - this.value.asJson.noSpaces + def merge(other: GfwProDashboardDataDateCount): GfwProDashboardDataDateCount = { + GfwProDashboardDataDateCount(value combine other.value) } + + def toJson: String = this.value.asJson.noSpaces } object GfwProDashboardDataDateCount { - def empty: GfwProDashboardDataDateCount = - GfwProDashboardDataDateCount(SortedMap()) - - def fill(alertDate: Option[String], - alertCount: Int, - weekly: Boolean = false, - monthly: Boolean = false, - viirs: Boolean = false): GfwProDashboardDataDateCount = { - - alertDate match { - case Some(date) => { - - val formatter = new java.text.SimpleDateFormat("yyyy-MM-dd") - val alertDateCalendar = Calendar.getInstance(); - alertDateCalendar.setTime(formatter.parse(date)) - - val now = Calendar.getInstance() - - val (maxDate, minDate, alertDateFmt) = { - if (monthly) { - val dayOfMonth = now.get(Calendar.DAY_OF_MONTH) - now.add(Calendar.DATE, -dayOfMonth) - val maxDate = now.getTime - now.add(Calendar.MONTH, -12) - val minDate = now.getTime + /** ex: 2016-1-1 => 2015-53 because the 1st of 2016 is Friday of the last week of 2015 */ + val WeekOfYear = + new DateTimeFormatterBuilder() + .parseCaseInsensitive() + .appendValue(IsoFields.WEEK_BASED_YEAR, 4, 10, SignStyle.EXCEEDS_PAD) + .appendLiteral("-") + .appendValue(IsoFields.WEEK_OF_WEEK_BASED_YEAR, 2) + .toFormatter(java.util.Locale.US); - val alertDateFmt = - f"${alertDateCalendar.get(Calendar.YEAR)}-${alertDateCalendar.get(Calendar.MONTH)}%02d" + val MonthOfYear = + new DateTimeFormatterBuilder() + .parseCaseInsensitive() + .appendValue(ChronoField.YEAR, 4, 10, SignStyle.EXCEEDS_PAD) + .appendLiteral("-") + .appendValue(ChronoField.MONTH_OF_YEAR, 2) + .toFormatter(java.util.Locale.US); - (maxDate, minDate, alertDateFmt) - } else if (weekly) { - val dayOfWeek = now.get(Calendar.DAY_OF_WEEK) - now.add(Calendar.DATE, -dayOfWeek) - val maxDate = now.getTime + implicit def injection: Injection[GfwProDashboardDataDateCount, String] = Injection(_.toJson, fromString) - now.add(Calendar.WEEK_OF_YEAR, -4) - val minDate = now.getTime + def empty: GfwProDashboardDataDateCount = GfwProDashboardDataDateCount(SortedMap()) - val alertDateFmt = - f"${alertDateCalendar.get(Calendar.YEAR)}-${alertDateCalendar.get(Calendar.WEEK_OF_YEAR)}%02d" + def fillDaily(alertDate: Option[LocalDate], alertCount: Int): GfwProDashboardDataDateCount = + fill(alertDate, alertCount, _.format(DateTimeFormatter.ISO_DATE)) - (maxDate, minDate, alertDateFmt) - } else if (viirs) { - val maxDate = now.getTime + def fillWeekly(alertDate: Option[LocalDate], alertCount: Int): GfwProDashboardDataDateCount = + fill(alertDate, alertCount, _.format(WeekOfYear)) - now.add(Calendar.DATE, -7) - val minDate = now.getTime + def fillMonthly(alertDate: Option[LocalDate], alertCount: Int): GfwProDashboardDataDateCount = + fill(alertDate, alertCount, _.format(MonthOfYear)) - val alertDateFmt = date + def fill( + alertDate: Option[LocalDate], + alertCount: Int, + formatter: LocalDate => String + ): GfwProDashboardDataDateCount = { - (maxDate, minDate, alertDateFmt) - } else { - val maxDate = now.getTime - - now.add(Calendar.DATE, -now.get(Calendar.DAY_OF_WEEK)) - now.add(Calendar.WEEK_OF_YEAR, -4) - val minDate = now.getTime - - val alertDateFmt = date - - (maxDate, minDate, alertDateFmt) - } - } - - if (((alertDateCalendar.getTime before maxDate) || (alertDateCalendar.getTime equals maxDate)) && - (alertDateCalendar.getTime after minDate)) - GfwProDashboardDataDateCount(SortedMap(alertDateFmt -> alertCount)) - else this.empty - } + alertDate match { + case Some(date) => + val dateKey: String = formatter(date) + GfwProDashboardDataDateCount(SortedMap(dateKey -> alertCount)) - case _ => this.empty + case _ => + this.empty } } + + def fromString(value: String): GfwProDashboardDataDateCount = { + val sortedMap = io.circe.parser.decode[SortedMap[String, Int]](value) + GfwProDashboardDataDateCount(sortedMap.getOrElse(SortedMap())) + } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardExport.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardExport.scala index 5814458f..178e5d35 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardExport.scala @@ -1,7 +1,8 @@ package org.globalforestwatch.summarystats.gfwpro_dashboard -import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.{DataFrame, SaveMode} import org.globalforestwatch.summarystats.SummaryExport +import org.globalforestwatch.util.Util.getAnyMapValue object GfwProDashboardExport extends SummaryExport { @@ -15,16 +16,21 @@ object GfwProDashboardExport extends SummaryExport { "emptyValue" -> null ) - override protected def exportFeature(summaryDF: DataFrame, - outputUrl: String, - kwargs: Map[String, Any]): Unit = { + override protected def exportGfwPro(summaryDF: DataFrame, + outputUrl: String, + kwargs: Map[String, Any]): Unit = { + val saveMode = + if (getAnyMapValue[Boolean](kwargs, "overwriteOutput")) + SaveMode.Overwrite + else + SaveMode.ErrorIfExists summaryDF - .coalesce(1) + .repartition(1) .write + .mode(saveMode) .options(csvOptions) - .csv(path = outputUrl) - + .csv(path = outputUrl + "/final") } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardGridSources.scala index 40c11bfe..681cb853 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardGridSources.scala @@ -12,6 +12,7 @@ import org.globalforestwatch.layers._ case class GfwProDashboardGridSources(gridTile: GridTile) extends GridSources { val gladAlerts = GladAlerts(gridTile) + val treeCoverDensity2000 = TreeCoverDensityPercent2000(gridTile) def readWindow( windowKey: SpatialKey, @@ -23,12 +24,11 @@ case class GfwProDashboardGridSources(gridTile: GridTile) extends GridSources { gladAlertsTile <- Either .catchNonFatal(gladAlerts.fetchWindow(windowKey, windowLayout)) .right - + tcd2000Tile <- Either + .catchNonFatal(treeCoverDensity2000.fetchWindow(windowKey, windowLayout)) + .right } yield { - - - val tile = GfwProDashboardTile(gladAlertsTile) - + val tile = GfwProDashboardTile(gladAlertsTile, tcd2000Tile) Raster(tile, windowKey.extent(windowLayout)) } } @@ -42,9 +42,6 @@ object GfwProDashboardGridSources { .empty[String, GfwProDashboardGridSources] def getCachedSources(gridTile: GridTile): GfwProDashboardGridSources = { - cache.getOrElseUpdate(gridTile.tileId, GfwProDashboardGridSources(gridTile)) - } - } diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRDD.scala index 1d55551a..68d10ffc 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRDD.scala @@ -6,9 +6,9 @@ import geotrellis.raster._ import geotrellis.raster.rasterize.Rasterizer import geotrellis.raster.summary.polygonal._ import geotrellis.vector._ -import org.globalforestwatch.summarystats.SummaryRDD +import org.globalforestwatch.summarystats.ErrorSummaryRDD -object GfwProDashboardRDD extends SummaryRDD { +object GfwProDashboardRDD extends ErrorSummaryRDD { type SOURCES = GfwProDashboardGridSources type SUMMARY = GfwProDashboardSummary diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRawData.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRawData.scala index 816ef0d2..f969d92f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRawData.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRawData.scala @@ -6,28 +6,12 @@ import cats.Semigroup * * Note: This case class contains mutable values */ -case class GfwProDashboardRawData( - var alertCount: Int, - - ) { +case class GfwProDashboardRawData(var treeCoverExtentArea: Double, var alertCount: Int) { def merge(other: GfwProDashboardRawData): GfwProDashboardRawData = { - - GfwProDashboardRawData( - - alertCount + other.alertCount, - - ) + GfwProDashboardRawData(treeCoverExtentArea + other.treeCoverExtentArea, alertCount + other.alertCount) } } - object GfwProDashboardRawData { - implicit val lossDataSemigroup: Semigroup[GfwProDashboardRawData] = - new Semigroup[GfwProDashboardRawData] { - def combine(x: GfwProDashboardRawData, - y: GfwProDashboardRawData): GfwProDashboardRawData = - x.merge(y) - } - + implicit val lossDataSemigroup: Semigroup[GfwProDashboardRawData] = Semigroup.instance(_ merge _) } - diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRawDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRawDataGroup.scala index 4a6c689b..3274db4a 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRawDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRawDataGroup.scala @@ -1,6 +1,19 @@ package org.globalforestwatch.summarystats.gfwpro_dashboard -case class GfwProDashboardRawDataGroup(alertCoverage: Boolean, - alertDate: Option[String]) - +import org.globalforestwatch.summarystats.forest_change_diagnostic.ForestChangeDiagnosticDataDouble +import java.time.LocalDate +case class GfwProDashboardRawDataGroup( + alertDate: Option[LocalDate], + gladAlertsCoverage: Boolean +) { + def toGfwProDashboardData(alertCount: Int, totalArea: Double): GfwProDashboardData = { + GfwProDashboardData( + glad_alerts_coverage = gladAlertsCoverage, + glad_alerts_daily = GfwProDashboardDataDateCount.fillDaily(alertDate, alertCount), + glad_alerts_weekly = GfwProDashboardDataDateCount.fillWeekly(alertDate, alertCount), + glad_alerts_monthly = GfwProDashboardDataDateCount.fillMonthly(alertDate, alertCount), + viirs_alerts_daily = GfwProDashboardDataDateCount.empty, + tree_cover_extent_total = ForestChangeDiagnosticDataDouble.fill(totalArea)) + } +} diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRowSimple.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRowSimple.scala deleted file mode 100644 index d429db6b..00000000 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardRowSimple.scala +++ /dev/null @@ -1,9 +0,0 @@ -package org.globalforestwatch.summarystats.gfwpro_dashboard - -case class GfwProDashboardRowSimple(id: String, - gladAlertsCoverage: String, - gladAlertsDaily: String, - gladAlertsWeekly: String, - gladAlertsMonthly: String, - viirsAlertsDaily: String - ) diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardSummary.scala index 57657d11..a41e9cbc 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardSummary.scala @@ -6,6 +6,7 @@ import geotrellis.raster.Raster import geotrellis.raster.summary.GridVisitor import org.globalforestwatch.summarystats.Summary import org.globalforestwatch.util.Geodesy +import java.time.LocalDate /** LossData Summary by year */ case class GfwProDashboardSummary( @@ -17,52 +18,44 @@ case class GfwProDashboardSummary( // the years.combine method uses LossData.lossDataSemigroup instance to perform per value combine on the map GfwProDashboardSummary(stats.combine(other.stats)) } + def isEmpty = stats.isEmpty + + def toGfwProDashboardData(): GfwProDashboardData = { + stats + .map { case (group, data) => group.toGfwProDashboardData(data.alertCount, data.treeCoverExtentArea) } + .foldLeft(GfwProDashboardData.empty)( _ merge _) + } } object GfwProDashboardSummary { - // GfwProDashboardSummary from Raster[GfwProDashboardTile] -- cell types may not be the same - def getGridVisitor( - kwargs: Map[String, Any] - ): GridVisitor[Raster[GfwProDashboardTile], GfwProDashboardSummary] = + def getGridVisitor(kwargs: Map[String, Any]): GridVisitor[Raster[GfwProDashboardTile], GfwProDashboardSummary] = new GridVisitor[Raster[GfwProDashboardTile], GfwProDashboardSummary] { private var acc: GfwProDashboardSummary = new GfwProDashboardSummary() def result: GfwProDashboardSummary = acc - def visit(raster: Raster[GfwProDashboardTile], - col: Int, - row: Int): Unit = { + def visit(raster: Raster[GfwProDashboardTile], col: Int, row: Int): Unit = { + val tcd2000: Integer = raster.tile.tcd2000.getData(col, row) + val gladAlertDate: Option[LocalDate] = raster.tile.gladAlerts.getData(col, row).map { case (date, _) => date } + val gladAlertCoverage = raster.tile.gladAlerts.t.isDefined + val isTreeCoverExtent30: Boolean = tcd2000 > 30 - // This is a pixel by pixel operation - val gladAlertsCoverage: Boolean = - raster.tile.gladAlerts.getCoverage(col, row) - raster.tile.gladAlerts.getData(col, row) - val gladAlerts: Option[(String, Boolean)] = - raster.tile.gladAlerts.getData(col, row) + val groupKey = GfwProDashboardRawDataGroup(gladAlertDate, gladAlertsCoverage = gladAlertCoverage) + val summaryData = acc.stats.getOrElse(groupKey, GfwProDashboardRawData(treeCoverExtentArea = 0.0, alertCount = 0)) - val alertDate: Option[String] = { - gladAlerts match { - case Some((date, _)) => Some(date) - case _ => None - } + if (isTreeCoverExtent30) { + val areaHa = Geodesy.pixelArea(lat = raster.rasterExtent.gridRowToMap(row), raster.cellSize) / 10000.0 + summaryData.treeCoverExtentArea += areaHa } - val groupKey = - GfwProDashboardRawDataGroup(gladAlertsCoverage, alertDate) - - val summaryData: GfwProDashboardRawData = - acc.stats - .getOrElse(key = groupKey, default = GfwProDashboardRawData(0)) - - if (alertDate.isDefined) summaryData.alertCount += 1 + if (gladAlertDate.isDefined) { + summaryData.alertCount += 1 + } val new_stats = acc.stats.updated(groupKey, summaryData) acc = GfwProDashboardSummary(new_stats) - } - } - } diff --git a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardTile.scala b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardTile.scala index 5a36234f..045b86c2 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gfwpro_dashboard/GfwProDashboardTile.scala @@ -8,8 +8,10 @@ import org.globalforestwatch.layers._ * Tile-like structure to hold tiles from datasets required for our summary. * We can not use GeoTrellis MultibandTile because it requires all bands share a CellType. */ -case class GfwProDashboardTile(gladAlerts: GladAlerts#OptionalITile) - extends CellGrid[Int] { +case class GfwProDashboardTile( + gladAlerts: GladAlerts#OptionalITile, + tcd2000: TreeCoverDensityPercent2000#ITile +) extends CellGrid[Int] { def cellType: CellType = gladAlerts.cellType.getOrElse(IntCellType) diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsAnalysis.scala index 35815fb0..8c6a0b63 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsAnalysis.scala @@ -1,15 +1,14 @@ package org.globalforestwatch.summarystats.gladalerts -import java.time.LocalDateTime -import java.time.format.DateTimeFormatter - import geotrellis.vector.{Feature, Geometry} import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.globalforestwatch.features.FeatureId -import org.globalforestwatch.util.Util._ +import org.globalforestwatch.summarystats.SummaryAnalysis + +object GladAlertsAnalysis extends SummaryAnalysis { + val name = "gladalerts" -object GladAlertsAnalysis { def apply(featureRDD: RDD[Feature[Geometry, FeatureId]], featureType: String, spark: SparkSession, @@ -29,10 +28,7 @@ object GladAlertsAnalysis { summaryDF.repartition($"id", $"data_group") - val runOutputUrl: String = getAnyMapValue[String](kwargs, "outputUrl") + - "/gladalerts_" + DateTimeFormatter - .ofPattern("yyyyMMdd_HHmm") - .format(LocalDateTime.now) + val runOutputUrl: String = getOutputUrl(kwargs) GladAlertsExport.export( featureType, diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsCommand.scala index 64e55477..83fbc73d 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsCommand.scala @@ -3,6 +3,7 @@ package org.globalforestwatch.summarystats.gladalerts import org.globalforestwatch.summarystats.SummaryCommand import cats.implicits._ import com.monovore.decline.Opts +import org.globalforestwatch.features._ object GladAlertsCommand extends SummaryCommand { @@ -10,45 +11,26 @@ object GladAlertsCommand extends SummaryCommand { Opts.flag("change_only", "Process change only").orFalse val gladAlertsCommand: Opts[Unit] = Opts.subcommand( - name = "gladalerts", + name = GladAlertsAnalysis.name, help = "Compute GLAD summary statistics for GFW dashboards." ) { ( defaultOptions, changeOnlyOpt, - defaultFilterOptions, - gdamFilterOptions, - wdpaFilterOptions, - featureFilterOptions, - ).mapN { - (default, - changeOnly, - defaultFilter, - gadmFilter, - wdpaFilter, - featureFilter) => - val kwargs = Map( - "outputUrl" -> default._3, - "splitFeatures" -> default._4, - "changeOnly" -> changeOnly, - "iso" -> gadmFilter._1, - "isoFirst" -> gadmFilter._2, - "isoStart" -> gadmFilter._3, - "isoEnd" -> gadmFilter._4, - "admin1" -> gadmFilter._5, - "admin2" -> gadmFilter._6, - "idStart" -> featureFilter._1, - "idEnd" -> featureFilter._2, - "wdpaStatus" -> wdpaFilter._1, - "iucnCat" -> wdpaFilter._2, - "limit" -> defaultFilter._1, - "tcl" -> defaultFilter._2, - "glad" -> defaultFilter._3 - ) + featureFilterOptions + ).mapN { (default, changeOnly, filterOptions) => + val kwargs = Map( + "outputUrl" -> default.outputUrl, + "noOutputPathSuffix" -> default.noOutputPathSuffix, + "changeOnly" -> changeOnly + ) - runAnalysis("gladalerts", default._1, default._2, kwargs) + val featureFilter = FeatureFilter.fromOptions(default.featureType, filterOptions) + runAnalysis { spark => + val featureRDD = FeatureRDD(default.featureUris, default.featureType, featureFilter, default.splitFeatures, spark) + GladAlertsAnalysis(featureRDD, default.featureType, spark, kwargs) + } } } - } diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala index bb100a7e..7ce4f1cf 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala @@ -8,23 +8,23 @@ object GladAlertsDF { val contextualLayers: List[String] = List( "is__umd_regional_primary_forest_2001", - "is__birdlife_alliance_for_zero_extinction_site", - "is__birdlife_key_biodiversity_area", - "is__landmark_land_right", - "gfw_plantation__type", - "is__gfw_mining", - "is__gfw_managed_forest", + "is__birdlife_alliance_for_zero_extinction_sites", + "is__birdlife_key_biodiversity_areas", + "is__landmark_indigenous_and_community_lands", + "gfw_plantations__type", + "is__gfw_mining_concessions", + "is__gfw_managed_forests", "rspo_oil_palm__certification_status", "is__gfw_wood_fiber", - "is__peatland", + "is__gfw_peatlands", "is__idn_forest_moratorium", "is__gfw_oil_palm", - "idn_forest_area__type", - "per_forest_concession__type", + "idn_forest_area__class", + "per_forest_concessions__type", "is__gfw_oil_gas", - "is__gmw_mangroves_2016", - "is__ifl_intact_forest_landscape_2016", - "bra_biome__name" + "is__gmw_global_mangrove_extent_2016", + "is__ifl_intact_forest_landscapes_2016", + "ibge_bra_biomes__name" ) def unpackValues(unpackCols: List[Column], @@ -41,23 +41,23 @@ object GladAlertsDF { $"data_group.alertDate" as "alert__date", $"data_group.isConfirmed" as "is__confirmed_alert", $"data_group.primaryForest" as "is__umd_regional_primary_forest_2001", - $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_site", - $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_area", - $"data_group.landmark" as "is__landmark_land_right", - $"data_group.plantations" as "gfw_plantation__type", - $"data_group.mining" as "is__gfw_mining", - $"data_group.logging" as "is__gfw_managed_forest", + $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_sites", + $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas", + $"data_group.landmark" as "is__landmark_indigenous_and_community_lands", + $"data_group.plantations" as "gfw_plantations__type", + $"data_group.mining" as "is__gfw_mining_concessions", + $"data_group.logging" as "is__gfw_managed_forests", $"data_group.rspo" as "rspo_oil_palm__certification_status", $"data_group.woodFiber" as "is__gfw_wood_fiber", - $"data_group.peatlands" as "is__peatland", + $"data_group.peatlands" as "is__gfw_peatlands", $"data_group.indonesiaForestMoratorium" as "is__idn_forest_moratorium", $"data_group.oilPalm" as "is__gfw_oil_palm", - $"data_group.indonesiaForestArea" as "idn_forest_area__type", - $"data_group.peruForestConcessions" as "per_forest_concession__type", + $"data_group.indonesiaForestArea" as "idn_forest_area__class", + $"data_group.peruForestConcessions" as "per_forest_concessions__type", $"data_group.oilGas" as "is__gfw_oil_gas", - $"data_group.mangroves2016" as "is__gmw_mangroves_2016", - $"data_group.intactForestLandscapes2016" as "is__ifl_intact_forest_landscape_2016", - $"data_group.braBiomes" as "bra_biome__name", + $"data_group.mangroves2016" as "is__gmw_global_mangrove_extent_2016", + $"data_group.intactForestLandscapes2016" as "is__ifl_intact_forest_landscapes_2016", + $"data_group.braBiomes" as "ibge_bra_biomes__name", $"data.totalAlerts" as "alert__count", $"data.alertArea" as "alert_area__ha", $"data.co2Emissions" as "whrc_aboveground_co2_emissions__Mg", @@ -66,7 +66,7 @@ object GladAlertsDF { val cols = if (!wdpa) - unpackCols ::: ($"data_group.protectedAreas" as "wdpa_protected_area__iucn_cat") :: defaultCols + unpackCols ::: ($"data_group.protectedAreas" as "wdpa_protected_areas__iucn_cat") :: defaultCols else unpackCols ::: defaultCols df.filter($"data_group.tile.z" === minZoom) @@ -83,7 +83,7 @@ object GladAlertsDF { val cols = if (!wdpa) - groupByCols ::: gladCols ::: "wdpa_protected_area__iucn_cat" :: contextualLayers + groupByCols ::: gladCols ::: "wdpa_protected_areas__iucn_cat" :: contextualLayers else groupByCols ::: gladCols ::: contextualLayers @@ -133,7 +133,7 @@ object GladAlertsDF { List($"alert__count", $"alert_area__ha", $"whrc_aboveground_co2_emissions__Mg") val contextLayers: List[String] = - if (!wdpa) "wdpa_protected_area__iucn_cat" :: contextualLayers + if (!wdpa) "wdpa_protected_areas__iucn_cat" :: contextualLayers else contextualLayers val selectCols: List[Column] = cols.foldRight(Nil: List[Column])( @@ -157,12 +157,12 @@ object GladAlertsDF { val cols = if (!wdpa) - groupByCols ::: "wdpa_protected_area__iucn_cat" :: contextualLayers + groupByCols ::: "wdpa_protected_areas__iucn_cat" :: contextualLayers else groupByCols ::: contextualLayers df.groupBy(cols.head, cols.tail: _*) - .agg(sum("area__ha") as "area__ha") + .agg(sum("alert_area__ha") as "alert_area__ha") } def whitelist(groupByCols: List[String], @@ -173,33 +173,33 @@ object GladAlertsDF { val defaultAggCols = List( max("is__umd_regional_primary_forest_2001") as "is__umd_regional_primary_forest_2001", - max("is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", - max("is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", - max("is__landmark_land_right") as "is__landmark_land_right", - max(length($"gfw_plantation__type")) - .cast("boolean") as "gfw_plantation__type", - max("is__gfw_mining") as "is__gfw_mining", - max("is__gfw_managed_forest") as "is__gfw_managed_forest", + max("is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites", + max("is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas", + max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", + max(length($"gfw_plantations__type")) + .cast("boolean") as "gfw_plantations__type", + max("is__gfw_mining_concessions") as "is__gfw_mining_concessions", + max("is__gfw_managed_forests") as "is__gfw_managed_forests", max(length($"rspo_oil_palm__certification_status")) .cast("boolean") as "rspo_oil_palm__certification_status", max("is__gfw_wood_fiber") as "is__gfw_wood_fiber", - max("is__peatland") as "is__peatland", + max("is__gfw_peatlands") as "is__gfw_peatlands", max("is__idn_forest_moratorium") as "is__idn_forest_moratorium", max("is__gfw_oil_palm") as "is__gfw_oil_palm", - max(length($"idn_forest_area__type")) - .cast("boolean") as "idn_forest_area__type", - max(length($"per_forest_concession__type")) - .cast("boolean") as "per_forest_concession__type", + max(length($"idn_forest_area__class")) + .cast("boolean") as "idn_forest_area__class", + max(length($"per_forest_concessions__type")) + .cast("boolean") as "per_forest_concessions__type", max("is__gfw_oil_gas") as "is__gfw_oil_gas", - max("is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", - max("is__ifl_intact_forest_landscape_2016") as "is__ifl_intact_forest_landscape_2016", - max(length($"bra_biome__name")).cast("boolean") as "bra_biome__name" + max("is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", + max("is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", + max(length($"ibge_bra_biomes__name")).cast("boolean") as "ibge_bra_biomes__name" ) val aggCols = if (!wdpa) - (max(length($"wdpa_protected_area__iucn_cat")) - .cast("boolean") as "wdpa_protected_area__iucn_cat") :: defaultAggCols + (max(length($"wdpa_protected_areas__iucn_cat")) + .cast("boolean") as "wdpa_protected_areas__iucn_cat") :: defaultAggCols else defaultAggCols df.groupBy(groupByCols.head, groupByCols.tail: _*) @@ -211,28 +211,28 @@ object GladAlertsDF { val defaultAggCols = List( max("is__umd_regional_primary_forest_2001") as "is__umd_regional_primary_forest_2001", - max("is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", - max("is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", - max("is__landmark_land_right") as "is__landmark_land_right", - max("gfw_plantation__type") as "gfw_plantation__type", - max("is__gfw_mining") as "is__gfw_mining", - max("is__gfw_managed_forest") as "is__gfw_managed_forest", + max("is__birdlife_alliance_for_zero_extinction_sites") as "is__birdlife_alliance_for_zero_extinction_sites", + max("is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas", + max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", + max("gfw_plantations__type") as "gfw_plantations__type", + max("is__gfw_mining_concessions") as "is__gfw_mining_concessions", + max("is__gfw_managed_forests") as "is__gfw_managed_forests", max("rspo_oil_palm__certification_status") as "rspo_oil_palm__certification_status", max("is__gfw_wood_fiber") as "is__gfw_wood_fiber", - max("is__peatland") as "is__peatland", + max("is__gfw_peatlands") as "is__gfw_peatlands", max("is__idn_forest_moratorium") as "is__idn_forest_moratorium", max("is__gfw_oil_palm") as "is__gfw_oil_palm", - max("idn_forest_area__type") as "idn_forest_area__type", - max("per_forest_concession__type") as "per_forest_concession__type", + max("idn_forest_area__class") as "idn_forest_area__class", + max("per_forest_concessions__type") as "per_forest_concessions__type", max("is__gfw_oil_gas") as "is__gfw_oil_gas", - max("is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", - max("is__ifl_intact_forest_landscape_2016") as "is__ifl_intact_forest_landscape_2016", - max("bra_biome__name") as "bra_biome__name" + max("is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", + max("is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", + max("ibge_bra_biomes__name") as "ibge_bra_biomes__name" ) val aggCols = if (!wdpa) - (max("wdpa_protected_area__iucn_cat") as "wdpa_protected_area__iucn_cat") :: defaultAggCols + (max("wdpa_protected_areas__iucn_cat") as "wdpa_protected_areas__iucn_cat") :: defaultAggCols else defaultAggCols df.groupBy(groupByCols.head, groupByCols.tail: _*) diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDataGroup.scala index 3c03ed39..f6481f7f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDataGroup.scala @@ -1,7 +1,5 @@ package org.globalforestwatch.summarystats.gladalerts -import java.time.LocalDate - import org.globalforestwatch.util.Mercantile case class GladAlertsDataGroup(alertDate: String, diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsExport.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsExport.scala index 918bf737..09a494bc 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsExport.scala @@ -168,18 +168,18 @@ object GladAlertsExport extends SummaryExport { import spark.implicits._ val groupByCols = List( - "wdpa_protected_area__id", - "wdpa_protected_area__name", - "wdpa_protected_area__iucn_cat", - "wdpa_protected_area__iso", - "wdpa_protected_area__status" + "wdpa_protected_areas__id", + "wdpa_protected_areas__name", + "wdpa_protected_areas__iucn_cat", + "wdpa_protected_areas__iso", + "wdpa_protected_areas__status" ) val unpackCols = List( - $"id.wdpaId" as "wdpa_protected_area__id", - $"id.name" as "wdpa_protected_area__name", - $"id.iucnCat" as "wdpa_protected_area__iucn_cat", - $"id.iso" as "wdpa_protected_area__iso", - $"id.status" as "wdpa_protected_area__status" + $"id.wdpaId" as "wdpa_protected_areas__id", + $"id.name" as "wdpa_protected_areas__name", + $"id.iucnCat" as "wdpa_protected_areas__iucn_cat", + $"id.iso" as "wdpa_protected_areas__iso", + $"id.status" as "wdpa_protected_areas__status" ) _export(summaryDF, outputUrl + "/wdpa", kwargs, groupByCols, unpackCols, wdpa = true) diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGrid.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGrid.scala index efcdb85f..d8c03688 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGrid.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGrid.scala @@ -1,7 +1,7 @@ package org.globalforestwatch.summarystats.gladalerts import geotrellis.vector.Extent -import org.globalforestwatch.grids.{GridId, GridTile, TenByTen30mGrid} +import org.globalforestwatch.grids.{GridTile, TenByTen30mGrid} object GladAlertsGrid extends TenByTen30mGrid[GladAlertsGridSources] { diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGridSources.scala index 484680d8..72385cfa 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGridSources.scala @@ -3,8 +3,7 @@ package org.globalforestwatch.summarystats.gladalerts import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.Raster -import geotrellis.vector.Extent -import org.globalforestwatch.grids.{GridId, GridSources, GridTile} +import org.globalforestwatch.grids.{GridSources, GridTile} import org.globalforestwatch.layers._ /** diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsRDD.scala index 375246eb..38369257 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsRDD.scala @@ -3,15 +3,11 @@ package org.globalforestwatch.summarystats.gladalerts import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.summary.polygonal._ -import geotrellis.raster.summary.GridVisitor import geotrellis.raster._ import geotrellis.raster.rasterize.Rasterizer import geotrellis.vector._ import org.globalforestwatch.summarystats.SummaryRDD -import org.globalforestwatch.util.Util.getAnyMapValue -import org.globalforestwatch.util.{Geodesy, Mercantile} -import scala.annotation.tailrec object GladAlertsRDD extends SummaryRDD { diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsRowGeostore.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsRowGeostore.scala index c6f3d422..c1e374fa 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsRowGeostore.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsRowGeostore.scala @@ -1,6 +1,6 @@ package org.globalforestwatch.summarystats.gladalerts -import org.globalforestwatch.features.{GeostoreFeatureId, SimpleFeatureId} +import org.globalforestwatch.features.GeostoreFeatureId case class GladAlertsRowGeostore(id: GeostoreFeatureId, dataGroup: GladAlertsDataGroup, diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsSummary.scala index 95831dcf..9afb4ad9 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsSummary.scala @@ -8,6 +8,8 @@ import org.globalforestwatch.util.Util.getAnyMapValue import org.globalforestwatch.util.{Geodesy, Mercantile} import scala.annotation.tailrec +import java.time.format.DateTimeFormatter +import java.time.LocalDate /** LossData Summary by year */ case class GladAlertsSummary(stats: Map[GladAlertsDataGroup, GladAlertsData] = Map.empty) @@ -18,6 +20,7 @@ case class GladAlertsSummary(stats: Map[GladAlertsDataGroup, GladAlertsData] = M // the years.combine method uses LossData.lossDataSemigroup instance to perform per value combine on the map GladAlertsSummary(stats.combine(other.stats)) } + def isEmpty = stats.isEmpty } object GladAlertsSummary { @@ -39,7 +42,7 @@ object GladAlertsSummary { val maxZoom = 12 // This is a pixel by pixel operation - val glad: Option[(String, Boolean)] = + val glad: Option[(LocalDate, Boolean)] = raster.tile.glad.getData(col, row) if (!(changeOnly && glad.isEmpty)) { @@ -100,7 +103,7 @@ object GladAlertsSummary { else { val alertDate: String = { glad match { - case Some((date, _)) => date + case Some((date, _)) => date.format(DateTimeFormatter.ISO_DATE) case _ => null } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/package.scala b/src/main/scala/org/globalforestwatch/summarystats/package.scala new file mode 100644 index 00000000..e7998778 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/summarystats/package.scala @@ -0,0 +1,19 @@ +package org.globalforestwatch + +import cats.kernel.Semigroup +import cats.data.Validated +import geotrellis.raster.summary.polygonal.PolygonalSummaryResult +import org.globalforestwatch.features._ + +package object summarystats { + type Location[A] = Tuple2[FeatureId, A] + type ValidatedRow[A] = Validated[JobError, A] + type ValidatedSummary[A] = Validated[JobError, PolygonalSummaryResult[A]] + + type ValidatedLocation[A] = Validated[Location[JobError], Location[A]] + type ValidatedLocationSummary[A] = Validated[Location[JobError], Location[PolygonalSummaryResult[A]]] + + implicit def summarySemigroup[SUMMARY <: Summary[SUMMARY]]: Semigroup[SUMMARY] = Semigroup.instance { + case (s1, s2) => s1.merge(s2) + } +} diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeCoverLossCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeCoverLossCommand.scala index 02d48184..0837f485 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeCoverLossCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeCoverLossCommand.scala @@ -4,6 +4,7 @@ import cats.data.NonEmptyList import org.globalforestwatch.summarystats.SummaryCommand import cats.implicits._ import com.monovore.decline.Opts +import org.globalforestwatch.features._ object TreeCoverLossCommand extends SummaryCommand { @@ -23,35 +24,39 @@ object TreeCoverLossCommand extends SummaryCommand { ) .withDefault(NonEmptyList.of("")) - val treeCoverLossOptions - : Opts[(NonEmptyList[String], Int, Product with Serializable)] = - (contextualLayersOpts, tcdOpt, thresholdOpts).tupled + val carbonPoolOpts: Opts[Boolean] = Opts + .flag( + "carbon_pools", + "Carbon pools to optionally include. Currently can include: gfw_aboveground_carbon_stock_2000__Mg, gfw_belowground_carbon_stock_2000__Mg, gfw_soil_carbon_stock_2000__Mg" + ) + .orFalse + + val treeCoverLossOptions: Opts[(NonEmptyList[String], Int, Product with Serializable, Boolean)] = + (contextualLayersOpts, tcdOpt, thresholdOpts, carbonPoolOpts).tupled val treeCoverLossCommand: Opts[Unit] = Opts.subcommand( - name = "treecoverloss", + name = TreeLossAnalysis.name, help = "Compute Tree Cover Loss Statistics." ) { ( defaultOptions, treeCoverLossOptions, - defaultFilterOptions, featureFilterOptions - ).mapN { (default, treeCoverLoss, defaultFilter, featureFilter) => + ).mapN { (default, treeCoverLoss, filterOptions) => val kwargs = Map( - "outputUrl" -> default._3, - "splitFeatures" -> default._4, + "outputUrl" -> default.outputUrl, + "noOutputPathSuffix" -> default.noOutputPathSuffix, "contextualLayers" -> treeCoverLoss._1, "tcdYear" -> treeCoverLoss._2, "thresholdFilter" -> treeCoverLoss._3, - "idStart" -> featureFilter._1, - "idEnd" -> featureFilter._2, - "limit" -> defaultFilter._1, - "tcl" -> defaultFilter._2, - "glad" -> defaultFilter._3 + "carbonPools" -> treeCoverLoss._4 ) + val featureFilter = FeatureFilter.fromOptions(default.featureType, filterOptions) - runAnalysis("gladalerts", default._1, default._2, kwargs) - + runAnalysis { spark => + val featureRDD = FeatureRDD(default.featureUris, default.featureType, featureFilter, default.splitFeatures, spark) + TreeLossAnalysis(featureRDD, default.featureType, spark, kwargs) + } } } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossAnalysis.scala index cd87fd16..21090387 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossAnalysis.scala @@ -1,15 +1,14 @@ package org.globalforestwatch.summarystats.treecoverloss -import java.time.LocalDateTime -import java.time.format.DateTimeFormatter - import geotrellis.vector.{Feature, Geometry} import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.globalforestwatch.features.FeatureId -import org.globalforestwatch.util.Util.getAnyMapValue +import org.globalforestwatch.summarystats.SummaryAnalysis + +object TreeLossAnalysis extends SummaryAnalysis { + val name = "treecoverloss" -object TreeLossAnalysis { def apply(featureRDD: RDD[Feature[Geometry, FeatureId]], featureType: String, spark: SparkSession, @@ -28,10 +27,7 @@ object TreeLossAnalysis { summaryDF.repartition(partitionExprs = $"id") - val runOutputUrl: String = getAnyMapValue[String](kwargs, "outputUrl") + - "/treecoverloss_" + DateTimeFormatter - .ofPattern("yyyyMMdd_HHmm") - .format(LocalDateTime.now) + val runOutputUrl: String = getOutputUrl(kwargs) TreeLossExport.export(featureType, summaryDF, runOutputUrl, kwargs) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossDF.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossDF.scala index 7e977c87..72f9b7d9 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossDF.scala @@ -2,6 +2,7 @@ package org.globalforestwatch.summarystats.treecoverloss import com.github.mrpowers.spark.daria.sql.DataFrameHelpers.validatePresenceOfColumns import org.apache.spark.sql.functions.sum +import org.apache.spark.sql.functions._ import org.apache.spark.sql.{DataFrame, SparkSession} object TreeLossDF { @@ -9,7 +10,7 @@ object TreeLossDF { val treecoverLossMinYear = 2001 val treecoverLossMaxYear = 2020 - def unpackValues(df: DataFrame): DataFrame = { + def unpackValues(carbonPools: Boolean)(df: DataFrame): DataFrame = { val spark: SparkSession = df.sparkSession import spark.implicits._ @@ -21,6 +22,7 @@ object TreeLossDF { .getItem(i) .getItem("treecoverLoss") as s"umd_tree_cover_loss_${i}__ha" }).toList + val abovegroundBiomassLossCols = (for (i <- treecoverLossMinYear to treecoverLossMaxYear) yield { $"data.lossYear" @@ -41,7 +43,6 @@ object TreeLossDF { $"data_group.tcdYear" as "umd_tree_cover_extent__year", $"data_group.isPrimaryForest" as "is__umd_regional_primary_forest_2001", $"data_group.isPlantations" as "is__gfw_plantations", -// $"data_group.isGain" as "is__umd_tree_cover_gain_2000-2012", $"data.treecoverExtent2000" as "umd_tree_cover_extent_2000__ha", $"data.treecoverExtent2010" as "umd_tree_cover_extent_2010__ha", $"data.totalArea" as "area__ha", @@ -58,8 +59,19 @@ object TreeLossDF { $"data.totalFluxModelExtentArea" as "gfw_flux_model_extent__ha" ) + val carbonPoolCols = if (carbonPools) { + List( + $"data.totalAgc2000" as "gfw_aboveground_carbon_stock_2000__Mg", + $"data.totalBgc2000" as "gfw_belowground_carbon_stock_2000__Mg", + $"data.totalSoilCarbon2000" as "gfw_soil_carbon_stock_2000__Mg" + ) + } else { + List() + } + + df.select( - cols ::: treecoverLossCols ::: abovegroundBiomassLossCols ::: totalGrossEmissionsCo2eAllGasesCols: _* + cols ::: carbonPoolCols ::: treecoverLossCols ::: abovegroundBiomassLossCols ::: totalGrossEmissionsCo2eAllGasesCols : _* ) } @@ -67,6 +79,7 @@ object TreeLossDF { def contextualLayerFilter( includePrimaryForest: Boolean, includePlantations: Boolean, + carbonPools: Boolean )(df: DataFrame): DataFrame = { val spark: SparkSession = df.sparkSession @@ -94,6 +107,7 @@ object TreeLossDF { sum("whrc_aboveground_biomass_stock_2000__Mg") as "whrc_aboveground_biomass_stock_2000__Mg", sum($"avg_whrc_aboveground_biomass_stock_2000__Mg_ha-1" * $"umd_tree_cover_extent_2000__ha") / sum($"umd_tree_cover_extent_2000__ha") as "avg_whrc_aboveground_biomass_density_2000__Mg_ha-1", + sum("gfw_gross_cumulative_aboveground_co2_removals_2001-2020__Mg") as "gfw_gross_cumulative_aboveground_co2_removals_2001-2020__Mg", sum("gfw_gross_cumulative_belowground_co2_removals_2001-2020__Mg") @@ -110,6 +124,16 @@ object TreeLossDF { sum("gfw_flux_model_extent__ha") as "gfw_flux_model_extent__ha" ) + val carbonPoolCols = if (carbonPools) { + List( + sum("gfw_aboveground_carbon_stock_2000__Mg") as "gfw_aboveground_carbon_stock_2000__Mg", + sum("gfw_belowground_carbon_stock_2000__Mg") as "gfw_belowground_carbon_stock_2000__Mg", + sum("gfw_soil_carbon_stock_2000__Mg") as "gfw_soil_carbon_stock_2000__Mg" + ) + } else { + List() + } + val groupByCols = List( $"feature__id", $"umd_tree_cover_density__threshold", @@ -130,9 +154,11 @@ object TreeLossDF { df.groupBy(groupByCols ::: pfGroupByCol ::: plGroupByCol: _*) .agg( cols.head, - cols.tail ::: treecoverLossCols ::: abovegroundBiomassLossCols ::: totalGrossEmissionsCo2eAllGasesCols: _* + cols.tail ::: carbonPoolCols ::: treecoverLossCols ::: abovegroundBiomassLossCols ::: totalGrossEmissionsCo2eAllGasesCols: _* ) } + + } diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossData.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossData.scala index 8e7398b9..b87c898f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossData.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossData.scala @@ -14,6 +14,9 @@ case class TreeLossData( var totalGainArea: Double, var totalBiomass: Double, var avgBiomass: Double, + var totalAgc2000: Double, + var totalBgc2000: Double, + var totalSoilCarbon2000: Double, var totalGrossCumulAbovegroundRemovalsCo2: Double, var totalGrossCumulBelowgroundRemovalsCo2: Double, var totalGrossCumulAboveBelowgroundRemovalsCo2: Double, @@ -50,6 +53,9 @@ case class TreeLossData( totalBiomass + other.totalBiomass, // TODO: use extent2010 to calculate avg biomass incase year is selected avgBiomass, + totalAgc2000 + other.totalAgc2000, + totalBgc2000 + other.totalBgc2000, + totalSoilCarbon2000 + other.totalSoilCarbon2000, totalGrossCumulAbovegroundRemovalsCo2 + other.totalGrossCumulAbovegroundRemovalsCo2, totalGrossCumulBelowgroundRemovalsCo2 + other.totalGrossCumulBelowgroundRemovalsCo2, totalGrossCumulAboveBelowgroundRemovalsCo2 + other.totalGrossCumulAboveBelowgroundRemovalsCo2, diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossExport.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossExport.scala index e4e90dc7..edca0f8a 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossExport.scala @@ -24,12 +24,13 @@ object TreeLossExport extends SummaryExport { ) } + val carbonPools: Boolean = + getAnyMapValue[Boolean](kwargs, "carbonPools") + + summaryDF - .transform(TreeLossDF.unpackValues) - .transform( - TreeLossDF - .contextualLayerFilter(includePrimaryForest, includePlantations) - ) + .transform(TreeLossDF.unpackValues(carbonPools)) + .transform(TreeLossDF.contextualLayerFilter(includePrimaryForest, includePlantations, carbonPools)) .coalesce(1) .orderBy($"feature__id", $"umd_tree_cover_density__threshold") .write diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala index cc4870f6..d2657d83 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala @@ -1,7 +1,6 @@ package org.globalforestwatch.summarystats.treecoverloss import geotrellis.raster.Raster -import geotrellis.vector.Extent import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import org.globalforestwatch.grids.{GridSources, GridTile} @@ -17,6 +16,10 @@ case class TreeLossGridSources(gridTile: GridTile) extends GridSources { val treeCoverDensity2000 = TreeCoverDensityPercent2000(gridTile) val treeCoverDensity2010 = TreeCoverDensityPercent2010(gridTile) val biomassPerHectar = BiomassPerHectar(gridTile) + val agc2000 = Agc2000(gridTile) + val bgc2000 = Bgc2000(gridTile) + val soilCarbon2000 = SoilCarbon2000(gridTile) + val primaryForest = PrimaryForest(gridTile) val plantationsBool = PlantationsBool(gridTile) @@ -44,10 +47,14 @@ case class TreeLossGridSources(gridTile: GridTile) extends GridSources { } yield { // Failure for these will be converted to optional result and propagated with TreeLossTile - val biomassTile = biomassPerHectar.fetchWindow(windowKey, windowLayout) val primaryForestTile = primaryForest.fetchWindow(windowKey, windowLayout) val plantationsBoolTile = plantationsBool.fetchWindow(windowKey, windowLayout) + val biomassTile = biomassPerHectar.fetchWindow(windowKey, windowLayout) + val agc2000Tile = agc2000.fetchWindow(windowKey, windowLayout) + val bgc2000Tile = bgc2000.fetchWindow(windowKey, windowLayout) + val soilCarbon2000Tile = soilCarbon2000.fetchWindow(windowKey, windowLayout) + val grossCumulAbovegroundRemovalsCo2Tile = grossCumulAbovegroundRemovalsCo2.fetchWindow(windowKey, windowLayout) val grossCumulBelowgroundRemovalsCo2Tile = grossCumulBelowgroundRemovalsCo2.fetchWindow(windowKey, windowLayout) val grossEmissionsCo2eNonCo2Tile = grossEmissionsCo2eNonCo2.fetchWindow(windowKey, windowLayout) @@ -61,6 +68,9 @@ case class TreeLossGridSources(gridTile: GridTile) extends GridSources { tcd2000Tile, tcd2010Tile, biomassTile, + agc2000Tile, + bgc2000Tile, + soilCarbon2000Tile, primaryForestTile, plantationsBoolTile, grossCumulAbovegroundRemovalsCo2Tile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossRDD.scala index 33ccd62b..248bea1e 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossRDD.scala @@ -3,7 +3,6 @@ package org.globalforestwatch.summarystats.treecoverloss import cats.implicits._ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.raster.summary.polygonal._ -import geotrellis.raster.summary.GridVisitor import geotrellis.raster._ import geotrellis.raster.rasterize.Rasterizer import geotrellis.vector._ diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossSummary.scala index 26f9cff5..eb3bd9df 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossSummary.scala @@ -21,6 +21,7 @@ case class TreeLossSummary(stats: Map[TreeLossDataGroup, TreeLossData] = // the years.combine method uses LossData.lossDataSemigroup instance to perform per value combine on the map TreeLossSummary(stats.combine(other.stats)) } + def isEmpty = stats.isEmpty } object TreeLossSummary { @@ -45,6 +46,15 @@ object TreeLossSummary { val tcd2010: Integer = raster.tile.tcd2010.getData(col, row) val biomass: Double = raster.tile.biomass.getData(col, row) + + // Optionally calculate stocks in carbon pools in 2000 + val carbonPools: Boolean = + getAnyMapValue[Boolean](kwargs, "carbonPools") + + val agc2000: Float = raster.tile.agc2000.getData(col, row) + val bgc2000: Float = raster.tile.bgc2000.getData(col, row) + val soilCarbon2000: Float = raster.tile.soilCarbon2000.getData(col, row) + val grossCumulAbovegroundRemovalsCo2: Float = raster.tile.grossCumulAbovegroundRemovalsCo2.getData(col, row) val grossCumulBelowgroundRemovalsCo2: Float = raster.tile.grossCumulBelowgroundRemovalsCo2.getData(col, row) val grossEmissionsCo2eNonCo2: Float = raster.tile.grossEmissionsCo2eNonCo2.getData(col, row) @@ -75,6 +85,9 @@ object TreeLossSummary { val gainArea: Double = gain * areaHa val biomassPixel = biomass * areaHa + val agc2000Pixel = agc2000 * areaHa + val bgc2000Pixel = bgc2000 * areaHa + val soilCarbon2000Pixel = soilCarbon2000 * areaHa val grossCumulAbovegroundRemovalsCo2Pixel = grossCumulAbovegroundRemovalsCo2 * areaHa val grossCumulBelowgroundRemovalsCo2Pixel = grossCumulBelowgroundRemovalsCo2 * areaHa @@ -112,7 +125,7 @@ object TreeLossSummary { stats.getOrElse( key = pKey, default = - TreeLossData(TreeLossYearDataMap.empty, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + TreeLossData(TreeLossYearDataMap.empty, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) ) summary.totalArea += areaHa @@ -137,6 +150,12 @@ object TreeLossSummary { } summary.totalBiomass += biomassPixel + if (carbonPools) { + summary.totalAgc2000 += agc2000Pixel + summary.totalBgc2000 += bgc2000Pixel + summary.totalSoilCarbon2000 += soilCarbon2000Pixel + } + summary.totalGrossCumulAbovegroundRemovalsCo2 += grossCumulAbovegroundRemovalsCo2Pixel summary.totalGrossCumulBelowgroundRemovalsCo2 += grossCumulBelowgroundRemovalsCo2Pixel summary.totalGrossCumulAboveBelowgroundRemovalsCo2 += grossCumulAboveBelowgroundRemovalsCo2Pixel diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossTile.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossTile.scala index a6fd6863..50063115 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossTile.scala @@ -13,6 +13,9 @@ case class TreeLossTile(loss: TreeCoverLoss#ITile, tcd2000: TreeCoverDensityPercent2000#ITile, tcd2010: TreeCoverDensityPercent2010#ITile, biomass: BiomassPerHectar#OptionalDTile, + agc2000: Agc2000#OptionalFTile, + bgc2000: Bgc2000#OptionalFTile, + soilCarbon2000: SoilCarbon2000#OptionalFTile, primaryForest: PrimaryForest#OptionalITile, plantationsBool: PlantationsBool#OptionalITile, grossCumulAbovegroundRemovalsCo2: GrossCumulAbovegroundRemovalsCo2#OptionalFTile, diff --git a/src/main/scala/org/globalforestwatch/util/CompareRastersApp.scala b/src/main/scala/org/globalforestwatch/util/CompareRastersApp.scala new file mode 100644 index 00000000..4fda53b4 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/util/CompareRastersApp.scala @@ -0,0 +1,73 @@ +package org.globalforestwatch + +import cats.implicits._ +import com.monovore.decline._ +import java.io.File + +import geotrellis.raster._ +import geotrellis.layer._ + +/** Utility CLI to compare to sets of rasters that share their name. + * This was once useful to verify that GLAD rasters used by this app are the same that are used in production. + * Turns out they are so nothing needed to be done, but next step might have been to produce GeoJSON of tiles where mismatch happens. + */ +object CompareRastersApp + extends CommandApp( + name = "compare-rasters", + header = "Compare rasters with same name by pixels", + main = { + ( + Opts.option[String]("left", help = "The path that points to data that will be read"), + Opts.option[String]("right", help = "The path that points to data that will be read") + ).mapN { (left, right) => + import CompareRasters._ + val matched: List[(File, File)] = { + def makeMap(files: Array[File]): Map[String, List[File]] = + files.map({ f => f.getName -> List(f) }).toMap + + val leftFiles = listTiffs(left) + val rightFiles = listTiffs(right) + + val pairs = makeMap(leftFiles) |+| makeMap(rightFiles) + + pairs.values + .filter{ files => + val matched = files.length == 2 + if (! matched) println(s"Unmatched: ${files.head}") + matched + } + .map { case List(l, r) => (l, r) } + .toList + } + + matched.par.foreach { case (leftFile, rightFile) => + println(s"Comparing: $leftFile with $rightFile") + val leftRaster = RasterSource(leftFile.toString) + val rightRaster = RasterSource(rightFile.toString) + compareRasters(leftRaster, rightRaster) + } + } + } + ) + +object CompareRasters { + def listTiffs(path: String): Array[File] = { + new File(path).listFiles + .filter(_.isFile) + .filter(_.getName().endsWith(".tif")) + } + + def compareRasters(left: RasterSource, right: RasterSource) = { + require(left.extent == right.extent) + require(left.dimensions == right.dimensions) + val layout = LayoutDefinition(left.gridExtent, tileSize=4000) + val leftTiles = LayoutTileSource.spatial(left, layout) + val rightTiles = LayoutTileSource.spatial(right, layout) + leftTiles.keys.foreach { key => + val a = leftTiles.read(key).get + val b = rightTiles.read(key).get + + require(a == b, s"Not equal at $key!") + } + } +} diff --git a/src/main/scala/org/globalforestwatch/util/GeoSparkGeometryConstructor.scala b/src/main/scala/org/globalforestwatch/util/GeoSparkGeometryConstructor.scala new file mode 100644 index 00000000..5ab95ee8 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/util/GeoSparkGeometryConstructor.scala @@ -0,0 +1,53 @@ +package org.globalforestwatch.util + +import org.locationtech.jts.geom.{Coordinate, CoordinateSequenceFactory, Geometry, GeometryFactory, MultiPolygon, Point, Polygon, PrecisionModel} +import org.locationtech.jts.io.WKBWriter +import org.globalforestwatch.util.Util.convertBytesToHex + +object GeometryConstructor { + val geomFactory = new GeometryFactory(new PrecisionModel(), 0) + val coordSeqFactory: CoordinateSequenceFactory = + geomFactory.getCoordinateSequenceFactory + + def createMultiPolygon(polygons: Array[Polygon], + srid: Int = 0): MultiPolygon = { + + val multiPolygon = new MultiPolygon(polygons, geomFactory) + multiPolygon.setSRID(srid) + multiPolygon + } + + def createPolygon1x1(minX: Double, minY: Double, srid: Int = 0): Polygon = { + + val polygon = geomFactory.createPolygon( + coordSeqFactory.create( + Array( + new Coordinate(minX, minY), + new Coordinate(minX + 1, minY), + new Coordinate(minX + 1, minY + 1), + new Coordinate(minX, minY + 1), + new Coordinate(minX, minY) + ) + ) + ) + + polygon.setSRID(srid) + polygon + } + + def createPoint(x: Double, y: Double, srid: Int = 0): Point = { + + val point = new Point( + coordSeqFactory.create(Array(new Coordinate(x, y))), + geomFactory + ) + + point.setSRID(srid) + point + } + + def toWKB(geom: Geometry): String = { + val writer = new WKBWriter(2) + convertBytesToHex(writer.write(geom)) + } +} diff --git a/src/main/scala/org/globalforestwatch/util/GeometryReducer.scala b/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryReducer.scala similarity index 51% rename from src/main/scala/org/globalforestwatch/util/GeometryReducer.scala rename to src/main/scala/org/globalforestwatch/util/GeotrellisGeometryReducer.scala index f541fd2b..3b8412a6 100644 --- a/src/main/scala/org/globalforestwatch/util/GeometryReducer.scala +++ b/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryReducer.scala @@ -1,10 +1,8 @@ package org.globalforestwatch.util -import geotrellis.vector.Geometry -import geotrellis.vector.io.wkb.WKB import org.locationtech.jts.precision.GeometryPrecisionReducer -object GeometryReducer extends java.io.Serializable { +object GeotrellisGeometryReducer extends java.io.Serializable { // We need to reduce geometry precision a bit to avoid issues like reported here // https://github.com/locationtech/geotrellis/issues/2951 @@ -19,23 +17,7 @@ object GeometryReducer extends java.io.Serializable { ) def reduce( - gpr: org.locationtech.jts.precision.GeometryPrecisionReducer - )(g: geotrellis.vector.Geometry): geotrellis.vector.Geometry = + gpr: org.locationtech.jts.precision.GeometryPrecisionReducer + )(g: geotrellis.vector.Geometry): geotrellis.vector.Geometry = geotrellis.vector.GeomFactory.factory.createGeometry(gpr.reduce(g)) - - def isValidGeom(wkb: String): Boolean = { - val geom: Option[Geometry] = try { - Some(reduce(gpr)(WKB.read(wkb))) - } catch { - case ae: java.lang.AssertionError => - println("There was an empty geometry") - None - case t: Throwable => throw t - } - - geom match { - case Some(g) => true - case None => false - } - } } diff --git a/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryValidator.scala b/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryValidator.scala new file mode 100644 index 00000000..8e686a40 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryValidator.scala @@ -0,0 +1,90 @@ +package org.globalforestwatch.util +import org.apache.log4j.Logger +import geotrellis.vector.{ + Geometry, + GeomFactory, + LineString, + MultiPoint, + Point, + Polygon, + MultiLineString, + MultiPolygon +} +import geotrellis.vector.io.wkb.WKB +import org.globalforestwatch.util.GeotrellisGeometryReducer.{gpr, reduce} +import scala.util.Try + +object GeotrellisGeometryValidator extends java.io.Serializable { + val logger = Logger.getLogger("Geotrellis Geometry Validator") + + def isNonEmptyGeom(geom: Geometry): Boolean = { + val maybeGeom: Option[Geometry] = try { + Some(reduce(gpr)(geom)) + } catch { + case _: java.lang.AssertionError => + println("There was an empty geometry") + None + } + + maybeGeom match { + case Some(_) => true + case None => false + } + } + + def isNonEmptyGeom(wkb: String): Boolean = { + isNonEmptyGeom(makeValidGeom(wkb)) + } + + def makeValidGeom(geom: Geometry): Geometry = { + val validGeom = { + if (!geom.isValid) { + + val fixedGeom = GfwGeometryFixer.fix(geom) + + // Geometry fixer may alter the geometry type or even return an empty geometry + // We want to try to preserve the geometry type if possible + + preserveGeometryType(fixedGeom, geom.getGeometryType) + + } else preserveGeometryType(geom, geom.getGeometryType) + } + + validGeom.normalize() + validGeom + } + + def makeValidGeom(wkb: String): Geometry = { + val geom: Option[Geometry] = Try(WKB.read(wkb)).toOption + geom.map(makeValidGeom(_)).getOrElse(GeomFactory.factory.createPoint()) + } + + def makeMultiGeom(geom: Geometry): Geometry = { + geom match { + case point: Point => MultiPoint(point) + case line: LineString => MultiLineString(line) + case polygon: Polygon => MultiPolygon(polygon) + case _ => + throw new IllegalArgumentException( + "Can only convert Point, LineString and Polygon to Multipart Geometries." + ) + } + } + + private def preserveGeometryType(geom: Geometry, + desiredGeometryType: String): Geometry = { + if (desiredGeometryType != geom.getGeometryType && desiredGeometryType + .contains(geom.getGeometryType)) { + logger.warn( + s"Fixed geometry of type ${geom.getGeometryType}. Cast to ${desiredGeometryType}." + ) + makeMultiGeom(geom) + } else if (desiredGeometryType != geom.getGeometryType) { + logger.warn( + s"Not able to preserve geometry type. Return ${geom.getGeometryType} instead of ${desiredGeometryType}" + ) + geom + } else geom + } + +} diff --git a/src/main/scala/org/globalforestwatch/util/GfwGeometryFixer.scala b/src/main/scala/org/globalforestwatch/util/GfwGeometryFixer.scala new file mode 100644 index 00000000..cec71539 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/util/GfwGeometryFixer.scala @@ -0,0 +1,150 @@ +/* + * This is a partial scala port of the JTS GeometryFixer. + * This class is was not available in the JTS version currently used in Geotrellis. + * Now it may perform additional fixes specific to this project. + * + * + * https://github.com/locationtech/jts/blob/master/modules/core/src/main/java/org/locationtech/jts/geom/util/GeometryFixer.java + * Copyright (c) 2021 Martin Davis. + */ + +package org.globalforestwatch.util + +import org.apache.log4j.Logger +import org.globalforestwatch.util.GeotrellisGeometryReducer.{gpr, reduce} +import org.locationtech.jts.geom.{Geometry, Polygon, MultiPolygon, GeometryFactory, TopologyException} +import org.locationtech.jts.geom.util.GeometryFixer +import org.locationtech.jts.operation.overlay.snap.GeometrySnapper + +import scala.annotation.tailrec + +case class GfwGeometryFixer(geom: Geometry, keepCollapsed: Boolean = false) { + private val logger: Logger = Logger.getLogger("GfwGeometryFixer") + private val factory: GeometryFactory = geom.getFactory + private val snapTolerance: Double = 1 / math.pow( + 10, + geom.getPrecisionModel.getMaximumSignificantDigits - 1 + ) + + private val maxSnapTolerance: Double = snapTolerance * 1000 + def fix(): Geometry = { + if (geom.getNumGeometries == 0) { + geom.copy() + } else { + + // doing a cheap trick here to eliminate sliver holes and other artifacts. However this might change geometry type. + // so we need to do this early on to avoid winding code. This block is not part of the original Java implementation. + val preFixedGeometry = + geom match { + case poly: Polygon => ironPolgons(poly) + case multi: MultiPolygon => ironPolgons(multi) + case _ => geom + } + + GeometryFixer.fix(preFixedGeometry) + } + } + + /** Ironing out potential sliver artifacts such as holes that resemble lines. Should only be used with Polygons or MultiPolygons. + */ + private def ironPolgons(geom: Geometry): Geometry = { + val bufferedGeom: Geometry = geom.buffer(0.0001).buffer(-0.0001) + val polygons: Geometry = extractPolygons(bufferedGeom) + reduce(gpr)(polygons) + } + + private def extractPolygons(multiGeometry: Geometry): Geometry = { + def loop(multiGeometry: Geometry): List[Option[Polygon]] = { + + val geomRange: List[Int] = + List.range(0, multiGeometry.getNumGeometries) + + val nested_polygons = for { + i <- geomRange + + } yield { + multiGeometry.getGeometryN(i) match { + case geom: Polygon => List(Some(geom)) + case multiGeom: MultiPolygon => loop(multiGeom) + case _ => List() + } + } + nested_polygons.flatten + } + + val polygons: Array[Option[Polygon]] = loop(multiGeometry).toArray + union(polygons) + } + + /** Poor man's implementation of JTS Overlay NG Robust difference (not part of current JTS version) + */ + @tailrec + private def difference( + geom1: Geometry, + geom2: Geometry, + adjustedSnapTolerance: Double = snapTolerance + ): Geometry = { + try { + val snappedGeometries: Array[Geometry] = + GeometrySnapper.snap(geom1, geom2, adjustedSnapTolerance) + snappedGeometries match { + case Array(snappedGeom1: Geometry, snappedGeom2: Geometry) => + snappedGeom1.difference(snappedGeom2) + } + } catch { + case e: TopologyException => + if (adjustedSnapTolerance >= maxSnapTolerance) throw e + else difference(geom1, geom2, adjustedSnapTolerance * 10) + } + } + + /** Poor man's implementation of JTS Overlay NG Robust union (not part of current JTS version) + */ + @tailrec + private def union[T <: Geometry]( + parts: Array[Option[T]], + baseGeometry: Geometry = factory.createPolygon(), + adjustedSnapTolerance: Double = snapTolerance + ): Geometry = { + try { + parts.foldLeft(factory.createGeometry(baseGeometry)) { (acc, part) => + part match { + + // Snap the first geometry to itself + case Some(geom) if acc.isEmpty && !geom.isEmpty => + GeometrySnapper.snapToSelf(geom, adjustedSnapTolerance, true) + + // Afterwards snap geometries to each other + case Some(geom) if !geom.isEmpty => + val snappedGeometries: Array[Geometry] = + GeometrySnapper.snap(geom, acc, adjustedSnapTolerance) + snappedGeometries match { + case Array(snappedGeom: Geometry, snappedAcc: Geometry) => + snappedAcc.union(snappedGeom) + } + + // Or simply return accumulator in case part is emtpy + case _ => acc + } + } + } catch { + case e: TopologyException => + // In case there is a topology error increase snap Tolerance by factor 10 + if (adjustedSnapTolerance > maxSnapTolerance) throw e + else { + logger.debug( + s"Adjust snap tolerance to ${adjustedSnapTolerance * 10}" + ) + union(parts, baseGeometry, adjustedSnapTolerance * 10) + } + + } + + } +} + +object GfwGeometryFixer { + def fix(geom: Geometry): Geometry = { + GfwGeometryFixer(geom).fix() + } +} \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/util/GridDF.scala b/src/main/scala/org/globalforestwatch/util/GridDF.scala new file mode 100644 index 00000000..fd916513 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/util/GridDF.scala @@ -0,0 +1,27 @@ +package org.globalforestwatch.util + +import org.locationtech.jts.geom.Envelope +import geotrellis.vector.Point +import org.apache.spark.sql.functions.{col, lit, udf} +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.sedona.core.spatialRDD.PolygonRDD +import org.apache.sedona.sql.utils.Adapter +import org.globalforestwatch.grids.GridId.pointGridId + +object GridDF { + def apply(envelope: Envelope, spark: SparkSession): DataFrame = { + + val pointToGridId = udf((x: Double, y: Double) => pointGridId(Point(x, y), 1)) + + // spark.udf.register("pointToGridId", pointToGridId) + + val polygonRDD: PolygonRDD = GridRDD(envelope, spark) + // TODO: fix gridID + Adapter.toDf(polygonRDD, spark) + // .withColumn("x", expr("ST_X(ST_Centroid(geometry))")) + // .withColumn("y", expr("ST_Y(ST_Centroid(geometry))")) + .withColumn("featureId", lit(0)) + // pointToGridId(col("x"), col("y")) as "featureId" + .select(col("geometry") as "polyshape", col("featureId")) + } +} diff --git a/src/main/scala/org/globalforestwatch/util/GridRDD.scala b/src/main/scala/org/globalforestwatch/util/GridRDD.scala new file mode 100644 index 00000000..d5f094f5 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/util/GridRDD.scala @@ -0,0 +1,44 @@ +package org.globalforestwatch.util + +import org.locationtech.jts.geom.{Envelope, MultiPolygon, Polygon} +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.SparkSession +import org.apache.sedona.core.spatialRDD.PolygonRDD +import org.globalforestwatch.grids.GridId.pointGridId +import org.globalforestwatch.util.GeometryConstructor.createPolygon1x1 + +object GridRDD { + def apply(envelope: Envelope, spark: SparkSession, clip: Boolean = false): PolygonRDD = { + + val gridCells = getGridCells(envelope) + + val tcl_geom: MultiPolygon = TreeCoverLossExtent.geometry + + val gridRDD: RDD[Polygon] = { + spark.sparkContext + .parallelize(gridCells) + .flatMap { p => + val poly = createPolygon1x1(minX = p._1, minY = p._2) + val gridId = pointGridId(p._1, p._2 + 1, 1) + poly.setUserData(gridId) + if (clip) + if (poly coveredBy tcl_geom) List(poly) + else List() + else List(poly) + } + + } + val spatialGridRDD = new PolygonRDD(gridRDD) + spatialGridRDD.analyze() + spatialGridRDD + } + + private def getGridCells(envelope: Envelope): IndexedSeq[(Int, Int)] = { + { + for (x <- envelope.getMinX.floor.toInt until envelope.getMaxX.ceil.toInt; + y <- envelope.getMinY.floor.toInt until envelope.getMaxY.ceil.toInt) + yield (x, y) + } + } + +} diff --git a/src/main/scala/org/globalforestwatch/util/IntersectGeometry.scala b/src/main/scala/org/globalforestwatch/util/IntersectGeometry.scala index 15c74e38..d936baf2 100644 --- a/src/main/scala/org/globalforestwatch/util/IntersectGeometry.scala +++ b/src/main/scala/org/globalforestwatch/util/IntersectGeometry.scala @@ -1,81 +1,109 @@ package org.globalforestwatch.util -import geotrellis.vector.{Geometry, Polygon, Point} -import org.locationtech.jts.geom.Coordinate +import cats.data.Validated.{Valid, Invalid} +import org.locationtech.jts.geom.{ + Geometry, + GeometryCollection, + MultiPolygon, + Polygon +} +import scala.util.{Try, Success, Failure} -object IntersectGeometry { +import org.globalforestwatch.util.GeometryConstructor.createMultiPolygon +import org.globalforestwatch.summarystats.{GeometryError, ValidatedRow} - def createPolygon(minX: Double, minY: Double): Polygon = { - val polygon: Polygon = Polygon( - Point(minX, minY), - Point(minX + 1, minY), - Point(minX + 1, minY + 1), - Point(minX, minY + 1), - Point(minX, minY) - ) - polygon.setSRID(4326) - polygon - } +object IntersectGeometry { - def getIntersecting1x1Grid(geometry: Geometry): IndexedSeq[Polygon] = { + def intersectGeometries(thisGeom: Geometry, + thatGeom: Geometry): List[MultiPolygon] = { /** - * Return grid of 1x1 degree polygons intersecting with input geometry + * Intersection can return GeometryCollections + * Here we filter resulting geometries and only return those of the same type as thisGeom * */ - val coords: Array[Coordinate] = geometry.getEnvelope.getCoordinates - val (minX, minY, maxX, maxY) = coords.foldLeft(180.0, 90.0, -180.0, -90.0)( - (z, coord) => - (coord.x min z._1, coord.y min z._2, coord.x max z._3, coord.y max z._4) - ) - - for { - x <- minX.floor.toInt until maxX.ceil.toInt - y <- minY.floor.toInt until maxY.ceil.toInt - if { - val polygon = createPolygon(x, y) - polygon.within(TreeCoverLossExtent.geometry) && polygon.intersects( - geometry - ) - } - - } yield { - createPolygon(x, y) + val userData = thisGeom.getUserData + val intersection: Geometry = thisGeom intersection thatGeom + intersection match { + case poly: Polygon => + val multi = createMultiPolygon(Array(poly)) + multi.setUserData(userData) + List(multi) + case multi: MultiPolygon => + multi.setUserData(userData) + List(multi) + case collection: GeometryCollection => + extractPolygons(collection) + .filterNot(_.isEmpty) + .map{ geom => + geom.setUserData(userData) + geom + }.toList + case _ => List() } - } - def intersectGeometries(thisGeom: Geometry, - thatGeom: Geometry): List[Geometry] = { - + def validatedIntersection( + thisGeom: Geometry, + thatGeom: Geometry + ): ValidatedRow[List[MultiPolygon]] = { /** * Intersection can return GeometryCollections * Here we filter resulting geometries and only return those of the same type as thisGeom - * This will also explode MultiPolygons into separate features. * */ - val intersection: Geometry = thisGeom intersection thatGeom + val userData = thisGeom.getUserData - val geomRange: List[Int] = List.range(0, intersection.getNumGeometries) - - for { - i <- geomRange - // GeometryType should be the same or a `non-Multi` type. Ie Polygon instead of MultiPolygon. - if thisGeom.getGeometryType contains intersection - .getGeometryN(i) - .getGeometryType - } yield { - val geom = intersection.getGeometryN(i) - val normalizedGeom = { - // try to make geometry valid. This is a basic trick, we might need to make this more sophisticated - // There are some code samples here for JTS - // https://stackoverflow.com/a/31474580/1410317 - if (!geom.isValid) geom.buffer(0.0001).buffer(-0.0001) - else geom + val attempt = Try { + val intersection: Geometry = thisGeom intersection thatGeom + intersection match { + case poly: Polygon => + val multi = createMultiPolygon(Array(poly)) + multi.setUserData(userData) + List(multi).filterNot(_.isEmpty) + case multi: MultiPolygon => + multi.setUserData(userData) + List(multi).filterNot(_.isEmpty) + case collection: GeometryCollection => + extractPolygons(collection) + .filterNot(_.isEmpty) + .map{ geom => + geom.setUserData(userData) + geom + }.toList + case _ => List() } + } + + attempt match { + case Success(v) => Valid(v) + case Failure(e) => Invalid(GeometryError(s"Failed intersection with ${thatGeom}")) + } + } + + def extractPolygons(multiGeometry: Geometry): Option[MultiPolygon] = { + def loop(multiGeometry: Geometry): List[Polygon] = { - normalizedGeom.normalize() - normalizedGeom + val geomRange: List[Int] = + List.range(0, multiGeometry.getNumGeometries) + + val nested_polygons = for { + i <- geomRange + + } yield { + multiGeometry.getGeometryN(i) match { + case geom: Polygon => List(geom) + case multiGeom: MultiPolygon => loop(multiGeom) + case _ => List() + } + } + nested_polygons.flatten } + val polygons: List[Polygon] = loop(multiGeometry) + + if (polygons.isEmpty) None + else Some(createMultiPolygon(polygons.toArray)) + } + } diff --git a/src/main/scala/org/globalforestwatch/util/PartitionSkewedRDD.scala b/src/main/scala/org/globalforestwatch/util/PartitionSkewedRDD.scala new file mode 100644 index 00000000..0bfbf809 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/util/PartitionSkewedRDD.scala @@ -0,0 +1,48 @@ +package org.globalforestwatch.util + +import scala.reflect.ClassTag + +import org.apache.spark.rdd.RDD +import org.apache.spark.HashPartitioner + +object RepartitionSkewedRDD { + def bySparseId[A: ClassTag](rdd: RDD[(Long, A)], maxPartitionSize: Int): RDD[A] = { + val counts = rdd.map{ case (id, _) => (id, 1l) }.reduceByKey(_ + _).collect().sortBy(_._2) + val splits = PartitionSplit.fromCounts(counts, maxPartitionSize) + val paritionIndex: (Long, A) => Int = (id, v) => splits(id).partitionForRow(v) + val parallelism = rdd.sparkContext.defaultParallelism + val numPartitions: Int = math.max(parallelism, splits.size / 16).toInt + + rdd + .map { case (id, v) => (paritionIndex(id, v), v) } + .partitionBy(new HashPartitioner(numPartitions)) + .values + } +} + + +case class PartitionSplit(partitionIndex: Int, splits: Int) { + require(splits >= 0, s"Min of one split required: $splits") + + def maxPartitionIndex: Int = partitionIndex + splits - 1 + + def partitionForRow[A](row: A): Int = { + partitionIndex + row.hashCode() % splits + } +} + +object PartitionSplit { + def fromCounts(counts: Seq[(Long, Long)], maxPartitionSize: Int): Map[Long, PartitionSplit] = { + counts.foldLeft(List.empty[(Long, PartitionSplit)]) { + case (Nil, (id, count)) => + val index = 0 + val splits = math.ceil(count.toDouble / maxPartitionSize).toInt + id -> PartitionSplit(index, splits) :: Nil + + case (acc@((_, head) :: _), (id, count)) => + val index = head.partitionIndex + head.splits + val splits = math.ceil(count.toDouble / maxPartitionSize).toInt + id -> PartitionSplit(index, splits) :: acc + }.toMap + } +} \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/util/PartitionVisualization.scala b/src/main/scala/org/globalforestwatch/util/PartitionVisualization.scala new file mode 100644 index 00000000..b6a80470 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/util/PartitionVisualization.scala @@ -0,0 +1,35 @@ +package org.globalforestwatch.util + +import org.apache.spark.sql._ +import geotrellis.vector._ +import geotrellis.vector.io.json.JsonFeatureCollection +import _root_.io.circe.generic.auto._ +import _root_.io.circe.generic.auto._ +import _root_.io.circe.Json +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder + +object PartitionVisualization { + case class PartitionInfo(index: Int) + implicit val enc = ExpressionEncoder[Extent] + def asGeoJson[A](df: DataFrame): Json = { + val partitionFeatures = + df.mapPartitions { partition => + if (partition.nonEmpty) { + val p = partition + .map { row => + val long = row.getDouble(1) + val lat = row.getDouble(0) + val extent = Extent(long, lat, long, lat) + extent + } + .reduce(_ combine _) + Iterator.single(p) + } else Iterator.empty + } + + val features = partitionFeatures.collect.map { extent => + Feature(extent.toPolygon(), PartitionInfo(0)) + } + JsonFeatureCollection(features).asJson + } +} diff --git a/src/main/scala/org/globalforestwatch/util/RDDAdapter.scala b/src/main/scala/org/globalforestwatch/util/RDDAdapter.scala new file mode 100644 index 00000000..bde4063f --- /dev/null +++ b/src/main/scala/org/globalforestwatch/util/RDDAdapter.scala @@ -0,0 +1,32 @@ +package org.globalforestwatch.util + +import geotrellis.vector.{Feature, Geometry} +import org.apache.spark.rdd.RDD +import org.globalforestwatch.features.FeatureId +import org.locationtech.jts.geom.Geometry +import org.apache.sedona.core.spatialRDD.SpatialRDD +import org.globalforestwatch.summarystats.Location + +object RDDAdapter { + + + def toSpatialRDD(rdd: RDD[Feature[Geometry, FeatureId]]): SpatialRDD[Geometry] = { + val spatialRDD = new SpatialRDD[Geometry]() + spatialRDD.rawSpatialRDD = rdd.map { feature => + feature.geom.setUserData(feature.data) + feature.geom + }.toJavaRDD() + spatialRDD.analyze() + spatialRDD + } + + def toSpatialRDDfromLocationRdd(rdd: RDD[Location[Geometry]]): SpatialRDD[Geometry] = { + val spatialRDD = new SpatialRDD[Geometry]() + spatialRDD.rawSpatialRDD = rdd.map { case Location(id, geom) => + geom.setUserData(id) + geom + }.toJavaRDD() + spatialRDD.analyze() + spatialRDD + } +} diff --git a/src/main/scala/org/globalforestwatch/util/RequestPayerS3ClientFactory.scala b/src/main/scala/org/globalforestwatch/util/RequestPayerS3ClientFactory.scala index 607013ad..02aa5e49 100644 --- a/src/main/scala/org/globalforestwatch/util/RequestPayerS3ClientFactory.scala +++ b/src/main/scala/org/globalforestwatch/util/RequestPayerS3ClientFactory.scala @@ -2,7 +2,7 @@ package org.globalforestwatch.util import com.amazonaws.ClientConfiguration import com.amazonaws.auth.AWSCredentialsProvider -import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client, AmazonS3ClientBuilder} +import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder} import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory import org.apache.log4j.Logger @@ -30,7 +30,6 @@ class RequestPayerS3ClientFactory extends DefaultS3ClientFactory { awsConf.addHeader("x-amz-request-payer", "requester"); - new AmazonS3Client(credentials, awsConf) - + AmazonS3ClientBuilder.standard().withCredentials(credentials).withClientConfiguration(awsConf).build() } } diff --git a/src/main/scala/org/globalforestwatch/util/SpatialJoinRDD.scala b/src/main/scala/org/globalforestwatch/util/SpatialJoinRDD.scala new file mode 100644 index 00000000..e28df7e1 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/util/SpatialJoinRDD.scala @@ -0,0 +1,167 @@ +package org.globalforestwatch.util + +import org.locationtech.jts.geom.Geometry +import org.apache.spark.api.java.JavaPairRDD +import org.apache.sedona.core.enums.{GridType, IndexType} +import org.apache.sedona.core.spatialOperator.JoinQuery +import org.apache.sedona.core.spatialRDD.SpatialRDD +import org.globalforestwatch.summarystats.forest_change_diagnostic.ForestChangeDiagnosticAnalysis + +import java.util +import scala.reflect.ClassTag + +object SpatialJoinRDD { + + def spatialjoin[A <: Geometry : ClassTag, B <: Geometry : ClassTag]( + queryWindowRDD: SpatialRDD[A], + valueRDD: SpatialRDD[B], + buildOnSpatialPartitionedRDD: Boolean = true, // Set to TRUE only if run join query + considerBoundaryIntersection: Boolean = false, // Only return gemeotries fully covered by each query window in queryWindowRDD + usingIndex: Boolean = false + ): JavaPairRDD[A, util.List[B]] = { + + try { + queryWindowRDD.spatialPartitioning(GridType.QUADTREE) + + valueRDD.spatialPartitioning(queryWindowRDD.getPartitioner) + + if (usingIndex) + queryWindowRDD.buildIndex( + IndexType.QUADTREE, + buildOnSpatialPartitionedRDD + ) + + JoinQuery.SpatialJoinQuery( + valueRDD, + queryWindowRDD, + usingIndex, + considerBoundaryIntersection + ) + } catch { + case _: java.lang.IllegalArgumentException => + try { + valueRDD.spatialPartitioning(GridType.QUADTREE) + + queryWindowRDD.spatialPartitioning(valueRDD.getPartitioner) + + if (usingIndex) + valueRDD.buildIndex( + IndexType.QUADTREE, + buildOnSpatialPartitionedRDD + ) + JoinQuery.SpatialJoinQuery( + valueRDD, + queryWindowRDD, + usingIndex, + considerBoundaryIntersection + ) + } catch { + case _: java.lang.IllegalArgumentException => + ForestChangeDiagnosticAnalysis.logger.warn( + "Skip spatial partitioning. Dataset too small." + ) + // Use brute force + bruteForceJoin(queryWindowRDD, valueRDD) + } + } + } + + def flatSpatialJoin[A <: Geometry : ClassTag, B <: Geometry : ClassTag]( + queryWindowRDD: SpatialRDD[A], + valueRDD: SpatialRDD[B], + buildOnSpatialPartitionedRDD: Boolean = true, // Set to TRUE only if run join query + considerBoundaryIntersection: Boolean = false, // Only return gemeotries fully covered by each query window in queryWindowRDD + usingIndex: Boolean = false + ): JavaPairRDD[B, A] = { + + val queryWindowCount = queryWindowRDD.approximateTotalCount + val queryWindowPartitions = queryWindowRDD.rawSpatialRDD.getNumPartitions + val valueCount = valueRDD.approximateTotalCount + val valuePartitions = valueRDD.rawSpatialRDD.getNumPartitions + + try { + queryWindowRDD.spatialPartitioning( + GridType.QUADTREE, + Seq(queryWindowPartitions, (queryWindowCount / 2).toInt).min + ) + valueRDD.spatialPartitioning(queryWindowRDD.getPartitioner) + if (usingIndex) + queryWindowRDD.buildIndex( + IndexType.QUADTREE, + buildOnSpatialPartitionedRDD + ) + + JoinQuery.SpatialJoinQueryFlat( + queryWindowRDD, + valueRDD, + usingIndex, + considerBoundaryIntersection + ) + } catch { + case _: java.lang.IllegalArgumentException => + ForestChangeDiagnosticAnalysis.logger.warn( + "Try to partition using valueRDD." + ) + try { + valueRDD.spatialPartitioning( + GridType.QUADTREE, + Seq(valuePartitions, (valueCount / 2).toInt).min + ) + queryWindowRDD.spatialPartitioning(valueRDD.getPartitioner) + if (usingIndex) + valueRDD.buildIndex( + IndexType.QUADTREE, + buildOnSpatialPartitionedRDD + ) + + JoinQuery.SpatialJoinQueryFlat( + queryWindowRDD, + valueRDD, + usingIndex, + considerBoundaryIntersection + ) + } catch { + case _: java.lang.IllegalArgumentException => + ForestChangeDiagnosticAnalysis.logger.warn( + "Skip spatial partitioning. Dataset too small." + ) + // need to flip rdd order in order to match response pattern + bruteForceFlatJoin(valueRDD, queryWindowRDD) + + } + + } + } + + private def bruteForceJoin[A <: Geometry : ClassTag, B <: Geometry : ClassTag]( + leftRDD: SpatialRDD[A], + rightRDD: SpatialRDD[B] + ): JavaPairRDD[A, util.List[B]] = { + JavaPairRDD.fromRDD( + leftRDD.rawSpatialRDD.rdd + .cartesian(rightRDD.rawSpatialRDD.rdd) + .filter((pair: (Geometry, Geometry)) => pair._1.intersects(pair._2)) + .aggregateByKey(new util.HashSet[B]())({ (h: util.HashSet[B], v: B) => + h.add(v) + h + }, { (left: util.HashSet[B], right: util.HashSet[B]) => + left.addAll(right) + left + }).mapValues( hs => new util.ArrayList(hs)) + ) + } + + private def bruteForceFlatJoin[A <: Geometry : ClassTag, + B <: Geometry : ClassTag]( + leftRDD: SpatialRDD[A], + rightRDD: SpatialRDD[B] + ): JavaPairRDD[A, B] = { + JavaPairRDD.fromRDD( + leftRDD.rawSpatialRDD.rdd + .cartesian(rightRDD.rawSpatialRDD.rdd) + .filter((pair: (Geometry, Geometry)) => pair._1.intersects(pair._2)) + ) + + } + +} diff --git a/src/main/scala/org/globalforestwatch/util/TreeCoverLossExtent.scala b/src/main/scala/org/globalforestwatch/util/TreeCoverLossExtent.scala index 17b13844..e0334e55 100644 --- a/src/main/scala/org/globalforestwatch/util/TreeCoverLossExtent.scala +++ b/src/main/scala/org/globalforestwatch/util/TreeCoverLossExtent.scala @@ -1,6 +1,5 @@ package org.globalforestwatch.util -import geotrellis.vector.MultiPolygon import geotrellis.vector._ object TreeCoverLossExtent { diff --git a/src/main/scala/org/globalforestwatch/util/Util.scala b/src/main/scala/org/globalforestwatch/util/Util.scala index 3770b34a..6ce47d75 100644 --- a/src/main/scala/org/globalforestwatch/util/Util.scala +++ b/src/main/scala/org/globalforestwatch/util/Util.scala @@ -1,7 +1,6 @@ package org.globalforestwatch.util import java.io.File - import com.amazonaws.services.s3.AmazonS3URI import geotrellis.raster.RasterExtent import geotrellis.store.index.zcurve.Z2 @@ -10,18 +9,22 @@ import geotrellis.layer.{LayoutDefinition, SpatialKey} import geotrellis.vector.{Extent, Feature, Geometry, Point} import org.apache.spark.Partitioner import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{Column, SparkSession} +import org.apache.spark.sql.functions.{col, struct} import org.globalforestwatch.features.FeatureId import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.services.s3.model.PutObjectRequest +import scala.reflect.runtime.universe._ + object Util { def uploadFile(file: File, uri: AmazonS3URI): Unit = { val body = RequestBody.fromFile(file) val putObjectRequest = PutObjectRequest.builder() - .bucket(uri.getBucket) - .key(uri.getKey) - .build() + .bucket(uri.getBucket) + .key(uri.getKey) + .build() S3ClientProducer.get().putObject(putObjectRequest, body) } @@ -54,9 +57,49 @@ object Util { .partitionBy(partitioner) } - def getAnyMapValue[T: Manifest](map: Map[String, Any], key: String): T = - map(key) match { + def getAnyMapValue[T: Manifest](map: Map[String, Any], key: String): T = { + map.getOrElse(key, None) match { case v: T => v case _ => throw new IllegalArgumentException("Wrong type") } + + } + + def convertBytesToHex(bytes: Seq[Byte]): String = { + val sb = new StringBuilder + for (b <- bytes) { + sb.append(String.format("%02x", Byte.box(b))) + } + sb.toString + } + + def countRecordsPerPartition[T](rdd: RDD[T], spark: SparkSession): Unit = { + import spark.implicits._ + + rdd + .mapPartitionsWithIndex { case (i, rows) => Iterator((i, rows.size)) } + .toDF("partition_number", "number_of_records") + .orderBy("number_of_records") + .show(100, false) + + + rdd + .mapPartitionsWithIndex { case (i, rows) => Iterator((i, rows.size)) } + .toDF("partition_number", "number_of_records") + .orderBy(col("number_of_records").desc) + .show(100, false) + } + + /** Select columns with same names as case class fields and group them into a struct */ + def colsFor[T: TypeTag]: Column = { + val cols = typeOf[T].members.collect { + case m: MethodSymbol if m.isCaseAccessor => col(m.name.toString) + }.toSeq + struct(cols: _*) + } + + /** Select given fields from a struct column */ + def fieldsFromCol(col: Column, fields: List[String]): List[Column] = + fields.map(name => col.getField(name).as(name)) + } diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties new file mode 100644 index 00000000..4ace141e --- /dev/null +++ b/src/test/resources/log4j.properties @@ -0,0 +1,17 @@ +# Set everything to be logged to the console +log4j.rootCategory=WARN, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.out +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss} %c{1}: %m%n + +log4j.logger.geotrellis.spark=INFO + +# Settings to quiet third party logs that are too verbose +log4j.logger.org.datasyslab=WARN +log4j.category.org.spark_project.jetty=WARN +log4j.logger.org.apache.spark=WARN +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.apache.spark.scheduler.DAGScheduler=WARN +log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=WARN +log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=WARN \ No newline at end of file diff --git a/src/test/resources/palm-32-fcd-output/._SUCCESS.crc b/src/test/resources/palm-32-fcd-output/._SUCCESS.crc new file mode 100644 index 00000000..3b7b0449 Binary files /dev/null and b/src/test/resources/palm-32-fcd-output/._SUCCESS.crc differ diff --git a/src/test/resources/palm-32-fcd-output/.part-00000-03e32aa8-6bf7-4cc6-b98b-5025619dad25-c000.csv.crc b/src/test/resources/palm-32-fcd-output/.part-00000-03e32aa8-6bf7-4cc6-b98b-5025619dad25-c000.csv.crc new file mode 100644 index 00000000..060244c0 Binary files /dev/null and b/src/test/resources/palm-32-fcd-output/.part-00000-03e32aa8-6bf7-4cc6-b98b-5025619dad25-c000.csv.crc differ diff --git a/src/test/resources/palm-32-fcd-output/_SUCCESS b/src/test/resources/palm-32-fcd-output/_SUCCESS new file mode 100644 index 00000000..e69de29b diff --git a/src/test/resources/palm-32-fcd-output/part-00000-03e32aa8-6bf7-4cc6-b98b-5025619dad25-c000.csv b/src/test/resources/palm-32-fcd-output/part-00000-03e32aa8-6bf7-4cc6-b98b-5025619dad25-c000.csv new file mode 100644 index 00000000..4528f890 --- /dev/null +++ b/src/test/resources/palm-32-fcd-output/part-00000-03e32aa8-6bf7-4cc6-b98b-5025619dad25-c000.csv @@ -0,0 +1,2 @@ +list_id location_id status_code location_error tree_cover_loss_total_yearly tree_cover_loss_primary_forest_yearly tree_cover_loss_peat_yearly tree_cover_loss_intact_forest_yearly tree_cover_loss_protected_areas_yearly tree_cover_loss_sea_landcover_yearly tree_cover_loss_idn_landcover_yearly tree_cover_loss_soy_yearly tree_cover_loss_idn_legal_yearly tree_cover_loss_idn_forest_moratorium_yearly tree_cover_loss_prodes_yearly tree_cover_loss_prodes_wdpa_yearly tree_cover_loss_prodes_primary_forest_yearly tree_cover_loss_brazil_biomes_yearly tree_cover_extent_total tree_cover_extent_primary_forest tree_cover_extent_protected_areas tree_cover_extent_peat tree_cover_extent_intact_forest natural_habitat_primary natural_habitat_intact_forest total_area protected_areas_area peat_area brazil_biomes idn_legal_area sea_landcover_area idn_landcover_area idn_forest_moratorium_area south_america_presence legal_amazon_presence brazil_biomes_presence cerrado_biome_presence southeast_asia_presence indonesia_presence commodity_value_forest_extent commodity_value_peat commodity_value_protected_areas commodity_threat_deforestation commodity_threat_peat commodity_threat_protected_areas commodity_threat_fires +1 31 2 {"2001":1021.7622,"2002":851.014,"2003":310.1835,"2004":2169.8398,"2005":2325.3843,"2006":4162.4968,"2007":2968.7863,"2008":4015.4403,"2009":2002.9194,"2010":1173.7001,"2011":1703.6902,"2012":2838.0498,"2013":1841.7568,"2014":2468.7732,"2015":2028.9672,"2016":3344.8135,"2017":1026.7609,"2018":525.5327,"2019":618.7052,"2020":924.699} {"2001":154.8617,"2002":306.7253,"2003":92.3781,"2004":717.7405,"2005":1202.6952,"2006":1831.5766,"2007":1668.2764,"2008":1753.2317,"2009":797.282,"2010":454.5023,"2011":872.3613,"2012":1251.8543,"2013":1083.6799,"2014":1290.2177,"2015":1360.2574,"2016":2313.5001,"2017":286.2809,"2018":159.8557,"2019":162.3929,"2020":134.2652} {"2001":593.0861,"2002":268.8422,"2003":84.1533,"2004":845.0813,"2005":1192.0045,"2006":1712.9994,"2007":1325.3277,"2008":1133.4089,"2009":671.6887,"2010":445.8113,"2011":561.1753,"2012":1236.2464,"2013":890.3066,"2014":1071.3146,"2015":1137.134,"2016":1868.6557,"2017":224.0977,"2018":153.0175,"2019":166.6993,"2020":169.4626} {} {"2001":42.2692,"2002":228.1732,"2003":11.3743,"2004":3.9196,"2005":1.614,"2006":15.3711,"2007":4.4576,"2008":2.8437,"2009":4.765,"2010":7.9931,"2011":10.7597,"2012":8.1466,"2013":0.1537,"2014":8.5307,"2015":18.6758,"2016":139.2616,"2017":10.7596,"2018":0.3843,"2019":0.2306,"2020":0.0} {"Rubber plantation":{"2001":3.0745,"2002":16.5256,"2003":36.0493,"2004":66.1791,"2005":73.4812,"2006":25.9797,"2007":5.9184,"2008":56.571,"2009":47.7317,"2010":33.3581,"2011":21.9825,"2012":52.9583,"2013":11.9137,"2014":42.2742,"2015":34.2038,"2016":63.4883,"2017":10.6839,"2018":24.4423,"2019":22.1363,"2020":10.4533},"Secondary forest":{"2001":240.1012,"2002":352.6874,"2003":51.186,"2004":522.8408,"2005":879.6014,"2006":1310.6826,"2007":981.6686,"2008":756.8744,"2009":359.2934,"2010":232.485,"2011":575.4717,"2012":1110.4372,"2013":787.2514,"2014":772.2979,"2015":966.528,"2016":1571.8466,"2017":149.9382,"2018":89.3794,"2019":136.8781,"2020":121.8915},"Agriculture":{"2001":3.151,"2002":9.1452,"2003":5.4563,"2004":53.8715,"2005":30.3561,"2006":22.9009,"2007":6.5323,"2008":10.9893,"2009":159.7649,"2010":38.7323,"2011":100.4403,"2012":104.3592,"2013":15.3698,"2014":35.8124,"2015":19.6734,"2016":38.1942,"2017":19.2886,"2018":10.5282,"2019":11.2197,"2020":7.9922},"Oil palm plantation":{"2001":389.5357,"2002":222.339,"2003":103.9797,"2004":96.4524,"2005":67.8614,"2006":368.7244,"2007":440.2632,"2008":428.9814,"2009":151.7946,"2010":184.5942,"2011":113.5139,"2012":263.0128,"2013":147.9443,"2014":88.3878,"2015":58.1061,"2016":70.7105,"2017":44.5029,"2018":31.2823,"2019":233.0475,"2020":526.052},"Swamp":{"2001":265.2372,"2002":112.4372,"2003":38.2726,"2004":648.495,"2005":548.2747,"2006":855.4703,"2007":1129.3025,"2008":2086.08,"2009":484.4962,"2010":300.1085,"2011":526.378,"2012":478.8799,"2013":482.4034,"2014":742.3953,"2015":446.518,"2016":539.8938,"2017":620.8959,"2018":204.1215,"2019":105.5176,"2020":122.2736},"Settlements":{"2001":0.1537,"2002":0.9992,"2003":0.0,"2004":0.6918,"2005":0.1537,"2006":1.1529,"2007":1.1529,"2008":0.538,"2009":1.0761,"2010":0.8455,"2011":1.1529,"2012":0.8454,"2013":0.0,"2014":0.6918,"2015":0.1537,"2016":0.2306,"2017":0.3843,"2018":0.0,"2019":0.1537,"2020":1.1529},"Grassland/shrub":{"2001":59.3337,"2002":89.231,"2003":37.5821,"2004":445.7701,"2005":432.4583,"2006":514.3995,"2007":235.9463,"2008":500.7963,"2009":334.6362,"2010":269.6786,"2011":186.2981,"2012":378.8895,"2013":330.4736,"2014":424.3189,"2015":165.2413,"2016":151.5619,"2017":77.7013,"2018":84.6964,"2019":29.59,"2020":91.3842},"Primary forest":{"2001":41.1934,"2002":30.6653,"2003":13.68,"2004":98.6793,"2005":209.8123,"2006":379.429,"2007":115.8962,"2008":96.2208,"2009":368.2156,"2010":47.8819,"2011":42.0413,"2012":228.795,"2013":26.1305,"2014":255.8481,"2015":270.3755,"2016":823.8133,"2017":81.5399,"2018":47.9595,"2019":64.4845,"2020":22.7495},"Water bodies":{"2001":0.8454,"2002":0.0768,"2003":0.0,"2004":0.1537,"2005":0.0,"2006":0.0,"2007":0.0769,"2008":0.0,"2009":0.0,"2010":0.1537,"2011":0.2306,"2012":0.6916,"2013":0.6917,"2014":0.6148,"2015":0.0,"2016":0.2306,"2017":0.0768,"2018":0.0,"2019":0.0,"2020":0.0},"Mixed tree crops":{"2001":19.1363,"2002":16.9073,"2003":23.9776,"2004":236.7062,"2005":83.3852,"2006":683.7575,"2007":52.029,"2008":78.3891,"2009":95.9108,"2010":65.8624,"2011":136.1808,"2012":219.1809,"2013":39.5784,"2014":106.132,"2015":68.1674,"2016":84.8439,"2017":21.749,"2018":33.1229,"2019":15.6777,"2020":20.7498}} {"Secondary Mangrove Forest":{"2001":1.6907,"2002":0.4611,"2003":0.0,"2004":16.0618,"2005":37.1973,"2006":5.0726,"2007":11.2977,"2008":55.7983,"2009":29.4346,"2010":5.4566,"2011":15.0628,"2012":87.9151,"2013":66.4044,"2014":232.3278,"2015":83.5359,"2016":223.5551,"2017":73.0839,"2018":9.3755,"2019":15.0624,"2020":12.5263},"Airport\t/ Harbour":{"2001":0.0,"2002":0.0,"2003":0.0,"2004":0.0,"2005":0.0,"2006":0.0,"2007":0.1537,"2008":0.0,"2009":0.0,"2010":0.0,"2011":0.8453,"2012":0.0,"2013":0.2305,"2014":0.3842,"2015":0.1537,"2016":0.6916,"2017":0.0,"2018":0.0,"2019":0.2305,"2020":0.0},"Bush / Shrub":{"2001":4.9185,"2002":20.2891,"2003":11.7584,"2004":38.7334,"2005":22.748,"2006":135.7978,"2007":15.2937,"2008":74.7011,"2009":35.6594,"2010":20.4429,"2011":50.7993,"2012":105.518,"2013":11.4509,"2014":46.2648,"2015":55.7949,"2016":62.8662,"2017":264.4497,"2018":34.5065,"2019":9.8372,"2020":5.1492},"Dryland Agriculture":{"2001":84.8447,"2002":33.5843,"2003":11.1431,"2004":252.604,"2005":242.9223,"2006":627.0307,"2007":369.7346,"2008":133.182,"2009":250.2177,"2010":94.1414,"2011":239.3058,"2012":573.8395,"2013":135.7163,"2014":221.7118,"2015":197.2781,"2016":355.0518,"2017":95.8305,"2018":45.6485,"2019":37.0411,"2020":45.3409},"Bodies of Water":{"2001":2.4593,"2002":0.0,"2003":0.538,"2004":0.7685,"2005":0.2306,"2006":2.6132,"2007":3.228,"2008":7.7625,"2009":0.2306,"2010":0.4611,"2011":3.0743,"2012":3.689,"2013":1.3834,"2014":2.7668,"2015":0.8454,"2016":5.0725,"2017":2.3057,"2018":1.9215,"2019":0.6148,"2020":0.3074},"Estate Crop Plantation":{"2001":759.8369,"2002":469.6682,"2003":221.0338,"2004":1396.0776,"2005":1569.454,"2006":2808.3496,"2007":2218.0015,"2008":2990.2754,"2009":1359.0495,"2010":854.7606,"2011":983.3553,"2012":1269.5731,"2013":1053.2509,"2014":1138.9796,"2015":466.1251,"2016":509.328,"2017":239.7129,"2018":205.5913,"2019":312.8252,"2020":676.3081},"Swamp Shrub":{"2001":108.6809,"2002":161.2393,"2003":29.9738,"2004":349.2387,"2005":343.4668,"2006":343.8502,"2007":161.7821,"2008":217.8087,"2009":144.181,"2010":91.8383,"2011":132.4202,"2012":376.0483,"2013":159.6257,"2014":316.9527,"2015":293.3632,"2016":416.561,"2017":115.2812,"2018":74.3201,"2019":84.0036,"2020":76.6249},"Secondary Swamp Forest":{"2001":13.1422,"2002":34.0466,"2003":10.3753,"2004":47.3408,"2005":49.3408,"2006":53.4901,"2007":69.7074,"2008":167.0823,"2009":62.716,"2010":41.5034,"2011":90.6095,"2012":170.2307,"2013":292.1891,"2014":371.8946,"2015":608.8459,"2016":984.8287,"2017":186.3736,"2018":101.3724,"2019":136.1879,"2020":80.2367},"Settlement Area":{"2001":29.5885,"2002":83.6144,"2003":6.7627,"2004":4.688,"2005":1.7676,"2006":14.6784,"2007":4.9187,"2008":145.0268,"2009":7.839,"2010":5.687,"2011":6.9934,"2012":17.7525,"2013":5.3028,"2014":8.9916,"2015":9.4526,"2016":18.0599,"2017":7.7619,"2018":5.0722,"2019":4.611,"2020":4.7648},"Swamp":{"2001":1.9214,"2002":0.3843,"2003":0.1537,"2004":0.0769,"2005":1.9983,"2006":2.4594,"2007":1.1528,"2008":0.9223,"2009":2.7668,"2010":3.3817,"2011":0.0,"2012":6.8402,"2013":0.0,"2014":0.4611,"2015":3.228,"2016":1.9983,"2017":0.0,"2018":0.0,"2019":1.1529,"2020":1.8446},"Shrub-Mixed Dryland Farm":{"2001":2.8435,"2002":8.8381,"2003":7.8389,"2004":37.274,"2005":23.2102,"2006":119.3521,"2007":6.5326,"2008":44.7298,"2009":32.6628,"2010":25.9771,"2011":31.9713,"2012":64.6331,"2013":19.9053,"2014":27.0522,"2015":23.2862,"2016":27.8205,"2017":9.2994,"2018":9.914,"2019":7.6083,"2020":11.2203},"Mining Area":{"2001":7.301,"2002":2.7666,"2003":5.2258,"2004":11.9889,"2005":15.2172,"2006":9.1456,"2007":7.6082,"2008":34.8914,"2009":16.9072,"2010":8.9918,"2011":12.4502,"2012":29.5112,"2013":1.0759,"2014":16.8304,"2015":2.5362,"2016":1.9982,"2017":1.7676,"2018":0.7685,"2019":0.3074,"2020":0.4611},"Bare Land":{"2001":3.8428,"2002":35.2766,"2003":5.3801,"2004":14.4491,"2005":17.6005,"2006":39.8116,"2007":99.1447,"2008":141.5687,"2009":59.9482,"2010":20.7508,"2011":136.3415,"2012":129.3478,"2013":94.2991,"2014":83.0794,"2015":280.0642,"2016":735.1371,"2017":28.9729,"2018":36.1198,"2019":8.3774,"2020":7.1477},"Transmigration Area":{"2001":0.6917,"2002":0.8454,"2003":0.0,"2004":0.538,"2005":0.2306,"2006":0.8455,"2007":0.2306,"2008":1.691,"2009":1.3066,"2010":0.3074,"2011":0.4611,"2012":3.1513,"2013":0.9223,"2014":1.0761,"2015":4.4578,"2016":1.8446,"2017":1.9215,"2018":0.9223,"2019":0.8455,"2020":2.767}} {} {"Other Utilization Area":{"2001":712.0267,"2002":482.1867,"2003":221.5682,"2004":1414.5116,"2005":1126.5942,"2006":2837.7298,"2007":1853.8397,"2008":3013.7624,"2009":1165.5631,"2010":833.4598,"2011":1098.1437,"2012":1865.7614,"2013":971.0994,"2014":1259.078,"2015":657.4037,"2016":999.5492,"2017":622.2031,"2018":279.4429,"2019":376.9137,"2020":705.8893},"Converted Production Forest":{"2001":151.8635,"2002":60.1778,"2003":69.0172,"2004":724.5834,"2005":1148.2139,"2006":1123.3127,"2007":1023.561,"2008":844.268,"2009":747.8878,"2010":275.2161,"2011":481.5731,"2012":804.1156,"2013":820.0067,"2014":1024.9196,"2015":1197.936,"2016":1866.668,"2017":356.367,"2018":233.3308,"2019":223.499,"2020":193.5994},"Production Forest":{"2001":113.1434,"2002":80.4763,"2003":7.6858,"2004":26.0567,"2005":48.7316,"2006":183.5469,"2007":84.3149,"2008":147.2649,"2009":84.4729,"2010":56.57,"2011":110.1392,"2012":156.3372,"2013":49.1136,"2014":173.4782,"2015":154.1063,"2016":334.2621,"2017":35.1256,"2018":10.4532,"2019":17.4472,"2020":24.9028},"Sanctuary Reserves/Nature Conservation Area":{"2001":42.2692,"2002":228.1732,"2003":11.3743,"2004":3.9196,"2005":1.614,"2006":15.3711,"2007":4.4576,"2008":2.8437,"2009":4.765,"2010":7.9931,"2011":10.7597,"2012":8.1466,"2013":0.1537,"2014":8.5307,"2015":18.6758,"2016":139.2616,"2017":10.7596,"2018":0.3843,"2019":0.2306,"2020":0.0}} {"2001":85.0014,"2002":248.2325,"2003":18.829,"2004":97.8293,"2005":96.2941,"2006":176.9875,"2007":138.7928,"2008":129.4126,"2009":109.4342,"2010":65.0144,"2011":100.5959,"2012":428.132,"2013":566.3779,"2014":467.2467,"2015":304.2577,"2016":712.6515,"2017":145.3232,"2018":56.2574,"2019":82.8502,"2020":54.0272} {} {} {} {} 76338.8266 34513.678 6014.0986 27160.3808 0.0 34530.8164 0.0 125583.7284 6614.0107 36410.7324 {} {"Other Utilization Area":81822.5991,"Converted Production Forest":28600.5448,"Production Forest":4848.8827,"Sanctuary Reserves/Nature Conservation Area":6614.0107} {"Rubber plantation":3542.3364,"Secondary forest":24896.0268,"Agriculture":2763.3729,"Oil palm plantation":24398.5485,"Swamp":29043.6031,"Settlements":851.2415,"Grassland/shrub":22752.6953,"Primary forest":8261.1709,"Water bodies":3133.0348,"Mixed tree crops":5941.6982} {"Secondary Mangrove Forest":2171.7293,"Airport\t/ Harbour":102.8227,"Bush / Shrub":2875.049,"Dryland Agriculture":8275.6887,"Bodies of Water":3707.2217,"Estate Crop Plantation":69353.0242,"Swamp Shrub":7538.7062,"Secondary Swamp Forest":19794.8549,"Settlement Area":4241.1759,"Swamp":87.0785,"Shrub-Mixed Dryland Farm":1687.8706,"Mining Area":1392.3433,"Bare Land":2902.1353,"Transmigration Area":1454.0282} 13342.6717 false false false false true true {"2002":24579.8084,"2003":24451.4643,"2004":24189.9373,"2005":24132.0676,"2006":23677.7243,"2007":23217.4612,"2008":22247.8139,"2009":21697.0839,"2010":21235.5008,"2011":20791.3746,"2012":20560.122,"2013":20057.1275,"2014":19054.2097,"2015":18742.9578,"2016":18210.5965,"2017":17209.7962,"2018":15452.9753,"2019":15189.6774} {"2002":36410.7324,"2003":36410.7324,"2004":36410.7324,"2005":36410.7324,"2006":36410.7324,"2007":36410.7324,"2008":36410.7324,"2009":36410.7324,"2010":36410.7324,"2011":36410.7324,"2012":36410.7324,"2013":36410.7324,"2014":36410.7324,"2015":36410.7324,"2016":36410.7324,"2017":36410.7324,"2018":36410.7324,"2019":36410.7324} {"2002":6614.0107,"2003":6614.0107,"2004":6614.0107,"2005":6614.0107,"2006":6614.0107,"2007":6614.0107,"2008":6614.0107,"2009":6614.0107,"2010":6614.0107,"2011":6614.0107,"2012":6614.0107,"2013":6614.0107,"2014":6614.0107,"2015":6614.0107,"2016":6614.0107,"2017":6614.0107,"2018":6614.0107,"2019":6614.0107} {"2002":389.8711,"2003":319.3967,"2004":512.213,"2005":914.6063,"2006":1429.9104,"2007":1520.3773,"2008":1012.313,"2009":905.7094,"2010":675.3788,"2011":734.247,"2012":1505.9123,"2013":1314.1698,"2014":843.6132,"2015":1533.1615,"2016":2757.6213,"2017":2020.1188,"2018":381.1145,"2019":273.6768} {"2002":15797.459,"2003":15736.5917,"2004":15911.8092,"2005":16150.8924,"2006":16546.1487,"2007":16716.3841,"2008":16301.6907,"2009":16125.7686,"2010":16064.9017,"2011":15977.2141,"2012":16403.1978,"2013":16377.6838,"2014":16055.4484,"2015":16611.0362,"2016":17499.8752,"2017":16888.2637,"2018":15818.4407,"2019":15819.1363} {"2002":222.102,"2003":200.4294,"2004":9.6067,"2005":0.4611,"2006":3.7659,"2007":3.8427,"2008":0.1537,"2009":1.2297,"2010":2.8437,"2011":2.9205,"2012":3.2279,"2013":1.9982,"2014":0.3843,"2015":13.7571,"2016":74.1656,"2017":63.0214,"2018":2.2287,"2019":0.0} {} diff --git a/src/test/resources/palm-oil-32.tsv b/src/test/resources/palm-oil-32.tsv new file mode 100644 index 00000000..e1b03583 --- /dev/null +++ b/src/test/resources/palm-oil-32.tsv @@ -0,0 +1,33 @@ +list_id location_id geom +1 1 0103000000010000009A0000005CD266EA631C54400041BDD7180F1840DA7ABBF7E51B54401C2ABEB3AB0818408DD55557501B5440AC9B65BDF50118401BB5FD56B41A5440816F0B10D8FB17402DEC1F91121A5440519914B958F61740C6B8DBA56B1954406FDF2A297DF117400F93643AC0185440AD17DE2E4AED17407C055FF810185440D785E5F1C3E91740222E398D5E175440B4FF05EFEDE61740C32793D0F616544095AE6640B4E51740D8370080FB155440C04C0460F2EB1740243C0000A515544080C50320A7ED1740880700A08015544080AF0200F0ED17409C08000053155440408300C081EE1740AC09006045155440006DFF9FCAEE174074A0FF9FFC14544080B9FC1F11F11740201000A0D31454408048FDFF7CF21740ACA3FFBFB31454404072FD9FE9F31740C0A4FF1FA614544000ABFB9F7AF41740DC8000009D1454400095FA7FC3F41740901600E061145440C00DFA3F78F61740A41700405414544080F7F81FC1F61740B81800A026145440804DFBFF9BF71740B8FBFF7F56125440C06F050006FB1740F80D00E0EE105440004B0360B8FE1740E04600C0DC1054400035024001FF174070DAFFDFBC105440C008000093FF1740ACA3FFBFB3105440C0230240DBFF1740DC1A00600B105440000907E03E09184078E9FF1F9E0F544080D0FBBF1D061840608800A09D0F54400084FE3FD7051840A86000C0950F54404049FB7FD2051840C8B3FF5F870F5440C0E6FCDFD4051840187EFF9F700F5440808BFEDFF7061840742300605E0F54404033FA5F1B081840485B00E0590F5440401DF93F64081840E8EFFF5F2C0F5440800EF8BFCD0B184028B9FF3F230F544040E40440F10C1840E4412AFA200F54400041BDD7180F1840FCF0FFBF1E0F54404093006035111840742300605E0F54408091F87F4F1A1840345A0080670F544040670500731B1840C8B3FF5F870F544040AF0020D31C1840187EFF9F700F5440003607A0021E184084BEFF1FBF0E5440C0470160442418402C680060760E544080D60140B02518407C3200A05F0E544080D60140B0251840208DFFDF510E544080EC026067251840F0FEFF9F0D0E544000A005E02023184030C8FF7F040E544040FB03E0FD211840D8370080FB0D544040D103409120184040C9FFDFF60D5440C0AE0660B71F18405C3F00205C0D5440801BFF1FD6241840F80D00E0EE0C5440C0F6FC7F88281840A47D00E0E50C5440C0E0FB5FD128184048D8FF1FD80C5440C0CAFA3F1A2918405C5C0040CC0C5440C0CAFA3F1A29184094DCFF9FA10C544080B4F91F63291840544D00006B0C544040A3F91F3D2A184030ABFF5F340C5440C0070780CC2C1840C8B3FF5F870B54400001FE7F553518409CEBFFDF820B544040E802E09D351840B0ECFF3F750B54400090FE5FC1361840187EFF9F700B5440C063FC1F533718402C7FFFFF620B544040B0F99F99391840ECB5FF1F6C0B54404055FB9FBC3A1840B0ECFF3F750B544040C1FE9F4E3B1840B0ECFF3F750B54400095FC5FE03B1840187EFF9F700B54400050FF7FBA3C1840ECB5FF1F6C0B54400024FD3F4C3D1840345A0080670B5440C00DFC1F953D184098250020430B544040A4F79F923F1840AC260080350B54404044F87F24401840E8EFFF5F2C0B5440405C08606D4018407C3200A01F0B5440405C08606D401840F82A0000DF0A5440808104C0D643184048F5FF3FC80A544000290040FA441840ACC0FFDFA30A5440C0A1FFFFAE4618402ABE1ED2950A5440A4CAB7893F4718400283B844500A5440B90704DB395018408FAEDC6A0A0A5440A547DF44E25A18404E49414ACF095440B5D582F2CA651840B4909A1D9F095440E2A7EB16E97018406011C7147A09544095ECFDAF317C1840AAA29F54600954400F106991998718402A41D2F651095440F429B66F159318407CEFC7094F095440A9EA75EB999E1840D6B995905709544020FE829C1BAA184006EEF8826B09544050C34D1D8FB51840778E5ECD8A095440352B2616E9C01840D1FBF550B5095440749178481ECC184095C5CEE3EA0954408A6AF29923D718401B8701512B0A5440C3B8831FEEE1184031ABE358760A5440D058322873EC1840C6E545B1CB0A54407862B447A8F618402528BD052B0B5440C90EC76083001940BCC9F5F7930B544086D637AFFA0919400C951020060C5440A7C595D104131940235E090D810C54404B5681D2981B19407ABE2645040D544059849131AE2319404E8872468F0D54404033C5EB3C2B1940E97A3A87210E5440977178833D32194061B89876BA0E544051A0D507A93819408375037D590F54403D06BB1B793E1940E954E3FCFD0F544033E30DFCA743194051D72F53A710544018A8748530481940D84411D8541154406B9172390E4C1940EA6C87DF05125440D480DF423D4F1940C09514BAB91254404896B779BA511940A7EF6BB56F13544009B23E6683531940A7DC221D2714544026B67443965419401D5A643BDF145440F60ED800F254194093D7A5599715544026B674439654194093C45CC14E16544009B23E66835319407A1EB4BC041754404896B779BA51194050474197B8175440D380DF423D4F1940626FB79E691854406B9172390E4C1940E9DC98231719544018A8748530481940515FE579C019544032E30DFCA7431940B73EC5F9641A54403D06BB1B793E1940D9FB2F00041B544052A0D507A938194051398EEF9C1B5440977178833D321940EC2B56302F1C54404033C5EB3C2B1940C0F5A131BA1C544059849131AE2319401756BF693D1D54404B5681D2981B19402E1FB856B81D5440A7C595D1041319407EEAD27E2A1E544087D637AFFA091940158C0B71931E5440CA0EC7608300194074CE82C5F21E54407862B447A8F618400909E51D481F5440D058322873EC18401F2DC725931F5440C2B8831FEEE11840A5EEF992D31F54408B6AF29923D718407045DA47002054407E95AF3EF1CD184069B8D22509205440749178481ECC1840C3256AA933205440352B2616E9C0184034C6CFF35220544050C34D1D8FB5184064FA32E66620544020FE829C1BAA1840BEC4006D6F205440A9EA75EB999E18401073F67F6C205440F429B66F15931840901129225E2054400E10699199871840DAA201624420544095ECFDAF317C184086232E591F205440E2A7EB16E97018407045DA470020544046C7BA9CBD691840EC6A872CEF1F5440B5D582F2CA651840AB05EC0BB41F5440A547DF44E25A1840383110326E1F5440B90704DB395018403F2A43E41D1F54405E772C42DC451840120A2A72C31E54406A185DBDD33B18406C7970355F1E5440E6443E3B2A3218407F886F91F11D5440BBCF4A4CE9281840E805CBF27A1D544011345C191A20184005B705CFFB1C5440979B9D5AC51718401DDD0CA4741C5440AD9DEE4EF30F18405CD266EA631C54400041BDD7180F1840 +1 2 0103000000010000006A0000007045DA4700E058409D0075D605BE1F4021A67295C3DF5840A9F02522CDB41F402F08B1F272DF58401175ECDC6FAA1F40A3FCC82118DF5840A6F1B1AC67A01F40299ACB7CB3DE58406F8C6A7FBE961F40CF207F6745DE58402F2BD6E47D8D1F4086CDFB4ECEDD5840D1F20E05AF841F40A48E3FA94EDD58408AB57E975A7C1F407105B9F4C6DC58404C2844DA88741F400749CAB737DC58402A49108A416D1F407CE64380A1DB58407DE983DA8B661F4096A1D8E204DB5840D9CF146F6E601F400D808A7A62DA584047608154EF5A1F408DAF11E8BAD95840342DD9FA13561F407BDC3DD10ED95840DE412030E1511F40929452E05ED858400D5F921B5B4E1F4001555FC3ABD75840F8C68A39854B1F40C8E7932BF6D658407B94145862491F405DB791CC3ED658409DF52694F4471F4094C2BA5B86D55840AFF98F573D471F40C9DC7E8FCDD45840AFF98F573D471F4000E8A71E15D458409DF52694F4471F4096B7A5BF5DD358407B94145862491F405C4ADA27A8D25840FAC68A39854B1F40CB0AE70AF5D158400D5F921B5B4E1F40E2C2FB1945D15840DE412030E1511F40D1EF270399D05840322DD9FA13561F40501FAF70F1CF584047608154EF5A1F40C7FD60084FCF5840D9CF146F6E601F40E1B8F56AB2CE58407DE983DA8B661F4056566F331CCE58402A49108A416D1F40EC9980F68CCD58404A2844DA88741F40B910FA4105CD58408AB57E975A7C1F40D8D13D9C85CC5840CFF20E05AF841F408F7EBA830ECC58402F2BD6E47D8D1F4034056E6EA0CB58406F8C6A7FBE961F40BAA270C93BCB5840A6F1B1AC67A01F402E9788F8E0CA58401175ECDC6FAA1F403CF9C65590CA5840A9F02522CDB41F402A0C2E314ACA58401E2D053A75BF1F40C67261D00ECA58403C22EF975DCA1F404D90606EDEC958407555746F7BD51F40C45E4B3BB9C95840C818FEBEC3E01F40FEF7315C9FC958402C2AB15A2BEC1F407204EFEA90C9584037FC7AF7A6F71F4056380DF68DC95840A2611F9B95012040CAFCB78096C9584005A08B5756072040EB9BB53D97C9584080A0DE6B8C072040F057B782AAC958400E1451FE0F0D2040BD1B77E8C9C958400500A5E3BC1220403A581993F4C95840459C416857182040BF0294582ACA58407FB4F8FED91D204070B8D9036BCA5840CFCA34323F23204050770D55B6CA58402A4763A981282040ED1DC1010CCB5840F74E422E9C2D2040F9763EB56BCB584095FA0CB289322040428BDA10D5CB5840EFBE805245372040FDE952AC47CC5840C205B75ECA3B2040FF8C3416C3CC5840E919CE5B14402040C2F44BD446CD5840EEBB5C091F4420402A0E1E64D2CD584070E7AB65E64720400E6B693B65CE5840698AB2B1664B2040BB4DAFC8FECE5840E82CCF749C4E204038FFC3739ECF58400EC93B80845120409EE0659E43D05840685838F21B54204064A0DAA4EDD058404AE5E8386056204037F691DE9BD15840CC3AE4144F582040B541CD9E4DD258401F9E709BE65920407B634B3502D358401D4E6C38255B20402023F8EEB8D35840DDDADFAF095C204001719E1671D4584027BE381F935C2040AFCF9CF529D55840B5F62AFEC05C20405C2E9BD4E2D5584027BE381F935C20403E7C41FC9AD65840DDDADFAF095C2040E23BEEB551D758401D4E6C38255B2040A85D6C4C06D858401E9E709BE659204026A9A70CB8D85840CC3AE4144F582040F9FE5E4666D958404AE5E83860562040C0BED34C10DA5840685838F21B54204025A07577B5DA58400EC93B8084512040A2518A2255DB5840E82CCF749C4E20404E34D0AFEEDB5840678AB2B1664B204034911B8781DC584070E7AB65E64720409BAAED160DDD5840EEBB5C091F4420405E1205D590DD5840E819CE5B1440204061B5E63E0CDE5840C205B75ECA3B20401A145FDA7EDE5840EFBE8052453720406528FB35E8DE584095FA0CB289322040708178E947DF5840F74E422E9C2D20400D282C969DDF58402A4763A981282040EDE65FE7E8DF5840CFCA34323F2320407045DA4700E05840ECB0D2E94B2120409F9CA59229E058407FB4F8FED91D2040234720585FE05840459C416857182040A083C2028AE058400500A5E3BC1220406D478268A9E058400E1451FE0F0D2040720384ADBCE0584080A0DE6B8C07204093A2816ABDE0584005A08B575607204007672CF5C5E05840A2611F9B95012040EB9A4A00C3E0584037FC7AF7A6F71F405FA7078FB4E058402C2AB15A2BEC1F409940EEAF9AE05840C818FEBEC3E01F40100FD97C75E058407555746F7BD51F40972CD81A45E058403C22EF975DCA1F4033930BBA09E058401D2D053A75BF1F407045DA4700E058409D0075D605BE1F40 +1 3 01060000001100000001030000000100000011000000608800A0BDBA5840C00702803D1620405C5C0040ACBA5840C00702803D16204038D7FFBFA5BA5840008603004B1620400C0F0040A1BA5840C0C2FDBF6B162040FC730080A0BA5840C0F701E08916204008490080A1BA584020ED03002F172040C0A4FF1FA6BA5840C0D9FC5F7817204084DBFF3FAFBA58404027FF3FC017204008E3FFDFCFBA584040C501408A182040201000A0D3BA584040E201609A1820406482FF1FDABA5840804303C097182040CC620080DABA5840A01BFCFF63182040BC440000D8BA584080630120E216204024D6FF5FD3BA5840A079024099162040D0450060CABA5840C08F03605016204028850080C6BA5840201B024048162040608800A0BDBA5840C00702803D1620400103000000010000000D00000098A2FF5F01BA5840C0F4FE9FC00E204050E7FF5FF9B95840C0F4FE9FC00E2040C481FE7FF2B9584020FDFCBF360F2040A494FF7FF2B95840C09E03A0910F204048D8FF1FF8B9584040A90180EC0F2040B086FF9F03BA584080DCFDBF241020405CD9FF7F0ABA5840E084FCBF2C10204034F4FFDF15BA5840A0E0FB5FD10F2040B06F000017BA5840C0F6FC7F880F2040B878FFBF14BA584020FDFCBF360F20400C2C006011BA584040CC0220000F2040D85400A00BBA5840C005FDBFC90E204098A2FF5F01BA5840C0F4FE9FC00E204001030000000100000012000000E07AFF7F59C5584040B303C00EC81F402CE5FF9F54C5584080ADF83FEDC71F40581901404EC5584080F5044017C81F40609FFF3F4AC55840006CFF3F58C81F4048F5FF3F48C55840008FFBDFDCC81F40C4EDFF9F47C5584000FA03808BC91F40BC61002048C55840806A00203BCA1F40609FFF3F4AC5584080DEF9BFCFCA1F4070BDFFBF4CC55840401C07A049CB1F40208DFFDF51C55840C00DFC1F95CB1F40F4C4FF5F6DC55840C0A3FDDF76CC1F40CCDFFFBF78C55840C0A3FDDF76CC1F4000B7FF7F7EC5584000D900C03FCC1F407C2CFF1F7CC558404034FFBF1CCB1F40B8FBFF7F76C55840C06BFB7F1ECA1F4088A1FFFF6EC55840804902607EC91F406C1400205DC558408080FA5F46C81F40E07AFF7F59C5584040B303C00EC81F40010300000001000000260000004CBBFFFFC7C55840003F0160B1C81F409093FF1FC0C558400057F97F84C81F4090B0FF3F90C558404005012091C81F403C20004087C5584080E0F79F97C81F4028B9FF3F83C55840C02B01A0A6C81F40FC73008080C5584080160300D4C81F40F4E1FF7F7DC5584040BE006014C91F4068CBFF9F7BC5584040F106A06AC91F400400000080C55840807800000ACA1F404C3E00C089C55840C088FD7F4BCB1F408092FFBF8DC558400067FE1FC7CB1F40208DFFDF91C55840403A004020CC1F402C68006096C55840005300E066CC1F409093FF1FA0C5584080930740E1CC1F40CC96FF3FB7C558400003FA7F00CE1F408884FFDFBEC55840807BFE3F44CE1F4070400080CEC55840C0BEFF1FBFCE1F40A0D40040D6C5584000F8FCDFFACE1F4018010060F2C55840C0490620B8CF1F40F8A7FF3FFDC55840806206C0FECF1F405081FFBF07C658404074008040D01F407C3200A01FC6584040DB0760D0D01F40D8D1FFDF29C658400082FE7FF2D01F40187EFF9F30C658408008032005D11F40187EFF9F30C65840C0A9004097D01F40D0C2FF9F28C65840C03603403BD01F40D47100C01BC65840808607C084CF1F40C4EDFF9F07C65840C04808A062CE1F403C030020F7C55840C05408C04CCD1F40A494FF7FF2C55840C01C0660F4CC1F40F4470020EFC55840C05D07808ACC1F40F47BFFDFEBC55840404E03800FCC1F4050040080E9C55840409102806DCB1F40406900C0E8C55840806001008BCA1F40B05200E0E6C55840C0F007E0BFC91F40347700A0D7C55840C076F81F24C91F4074890000D0C55840803DFB5FE8C81F404CBBFFFFC7C55840003F0160B1C81F400103000000010000001B000000BC61002008C65840C04EFD3F2BC91F400C0F004001C65840006300801AC91F40D47100C0FBC55840809B01A01DC91F40BCC1FF3FF6C55840408CFA7F30C91F408CEAFF7FF0C55840C0D8F7FF76C91F40882400C0F0C5584080D3FC1FCAC91F40B878FFBFF4C55840C0AFFFDF7DCA1F40C4530040F9C558408027FA3F31CB1F407CAFFFDFFDC55840C0770460D1CB1F4018E4FF3F02C65840806CF71F57CC1F40B05200E006C6584000A1FC7FCACC1F4068AEFF7F0BC658404093006035CD1F40B09DFE3F10C65840805401E0A0CD1F40A86000C015C658404076FE5F08CE1F40DC97FF9F69C65840C007F9BF74D31F4094F9FFBF71C65840003FFF7F94D31F4048D8FF1F78C65840C068F83F55D31F40E44600C07CC65840406204E0E1D21F4030C8FF7F84C6584000EB03404AD11F4028B9FF3F83C6584040B0FE9F28D01F40807B002081C6584080CCFD1F71CF1F405098FE5F54C6584040C0052088CC1F40E889FFBF3AC65840404E01A0F2CA1F40A8DDFFFF33C65840401BFB5F9CCA1F4038D7FFBF25C6584080E2FC5F0BCA1F40ACA3FFBF13C65840004204A07AC91F40BC61002008C65840C04EFD3F2BC91F400103000000010000000C000000B05200E0E6C55840C0B006209CB61F407CAFFFDFDDC55840003104A054B61F40CC620080DAC55840007EFDBFD3B61F4040E6FFFFE6C5584080DD004009B81F402CE5FF9FF4C55840808602C0F5B81F4058ADFF1FF9C55840808602C0F5B81F4018E4FF3F02C65840006AFA7FE4B81F4018E4FF3F02C658400080FB9F9BB81F407CAFFFDFFDC55840C0DA06C008B81F4058ADFF1FF9C558408009038077B71F402CE5FF9FF4C55840C01F04A02EB71F40B05200E0E6C55840C0B006209CB61F400103000000010000001D0000009C85FF3F11C05840C0C10440A5BA1F405CD9FF7F0AC05840C070F79F20BA1F407045DA4700C058404076714B46BA1F4050E7FF5FF9BF5840C07AFBBF5FBA1F40A0510080D4BF5840C0400260EBBB1F40049AFF5FCEBF584000BEF79F4BBC1F40F80D00E0AEBF5840804C078064BF1F40D85400A0ABBF584040F1FFBFBEBF1F4074A0FF9F9CBF5840C0EBFFDF82C11F40485B00E099BF5840005FFDDFDEC11F40543601609EBF5840401E07602EC21F40CCFCFFDFA8BF58404088FCDF83C21F40A8DDFFFFB3BF5840408D04E0C0C21F40F82A0000BFBF584000DE01E0D0C21F4048F5FF3FC8BF584080160300D4C21F4060BCFF5FDABF58404043FAFFCEC21F40B895FFDFE4BF584000680380C8C21F4008E3FFDFEFBF58404038FB7FACC21F40046600A0F1BF5840405304A0A0C21F40404C00A0F8BF58400030012070C21F407045DA4700C058408073295DE3C11F406CF7FFFF0CC05840008E0080F9C01F407C98004011C058404094FEDF8AC01F408C50002022C0584080D3F71F3BBD1F4028B9FF3F23C0584080BA0680A1BC1F4080F8FF5F1FC0584000E104209ABB1F4048DE00A01BC0584000FBFA1F35BB1F40488FFF9F16C05840C067F8DFE2BA1F409C85FF3F11C05840C0C10440A5BA1F4001030000000100000023000000880700A080BF584065A57BD750851F403378A06108BF58407E686C1438871F40E7E7D511A0BE5840FF955ED44E891F40A8DDFFFF73BE5840C0BEF83F138F1F4008E3FFDF0FBE5840000E030041A71F40001D0020F0BD584000380580CAAF1F40001D0020F0BD584080C9FFDF36B11F40B878FFBFF4BD5840406E01E059B21F40946501E006BE5840401303E07CB31F40B81800A026BE584080550640A2B41F405098FE5F34BE5840C0A4FD3FE9B41F403C8600E038BE5840C0A4FD3FE9B41F400CA9FF9F4FBE584080550640A2B41F40CCDFFFBF58BE584080550640A2B41F40048300C061BE584080550640A2B41F404412006078BE584080550640A2B41F40ACA3FFBFB3BE58404089082031B51F4018FBFEDFEEBE5840005D06E0C2B51F40B87E0040F8BE584080A9036009B61F40B009006025BF5840005D06E0C2B51F409842004033BF5840407307007AB51F40347700A057BF584080550640A2B41F40E87200206EBF5840408E044033B31F408CCDFF5FA0BF5840C035FEDF39AD1F4068CBFF9FDBBF5840C0F7FAFFDD9D1F4068CBFF9FDBBF5840C0EF00A0A1931F40143B00A0D2BF5840809EFFDF57931F40003A0040C0BF5840405CFC7F32921F40E8720020AEBF584000680380C8901F4050040080A9BF5840409405C036901F40243C0000A5BF584080C00700A58F1F408CCDFF5FA0BF5840C0D4F95FCA8E1F40383D006097BF5840005CFA9F158D1F40880700A080BF584080EC026067851F40880700A080BF584065A57BD750851F40010300000001000000120000001CC7FF1F52C55840805103A066A31F40FC8AFF1F4DC55840C0B8F99F2CA31F405081FFBF47C55840C068F83F55A31F40A4FAFF1F44C55840C0200400A1A31F40509EFFDF37C5584040030340C9A41F40BCC1FF3F36C55840C0AE016028A51F409033000032C55840008B0240BFA61F407046010032C5584000A2FCDF3CA71F40B4B2FFFF34C5584040BD02E0BEA71F40F098FFFF3BC558408041FEFF23A81F40243C000045C5584080E1F9DF26A81F40F47BFFDF4BC55840009408E0A8A71F40683701C050C558400015FDFF0AA71F402CE5FF9F54C558408052080068A61F40205F01A058C55840007207A007A51F401C9901E058C5584040A904A05EA41F4058A7FE9F55C558404018F81FD3A31F401CC7FF1F52C55840805103A066A31F400103000000010000001B000000B5D80C6453C458404649E6471D831F405019DBAD09C458400928C6523F821F40F82D5895C5C35840EE137083B7811F40880700A0C0C3584000C2FF3F16821F4058790060BCC3584040AAF81F96821F40640500E0BBC35840C00204801E831F404C270120BDC35840404D03209D831F407CCCFFFFCDC358400023FFBFF6871F40886D004012C45840406BFCBF739A1F40E0FDFF3F1BC458408046FF1FB5A01F40E0FDFF3F1BC458400084FE3FD7A31F40E0FDFF3F1BC4584040B3F7DFD3A51F401801006072C45840C0BE06006BAA1F405CF6FF9F9AC458400001088073A81F403854FFFF03C5584000F302A015AA1F400CA9FF9F0FC5584040DC05E025AA1F4048D8FF1F18C55840809E04E0E6A91F40047DFF3F1EC55840C033000072A91F400C0F004021C5584000F4FB1FDCA81F40E8EFFF5F2CC5584000290040FAA41F40E8EFFF5F2CC55840406B03A01FA41F40E8EFFF5F2CC55840C01E0620D9A11F40E8EFFF5F2CC5584080CBF8BF6F9E1F40C02700E027C55840003AFE5F039D1F40FCF0FFBFFEC4584040D6FD7F76971F409C85FF3F71C4584040F4020088861F40B5D80C6453C458404649E6471D831F40010300000001000000300000006C140020BDC0584040A9F8BF239C1F40488FFF9FB6C05840801AFABFD49B1F40EC5500007EC05840804D00002B9C1F40E0F7FEBF77C05840C03BFF5F3D9C1F40DC97FF9F69C0584080A6F75F779C1F4094DCFF9F61C0584040AC02E0989C1F4000B7FF7F5EC05840C0C60060A79C1F40C453004059C0584040BE0740C09C1F4010F2FF1F51C05840C0D2FB7F029D1F40D0C2FF9F48C0584080CA0620559D1F400849008041C058404096F7BFC39D1F406C91FF5F3BC05840C0D6FE1F3E9E1F40C836002029C05840003DFABF20A01F40A47D00E025C0584040BD0000A2A01F407045DA4700C0584040FDCBF583A61F40844100E0C0BF5840C097FBDF6FB01F40608800A0BDBF5840C02D0480FDB01F4074A0FF9FBCBF58404063FD5FA8B11F40880700A0C0BF5840C051F99F48B21F40E4A6FFDFCABF584040AAF81F96B21F402C680060D6BF58404078050099B21F4004000000E0BF58400071FB5F5AB21F40CCFCFFDFE8BF5840C021046013B21F407045DA4700C058404031E20588B11F40ECD2FF3F1CC05840C0FDFB7FE1B01F40C453004059C05840C0D7F79F04AF1F40C836002069C058404053F8BF65AE1F4094F9FFBF71C05840C01006400AAE1F403C20004087C05840003EFA1F93AC1F40E42900A08CC05840403A004020AC1F4014B8FFDF90C05840C058FA7FBEAB1F40E07AFF7F99C0584080270120DDAA1F40945F0060A3C05840805802A0BFA91F404CBBFFFFC7C05840801801E09BA41F40DC1A0060CBC0584000ECFCBF10A41F40F89000A0D0C05840C0DB06207BA21F400C2C0060D1C0584080C5FC3FFBA11F400C2C0060D1C0584080DDFB3F7AA11F400C2C0060D1C05840805AFB7FF8A01F400C2C0060D1C058408072FA7F77A01F400C2C0060D1C0584080EFF9BFF59F1F400C2C0060D1C058408067FDDF719F1F400C2C0060D1C05840400906A0E99E1F400C2C0060D1C058408013F9DF5E9E1F40882400C0D0C0584040B501A0D69D1F4060220000CCC05840C0BC0140F79C1F40345A0080C7C0584000A2F7DFAD9C1F406C140020BDC0584040A9F8BF239C1F4001030000000100000010000000404C00A0B8C2584080F3FC5F31811F40106FFF5FAFC25840C05007002E811F40B81800A0A6C25840002A00A06C811F40047DFF3F9EC2584040EC03A0BC811F4040C9FFDF96C25840C0E9FA1F0F821F4094F9FFBF91C2584040C0F93F4D821F402C4B004086C2584000A005E020831F402C4B004086C25840007403A0B2831F40F4C4FF5F8DC258404041062025841F40F08100608FC25840804AFBDF44841F40947C0080F3C25840803100408D841F4038D7FFBF05C35840C047016044841F40D4A5FF7F18C358400023FFBFF6811F40BCC1FF3F16C35840803D006077811F40FC8AFF1F0DC35840001EFE9F65811F40404C00A0B8C2584080F3FC5F31811F400103000000010000000B00000010F2FF1F11C1584040A3F91F3D8A1F407C980040F1C0584040A3F91F3D8A1F40E42900A0ECC058404048FB1F608B1F409C85FF3FF1C058404083FBBFF28B1F40A86000C0F5C058400006F8BF3A8C1F40945F006003C158408025FE3F868C1F407883FF7F0CC158408025FE3F868C1F4010F2FF1F11C158404003FE3F3A8C1F4038BAFF9F15C158408099FCDFA98B1F40AC26008015C15840400F0540D08A1F4010F2FF1F11C1584040A3F91F3D8A1F400103000000010000000F000000456FE5BABFC158403DE5635E3B801F40C27DE38728C15840CF4D618FD1801F40249DA52271C058400928C6523F821F409F1174D83AC058405A2A6CCCE2821F40EC3800E04DC05840C058F89FA1861F4024D6FF5F53C05840008BF83FA1871F4040C9FFDF56C05840C0F0000014881F40D83700805BC0584040AF07007F881F40149BFFBF60C0584040C5FA5FDE881F40143B00A072C0584000F00460DB891F40FCF0FFBF7EC0584080D1FE3F028A1F40E889FFBFBAC05840401C00C09D891F40B009006025C15840800B01603F891F40882400C090C1584000ABFB9F7A821F40456FE5BABFC158403DE5635E3B801F4001030000000100000011000000786C00E05FC358408059020032861F40507001A03EC35840405B03006C851F409C85FF3FF1C25840404F010065871F40D0280040FAC25840805108A0F5891F40C4D0FF7F77C35840C04AFF9F7E921F40CC96FF3F97C35840804301E07A941F40640500E09BC35840804301E07A941F40243C0000A5C35840C0A3F8DFE7931F40E8720020AEC3584000B206800E931F40E8720020AEC35840C0CA0120C6921F40003A0040A0C35840C0470840F08D1F409C6E00A084C35840402DF9DF17891F40003A004080C358408065040039881F402CE5FF9F74C35840407BFC5F27871F4000D4FF9F6EC35840C06005C0C4861F40BC61002068C35840C088078069861F40786C00E05FC358408059020032861F400103000000010000002B000000FC8AFF1F8DC45840807D0300B8BB1F40880700A080C4584080A5FEDFB0BB1F405CF6FF9F7AC4584080BBF81FBCBB1F40A86000C075C458404049FB7FD2BB1F40F4C4FF5F4DC4584000CCFE5FC6BD1F40D488FF5F48C45840806E08C005BE1F402885008046C4584000D900C03FBE1F40A034006044C458404088FCDF83BE1F40048300C041C458408049FD5FEFBE1F40844100E040C4584000B8020083BF1F4010D5FFFF40C45840002101E02EC01F408C50002042C45840C0820100D7C01F40280200C044C4584080EC026067C11F40BC61002048C458404041FA3FEAC11F40ECB5FF1F4CC45840C01C016065C21F408092FFBF6DC458408039FF9F58C51F4030ABFF5F74C4584000D30440CBC51F407883FF7F8CC45840407505E041C71F40D4A5FF7FD8C458400098FDBF8CCB1F40A877FF5FE2C4584000B101200DCC1F40E8720020EEC45840807900607CCC1F4098BFFF7FF1C4584000E9038065CC1F40A4170040F4C45840C0B9FEFF2DCC1F40C8190000F9C45840C0A3FFBF93CB1F40F4E1FF7FFDC45840402AFB9FDDCA1F40FCF0FFBFFEC458400041FF3F79CA1F40BCD8FEDF02C55840804A07C07FC81F4028B9FF3F03C558404027F85F14C81F406C63012002C55840808302A09EC71F40608800A0FDC45840008E0760A5C61F40C4530040F9C45840004903A0D3C51F4024590020F5C458408094FBBF18C51F409476FFFFEFC458408063FA3F36C41F40200AFF1FD0C45840401E058011BF1F40E8EFFF5FCCC45840401000A0B3BE1F403C9DFF7FC5C45840C0DDF81F08BE1F40880700A0C0C458404043F81FB2BD1F40CC620080BAC45840405702404DBD1F40947C0080B3C4584080220500DBBC1F40ECB5FF1FACC4584080BAFA9F66BC1F40945F0060A3C45840C042F95F07BC1F4058ADFF1F99C45840C0CEF8DFC6BB1F40FC8AFF1F8DC45840807D0300B8BB1F4001030000000100000015030000B34493CE8BCD584080A0DE6B8C07204052CC7B5389CD5840496235202C072040D7E9ABEF58CD58404FF9B7359D012040EF0EB18C1DCD5840B337251052F81F40AA218D65D7CC58408363F5FAA9ED1F40389AE6BF86CC5840EE7176B84CE31F40C9DFC2EB2BCC5840CC3AFC8A44D91F4047773643C7CB5840A64175609BCF1F400A550B2A59CB5840A9979BC85AC61F40F7AB5D0DE2CA5840286D83EB8BBD1F40939D2F6362CA58409992908037B51F402037F4A9DAC9584013C1DBC565AD1F400A3012684BC95840261410781EA61F409CE55E2BB5C858402CA7C8CA689F1F404F17928818C8584006CA75614B991F40ACEDB21A76C758406DBBD048CC931F40FCDB7E82CEC65840244BE4F0F08E1F40C9F2CA6522C658404434B027BE8A1F40423EE06E72C558407E686C1438871F4035D0D34BBFC45840DBE8703362841F405DB503FDA0C45840301E6FF006841F400C2C0060F1C45840C01EFA3F9E8B1F40E42900A00CC55840C094FF7F528F1F40B81800A026C558404047FD9F0A941F40C4EDFF9F47C5584040140340EF9B1F40B0ECFF3F75C55840C035FEDF39AD1F40742300607EC5584000620200C5AF1F40880700A080C5584080FEF9FF36B01F4044A6FE3F83C5584040C7F83FA6B01F40508700408BC55840C064036071B11F40FC56006090C55840C050F93FD6B11F40302E002096C55840C0AC01A043B21F40FC3F01C0A3C5584000A005E020B31F4064E8FFBFABC558404009F8DF91B31F4018840020B4C5584040910780FCB31F40383D006037C658404030FC1FE1B91F40DC8000003DC65840807B0520F0B91F4078E9FF1FDEC5584000D700005BB91F402C4B004026C5584080D000C0ACAE1F40F8A7FF3FFDC4584040C604C06EAC1F409476FFFFEFC45840804105E0CFAB1F4054CAFF3FE9C4584000BDFE1F85AB1F4018E4FF3FE2C45840C06D07203EAB1F4060220000CCC45840C09E01C074AA1F40A0340060C4C45840008D070033AA1F40BCDEFF5FA6C45840C06FF73FAEA91F408CCDFF5FA0C458404046FD3F98A91F405CF6FF9F9AC458400097FA3FA8A91F40281F00E094C45840C0CB0680C7A91F4084DBFF3F8FC45840C03B0640E9A91F40487800008AC45840C03E048023AA1F407C8101A084C458408003020074AA1F4080F8FF5F7FC45840C0A60700ECAA1F40CC6200807AC45840C057060087AB1F40543000E07AC4584040CEF91F1CAC1F40E80C00807CC4584000B003A0B7AC1F40383D006097C45840404001C023B11F401C2D00C023C5584000D9F9DF93C01F40C02700E027C55840C0BC0140F7C01F40805E000031C55840C03503E0C8C11F40C087FFFF35C55840409EFDFF3AC21F40D83700803BC55840C0FAFD3FA7C21F403C80FF5F55C55840C0E2002045C41F402C7FFFFF62C558404083F9DFD5C41F40602200006CC55840006DF8BF1EC51F4064E8FFBFABC558404019FD7FD4C61F4034110000C6C55840402606C0F9C61F4074A0FF9FDCC55840802CFD3FDFC61F40E4DAFE9FE7C55840C0E5F77FD3C61F40A0CEFFBFF2C5584080AC06A0D2C61F40F8A7FF3FFDC558404054F81FD8C61F402885008006C65840C01FF8BFF3C61F4000D4FF9F0EC658400059084016C71F4038BAFF9F15C6584080C403C034C71F4068CBFF9F1BC6584000EDFA3F66C71F4098A2FF5F21C658404026F8FFA1C71F40EC3800E02DC6584000DA034024C81F4030ABFF5F34C65840C06508C072C81F40E4C3FFFF3AC6584080DEF9BFCFC81F40901600E041C658400093088036C91F40B05200E066C65840C098004071CB1F40A051008094C65840C07308A041CE1F40C819000099C6584000C8FBBF8ACE1F40AC2C010099C65840805AFB7FF8CE1F40D4A5FF7F98C6584080B9FC1F11D11F404412006098C65840C09E03A091D11F40CC96FF3F97C6584080E9024010D21F40A8DDFFFF93C65840006505408ED21F40FC56006090C658408046F83F09D31F400CA9FF9F8FC65840001201A0EDD31F40F08100608FC65840407A03E060D41F40A0C8FE3F8FC65840007701E0ECDB1F407C1500808FC65840800A08E078DC1F40FC56006090C658408068074002DD1F40A0CEFFBF92C65840C0BFF89F85DD1F40C087FFFF95C65840807304E007DE1F40543000E09AC6584000D206C075DE1F40141E0080A2C65840C04DFDDFB8DE1F40D85400A0ABC65840802E0700E2DE1F4084DBFF3FAFC658408038FF3FE6DE1F4038BAFF9FB5C6584080440140EDDE1F40543000E0BAC65840C04107C0ECDE1F4080F8FF5FBFC658400011FC3FECDE1F406482FF1FDAC65840806902A0E5DE1F40A877FF5FE2C65840404FFCFFD5DE1F40C4700060E9C65840406B08A0AEDE1F400CA9FF9FEFC6584000BE038086DE1F40AC260080F5C658408066F87F70DE1F40B81800A006C75840C0CAFC1F37DE1F404C8D01C00EC75840C012FD3F26DE1F403C9DFF7F25C7584080E6FAFFB7DE1F40D40B00202AC758400035024001DF1F407855014033C75840C008000093DF1F40D40B00202AC7584080DCFDBF24E01F4030C8FF7F24C7584080DCFDBF24E01F40587900601CC7584080DCFDBF24E01F40DC1A0060CBC658404024060015E01F40D4A5FF7F98C65840C0F2FEDFDBDF1F40581300C08AC65840C008000093DF1F40C02700E087C6584040B4014064DF1F40587900607CC65840803601601ED91F40F44700206FC658404094F7FFDED61F40FC73008060C65840800CF8FFE8D41F40ECD2FF3F5CC65840C0FBF89F8AD41F40CC7F00A04AC65840006BFADF56D31F40A4FAFF1F44C65840C0ED04A0F6D21F4070DAFFDF3CC6584080E6FAFFB7D21F40BCC1FF3F36C65840408F02C088D21F40FC56006070C5584080B1FBDF28CD1F40E0FDFF3F5BC5584080D5FCDFAECC1F40882400C050C55840C04AFF9F7ECC1F40E8EFFF5F4CC558404088FCDF83CC1F404C3E00C049C55840407004C0B0CC1F4070BDFFBF2CC55840C09CFEDF1DD01F40C02700E027C558404037F8FFC7D01F40C02700E027C558404032FCDFC5D21F40E8EFFF5F2CC55840401CFBBF0ED31F4014B8FFDF30C5584000F0F87FA0D31F4010F2FF1F31C5584080520120BCD41F4010F2FF1F31C5584040BAF8BF49D51F4014B8FFDF30C55840C02F0440E2D51F40001D002030C55840C00DFA3F78D61F40FCF0FFBF1EC5584000E600409CD91F4038BAFF9F15C5584080ED00E0BCDA1F40945F006003C55840809202E0DFDB1F408884FFDFFEC45840809202E0DFDB1F40640500E0FBC45840C092FABFDEDB1F40886D0040F2C458404082FB5F80DB1F4000D4FF9FEEC4584080FCF93F52DB1F40DC1A0060EBC45840009804A038DB1F404C38FF3FE6C45840C099FB9F54DB1F407CAFFFDFDDC4584040A40380CDDB1F40302E0020D6C458404074008040DC1F400CA9FF9FCFC45840C0CD0640ACDC1F40E05DFF5FC9C4584080CC04001DDD1F4018670000C4C45840008100009DDD1F408884FFDFBEC4584040E807E02CDE1F40846FFE1FBAC4584000320400C7DE1F40302E0020B6C45840C0C407806EDF1F409C080000B3C45840009A04601DE01F4090330000B2C45840007EFDBFD3E01F40886D0040B2C4584000B6F83F80E11F40B06F0000B7C45840C0250020A3E31F40640500E0BBC4584000E0F8DFECE41F4094DCFF9FC1C45840C0EE02204CE61F40ACC0FFDFC3C45840807CFE9FB6E61F40C0A4FF1FC6C458400003FC5F1DE71F4054CAFF3FC9C4584080E401207FE71F40E06300E0CCC45840809CFA1FE4E71F40FC560060D0C45840C06EF9BF58E81F407883FF7FCCC4584040E90060F3E81F403C200040C7C45840403704E002E91F40807B0020C1C458404043FAFFCEE81F40289CFF1FB3C45840804C00A0B8E71F400CA9FF9FAFC4584080DAFDFF3FE71F4070BDFFBFACC45840409906E0C7E61F4048780000AAC45840003104A054E61F40E44600C09CC4584000E700A00EE41F40E0F7FEBF97C458404056FBFF2EE31F40B43500C096C4584080F6FDBFDDE21F40E80C00809CC45840803005E0A9E01F40003A0040A0C45840808B03E086DF1F4030C8FF7FA4C4584040C7F83FA6DE1F4040E6FFFFA6C45840808CFC5F4DDE1F40C4700060A9C458400092F73FFADD1F4098BFFF7FB1C458408022FE1F2FDD1F402CE5FF9FB4C458404013FEDFEDDC1F4044120060B8C4584000F803C0A6DC1F40901600E0C1C4584040F4FDFFF8DB1F40345A0080C7C4584040ABFD7F97DB1F40049AFF5FCEC4584000C9070038DB1F40B43500C0D6C4584080EFF7DFD8DA1F40880700A0E0C45840C08208E082DA1F4050870040EBC458404092FBFF33DA1F40947C0080F3C4584080E4FA3FD3D91F400400000000C55840403704E002D91F40B009006005C55840C0B006209CD81F40C470006009C55840C023FB5F2FD81F40001D002010C55840805D05A06DD71F4030ABFF5F14C5584080800140F2D51F40AC26008015C5584040BF00C086D51F4038BAFF9F15C55840C0F4F7BF14D51F40CC13FF7F15C558408022F91FA0D41F40882400C010C5584080E9024010D21F40F44700200FC55840000F0360B3D11F407883FF7F0CC55840808805A04CD11F40B81800A006C5584080C100806BD01F4028B9FF3F03C5584080870720F7CF1F40D0280040FAC45840C0D4F95FCACE1F40E42900A0ECC45840C02FF85FA7CD1F40BC610020E8C458400046F97F5ECD1F40CC620080DAC458404072FBBFCCCC1F400C2C0060D1C458404072FBBFCCCC1F4098BFFF7FB1C4584080AFFDFF60CB1F40D0C2FF9F68C45840C0FD07601CC71F4008C6FFBF5FC45840808C084088C61F408884FFDF5EC4584000A6FF7F78C61F40740600404EC45840805A076033C51F40C836002049C45840C05403C0BDC41F408CEAFF7F30C45840C058FF7F4DC21F4070BDFFBF2CC458404041FA3FEAC11F40D40B00202AC4584040B8FADF81C11F40406900C028C4584000DC030009C11F40C8B3FF5F27C45840C0C5F91F89C01F40BCDEFF5F26C4584080A2FD7F04C01F40345A008027C45840004E06A081BF1F40406900C028C45840800DFF3F07BF1F40D04500602AC458408076FB3F96BE1F4000D4FF9F2EC45840C0A0FF9F3CBE1F40404C00A038C4584000B8020083BD1F4084BEFF1F3FC4584000C1FADF14BD1F409CEBFFDF42C4584000CDF7DF8CBC1F40581300C04AC45840C09304206FBB1F4038BAFF9F55C45840C011FDDFB3B91F40B8FBFF7F56C45840C05B02C016B91F4034F4FFDF55C45840807EFC7F7EB81F409C85FF3F51C45840403003008DB71F40D85400A04BC4584000DE01E0D0B61F40B878FFBF14C4584000CCF77F1AB21F40E8720020AEC358408001FD3F00A51F40EC3800E0ADC35840C086FDBF66A41F405890FFFFA8C3584040170680B8A21F40A4FAFF1FA4C35840C0E1FBBF43A11F40F8A7FF3F9DC3584080D9F89F3EA01F40C00A00C097C358404020F95FBB9F1F401CC7FF1F92C35840406903E03A9F1F403C9DFF7F85C35840009AFD7F719E1F400400000080C3584040CEF91F1C9E1F405CF6FF9F7AC35840C03FFB1FCD9D1F40105800C062C35840C019FC3F7F9C1F40CC6200803AC3584080F902E0C39A1F40B086FF9F23C3584000A201E0CB991F40ECD2FF3F1CC3584040ECF7BF81991F4040E6FFFF06C35840C0FB0480C5981F4050040080E9C15840801CFF7F488F1F402C7FFFFFE2C1584080D100201F8F1F4050040080C9C1584040E60220B98E1F40A877FF5FC2C15840C07F03C09C8E1F4060BCFF5FBAC15840C0230060BE8E1F4018010060B2C1584040FAFE7FFC8E1F4050870040ABC15840C094FF7F528F1F40280200C0A4C15840806003E0A78F1F40F82A00009FC15840806206C0FE8F1F40485B00E099C15840C0FF004055901F40302E002096C15840C0D20260AE901F4010F2FF1F71C1584080F2FCFFBE941F403C80FF5F55C15840002A05A0FB971F40208DFFDF51C15840C092FFBF6D981F406CF7FFFF4CC15840800F02205E991F40D488FF5F48C15840002BFE1FC29A1F405081FFBF47C15840806B05803C9B1F40C4EDFF9F47C15840C084FFDF9E9E1F40C4EDFF9F47C1584040F3FD9F869F1F40BC61002048C1584080C5FC3FFB9F1F405C5C00404CC158404003F93FABA11F400CA9FF9F4FC15840005EFF5F89A21F40187EFF9F50C1584000CB05E0FFA21F40882400C050C158408073FFDF78A31F4010F2FF1F51C15840404001C023A51F4010F2FF1F51C15840404DFC3FF1A51F40882400C050C15840801AFFBF63A81F405C5C00404CC15840807001A03EA91F40AC26008035C1584000620020A8AC1F408884FFDFFEC05840407803207CB31F40A86000C0F5C05840007D04400DB41F40A0510080D4C0584080FC05208DB51F40D0C2FF9FC8C0584080A9036009B61F40A877FF5FC2C05840C06C02C03CB61F40201000A033C058404010F9BF07BF1F409C85FF3F31C05840C00006A056BF1F40442F008028C05840407202A078C01F4030C8FF7F24C0584000B8FDFFF3C01F4008C6FFBF1FC0584080E2F75F7CC11F405CF6FF9F1AC05840C0DDF81F08C21F403C80FF5F15C0584000CA05808DC21F4008E3FFDF0FC058408035FA1F00C31F407045DA4700C05840C0363734D2C31F40C00A00C0F7BF5840C0E2002045C41F4090B0FF3FD0BF5840C04007607AC41F4088A1FFFFAEBF5840C0AD0100B6CA1F40404C00A0B8BF5840C02400C030CB1F40E80C0080BCBF5840C0C2FDBF6BCB1F40786C00E0BFBF584000FCFA7FA7CB1F40105800C0C2BF584040D907A0EBCB1F40A84300A0C5BF5840806008E036CC1F40406900C0C8BF5840005EFF5F89CC1F40F4C4FF5FCDBF5840807C03A045CD1F400CA9FF9FCFBF5840C0C705C0A8CD1F40302E0020D6BF5840C060F9DF89D01F40383D0060D7BF584040C0F93F4DD61F40C087FFFFD5BF584080D2F7BFC8D61F40A0CEFFBFD2BF584000A2FCDF3CD71F40F0FEFF9FCDBF5840C0200400A1D71F40E07AFF7F99BF5840C0F7FAFFDDD91F40B4CFFF1F85BF584080ED00E0BCDA1F40F098FFFF7BBF584080ED00E0BCDA1F407CAFFFDF7DBF584080D503C05ADA1F409093FF1FA0BF584000F2074032D81F40B05200E0A6BF584040CD05A0E4D71F4074060040AEBF584040590520A4D71F403C80FF5FB5BF58400020086068D71F4068CBFF9FBBBF5840407BFC5F27D71F4098A2FF5FC1BF58400059FC5FDBD61F403C9DFF7FC5BF5840002DFCFF89D61F40C4EDFF9FC7BF584000F2074032D61F4048F5FF3FC8BF5840801C0480D7D51F405081FFBFC7BF584000AC00007CD51F40640500E0BBBF5840C079024099CD1F40E0FDFF3FBBBF58400044028042CD1F4078E9FF1F7EBF58400084FE3FD7C31F40E4C3FFFF7ABF5840809C010090C31F4034F4FFDF75BF5840806201C06FC31F40A034006004BF5840C0B407E0BAC01F408C4AFF9FFEBE5840C07A07A09AC01F40947C0080F3BE5840803CFBFF75C01F40684E00605DBE584080BFFBBFF7BE1F4018E4FF3F02BE5840C08DF99F4DC31F408CCDFF5F00BE584040CDF9BFA9C31F4008E3FFDFEFBD5840C0C0F8FFF7C51F40807B0020E1BD5840C00A03E0E9C71F40608800A0DDBD584040A7FADF5BC81F403C8600E078BD584080EAFD9FF3D31F40485B00E059BD584000DCFC1F5DD71F409410FF5F3EBD5840C0C901C053DA1F40705D00A01EBD5840003FFF7F94DD1F401C4A00E013BD584040C7F83FA6DE1F406482FF1FFABC5840807E0860B9E01F40C8360020E9BC584040D7FDDFE8E11F40945F0060E3BC5840808E062050E21F407CAFFFDFDDBC5840C0DE0460B5E21F407C3200A0BFBC5840808FFF9F16E51F405CF6FF9FBABC5840C001FA1F8EE51F408884FFDF9EBC584080B3078048E81F40D04500606ABC584080D3F71F3BEF1F40C02700E067BC5840806BFE9F90EF1F4018E4FF3F62BC5840C0A602005DF01F4048D8FF1F58BC5840C0BD06A0F8F11F40B069FF7F53BC58408085FB7FD7F21F401CC7FF1F52BC5840C03DFF1F22F31F4094F9FFBF51BC5840806201C06FF31F40046600A051BC584040220320BEF31F40208DFFDF51BC5840C06FF91FCBF41F40886D004052BC58404051FFDF2CF51F40488FFF9F56BC58408072FA7F77F61F403C8600E058BC584080DCFBDF07F71F40E80C00805CBC58400035074090F71F40681A01A060BC58404093FE7F18F81F40B4CFFF1F65BC58400011032098F81F40D8D1FFDF69BC5840C04F00C00FF91F407489000070BC5840806907A074F91F40302E002076BC5840007A0600D3F91F40F098FFFF7BBC58404012056027FA1F40CC96FF3F97BC584040E2066029FB1F4084BEFF1F9FBC584000FEF85F6FFB1F40CC79FF1FA7BC5840C0C705C0A8FB1F4080F8FF5FBFBC5840408708604CFC1F404CBBFFFFC7BC584040F2F83F85FC1F40F82A0000DFBC58400048FE3FD2FC1F40D0280040FABC584000F605E0DEFC1F406482FF1F1ABD5840005104E0BBFB1F4014B8FFDF30BD5840402704404FFA1F40D8EEFFFF39BD58408004076075F91F40705D00A03EBD584080980360E3F81F409825002043BD5840402704404FF81F409825002043BD5840C0F007E0BFF71F40C4EDFF9F47BD5840C01F04A02EF71F40882400C050BD5840007808200BF61F4000B7FF7F5EBD584000A203C0E8F41F40B0ECFF3F75BD584080F8F87F33F31F40DCB4FFBF79BD5840C00EFA9FEAF21F40105800C0A2BD5840C0C6056036F11F4064E8FFBFABBD5840C0DC0680EDF01F40E07AFF7FB9BD584040C2FC1FA4F01F40D85400A0CBBD584040C2FC1FA4F01F40B05200E0E6BD5840C0DC0680EDF01F4084BEFF1FBFBE5840007808200BF61F4098A2FF5FC1BE5840004C06E09CF61F4098A2FF5FC1BE5840C08BFEDFF7F61F4098A2FF5FC1BE5840402704404FF81F4098A2FF5FC1BE5840C01507609BF81F4070DAFFDFBCBE5840400200C0E4F81F402C4B004086BE58400011032098FA1F40901600E061BE58400011032098FA1F400CA9FF9F4FBE584040A701C007FA1F40B81800A026BE58408009038077F71F40C819000019BE58400062070054F61F40A051008014BE5840005D06E0C2F51F408064018014BE58404076F95F79F51F40001D0020F0BD584080F8F87F33F31F407CAFFFDFDDBD5840C0BAFE5FA0F21F4090B0FF3FD0BD5840003BFCDF58F21F40141E0080C2BD5840003BFCDF58F21F40B878FFBFB4BD5840C0BAFE5FA0F21F4058FC00209EBD5840C0B3FB9F0DF41F40047DFF3F7EBD5840C03505C0E5F61F40DCB4FFBF79BD58408009038077F71F40C4EDFF9F47BD58408042036025FF1F407838012043BD58402000000000002040C4EDFF9F47BD5840C09A000056052040C02700E047BD5840007002E093052040C470006049BD58408045FC9FD0052040E06300E04CBD5840005AFCBF4D062040281F00E054BD584040C1FE9F4E0720408059B4E256BD584080A0DE6B8C0720402CCE000088BD5840A0330000720D2040E0E0FF1F8BBD58408093FD3FC30D204008E3FFDF8FBD5840E04B01E00D0E2040281F00E0B4BD5840E01101A0ED0F2040B895FFDFE4BD5840005EFF5F89152040D8D1FFDFE9BD5840E03FFFDF061720408C50002022BE58406051FCBFBA1C2040B81800A026BE5840C048FCBF271D2040784F00C02FBE5840A0B30280B91D2040E0E0FF1F4BBE5840007002E0931E2040A8DDFFFF73BE584060E20340B71F20406C1400207DBE58406026FCBFDB1F20409825002083BE58406026FCBFDB1F204070BDFFBF8CBE58406026FCBFDB1F2040D4A5FF7F98BE58406026FCBFDB1F2040CC7F00A0AABE5840405101C0491F204064EE0040AFBE5840006AFC5F011F204070DAFFDFBCBE5840405101C0491F2040F4470020CFBE584040B601004920204014B8FFDFD0BE5840204F010065202040886D0040D2BE5840A0CCFFFF8D202040886D0040D2BE58402069FEDFAB202040F4470020CFBE5840608BFCFFDA202040F08100608FBE5840207B0160B6202040048300C081BE5840C09E03A0912020400CA9FF9F6FBE584080B7FE3F492020400CA9FF9F4FBE584060E20340B71F20403C8600E038BE5840405101C0491F2040508700402BBE5840002604E0DC1E204044ACFFBF06BE5840E07BFDFFEE1C2040508700400BBE5840C02402A04D1C2040DC1A00600BBE584040D7FFBF051B2040B05200E006BE584060ED00E0BC1A204068310040EDBD58408084FDFF811920408CEAFF7FD0BD5840A02CFF1FFC172040581300C0CABD5840C0C2FDBF6B172040C4EDFF9FC7BD584000AD0060EE162040B895FFDFC4BD5840E0D600005B152040348EFF3FC4BD5840A09EFCBFE514204030C8FF7FC4BD5840809BFC9F8E14204048F5FF3FC8BD5840A09D03401F142040D8D1FFDFC9BD584000220140A1132040348EFF3FC4BD5840C085FF3F111320406CE00060C0BD5840E0A0FC7FCA122040E4C3FFFFBABD5840E02D01608B122040C03EFF7FB4BD5840405CFE5F4F122040844100E060BD5840006DFF9FCA0F2040E889FFBF5ABD584060C50320A70F20407CAFFFDF1DBD5840C0BD01A0690F204030ABFF5F14BD58408056FFBF680F2040646B00800DBD5840E0FEFDBF700F20402076004005BD58402088FCDF830F204080F8FF5FFFBC58400086FC1F9F0F20402C680060F6BC5840209BFFBF00102040143B00A0F2BC5840009403E019112040302E0020F6BC584060F1FC9F4C122040C4D0FF7FF7BC58400098FF9FA91220408CCDFF5F00BD584000AD0060EE132040B009006005BD5840E0AE034045142040D488FF5F08BD584080EC026067142040602200000CBD58402094FEDF8A1420401CDEFEBF1EBD58402077FEBF7A152040901600E021BD5840A0FAFD3FA71520402885008026BD5840E03A03C004162040BC61002028BD58404098FC7F3716204068AEFF7F2BBD584080EF00A0A1162040C02700E027BD5840C0EA01602D182040FCF0FFBF1EBD5840206C0120751920406482FF1F1ABD584000560000BE19204010F2FF1F11BD5840001B00602B1A2040C00A00C0F7BC5840A0BBFCDFF51A2040C84DFFBFF5BC584040D7FFBF051B2040E8EFFF5FECBC5840204C03C02A1B2040C087FFFFB5BC5840A093FF1FE01B2040D488FF5FA8BC584080A2FD7F041C2040F46400409FBC584080A2FD7F041C2040FC8AFF1F8DBC5840608CFC5F4D1C2040600B01607FBC58400075FEFF951C2040D0C2FF9F68BC5840802200004C1D2040A4FAFF1F64BC5840C00DFC1F951D20407C3200A05FBC58402005FC1F021E20409C1FFF9F5FBC5840008603004B1E2040D0C2FF9F68BC5840002604E0DC1E2040F46400407FBC5840C0230240DB1F20406CF7FFFFACBC58406081FFBF472120405CF6FF9FBABC5840E038FEFF9021204020F3FF7FE3BC5840A027FEFF6A22204008DDFE5FECBC5840C002FF7F8F22204010F2FF1F11BD584020D5FFFF202320409C85FF3F91BD5840C056FC9FF6262040E0E0FF1F8BBD5840407402605D2720407C1500808FBD584060DC02C0B3272040BCDEFF5FA6BD584080E50180F1292040047DFF3F7EBD584040E00380D2272040A86000C0D5BC5840607C03A045232040F46400409FBC5840E0530040D921204080F8FF5F7FBC5840A0880280DA202040B8FBFF7F56BC5840402C02406E1F2040208DFFDF51BC584060420360251F2040F4C4FF5F4DBC5840002604E0DC1E2040CCFCFFDF48BC5840007002E0931E2040CCFCFFDF48BC584020E0FC9F261E2040208DFFDF51BC5840E023FD3F4C1D20402C68006076BC5840807DFEFF281C2040E889FFBF9ABC584040C1FE9F4E1B2040ACC0FFDFA3BC584020E6FD1F2A1B20406CF7FFFFACBC584020E6FD1F2A1B204084BEFF1FBFBC584040FCFE3FE11A2040FCF0FFBFFEBC58406057FD3FBE1920407883FF7F0CBD5840C08F03605019204010F2FF1F11BD5840A09900A0E318204010F2FF1F11BD5840C0EA01602D18204010F2FF1F11BD5840E0250200C017204010F2FF1F11BD5840005EFF5F8917204010F2FF1F11BD5840C094FF7F5217204010F2FF1F11BD584020CA02601B17204010F2FF1F11BD5840E005FF9FE616204014B8FFDF10BD5840A0420040B31620400CA9FF9F0FBD58406085026083162040DC1A00600BBD5840203504201E1620405081FFBF07BD584020D1FC5FE515204028B9FF3F03BD5840C0250020A3152040684E0060FDBC5840203E03E05B152040B06F0000F7BC5840E05703E01415204098BFFF7FF1BC5840200E00E0CE142040FC8AFF1FEDBC5840209AFF5F8E142040D8D1FFDFE9BC58406092FDDF501420409C6E00A0E4BC584080C7FCFFDF13204014B8FFDFD0BC5840407CFCBF99112040001D0020D0BC5840C04CFF5F6311204070400080CEBC5840A08EFC1F32102040F4470020CFBC5840E087FFFFF50F20400C2C0060D1BC584060040080C90F2040880700A0E0BC5840C068FF1F010F2040A0340060E4BC584080AB0140D10E2040D0C2FF9FE8BC5840E0610020A80E2040F4C4FF5FEDBC5840801FFDBF820E204098420040F3BC58402043FFFF5D0E20405CF6FF9FFABC5840409901E0380E204020F3FF7F03BD584080550140130E2040E06300E00CBD584060440140ED0D2040208DFFDF31BD584020D003E01E0A2040ECB9582D1DBD584080A0DE6B8C072040E80C0080FCBC5840201303E07C0320402C680060F6BC5840E041FDBFCE022040201000A0F3BC5840C0F0FBFF8402204068AEFF7FEBBC58404006FC7F74022040A84300A0E5BC5840E0F2FBBF69022040EC210140E1BC58408067FDDF71022040F4640040DFBC5840E0AB00007C02204018010060D2BC5840C0EF0280BE0220403C9DFF7FA5BC5840609C0100900420409C85FF3F91BC58404054FFFF83052040E42900A08CBC5840809402A0C4052040C4EDFF9F87BC5840C003FFDF01062040901600E081BC5840A03603403B0620405890FFFF68BC5840A029FCDF32072040FC73008060BC584040BF00C086072040DC3B37C85FBC584080A0DE6B8C072040ECD2FF3FDCBB5840E04A01809B0B2040D4A5FF7F98BB584080C901C0530D20400849008061BB584060A80300970E2040609FFF3F2ABB5840009601C0E10F2040345A008007BB5840A070FE7FCC10204000B7FF7FDEBA5840C0F1FB5FF71120403C8600E0D8BA584080D701A02212204018840020D4BA5840609903C055122040F0810060CFBA58402036048090122040F47BFFDFCBBA584040500160D712204064E8FFBFCBBA58400015FFDF27132040F4470020CFBA584040D2FCBF57132040B81800A0E6BA584000F2FD3F14142040ECB5FF1FECBA5840E0A7FD5F40142040F80D00E0EEBA5840A069FD9F56142040ACA3FFBFF3BA5840801A01A080142040B87E0040F8BA584060DBFD5FB214204070BDFFBF0CBB5840A0D60320CD152040F08100600FBB5840403704E002162040B086FF9F23BB584020A40380CD1720402076004025BB584020BE006014182040B009006025BB58400077FCDF5D182040243C000025BB584040DEFEBF5E1A2040886D004092BB5840006A0160901E20400CA9FF9FAFBB5840401BFD3FB91E204094F9FFBFD1BB584020000000001F20409093FF1FE0BB5840003800803B1F204048780000EABB584060DDFB3F7A1F20402CE5FF9FF4BB5840C0ACFE7FD11F2040D47100C0FBBB584000A5010023202040844100E000BC5840207B0160B6202040B086FF9F03BC5840405F01A0182120400400000000BC5840006A01609021204038EEFE5FF2BB5840A027FEFF6A222040608800A0DDBB584040FEFC1FA923204058ADFF1FD9BB5840E0B7FB1FD7232040243C0000C5BB5840600EFDBF5C2420409093FF1FC0BB58400018FD1F62242040D8EEFFFFB9BB5840C01800A0462420403C030020B7BB584060A0FDBF1F24204068CBFF9FBBBB5840E0B7FB1FD7232040B895FFDFC4BB584000E4FD5F4523204068CBFF9FDBBB5840C002FF7F8F22204054CAFF3FE9BB5840E02E01C0FD21204018010060F2BB5840805C00406C212040488FFF9FF6BB58404070FFBF2121204040C9FFDFF6BB5840A0880280DA20204018010060F2BB584020F602C06C20204088B8FE9FDBBB584060E20340B71F20407CCCFFFFCDBB5840402C02406E1F2040003A0040C0BB5840405101C0491F2040E8720020AEBB584060420360251F2040602200008CBB584060420360251F2040D0BCFE1F85BB584060420360251F2040C4D0FF7F77BB584060420360251F2040149BFFBF60BB584060420360251F204074A0FF9FFCBA584080B7FE3F49202040A47D00E0E5BA5840A0880280DA20204074A0FF9FDCBA58404070FFBF21212040201000A0D3BA5840006A0160902120405CD9FF7FCABA5840402601C06A2220405CD9FF7FCABA5840607C03A04523204048D8FF1FD8BA584020F4FDFFF82520400C0F0040E1BA5840C00D030041272040A47D00E0E5BA58408050FE3F6527204054CAFF3FE9BA584000B90260F527204024420180E8BA5840E0DEFF5F262820402C7FFFFFE2BA584060C9FFDF362820407CAFFFDFDDBA5840E0E9FCFF2B282040485B00E0D9BA5840A0C202C0FA272040ACA3FFBFB3BA5840A0E6FCDFD42520405CD9FF7FAABA5840607C03A0452320403CEC0080AABA584080520300D922204070DAFFDFBCBA5840008002804721204098A2FF5FC1BA5840809700E0FE202040D45A0120CFBA5840C09E03A091202040002301A0D3BA5840C09E03A091202040E44600C0DCBA584040B6010049202040A47D00E0E5BA5840803CFDDF921F2040A47D00E0E5BA5840405101C0491F2040A47D00E0E5BA5840006AFC5F011F2040A47D00E0E5BA5840007002E0931E2040A47D00E0E5BA5840C020FFFF111E2040A47D00E0E5BA5840E093FCFF6D1D204030C8FF7FE4BA5840E0F703C0A61C204028B9FF3FE3BA5840C077FF5F421C2040E44600C0DCBA58408050FE3F651A2040C087FFFFD5BA584040FEFC1FA91920403CB4FE1F72BA5840C04EFD3F2B17204040E6FFFF66BA584040DD004009172040E80C00805CBA584040DD00400917204030C8FF7F44BA5840E05CFFFF16172040D02800403ABA5840E0D2FD5F1F172040D488FF5F08BA5840E0250200C0172040348EFF3F04BA58406038FF3FE6172040FC8AFF1FEDB9584080630120E2182040DC97FF9FE9B9584040FFFC7F1B19204024590020D5B9584040D7FFBF051A2040FC560060D0B95840A01F04A02E1A2040DCB4FFBF99B9584060E3FCBF7D1B2040ACA3FFBF13B958404088FEBFA01D20406C91FF5FFBB8584020E5FDBFB71D20403C8C01607CB8584060A0FDBF1F1D2040B43500C076B85840E039FE5F031C2040807B0020A1B85840A00204801E1C20406CF7FFFFACB8584040110000261C2040B87E0040B8B85840C048FCBF271C2040A4B1FF9FC2B8584080E2FE3F281C20406CF7FFFFCCB8584080720160231C20409825002003B95840E0340240011C2040FC8AFF1F0DB9584000B2FC7FF01B204028B3FEBF1FB9584020E5FDBFB71B204040E6FFFF26B9584040E1FCFF981B2040ACA3FFBF33B9584080DFFBFF5E1B2040E0E0FF1F8BB95840C0E2FB1FB619204030ABFF5F94B958408002FD9F72192040280200C0A4B95840C0F200C0F8182040D85400A0ABB95840E083FC5FBA182040141E0080C2B9584080F50120A5172040CC79FF1FC7B95840001EFE9F65172040CC7F00A0CAB9584040C00020F916204008DDFE5FCCB95840A09C03E0AC16204094F9FFBFD1B95840C04102C05D15204010F2FF1FD1B95840A0D300E00315204088A1FFFFCEB95840C0D803E0B1142040609FFF3FCAB9584020DF032060142040B878FFBF54B9584000BEFE7FF70F2040740600404EB9584040CDFB9FC60F2040FCF0FFBF9EB85840600EFDBF5C0B2040E883FE3F97B85840800500E03B0B20401801006072B8584080230060BE0A20408092FFBF4DB8584040BEFB5F850A204008E3FFDF0FB8584080C10260880B20408092FFBFEDB75840602AFF5F170C20403C200040E7B75840C00D0300410C20408CCDFF5FE0B75840E093FCFF6D0C2040208DFFDFD1B7584080AAFCDFCF0C2040EC550000BEB75840207DFF3F7E0D20407CAFFFDF9DB75840A03002C0370E2040281F00E094B7584080E00060600E2040E80C00805CB7584040C00020F90E2040C087FFFF55B75840006AFC5F010F2040B05200E046B75840606D0360040F2040C8B3FF5F07B75840400002E01C0E204024590020D5B65840004502E0B40C20400C2C0060D1B6584060A80300970C2040B0E34B84CEB658404E904894790C204078268248CCB658400C9EC146D00C2040B86D5E68B2B6584028ED70138412204019DD75F6A3B6584046A9BBE041182040ED3C5401A1B6584040C814FF031E204004A7258CA9B65840FBBE8ABAC42320404B6CB28EBDB658405F346D607E292040984166F5DCB6584031EEF1442B2F204053B062A107B758409A60D3C8C534204086BC9C683DB758406A4DE35E483A2040C1A605167EB7584030EA8B91AD3F204003A5BE69C9B75840B7133A08F0442040DE6257191FB85840922AAB8C0A4A20400A0E17D07EB85840604E1910F84E2040F3A94F2FE8B85840DDCD40B0B35320402B5ABBCE5AB9584048C639BC385820406E49E33CD6B95840461721B9825C2040FAC88FFF59BA584067FF8B668D6020404A3A4194E5BA5840BFE9C1C254642040274CB17078BB5840802DB80ED56720403A0B5C0312BC5840F0BDCBD10A6B2040EF3D10B4B1BC58407D0A35DDF26D2040EE7B86E456BD5840E693324F8A702040FF69FEF000BE5840EA04E795CE722040E37CE130AFBE58400FE9E771BD742040819F6AF760BF58401D6C7AF8547620407045DA4700C0584099C6F6026E772040B213529415C0584006DF7B95937720404AE07C54CCC058400B14F40C787820402E1BAF8284C15840BAFA4F7C017920403A5B40683DC25840B93D435B2F792040469BD14DF6C25840BAFA4F7C017920402BD6037CAEC358400B14F40C78782040C2A22E3C65C4584006DF7B9593772040F31616D919C558401D6C7AF85476204091399F9FCBC558400FE9E771BD742040754C82DF79C65840EA04E795CE722040873AFAEB23C75840E693324F8A7020408678701CC9C758407D0A35DDF26D20403BAB24CD68C85840F0BDCBD10A6B20404D6ACF5F02C95840802DB80ED56720402A7C3F3C95C95840BFE9C1C2546420407AEDF0D020CA584067FF8B668D602040066D9D93A4CA5840471721B9825C2040495CC50120CB584048C639BC38582040810C31A192CB5840DDCD40B0B35320406AA86900FCCB5840604E1910F84E2040965329B75BCC5840922AAB8C0A4A20407111C266B1CC5840B7133A08F0442040B40F7BBAFCCC584030EA8B91AD3F2040EFF9E3673DCD58406B4DE35E483A204022061E2F73CD58409A60D3C8C5342040DD741ADB9DCD584031EEF1442B2F2040294ACE41BDCD58405F346D607E292040700F5B44D1CD5840FBBE8ABAC423204087792CCFD9CD584040C814FF031E20405CD90ADAD6CD584046A9BBE041182040BD482268C8CD584027ED701384122040FC8FFE87AECD58400C9EC146D00C2040B34493CE8BCD584080A0DE6B8C072040 +1 4 01060000000300000001030000000100000011000000608800A0BDBA5840C00702803D1620405C5C0040ACBA5840C00702803D16204038D7FFBFA5BA5840008603004B1620400C0F0040A1BA5840C0C2FDBF6B162040FC730080A0BA5840C0F701E08916204008490080A1BA584020ED03002F172040C0A4FF1FA6BA5840C0D9FC5F7817204084DBFF3FAFBA58404027FF3FC017204008E3FFDFCFBA584040C501408A182040201000A0D3BA584040E201609A1820406482FF1FDABA5840804303C097182040CC620080DABA5840A01BFCFF63182040BC440000D8BA584080630120E216204024D6FF5FD3BA5840A079024099162040D0450060CABA5840C08F03605016204028850080C6BA5840201B024048162040608800A0BDBA5840C00702803D1620400103000000010000000D00000098A2FF5F01BA5840C0F4FE9FC00E204050E7FF5FF9B95840C0F4FE9FC00E2040C481FE7FF2B9584020FDFCBF360F2040A494FF7FF2B95840C09E03A0910F204048D8FF1FF8B9584040A90180EC0F2040B086FF9F03BA584080DCFDBF241020405CD9FF7F0ABA5840E084FCBF2C10204034F4FFDF15BA5840A0E0FB5FD10F2040B06F000017BA5840C0F6FC7F880F2040B878FFBF14BA584020FDFCBF360F20400C2C006011BA584040CC0220000F2040D85400A00BBA5840C005FDBFC90E204098A2FF5F01BA5840C0F4FE9FC00E2040010300000001000000C0010000BE695959D1C8584080A0DE6B8C0720406810E63C75C8584003C94F8A350520405985FAF6DEC758405403AFB5DA01204037A4974A42C7584050C8C80598FD1F40AABCCDD29FC6584000FD8FF018F81F4059A76330F8C55840350AB69B3DF31F40CE1038094CC558404E0B33D50AEF1F404DD09D079CC45840723638C484EB1F4064EAB3D9E8C358400ECC17E5AEE81F40CFE2B93033C358408F8CD5058CE61F40CC0461C07BC25840C80B63431EE51F40C84B1B3EC3C158403C928A0767E41F405D9868600AC158403C928A0767E41F4058DF22DE51C05840C80B63431EE51F407045DA4700C058401744A4F0C0E51F405601CA6D9ABF58408F8CD5058CE61F40C1F9CFC4E4BE584010CC17E5AEE81F40D813E69631BE5840723638C484EB1F4057D34B9581BD58404E0B33D50AEF1F40CB3C206ED5BC5840350AB69B3DF31F4023D6E13958BC5840FC7FB13CDEF61F403C8600E058BC584080DCFBDF07F71F40E80C00805CBC58400035074090F71F40681A01A060BC58404093FE7F18F81F40B4CFFF1F65BC58400011032098F81F40D8D1FFDF69BC5840C04F00C00FF91F407489000070BC5840806907A074F91F40302E002076BC5840007A0600D3F91F40F098FFFF7BBC58404012056027FA1F40CC96FF3F97BC584040E2066029FB1F4084BEFF1F9FBC584000FEF85F6FFB1F40CC79FF1FA7BC5840C0C705C0A8FB1F4080F8FF5FBFBC5840408708604CFC1F404CBBFFFFC7BC584040F2F83F85FC1F40F82A0000DFBC58400048FE3FD2FC1F40D0280040FABC584000F605E0DEFC1F406482FF1F1ABD5840005104E0BBFB1F4014B8FFDF30BD5840402704404FFA1F40D8EEFFFF39BD58408004076075F91F40705D00A03EBD584080980360E3F81F409825002043BD5840402704404FF81F409825002043BD5840C0F007E0BFF71F40C4EDFF9F47BD5840C01F04A02EF71F40882400C050BD5840007808200BF61F4000B7FF7F5EBD584000A203C0E8F41F40B0ECFF3F75BD584080F8F87F33F31F40DCB4FFBF79BD5840C00EFA9FEAF21F40105800C0A2BD5840C0C6056036F11F4064E8FFBFABBD5840C0DC0680EDF01F40E07AFF7FB9BD584040C2FC1FA4F01F40D85400A0CBBD584040C2FC1FA4F01F40B05200E0E6BD5840C0DC0680EDF01F4084BEFF1FBFBE5840007808200BF61F4098A2FF5FC1BE5840004C06E09CF61F4098A2FF5FC1BE5840C08BFEDFF7F61F4098A2FF5FC1BE5840402704404FF81F4098A2FF5FC1BE5840C01507609BF81F4070DAFFDFBCBE5840400200C0E4F81F402C4B004086BE58400011032098FA1F40901600E061BE58400011032098FA1F400CA9FF9F4FBE584040A701C007FA1F40B81800A026BE58408009038077F71F40C819000019BE58400062070054F61F40A051008014BE5840005D06E0C2F51F408064018014BE58404076F95F79F51F40001D0020F0BD584080F8F87F33F31F407CAFFFDFDDBD5840C0BAFE5FA0F21F4090B0FF3FD0BD5840003BFCDF58F21F40141E0080C2BD5840003BFCDF58F21F40B878FFBFB4BD5840C0BAFE5FA0F21F4058FC00209EBD5840C0B3FB9F0DF41F40047DFF3F7EBD5840C03505C0E5F61F40DCB4FFBF79BD58408009038077F71F40C4EDFF9F47BD58408042036025FF1F407838012043BD58402000000000002040C4EDFF9F47BD5840C09A000056052040C02700E047BD5840007002E093052040C470006049BD58408045FC9FD0052040E06300E04CBD5840005AFCBF4D062040281F00E054BD584040C1FE9F4E0720408059B4E256BD584080A0DE6B8C0720402CCE000088BD5840A0330000720D2040E0E0FF1F8BBD58408093FD3FC30D204008E3FFDF8FBD5840E04B01E00D0E2040281F00E0B4BD5840E01101A0ED0F2040B895FFDFE4BD5840005EFF5F89152040D8D1FFDFE9BD5840E03FFFDF061720408C50002022BE58406051FCBFBA1C2040B81800A026BE5840C048FCBF271D2040784F00C02FBE5840A0B30280B91D2040E0E0FF1F4BBE5840007002E0931E2040A8DDFFFF73BE584060E20340B71F20406C1400207DBE58406026FCBFDB1F20409825002083BE58406026FCBFDB1F204070BDFFBF8CBE58406026FCBFDB1F2040D4A5FF7F98BE58406026FCBFDB1F2040CC7F00A0AABE5840405101C0491F204064EE0040AFBE5840006AFC5F011F204070DAFFDFBCBE5840405101C0491F2040F4470020CFBE584040B601004920204014B8FFDFD0BE5840204F010065202040886D0040D2BE5840A0CCFFFF8D202040886D0040D2BE58402069FEDFAB202040F4470020CFBE5840608BFCFFDA202040F08100608FBE5840207B0160B6202040048300C081BE5840C09E03A0912020400CA9FF9F6FBE584080B7FE3F492020400CA9FF9F4FBE584060E20340B71F20403C8600E038BE5840405101C0491F2040508700402BBE5840002604E0DC1E204044ACFFBF06BE5840E07BFDFFEE1C2040508700400BBE5840C02402A04D1C2040DC1A00600BBE584040D7FFBF051B2040B05200E006BE584060ED00E0BC1A204068310040EDBD58408084FDFF811920408CEAFF7FD0BD5840A02CFF1FFC172040581300C0CABD5840C0C2FDBF6B172040C4EDFF9FC7BD584000AD0060EE162040B895FFDFC4BD5840E0D600005B152040348EFF3FC4BD5840A09EFCBFE514204030C8FF7FC4BD5840809BFC9F8E14204048F5FF3FC8BD5840A09D03401F142040D8D1FFDFC9BD584000220140A1132040348EFF3FC4BD5840C085FF3F111320406CE00060C0BD5840E0A0FC7FCA122040E4C3FFFFBABD5840E02D01608B122040C03EFF7FB4BD5840405CFE5F4F122040844100E060BD5840006DFF9FCA0F2040E889FFBF5ABD584060C50320A70F20407CAFFFDF1DBD5840C0BD01A0690F204030ABFF5F14BD58408056FFBF680F2040646B00800DBD5840E0FEFDBF700F20402076004005BD58402088FCDF830F204080F8FF5FFFBC58400086FC1F9F0F20402C680060F6BC5840209BFFBF00102040143B00A0F2BC5840009403E019112040302E0020F6BC584060F1FC9F4C122040C4D0FF7FF7BC58400098FF9FA91220408CCDFF5F00BD584000AD0060EE132040B009006005BD5840E0AE034045142040D488FF5F08BD584080EC026067142040602200000CBD58402094FEDF8A1420401CDEFEBF1EBD58402077FEBF7A152040901600E021BD5840A0FAFD3FA71520402885008026BD5840E03A03C004162040BC61002028BD58404098FC7F3716204068AEFF7F2BBD584080EF00A0A1162040C02700E027BD5840C0EA01602D182040FCF0FFBF1EBD5840206C0120751920406482FF1F1ABD584000560000BE19204010F2FF1F11BD5840001B00602B1A2040C00A00C0F7BC5840A0BBFCDFF51A2040C84DFFBFF5BC584040D7FFBF051B2040E8EFFF5FECBC5840204C03C02A1B2040C087FFFFB5BC5840A093FF1FE01B2040D488FF5FA8BC584080A2FD7F041C2040F46400409FBC584080A2FD7F041C2040FC8AFF1F8DBC5840608CFC5F4D1C2040600B01607FBC58400075FEFF951C2040D0C2FF9F68BC5840802200004C1D2040A4FAFF1F64BC5840C00DFC1F951D20407C3200A05FBC58402005FC1F021E20409C1FFF9F5FBC5840008603004B1E2040D0C2FF9F68BC5840002604E0DC1E2040F46400407FBC5840C0230240DB1F20406CF7FFFFACBC58406081FFBF472120405CF6FF9FBABC5840E038FEFF9021204020F3FF7FE3BC5840A027FEFF6A22204008DDFE5FECBC5840C002FF7F8F22204010F2FF1F11BD584020D5FFFF202320409C85FF3F91BD5840C056FC9FF6262040E0E0FF1F8BBD5840407402605D2720407C1500808FBD584060DC02C0B3272040BCDEFF5FA6BD584080E50180F1292040047DFF3F7EBD584040E00380D2272040A86000C0D5BC5840607C03A045232040F46400409FBC5840E0530040D921204080F8FF5F7FBC5840A0880280DA202040B8FBFF7F56BC5840402C02406E1F2040208DFFDF51BC584060420360251F2040F4C4FF5F4DBC5840002604E0DC1E2040CCFCFFDF48BC5840007002E0931E2040CCFCFFDF48BC584020E0FC9F261E2040208DFFDF51BC5840E023FD3F4C1D20402C68006076BC5840807DFEFF281C2040E889FFBF9ABC584040C1FE9F4E1B2040ACC0FFDFA3BC584020E6FD1F2A1B20406CF7FFFFACBC584020E6FD1F2A1B204084BEFF1FBFBC584040FCFE3FE11A2040FCF0FFBFFEBC58406057FD3FBE1920407883FF7F0CBD5840C08F03605019204010F2FF1F11BD5840A09900A0E318204010F2FF1F11BD5840C0EA01602D18204010F2FF1F11BD5840E0250200C017204010F2FF1F11BD5840005EFF5F8917204010F2FF1F11BD5840C094FF7F5217204010F2FF1F11BD584020CA02601B17204010F2FF1F11BD5840E005FF9FE616204014B8FFDF10BD5840A0420040B31620400CA9FF9F0FBD58406085026083162040DC1A00600BBD5840203504201E1620405081FFBF07BD584020D1FC5FE515204028B9FF3F03BD5840C0250020A3152040684E0060FDBC5840203E03E05B152040B06F0000F7BC5840E05703E01415204098BFFF7FF1BC5840200E00E0CE142040FC8AFF1FEDBC5840209AFF5F8E142040D8D1FFDFE9BC58406092FDDF501420409C6E00A0E4BC584080C7FCFFDF13204014B8FFDFD0BC5840407CFCBF99112040001D0020D0BC5840C04CFF5F6311204070400080CEBC5840A08EFC1F32102040F4470020CFBC5840E087FFFFF50F20400C2C0060D1BC584060040080C90F2040880700A0E0BC5840C068FF1F010F2040A0340060E4BC584080AB0140D10E2040D0C2FF9FE8BC5840E0610020A80E2040F4C4FF5FEDBC5840801FFDBF820E204098420040F3BC58402043FFFF5D0E20405CF6FF9FFABC5840409901E0380E204020F3FF7F03BD584080550140130E2040E06300E00CBD584060440140ED0D2040208DFFDF31BD584020D003E01E0A2040ECB9582D1DBD584080A0DE6B8C072040E80C0080FCBC5840201303E07C0320402C680060F6BC5840E041FDBFCE022040201000A0F3BC5840C0F0FBFF8402204068AEFF7FEBBC58404006FC7F74022040A84300A0E5BC5840E0F2FBBF69022040EC210140E1BC58408067FDDF71022040F4640040DFBC5840E0AB00007C02204018010060D2BC5840C0EF0280BE0220403C9DFF7FA5BC5840609C0100900420409C85FF3F91BC58404054FFFF83052040E42900A08CBC5840809402A0C4052040C4EDFF9F87BC5840C003FFDF01062040901600E081BC5840A03603403B0620405890FFFF68BC5840A029FCDF32072040FC73008060BC584040BF00C086072040DC3B37C85FBC584080A0DE6B8C072040ECD2FF3FDCBB5840E04A01809B0B2040D4A5FF7F98BB584080C901C0530D20400849008061BB584060A80300970E2040609FFF3F2ABB5840009601C0E10F2040345A008007BB5840A070FE7FCC10204000B7FF7FDEBA5840C0F1FB5FF71120403C8600E0D8BA584080D701A02212204018840020D4BA5840609903C055122040F0810060CFBA58402036048090122040F47BFFDFCBBA584040500160D712204064E8FFBFCBBA58400015FFDF27132040F4470020CFBA584040D2FCBF57132040B81800A0E6BA584000F2FD3F14142040ECB5FF1FECBA5840E0A7FD5F40142040F80D00E0EEBA5840A069FD9F56142040ACA3FFBFF3BA5840801A01A080142040B87E0040F8BA584060DBFD5FB214204070BDFFBF0CBB5840A0D60320CD152040F08100600FBB5840403704E002162040B086FF9F23BB584020A40380CD1720402076004025BB584020BE006014182040B009006025BB58400077FCDF5D182040243C000025BB584040DEFEBF5E1A2040886D004092BB5840006A0160901E20400CA9FF9FAFBB5840401BFD3FB91E204094F9FFBFD1BB584020000000001F20409093FF1FE0BB5840003800803B1F204048780000EABB584060DDFB3F7A1F20402CE5FF9FF4BB5840C0ACFE7FD11F2040D47100C0FBBB584000A5010023202040844100E000BC5840207B0160B6202040B086FF9F03BC5840405F01A0182120400400000000BC5840006A01609021204038EEFE5FF2BB5840A027FEFF6A222040608800A0DDBB584040FEFC1FA923204058ADFF1FD9BB5840E0B7FB1FD7232040243C0000C5BB5840600EFDBF5C2420409093FF1FC0BB58400018FD1F62242040D8EEFFFFB9BB5840C01800A0462420403C030020B7BB584060A0FDBF1F24204068CBFF9FBBBB5840E0B7FB1FD7232040B895FFDFC4BB584000E4FD5F4523204068CBFF9FDBBB5840C002FF7F8F22204054CAFF3FE9BB5840E02E01C0FD21204018010060F2BB5840805C00406C212040488FFF9FF6BB58404070FFBF2121204040C9FFDFF6BB5840A0880280DA20204018010060F2BB584020F602C06C20204088B8FE9FDBBB584060E20340B71F20407CCCFFFFCDBB5840402C02406E1F2040003A0040C0BB5840405101C0491F2040E8720020AEBB584060420360251F2040602200008CBB584060420360251F2040D0BCFE1F85BB584060420360251F2040C4D0FF7F77BB584060420360251F2040149BFFBF60BB584060420360251F204074A0FF9FFCBA584080B7FE3F49202040A47D00E0E5BA5840A0880280DA20204074A0FF9FDCBA58404070FFBF21212040201000A0D3BA5840006A0160902120405CD9FF7FCABA5840402601C06A2220405CD9FF7FCABA5840607C03A04523204048D8FF1FD8BA584020F4FDFFF82520400C0F0040E1BA5840C00D030041272040A47D00E0E5BA58408050FE3F6527204054CAFF3FE9BA584000B90260F527204024420180E8BA5840E0DEFF5F262820402C7FFFFFE2BA584060C9FFDF362820407CAFFFDFDDBA5840E0E9FCFF2B282040485B00E0D9BA5840A0C202C0FA272040ACA3FFBFB3BA5840A0E6FCDFD42520405CD9FF7FAABA5840607C03A0452320403CEC0080AABA584080520300D922204070DAFFDFBCBA5840008002804721204098A2FF5FC1BA5840809700E0FE202040D45A0120CFBA5840C09E03A091202040002301A0D3BA5840C09E03A091202040E44600C0DCBA584040B6010049202040A47D00E0E5BA5840803CFDDF921F2040A47D00E0E5BA5840405101C0491F2040A47D00E0E5BA5840006AFC5F011F2040A47D00E0E5BA5840007002E0931E2040A47D00E0E5BA5840C020FFFF111E2040A47D00E0E5BA5840E093FCFF6D1D204030C8FF7FE4BA5840E0F703C0A61C204028B9FF3FE3BA5840C077FF5F421C2040E44600C0DCBA58408050FE3F651A2040C087FFFFD5BA584040FEFC1FA91920403CB4FE1F72BA5840C04EFD3F2B17204040E6FFFF66BA584040DD004009172040E80C00805CBA584040DD00400917204030C8FF7F44BA5840E05CFFFF16172040D02800403ABA5840E0D2FD5F1F172040D488FF5F08BA5840E0250200C0172040348EFF3F04BA58406038FF3FE6172040FC8AFF1FEDB9584080630120E2182040DC97FF9FE9B9584040FFFC7F1B19204024590020D5B9584040D7FFBF051A2040FC560060D0B95840A01F04A02E1A2040DCB4FFBF99B9584060E3FCBF7D1B2040ACA3FFBF13B958404088FEBFA01D20406C91FF5FFBB8584020E5FDBFB71D20403C8C01607CB8584060A0FDBF1F1D2040B43500C076B85840E039FE5F031C2040807B0020A1B85840A00204801E1C20406CF7FFFFACB8584040110000261C2040B87E0040B8B85840C048FCBF271C2040A4B1FF9FC2B8584080E2FE3F281C20406CF7FFFFCCB8584080720160231C20409825002003B95840E0340240011C2040FC8AFF1F0DB9584000B2FC7FF01B204028B3FEBF1FB9584020E5FDBFB71B204040E6FFFF26B9584040E1FCFF981B2040ACA3FFBF33B9584080DFFBFF5E1B2040E0E0FF1F8BB95840C0E2FB1FB619204030ABFF5F94B958408002FD9F72192040280200C0A4B95840C0F200C0F8182040D85400A0ABB95840E083FC5FBA182040141E0080C2B9584080F50120A5172040CC79FF1FC7B95840001EFE9F65172040CC7F00A0CAB9584040C00020F916204008DDFE5FCCB95840A09C03E0AC16204094F9FFBFD1B95840C04102C05D15204010F2FF1FD1B95840A0D300E00315204088A1FFFFCEB95840C0D803E0B1142040609FFF3FCAB9584020DF032060142040B878FFBF54B9584000BEFE7FF70F2040740600404EB9584040CDFB9FC60F2040A64210B612B95840898EE04F470E204049C233A3C1B85840362E593DEC10204046BC1A7F4AB8584026F683A953152040D31C0B5FDCB75840730B05F3F31920404A2428B077B75840449AD785C81E2040365F42D61CB758404CC2239ACC232040B26D742BCCB658400A3EF738FB282040754DC9FF85B65840B1E22C414F2E20401E81EC984AB658404E2A7E6CC3332040E265E4311AB6584012CFB9545239204028FFD6FAF4B558400A591A79F63E20406875D918DBB55840895EB743AA442040897BCAA5CCB55840F61B0C0F684A20405AC337B0C9B55840B2F08D2B2A502040FD9E4E3BD2B55840AD3D4DE5EA55204038E2D73EE6B5584008169B89A45B2040B00B3FA705B658409E2BAE6C51612040B8B2A45530B65840216041EFEB662040662BFC1F66B65840DB6726846E6C20405B4734D1A6B6584032F5C6B5D3712040820E6B29F2B6584031F38E2B16772040F63F2CDE47B75840C96A3AAF307C20407060BA9AA7B7584044C801321E8120409A10620011B85840B9569FD1D98520406F5BD7A683B858400AEC28DD5E8A204027A29C1CFFB8584018EAB8D9A88E20408FC172E782B9584066E9E186B3922040CA02D2840EBA58401695E8E27A96204071606B6AA1BA58403079BF2EFB99204030A0B1063BBB58402FC1C0F1309D2040A7B869C1DABB5840442622FD18A02040C2F341FC7FBC584053921F6FB0A2204061366F132ABD58407C45D9B5F4A42040A2CF4F5ED8BD5840FB98E291E3A620404A2C13308ABE58405EC97E187BA8204053C565D83EBF5840EE8489B5B9A92040499C20A4F5BF5840C64F092D9EAA20407045DA4700C05840B9D6F81CA6AA20401E94FBDDADC058402C266A9C27AB204012F241CF66C15840E81E5F7B55AB2040075088C01FC258402C266A9C27AB2040DB4763FAD7C25840C74F092D9EAA2040D21E1EC68EC35840EE8489B5B9A92040DBB7706E43C458405EC97E187BA8204082143440F5C45840FB98E291E3A62040C4AD148BA3C558407C45D9B5F4A4204062F041A24DC6584053921F6FB0A220407E2B1ADDF2C65840442622FD18A02040F543D29792C758402FC1C0F1309D2040B48318342CC858403079BF2EFB9920405BE1B119BFC858401695E8E27A962040962211B74AC9584065E9E186B3922040FE41E781CEC9584018EAB8D9A88E2040B688ACF749CA58400AEC28DD5E8A20408BD3219EBCCA5840B9569FD1D9852040B583C90326CB584044C801321E8120402EA457C085CB5840C96A3AAF307C2040A3D51875DBCB584032F38E2B16772040CA9C4FCD26CC584032F5C6B5D3712040BFB8877E67CC5840DC6726846E6C20406D31DF489DCC5840216041EFEB66204075D844F7C7CC58409E2BAE6C51612040ED01AC5FE7CC584007169B89A45B204028453563FBCC5840AD3D4DE5EA552040CA204CEE03CD5840B2F08D2B2A5020409C68B9F800CD5840F61B0C0F684A2040BD6EAA85F2CC5840895EB743AA442040FDE4ACA3D8CC58400A591A79F63E2040427E9F6CB3CC584011CFB954523920400763970583CC58404E2A7E6CC3332040B096BA9E47CC5840B1E22C414F2E204073760F7301CC58400B3EF738FB282040EF8441C8B0CB58404CC2239ACC232040DBBF5BEE55CB5840449AD785C81E204052C7783FF1CA5840720B05F3F3192040DF27691F83CA584026F683A953152040DC2150FB0BCA5840362E593DEC102040A25137498CC95840014C310AC20C2040BDA4998704C95840A561132FD9082040BE695959D1C8584080A0DE6B8C072040 +1 5 0103000000010000006A0000007045DA4700C0584084ED143512D52040DA61CA3480BF5840AA2A5DC637D3204088C8A8DED3BE5840D89D0A691ED12040184F1DAD23BE58409C6896655BCF204003B2734E70BD5840EF191D7AF0CD204028C01874BABC5840B94F960DDFCC20401482EBD102BC58402203732E28CC2040D3018C1D4ABB5840B2139391CCCB20405D5FA80D91BA5840B2139391CCCB20401DDF4859D8B958402203732E28CC204008A11BB720B95840B94F960DDFCC20402DAFC0DC6AB85840F0191D7AF0CD20401912177EB7B758409C6896655BCF2040A9988B4C07B75840D89D0A691ED1204057FF69F65AB65840AA2A5DC637D32040BB203126B3B55840C929046AA5D520405FD6EA8110B55840713614ED64D82040812D88AA73B4584034829E9773DB20407D8F423BDDB3584060DC5E63CEDE2040F97802C94DB35840DE0FB7FE71E22040DA55CCE1C5B2584075B0F4CF5AE620405612340C46B258409027DEF884EA20403CEBD7C6CEB15840CC8A835AECEE2040ED00E38760B15840AA834F998CF320401C2998BCFBB058408450542161F82040E473E5C8A0B0584071ABD02A65FD204078D0000750B05840092CE8BE930221405FFC067B0EB0584080A0DE6B8C07214041350EC709B058407F858ABCE707214025A6CF4ECEAF584098D384DD5B0D2140666A5FD99DAF584036FEB7BBEA122140D3B9F49678AF584090156FD68E182140151FB3AC5EAF58401768D097421E21406BC2843450AF58402BF7635A00242140DFC5FF3C4DAF5840F2D8A96EC229214004D156C955AF58401204BB20832F21408DDF54D169AF58400EF6EEBD3C352140E85A644189AF5840039E809AE93A2140F67BA1FAB3AF58407EF42C1784402140B2E6F7D2E9AF584061ADC5A6064621407E664B952AB058405A7AB2D36B4B214089A6AB0176B058401A645B45AE502140ACB692CDCBB0584022D576C5C855214045222EA42BB15840220B3645B65A21409353B22695B15840F3C34AE2715F2140AEF3B7EC07B25840BB1FC1EBF6632140CFECA28483B258402CDCA8E6406821409BAA127407B35840053C89924B6C21407E2A5B3893B35840A2219BED12702140E664064726B45840C51CC53893732140508F5D0EC0B458403F6855FBC8762140DFAEF9F55FB5584010167606B179214046EB5A5F05B65840E0ED5778487C2140C50A86A6AFB65840CCCA10BF8C7E21402779A7225EB758402A922A9B7B802140EB35BB2610B85840B72CE0211382214040003902C5B858400E3E06BF518321401713C4017CB958403BAB9D363684214081BFDD6F34BA5840A2580EA6BF84214098309A95EDBA584061E00885ED842140AFA156BBA6BB5840A3580EA6BF842140194E70295FBC58403BAB9D3636842140F060FB2816BD58400E3E06BF51832140452B7904CBBD5840B72CE0211382214009E88C087DBE584029922A9B7B8021406C56AE842BBF5840CCCA10BF8C7E2140EA75D9CBD5BF5840E0ED5778487C21407045DA4700C05840D44D17129E7B214052B23A357BC0584010167606B1792140E0D1D61C1BC158403F6855FBC87621404BFC2DE4B4C15840C51CC53893732140B236D9F247C25840A2219BED1270214095B621B7D3C25840053C89924B6C2140617491A657C358402CDCA8E640682140826D7C3ED3C35840BB1FC1EBF66321409D0D820446C45840F3C34AE2715F2140EB3E0687AFC45840210B3645B65A214084AAA15D0FC5584022D576C5C8552140A7BA882965C558401B645B45AE502140B2FAE895B0C558405A7AB2D36B4B21407F7A3C58F1C5584062ADC5A6064621403BE5923027C658407EF42C17844021404906D0E951C65840039E809AE93A2140A381DF5971C658400EF6EEBD3C3521402C90DD6185C658401204BB20832F2140519B34EE8DC65840F2D8A96EC2292140C59EAFF68AC658402BF7635A002421401C42817E7CC658401768D097421E21405DA73F9462C6584090156FD68E182140CAF6D4513DC6584036FEB7BBEA1221400BBB64DC0CC6584098D384DD5B0D2140EF2B2664D1C558407F858ABCE7072140D1642DB0CCC5584080A0DE6B8C072140B99033248BC55840092CE8BE930221404DED4E623AC5584071ABD02A65FD204014389C6EDFC458408450542161F82040436051A37AC45840A9834F998CF32040F5755C640CC45840CC8A835AECEE2040DB4E001F95C358409027DEF884EA2040560B684915C3584075B0F4CF5AE6204038E831628DC25840DE0FB7FE71E22040B4D1F1EFFDC158405FDC5E63CEDE2040AF33AC8067C1584034829E9773DB2040D18A49A9CAC05840713614ED64D820407640030528C05840C929046AA5D520407045DA4700C0584084ED143512D52040 +1 6 01060000001C00000001030000000100000011000000BC440000F8A45840A06A020058B62040886D0040F2A4584000FBFCFF51B620408092FFBFEDA45840E00504A075B62040581300C0EAA4584060A2028093B62040F82A0000DFA45840E04500600AB72040F82A0000DFA458402071FD3F77B72040B086FF9FE3A458408028FC7FC0B720402C4B0040E6A45840C01CFC5FD6B72040BC610020E8A45840009C0240E5B72040E42900A0ECA45840009C0240E5B72040945F006003A558404022FE1F2FB72040243C000005A55840404DFE1F0EB720402C4B004006A5584000A7FFDFEAB62040C4EDFF9F07A55840605802A0BFB6204038D7FFBF05A5584040B9FF3F83B6204078E9FF1FFEA4584080E5FC7F62B62040BC440000F8A45840A06A020058B620400103000000010000000E000000E42900A0ACA95840C080FE1F80BE204060BCFF5F9AA9584060A400605BBE20400C2C006071A958404078FE1FEDBE20401867000064A95840E02CFCFF89BF20401C2D00C063A95840A0A1FFFFAEBF20404C3E00C069A95840E0E30260D4BF204000B7FF7F7EA95840E05703E014C02040A494FF7F92A95840E05703E014C02040B8FBFF7F96A95840E05703E014C0204078E3FE9F9AA95840800DFF3F07C0204098BFFF7FB1A95840C0F1006086BF204098420040B3A9584020DF032060BF204010F2FF1FB1A95840408EFF3FA4BE2040E42900A0ACA95840C080FE1F80BE20400103000000010000001D000000CCDFFFBFD8A65840E07F00A02ABB20400494FEDFCAA65840C04300A025BB204018E4FF3FC2A6584000BBFB3F2EBB2040E80C0080BCA65840A01600E061BB204014B8FFDFB0A65840A03DFD3F05BC204084BEFF1F9FA65840A01F04A02EBD204098A2FF5F21A6584040CAFD5F8CC12040E889FFBF1AA65840C0D1FDFFACC12040E07AFF7F19A6584020E5FDBFB7C120402CE5FF9F14A6584080630120E2C1204098BFFF7F11A65840408803C02FC22040947C008013A65840204F010065C22040404C00A018A65840000AFD3F93C22040FCF0FFBF1EA65840209E0200CAC2204044ACFFBF26A65840C009006005C320400CA9FF9F2FA65840E0A802C041C32040B878FFBFD4A65840604804E028C62040149BFFBFE0A65840202704404FC62040C4EDFF9FE7A6584000FBFCFF51C62040D40B0020EAA65840201501C044C6204070BDFFBFECA6584020FDFCBF36C6204008E3FFDFEFA65840C0C40280DFC520400C0F004001A758408009038077C22040FC84FE9F09A75840C07303A0B2BF20403C9DFF7F05A7584060DC02C0B3BE2040348EFF3F04A75840207202A078BE2040ACA3FFBFF3A65840C0DDFFFFB3BC204094DCFF9FE1A65840A04B040080BB2040CCDFFFBFD8A65840E07F00A02ABB20400103000000010000000F000000F4E1FF7FFDA9584000BAFBDFBBC2204030ABFF5FF4A95840003404C0ABC22040EC3800E0EDA95840A03D0420B1C220402C4B0040E6A95840600A0100CDC22040243C0000E5A95840407BFC5F27C32040E4A6FFDFEAA95840204F010065C320403C8600E0F8A95840006DFF9FCAC32040D04500600AAA5840E006FD1F3CC42040882400C010AA5840C082FCFF47C42040AC26008015AA5840E006FD1F3CC42040302E002016AA5840A0D8FCFF05C42040302E002016AA584000EDFC1F83C32040A051008014AA5840C00101003AC32040740600400EAA5840006B01C002C32040F4E1FF7FFDA9584000BAFBDFBBC220400103000000010000000800000010F2FF1F51AA5840C05801606AC420404CBBFFFF47AA5840C05801606AC4204020F3FF7F43AA5840004502E0B4C42040A03A01E047AA584060A0FDBF1FC62040E42900A04CAA5840607C03A045C62040B086FF9F63AA5840607C03A045C62040B086FF9F63AA5840C0F7FCDFFAC5204010F2FF1F51AA5840C05801606AC4204001030000000100000011000000B05200E006AA58402069FEDFABC920408C50002002AA584020ABFD7F97C92040E44600C0FCA9584020C30180A5C92040C00A00C0F7A9584040C2FEFFC0C920400C2C0060F1A9584080A2FD7F04CA204028080140E8A95840006A016090CA204004000000E0A95840008DFDFF14CB204070DAFFDFDCA958402031FC7F53CB204070DAFFDFDCA9584060EA02A082CB2040E4120100E0A95840204CFEBF9BCB2040280200C0E4A95840E0BD038086CB204048780000EAA9584000C003406BCB2040D8EEFFFFF9A958404089FC3FF6CA2040048300C001AA584020DA0020B2CA2040B086FF9F03AA584000F3FD9F86CA2040BCDEFF5F06AA5840406502201CCA2040B05200E006AA58402069FEDFABC920400103000000010000000B000000187EFF9FB0AC5840C0C60060A7992040C02700E0A7AC5840C0C60060A7992040805E000091AC584040AF0200F099204038BAFF9F95AC5840208300C0819A204098250020C3AC5840604EFE7F809B204074230060DEAC5840604EFE7F809B204074230060DEAC58400092FE1FA69A2040485B00E0D9AC584000A8FF3F5D9A204020930060D5AC5840409901E0389A2040D8EEFFFFB9AC5840C0C60060A7992040187EFF9FB0AC5840C0C60060A79920400103000000010000000600000089304C1755A358409893603F07B12040955267E44FA3584006D535DBD0B120409842004053A35840204CFEBF9BB12040302E002056A3584000A8FF3F5DB120402C68006056A35840408202402CB1204089304C1755A358409893603F07B1204001030000000100000009000000640500E09BA75840400DFD5FEAB22040DC97FF9F89A75840400DFD5FEAB22040C4D0FF7F77A75840A0EEFB3FA0B32040F098FFFF7BA75840404301E07AB42040B4CFFF1F85A75840404301E07AB42040CC96FF3F97A75840603E00C0E9B32040640500E09BA75840A0EEFB3FA0B32040640500E09BA7584020B303C00EB32040640500E09BA75840400DFD5FEAB2204001030000000100000008000000B8E49F1145A35840021E879574B320405C77FF8B36A358406FBF6DBEA7B52040DC1BAF7A36A3584040AC438EABB52040F08100604FA3584040FEFC1FA9B420401884002054A3584060B0026062B4204040C9FFDF56A35840C005FDBFC9B32040CC90FEBF53A35840006A016090B32040B8E49F1145A35840021E879574B3204001030000000100000009000000B009006045A75840E066032056B42040844100E040A75840804A04A00DB42040C00A00C037A75840A0BFFF7F31B4204010D5FFFF20A75840402D00C0C3B4204010D5FFFF20A75840A02400C030B52040ECD2FF3F3CA75840800EFF9F79B52040704000804EA75840800EFF9F79B52040704000804EA75840A0FF004055B52040B009006045A75840E066032056B420400103000000010000002F00000071580FAB2DA35840D7606C289CB720406FD18BA51CA3584031130E845BBB20402D5EB82F0EA35840CFFFA84A19C12040616217390BA3584005B0B962DBC62040E916D9C413A35840EE2655189CCC20401AC79E0719A35840CD3D692B1DCE2040ACC0FFDF23A3584060B300A09CCD2040D04500602AA35840204EFC9F63CD204008E3FFDF2FA358404045FFBF42CD204060BCFF5F3AA3584020ED03002FCD20407883FF7F4CA35840009403E019CD2040A8DDFFFF53A35840C037FE9F1ECD2040485B00E059A35840605301802ECD20407C3200A05FA35840E0A7FD5F40CD2040B4CFFF1F65A35840A075FF9F5DCD2040ECD2FF3F1CA45840E0E9FCFF2BC820403C9DFF7F25A45840C0060220CBC72040D488FF5F28A4584080E403009CC7204070BDFFBF4CA45840408EFF3FA4C42040F4C4FF5F4DA45840A027FEFF6AC420407883FF7F4CA45840605F038035C42040C8B3FF5F47A45840A07103E0CDC320401C2D00C043A45840601DFDFF9DC3204054CAFF3F09A45840400CFDFF77C12040A0510080F4A35840208CFF7FBFC02040B0090060E5A3584060F201004EC02040F4E1FF7FDDA35840E0CD01401DC0204064E8FFBFCBA3584080D2008091BF2040CC79FF1FC7A358402055028068BF2040ACA3FFBFB3A358402096FCBF52BE204008E3FFDFAFA35840E051026011BE204068AEFF7FABA3584040CDFB9FC6BD2040C4EDFF9FA7A35840E0FF022072BD20403C1AFFBFA3A35840E067FC9F1CBD204070DAFFDF9CA35840E0B6FD9F81BC2040BC44000098A35840002FFCBF6EBB2040383D006097A35840C04300A025BB2040CC96FF3F97A35840C0ACFE7FD1BA2040383D006097A35840006AFC5F01B92040C00A00C097A35840002E0340A8B82040901600E0A1A3584040EEFE5F12B72040BCDEFF5FA6A35840E0F6FE5FA5B62040F80D00E0AEA35840004E01A0F2B5204078E9FF1FBEA35840800DFF3F07B52040BC610020C8A35840C0C8FE3F6FB4204008E3FFDF8FA358404016FC1F28B4204071580FAB2DA35840D7606C289CB72040010300000001000000070000005A5EBAC248AC5840C2D9724ED76A2040E1558B962EAC5840B1E29AADFE6A2040B06F000037AC5840207FFD1F466B2040E44600C03CAC5840E0BF01604E6B2040A4B1FF9F42AC5840C06BFD5F3B6B2040C8B3FF5F47AC5840408FFDBFF96A20405A5EBAC248AC5840C2D9724ED76A204001030000000100000009000000D8EEFFFFB9AE584060C0FB1F6A6A2040882400C0B0AE584060C0FB1F6A6A2040ECB5FF1FACAE584000A9FDBFB26A20404C6CFEFFA2AE5840400B0480B16C2040B0ECFF3FB5AE5840400B0480B16C2040345A0080C7AE584020F50260FA6B2040345A0080C7AE5840E06DFD1F206B204054360160BEAE584060C0FB1F6A6A2040D8EEFFFFB9AE584060C0FB1F6A6A20400103000000010000000E00000054CAFF3F09AC584020C2FC1FA46B2040ACC0FFDF03AC5840E0BD0380866B2040D0280040FAAB5840A0CCFFFF8D6B2040347700A0F7AB58402086FEFFBB6B2040CC96FF3FF7AB584060EB0200F56B2040347700A0F7AB5840600EFDBF5C6C20400849008001AC5840000804605A6D2040BCDEFF5F06AC5840E093FCFF6D6D204068AEFF7F0BAC5840C0CAFC1F376D2040E06300E00CAC58402011FE1F096D2040740600400EAC584080DE0280986C2040F4C4FF5F0DAC584020B8FFDF106C204070BDFFBF0CAC5840007F0220D56B204054CAFF3F09AC584020C2FC1FA46B20400103000000010000000A00000098BDDD37B0A8584073FEC5FA107620409F2D6E2C91A85840DA2C3BFAAB762040805E000091A8584040A5FEDFB0762040947C008093A85840E010FC3FEC762040449500209AA8584080F6FF9FFA762040CC79FF1FA7A85840402B02E0FB76204054CAFF3FA9A85840C0F1FB5FF776204050870040ABA85840C0E70320F3762040E42900A0ACA85840A0D60320CD76204098BDDD37B0A8584073FEC5FA107620400103000000010000000A00000010D5FFFF60AD5840A09900A0E378204024D6FF5F53AD5840A09900A0E378204074A0FF9F3CAD5840009100A05079204038D7FFBF45AD584000560000BE792040BC44000058AD5840E03FFFDF067A2040A84300A065AD5840E03FFFDF067A20406CF7FFFF6CAD584020A3FE1FCC7920406C7A00C06EAD584000560000BE7920406C7A00C06EAD5840806DFE5F7579204010D5FFFF60AD5840A09900A0E378204001030000000100000009000000CCDFFFBF78AE5840C06701A0AB6D20408884FFDF5EAE5840C06701A0AB6D204038BAFF9F55AE5840E0C6FB5F186E2040BC61002048AE5840A07103E0CD6F2040E42900A04CAE5840409EFDFF3A7020407883FF7F6CAE5840409EFDFF3A702040E8EFFF5F8CAE5840E006FD1F3C6F2040E8EFFF5F8CAE58408000FFBFAA6E2040CCDFFFBF78AE5840C06701A0AB6D20400103000000010000000C000000048300C0E1AE584020D7FDDFE86E20406039FF9FD8AE5840C03500C0566E2040544D0000CBAE5840A004FD5F576E20403C97FEFFA1AE5840C0FAFF1FC46E20403C8600E098AE584080F3FC5F316F20403C8600E098AE584080890000307020401CAAFFFFA1AE58404047FD9F0A7120409C6E00A0A4AE58404047FD9F0A7120400CA9FF9FAFAE58404047FD9F0A712040F0810060EFAE584000C6FDDFC26F2040F0810060EFAE584000DCFEFF796F2040048300C0E1AE584020D7FDDFE86E20400103000000010000001B000000043C12FF2FA5584072D0407BE68F2040C333143C1EA5584096B2D15FA59020405BA240CDE6A458408C2E77C64D932040406900C0E8A45840E0EDFF9F679320406C7A00C0EEA45840A01A03809D932040B0ECFF3FF5A45840E0D1FFDFC993204068CBFF9FFBA458404019FF5FF19320409CEBFFDF02A55840404E03800F9420405CD9FF7F0AA55840408202402C94204080F8FF5F7FA55840C0A70260CF95204044ACFFBFA6A55840202704404F96204024D6FF5FD3A5584080D6FC3F2196204030C8FF7F24A6584000C7FD3F3595204068B400002FA658404053FF9F11952040B06F000037A65840008AFFBFDA942040485B00E059A65840E0F901A06E9320405C3F00205CA65840E09C00C03A932040143B00A052A65840A079024099922040FC8AFF1F4DA65840205AFE9F6A922040B81800A046A658400049FE9F44922040786C00E03FA6584060420360259220404412006038A65840C07FFEBF0D92204088A1FFFF2EA65840604A02C0F0912040F01B00C01DA658408099FEBFC69120401C4A00E013A65840206FFF5FAF912040844100E060A5584000E502C046902040043C12FF2FA5584072D0407BE68F20400103000000010000000D000000D4A5FF7F98A7584000F4FB1FDC942040581300C08AA7584000F4FB1FDC942040E889FFBF7AA7584060BDFDDF2F952040BCC1FF3F76A7584020A2FEBF599520406C7A00C06EA758406049FD5FEF9520407489000070A75840A00700A0209620401C4A00E073A75840C0C302206D96204094DCFF9F81A75840C0C302206D962040CC7F00A08AA758404050FC5F48962040D4A5FF7F98A758406066FD7FFF952040DC8000009DA758408057FF1FDB952040DC8000009DA7584000AAFD1F25952040D4A5FF7F98A7584000F4FB1FDC942040010300000001000000150000008092FFBF0DA95840E0530040D97F20409410FF5FFEA85840C0560380A27F2040581300C08AA8584040250160F88120404412006078A85840A01C0160658220401CAAFFFF21A858408045FC9FD0842040F8A7FF3F1DA85840C056FC9FF6842040C8190000F9A75840206402C0A986204090B0FF3FB0A75840C07A02A00B9120403C200040A7A7584060B300A09C922040B0090060A5A75840604FFEDFF2932040D40B0020AAA75840E015044029942040187EFF9FB0A758408095020037942040048300C0C1A7584020E0FC9F269420408092FFBFCDA758404070FFBF2194204094F9FFBFD1A75840404DFE1F0E9420408C50002002A85840802EFDFFC38F2040BCDEFF5F46A85840A0D60320CD87204098BFFF7FF1A8584060A8030097832040B87E0040F8A8584000C003406B832040E80C0080FCA8584080CBFF9F1B8320408092FFBF0DA95840E0530040D97F20400103000000010000000B000000D04500606AA8584020B1FEFF9A8F2040B05801606AA85840206C0120758F2040C0A4FF1F66A85840806DFE5F758F2040E44600C05CA8584000D6FD7F768F2040A47D00E045A85840807E03602A902040C0A4FF1F46A8584080C2FBDF4E902040383D006057A8584080C2FBDF4E9020400C0F004061A8584080C2FBDF4E9020403411000066A8584080A302E005902040C0A4FF1F66A8584000870180A08F2040D04500606AA8584020B1FEFF9A8F20400103000000010000002600000060BCFF5F9AAC5840005302C083932040289CFF1F93AC584000C003406B932040001D002090AC58408061FC5F6E93204084DBFF3F8FAC58402035FF1F8F932040B43500C096AC5840E0C800208C94204074A0FF9F9CAC584020C9FDFF199520400C8CFF7F9FAC58408033FE1F559520409C6E00A0A4AC584080770280B4952040B8FBFF7FB6AC5840E04803A0D39620403C0901A0BAAC5840C01502600C972040C02700E0C7AC5840801E0440BC97204070BDFFBFCCAC5840204DFC3FF1972040046600A0D1AC5840A00CFCBF22982040B43500C0D6AC584040C1FE9F4E982040F098FFFFDBAC5840200701E07598204094DCFF9FE1AC5840C0AF01C09A982040243C0000E5AC584060D5FCDFAE982040C02700E0E7AC5840A059FFDFBF98204070400080EEAC584060330320E4982040AC260080F5AC5840A0C7FEDFFC9820405C3F0020FCAC58406032FEBFE29820400C0F004001AD5840C0DDFFFFB398204028B9FF3F03AD5840E0F901A06E98204020F3FF7F03AD5840A0FC02001B98204040E0FE7F03AD5840208CFF7FBF972040B44CFF5F03AD5840A07B0020619720409CEBFFDF02AD5840C009006005972040047DFF3FFEAC5840C0A602005D962040B878FFBFF4AC5840C0EA01602D9520406C7A00C0EEAC58402002FEDFC794204050870040EBAC5840C01FFF9F9F942040345A0080E7AC5840C06F000077942040A4B1FF9FE2AC5840609903C055942040E44600C0DCAC5840205CFC7F32942040A86000C0D5AC5840404DFE1F0E942040EC3800E0CDAC5840E0700060E99320409C6E00A0C4AC5840C095FFDFC493204060BCFF5F9AAC5840005302C08393204001030000000100000015000000001D0020D0A858406064FF9F377F2040406900C0C8A858408082FF1FBA7D2040DC97FF9FE9A858400024FF1F697E2040302E0020F6A8584040EE0360A17E204074A0FF9FFCA85840208102E0B97E20401CAAFFFF01A95840208CFF7FBF7E20402C68006016A95840E0B6FD9F817E2040E80C00801CA95840407A03E0607E20401801006032A9584060A6FE3F237D2040543000E03AA95840C0CA0300E3762040B06F000037A95840E0FF0220727620401801006032A95840401701802976204068CBFF9F1BA958404017018029762040B8FBFF7FB6A85840E0A0FC7FCA7D2040B878FFBFB4A85840E0E400E0297E2040543000E0BAA8584040D203A0037F2040608800A0BDA858406014FE3F607F2040901600E0C1A85840404902607E7F2040D40B0020CAA85840C097FDBF8C7F2040F0810060CFA85840404902607E7F2040001D0020D0A858406064FF9F377F20400103000000010000000E000000C00A00C037AD5840E041FDBFCE7B2040A494FF7F32AD58400033FF5FAA7B20408092FFBF2DAD58408011FDDFB37B2040487800002AAD5840E0FA0100E17B2040581300C00AAD58408082FF1FBA7D2040D488FF5F08AD5840404802000C7E20405081FFBF07AD58402096FCBF527E204068AEFF7F0BAD5840005302C0837E20409C08000013AD5840E0E500409C7E2040383D006017AD5840A09DFC5F737E2040CCFCFFDF28AD5840A09DFC5F737D2040DC8000003DAD5840C05AFF3F327C204080F8FF5F3FAD5840A0DF02E00A7C2040C00A00C037AD5840E041FDBFCE7B20400103000000010000001E000000B086FF9F63AC5840002E0340A895204084BEFF1F5FAC5840C0250020A3952040A8DDFFFF53AC58402002FEDFC7952040B4CFFF1F45AC5840C0F80140FC9520407C3200A03FAC5840C03B01405A9620408CCDFF5F40AC5840E0360020C996204028B9FF3F43AC5840200C020007972040348EFF3F64AC5840E0E7FE1F64982040487800006AAC584060A7FE9F9598204048D8FF1F78AC5840E039FE5F03992040E0FDFF3F7BAC5840E05CFFFF169920408CCDFF5F80AC58408095020037992040DC97FF9F89AC5840005AFCBF4D992040143B00A092AC5840005AFCBF4D992040CC6200809AAC58402003FE3F3A992040844100E0A0AC58408066FF5F1C99204030C8FF7FA4AC5840C0E6FEBFF1982040C4EDFF9FA7AC5840006DFF9FCA982040D488FF5FA8AC58402052FF3F9F982040D488FF5FA8AC5840200701E075982040C8B3FF5FA7AC584060F201004E982040945F0060A3AC5840606D036004982040FCF0FFBF9EAC5840401BFD3FB997204098BFFF7F91AC5840A01903202B972040D85400A08BAC584020DDFE5FEC962040CC6200807AAC5840200A04203F962040C087FFFF75AC584080BA0180129620409C85FF3F71AC5840A00FFFFFEB952040BC2D01606BAC5840A0CA0120C6952040B086FF9F63AC5840002E0340A8952040010300000001000000000300001E98BC5ECBB6584080A0DE6B8C0721405FAE8B0E13B75840CB36DFF359052140851AB8968EB7584073842DF80F0121409FE32F4E01B858402910BCED8AFC2040370165C36AB85840A74E764FCFF72040AFBEFC8DCAB8584018BF4BCEE1F220403BB3364F20B9584039CA864CC7ED2040A5194AB26BB95840C0F8F2D884E82040A22DB96CACB95840F85CD7A91FE32040AE3E9A3EE2B958402835CA179DDD20400933D6F20CBA5840FBF1629802D820402D405C5F2CBA58400EEBCFB855D220400EA94A6540BA5840EE2655189CCC2040965D0CF148BA584005B0B962DBC62040CB616BFA45BA5840CFFFA84A19C1204088EE978437BA584031130E845BBB20409B48249E1DBA58406FBF6DBEA7B520406856F560F8B95840CFDD459F03B02040310728F2C7B9584065DD76BC74AA204000AAEB818CB95840633EBD9600A520400F5D514B46B95840C9684094AC9F2040E4C81094F5B85840A3393CFB7D9A2040C66442AC9AB858401886CAEC79952040348C0FEE35B8584096B2D15FA590204093B658BDC7B75840BC561D1C058C2040CF2B528750B75840F4BBA5B59D872040A29917C2D0B65840B9D50A88738320409EF536EC48B65840071F47B28A7F20408520338CB9B55840CE909E12E77B2040FDC6FE2F23B55840D4A9CD428C782040EA02706C86B4584002417C947D752040A647ADDCE3B358406E98F70DBE722040442994213CB3584092E1366750702040E9941AE18FB25840E8192D07376E20405D15ABC5DFB15840BBDB6A01746C20407BC37C7D2CB15840DE701214096B20403386E7B976B05840972420A6F7692040C7C8EE62E8AF5840BAB0D8D369692040A877FF5F82AF5840006A0160906C20406C1400203DAF5840606D036004742040E06300E02CAF5840405101C049792040A0CEFFBFD2AE5840E00802E0AF792040A47D00E0C5AE5840C0CA0300E3792040340BFF7FC2AE5840A071FCFF217A204008490080A1AE5840A0960260A97C2040001D002090AE5840201AFDDF46802040F08100608FAE5840E03302E08E80204084DBFF3F8FAE58402019FD7FD48020400CA9FF9F8FAE584040F4FFDF15812040805E000091AE58406093024052812040A8DDFFFF93AE5840C09201A08A812040E889FFBF9AAE5840E08D0080F98120404CBBFFFFA7AE584020FCFC5FC4822040ACA3FFBFB3AE584080EBFDFF6584204008490080C1AE5840C04603E0EE862040ACC0FFDFC3AE5840E0F5FEFF328820401801006072AE5840C02A0320518A2040E889FFBF5AAE58406009FC9FCB89204038D7FFBF85AE5840E07D02C06286204084DBFF3F8FAE584060F1FC9F4C852040187EFF9F90AE5840600F02205E8420409C6E00A064AE5840E02CFCFF897F204040C9FFDF36AE584080B400000F7C2040F08100602FAE5840803A04005A7B2040508700402BAE584060EB0200F57A2040F47BFFDF2BAE5840401BFD3FB97920408092FFBF2DAE58408009038077792040187EFF9F30AE5840403401A039792040BCC1FF3F36AE5840E0A0FC7FCA7820407C3200A03FAE584000EAFEDF48782040A47D00E045AE5840001C00C09D772040A47D00E045AE5840E0000380E4762040201000A033AE584040E4FF3F62752040200AFF1F30AE584060BE02403175204004000000E0AD5840608D01C04E71204068CBFF9FDBAD5840603601601E712040488FFF9FD6AD584040310360FF702040143B00A0D2AD584040A90180EC70204038D7FFBFA5AD5840C054FEBF2E712040C00A00C057AD5840E0CD01401D722040A8D7FE7F50AD5840A01BFCFF63722040001D002050AD5840008100009D72204008E3FFDF4FAD5840205C0360DE7220401C4A00E053AD5840607800000A732040646B00806DAD584060330320E47320408CEAFF7F90AD5840E0CB0360557520404C2100A099AD584040C30360C27520403C200040A7AD5840008100009D7620407CAFFFDFDDAD5840C029FEBF4F7A2040DC1A00600BAE5840A0AD0100B6802040DC1A00600BAE5840E02E01C0FD81204054CAFF3F09AE5840C00F01E0088520402C4B004006AE58406006FE5F9186204020F3FF7F03AE584080E000606087204030C8FF7FC4AD58404071FF1F948D20400C8CFF7FBFAD5840606702E0008E2040B086FF9F83AD584040E301C00C922040646B00806DAD5840A010FF5F5E922040BC61002068AD5840E090FEBF33922040A877FF5F62AD5840E0B5FD3F0F92204070DAFFDF5CAD584060500340F4912040EC3E016051AD5840201804000E9220409C74012048AD5840802801804F922040A877FF5F42AD584000FA03808B922040F82A00003FAD5840208102E0B9922040F8A7FF3F3DAD5840C028FE5FDD922040543000E03AAD5840401701802993204058ADFF1F39AD5840C0B1FF9F62932040383D006037AD584040ABFF5FB4932040046600A031AD584060040080C998204094F9FFBF31AD584080C6038019992040501BFF1F56AD5840004502E0B49D2040543000E05AAD5840C0FBFF7F369E20408CCDFF5F60AD5840A0C400A0C29E2040C464FE5F62AD58406049FD5FEF9E2040ECB5FF1F6CAD5840A077FD7F259F2040DCB4FFBF79AD5840A077FD7F259F20409CEBFFDF82AD5840A037FCBF019F2040602200008CAD5840002604E0DC9E2040A494FF7F72AD58400010FEBF96A22040A411FFBF70AD584020B4FC3FD5A22040704000806EAD5840E0BBFEBF12A3204064E8FFBF6BAD584060F1FC9F4CA3204078E9FF1F5EAD5840005FFFBFFBA32040C453004019AD5840C07B02007EA62040A0CEFFBF12AD584040A002C0AEA62040602200000CAD584040EAFBBFD6A62040EC550000FEAC5840C031FD1F1BA72040B06F0000F7AC584000C802A036A72040DC97FF9FE9AC5840A08101A064A7204098250020E3AC5840C0DA01C079A720401C4A00E0D3AC5840603AFD1FAEA720408CCDFF5FC0AC58404024FCFFF6A72040C087FFFFB5AC5840000401C01EA82040E06300E0ACAC5840C029FEBF4FA820409C6E00A0A4AC5840A04AFF9F7EA8204080F8FF5F9FAC5840C07E0040B8A82040587900609CAC5840C0E6FEBFF1A82040705D00A09EAC5840605F038035A9204040E6FFFFA6AC58402086FEFFBBA92040D85400A0ABAC58406055FF5FF6A9204090B0FF3FB0AC5840A01F04A02EAA2040A86000C0B5AC5840A07B002061AA20406C91FF5FBBAC5840C09DFE3F90AA204098A2FF5FC1AC5840604C00A0B8AA2040C02700E0C7AC5840008AFFBFDAAA2040049AFF5FCEAC584040C601A0FCAA2040887301C0D5AC584020C9FDFF19AB2040F4E1FF7FDDAC5840205CFC7F32AB20403C9DFF7FE5AC58404039FDBF3BAB2040302E0020F6AC58402020FE5F4AAB204074230060FEAC5840405D03C050AB204040E6FFFF06AD584080AC01A043AB20408CEAFF7F10AD58404039FDBF3BAB2040345A008027AD5840E07F00A02AAB2040F015FF3F3AAD5840E0CD01401DAB20405890FFFF48AD5840806C00E01FAB2040D40B00206AAD584020E1010028AB2040ECD2FF3F7CAD5840E05400A04BAB2040A877FF5FA2AD5840A04801C0B6AB2040784F00C0AFAD5840C062FE9FFDAB20403C80FF5FB5AD58400009FDDF20AC2040D0280040BAAD58402074008040AC2040D8D1FFDFC9AD5840E007FD7FAEAC20409C7FFEBFEDAD5840E047FE3FD2AD20402C680060F6AD5840408202402CAE204068CBFF9FFBAD58408042FE5F96AE2040D8370080FBAD5840C0E70320F3AE20403C030020F7AD5840E0C800208CAF20400C2C0060F1AD58402086FEFFBBAF204004000000A0AD5840C0C8FE3F6FB1204058ADFF1F99AD58408067FDDF71B120401801006092AD5840A043FEBF08B12040187EFF9F90AD5840208102E0B9B0204010F2FF1F91AD5840E0EDFF9F67B02040A8DDFFFF93AD584000ECFCBF10B02040E07AFF7F99AD584000310180E2AF204098A2FF5FA1AD584040FEFC1FA9AF2040BC440000B8AD58402040FCBF94AF20405081FFBFC7AD58402040FCBF94AF2040ACA3FFBFD3AD584080A1FD1F92AF204098A2FF5FE1AD5840800EFF9F79AF2040D8D1FFDFE9AD5840005AFCBF4DAF2040F4C4FF5FEDAD5840809700E0FEAE204060A500C0EDAD5840C064FC7FC5AE2040544D0000EBAD5840E021FF5F84AE2040A877FF5FE2AD584020BE006014AE204090B0FF3FD0AD5840C0EBFFDF82AD204064E8FFBFABAD584000EDFC1F83AC2040C4EDFF9F87AD584020E8FBFFF1AB2040047DFF3F7EAD584000730020CEAB2040ECB5FF1F6CAD5840C06D0220AFAB204000B7FF7F5EAD58408065FFFFA9AB2040C4EDFF9F47AD58408065FFFFA9AB204014B8FFDFD0AC584040CDFB9FC6AB2040149BFFBF80AC584000CCFE5FC6AC2040640500E07BAC584040DFFE1FD1AC2040A051008074AC5840C047FC5FB5AC2040208DFFDF51AC584020C401E017AB2040F4C4FF5F4DAC584040DA0200CFAA2040C470006049AC5840C0BCFC3F68AA204060D3FEFF46AC58404017018029AA204048F5FF3F48AC58404019FF5FF1A920404C3E00C049AC5840808DFCBFBFA92040F47BFFDF4BAC5840A0DDFD1F97A9204000D4FF9F4EAC5840E00504A075A92040886D004052AC5840C09A000056A92040488FFF9F56AC584060C3FE5F33A9204068CBFF9F5BAC5840E0B001200DA92040048300C061AC5840C0D500A0E8A82040A8D7FE7F70AC5840A0960260A9A82040143B00A092AC58406084020011A82040243C0000A5AC5840205800C0A2A72040D8370080FBAC584060FB00C08BA5204018E4FF3F02AD5840401503A061A52040049AFF5F0EAD584000E700A00EA52040DCB4FFBF19AD584060520120BCA42040705D00A01EAD5840203003008DA42040FC56006030AD5840E0A0FC7FCAA32040C819000039AD5840E0F2FBBF69A32040DC8000003DAD584060D4FC7F3CA320405C5C00404CAD584040B3FEBF7FA2204084DBFF3F4FAD584060E6FFFF46A22040D85400A04BAD5840E00E0360B3A12040345A008047AD5840205B03006CA1204098A2FF5F41AD5840C04F02A02CA12040A4B1FF9FC2AC584040E301C00C9E2040CCE50040BCAC5840C0E0FD3FEE9D20405CF6FF9F9AAC5840200701E0759D204028B9FF3F83AC58404016FC1F289D2040E80C00807CAC5840A0EFFB9F129D2040F896012074AC5840006B01C0029D2040E0E0FF1F6BAC5840A0B5FB5FF29C204090330000F2AB584020F4FDFFF89B2040B06F0000B7AB5840C08B00C0149B2040A86000C0B5AB58408074FF3FEB9A20406C91FF5F9BAB584020ED03002F9920405004008089AB5840A05E0000519820401C2D00C083AB584060250340159820405C3F00207CAB584020D201C0E6972040201000A073AB5840809402A0C4972040F098FFFF3BAB5840C0B9FEFF2D9720400C7500E0F2AA5840203201E05496204074A0FF9FDCAA5840200BFD9F05962040B4B2FFFFD4AA5840E005FF9FE695204028850080C6AA5840E0F703C0A6952040740600404EAA5840404802000C932040ECD2FF3F3CAA58402052FF3F9F922040105800C002AA5840E0C800208C902040E42900A0ECA95840A06203A08C8F204098420040D3A958406052FC1F2D8E20404C3E00C0C9A9584000760360978D2040786C00E0BFA95840407BFC5F278D2040C4D0FF7F97A9584020E90060F38B2040FC56006090A95840C00BFE3FCD8B2040E4A6FFDF8AA95840A0B90300BD8B2040488FFF9F76A95840E0E500409C8B2040882400C070A958408048FFDF998B2040CCFCFFDF28A95840609EFFDF57902040C0A4FF1F26A9584040EE0360A1902040C0A4FF1F26A95840C0B2FFFFD49020402C4B004026A95840402E0020369120409C85FF3FF1A85840C07FFEBF0D922040D0450060AAA858402040FCBF94922040A84300A0A5A85840404902607E922040D0C2FF9FA8A85840803D0240949120407CCCFFFFADA85840609EFFDF5790204094F9FFBF71A8584080BA018012902040D0C2FF9F48A8584040DD00400991204018E4FF3F42A858402002FEDFC79020405CF6FF9F3AA8584080D2008091902040B0ECFF3F35A858402040FCBF94902040D8D1FFDF29A85840A092FFBF6D9120406C7A00C00EA85840A02202E068942040C8190000F9A7584040EF03C013972040B8ACFE7FF1A75840006DFF9FCA98204068AEFF7F0BA85840200403A03BA02040704000800EA85840A0A9FE5F7AA020409033000012A8584020E60220B9A02040345A008027A85840404301E07AA12040A494FF7F32A85840404301E07AA12040404C00A038A858406020004067A12040E44600C03CA85840C083016049A12040487800004AA858408017036046A120409476FFFF4FA858402031FC7F53A12040F46400405FA85840008D0200A4A12040ACC0FFDF63A85840A0BEFF1FBFA12040BCDEFF5F66A85840203F0340CEA12040D40B00206AA8584020D201C0E6A12040E06300E06CA85840800DFF3F07A2204058ADFF1F79A8584080A8FEFF07A32040ECD2FF3FBCA85840C0D500A0E8A82040E80C0080BCA85840604D00002BA92040E889FFBFBAA8584060100280D0AA2040C4530040B9A85840E05703E014AD2040D47100C0DBA85840E087FFFFF5B12040609FFF3FEAA8584040980360E3B2204070400080EEA85840C0E5FE5F7FB3204074890000F0A85840E001FCFFAAB5204074060040EEA85840C03FFDFFE9B52040609FFF3FEAA85840E050FDFF0FB62040A877FF5FE2A858404033FC3F38B620406482FF1FDAA85840E0DC01805EB62040F44D01A0D2A85840C0DA01C079B620404C3E00C0C9A858400016FF3F9AB620409C6E00A0C4A858402080FD7FB8B62040844100E0C0A8584080E50180F1B620406C140020BDA85840E084FCBF2CB72040CC620080BAA858408034038056B72040CC96FF3FB7A8584060A2028093B720401C4A00E0B3A85840A0E6FCDFD4B7204090B0FF3FB0A8584080D0FBBF1DB82040E4C3FFFF9AA858402074008040BB2040D83700809BA85840400CFDFF77BB20409093FF1FA0A85840405CFE5F4FBC2040B895FFDFA4A8584040AF0200F0BC2040D0C2FF9FA8A85840A0D102003CBD2040EC3800E0ADA85840201E008082BD2040AC260080B5A85840605802A0BFBD2040EC550000BEA85840005FFFBFFBBD204090330000D2A858408039FF9F58BE20407C150080EFA8584080AAFCDFCFBE204050E7FF5FF9A85840C0ECFF3FF5BE2040646B00800DA9584080D701A022BF204008C6FFBF1FA95840C07D00E045BF2040A0CEFFBF32A95840C0B1FF9F62BF20407CAFFFDF3DA958404090FD1F6CBF2040C470006049A958402055028068BF2040A417004054A9584000F0FF5F4CBF2040D02800405AA9584020E1010028BF204038D7FFBF85A9584080A6002040BE2040901600E0C1A95840C04102C05DBD20404C3E00C0C9A95840E043028042BD2040001D0020D0A95840808F018033BD2040A8DDFFFFD3A95840A0D102003CBD204018E4FF3F22AA5840000AFD3F93C02040E889FFBF9AAA584060C100806BC32040CC7F00A0CAAA5840C0B6FBBF64C42040243C0000C5AA5840609DFF7FE5C82040348EFF3FC4AA5840807802E026C9204098A2FF5FC1AA5840609600808CC92040E0FDFF3FBBAA5840C00D030041CA20409C570100B8AA5840602BFDDF6CCA2040947C0080B3AA5840E0EAFC5F9ECA20407CCCFFFFADAA58406080FF5FD5CA2040345A0080A7AA58402017FF9F0CCB20401884002094AA584080FB02A0A8CB2040048300C081AA584080400080CECB204008C6FFBF7FAA5840601E02609FCB20400C0F004061A958402088FCDF83D02040E4C3FFFF5AA958408000FFBFAAD02040B8FBFF7F56A958404061FF7FE0D020409C08000053A95840E0CCFCDF1BD120400CA9FF9F4FA958406008FC3F59D12040FC8AFF1F4DA958404071FF1F94D12040D0C2FF9F48A95840E040044008D22040BC61002048A95840009100A050D220404C3E00C049A95840804904409BD22040E06300E04CA95840400BFF7F22D320407C1500804FA95840608D01C04ED320401CC7FF1F52A95840200701E075D320409093FF1F60A95840C021046013D42040508700406BA95840808AFE7F85D42040B4B2FFFF94A95840000A024022D620405CF6FF9F9AA958406008FC3F59D62040D8D1FFDFA9A958400061FD9FC3D62040A877FF5FC2A9584060B603E065D7204090B0FF3FD0A95840E0C001C0C0D72040880D0120E4A95840C066014039D82040D85400A0EBA9584000A303205BD82040E80C0080FCA95840C013FD9F98D82040E87200200EAA58406015FE9FD2D82040105800C022AA5840606CFEFF02D920408458FF7F2DAA584000D5FD1F04D92040B06F000037AA584020E8FBFFF1D820407C3200A03FAA584000370200E6D82040345A008047AA584020CC0040E3D82040F44700204FAA5840C0CA0300E3D82040B8FBFF7F56AA584080CDFD7FE3D8204028B9FF3F63AA58400042FF9FEBD82040581300C06AAA58406060FCFFFBD8204090B0FF3F70AA5840607E01800DD92040B0ECFF3F75AA5840606A00203BD92040487800008AAA58400070FDDF04DA2040A0C8FE3F8FAA58400049FE9F44DA20401884002094AA584040CAFD5F8CDA2040CCDFFFBF98AA5840201FFEFFD7DA20406C1400209DAA58404070FFBF21DB2040807B0020A1AA58404090FD1F6CDB2040105E0140A6AA5840A09DFC5F73DD20402C4B0040A6AA5840A084FFDF9EDD2040A0340060A4AA5840609B01A01DE62040A4FAFF1FA4AA5840E0F8FC3F6DE6204018E4FF3FA2AA58408076FD1FB3E62040FC730080A0AA5840C04603E0EEE62040D47100C09BAA584060E9FD3F81E72040C4D0FF7F97AA584000200360D9E7204014B8FFDF90AA5840201AFDDF46E82040F44700208FAA5840803A04005AE82040581300C08AAA5840803A04005AE82040BC61002088AA5840C066014039E820408C50002082AA5840E0A7FD5F40E52040C0A4FF1F86AA5840A05403C0BDDF2040F4C4FF5F8DAA5840208EFD5F87DF20400C2C006091AA584000FC016053DF2040947C008093AA584020840020F4DE2040281F00E094AA5840E021FF5F84DE2040E4A6FFDF8AAA58408039FF9F58DB2040C02700E087AA584040C00020F9DA20401C2D00C083AA5840E078FFBFB4DA2040742300607EAA5840E0F300206BDA204058ADFF1F79AA5840403401A039DA204068AEFF7F6BAA58400059FE3FF8D92040ACC0FFDF63AA5840C03A01E0E7D92040A877FF5F42AA584040500160D7D92040B878FFBF34AA584080C1FB7FDCD920401867000024AA5840C0CA0300E3D92040E0E0FF1F0BAA584060D40360E8D9204040E6FFFFC6A95840600D044096D92040B4CFFF1F85A95840206EFFFF3CD82040AC26008075A9584080D9FF7FEAD72040D488FF5F68A95840C0BAFE5FA0D720409825002063A95840A0A3FDDF76D7204000B7FF7F5EA95840C082FCFF47D72040D8EEFFFF59A95840E05703E014D72040805E000051A958406023FE7FA1D620405C5C00404CA9584080E5FC7F62D62040E4DAFE9F47A95840A071FCFF21D620409825002043A958400095FC5FE0D52040F82A00003FA95840601DFDFF9DD52040581901402EA9584020CFFE7F1DD42040CC7F00A02AA9584080710100B1D32040406900C028A95840400AFF1FB0D22040406900C028A95840009100A050D2204018FBFEDF2EA9584020FE012038D12040046600A031A95840C0C3FD1FDED020402C68006036A9584000D6FD7F76D02040D83700803BA95840A03002C037D020402C7FFFFF42A95840803E02A006D02040D8D1FFDF49A95840C0E0FD3FEECF20401801006052A958408074FF3FEBCF2040B4B2FFFF54A95840C0E0FD3FEECF20406482FF1F5AA9584060500340F4CF2040149BFFBF60A9584060500340F4CF2040D0C2FF9F68A958404061FF7FE0CF204024D6FF5F73A95840C00E018096CF204058ADFF1F99A95840E08AFD3F30CE204030C8FF7FC4A95840202F03A01ACC2040784F00C00FAA5840805C00406CC82040347700A017AA5840C0F1FB5FF7C7204074A0FF9F1CAA5840A0D1FB1F90C720400C8CFF7F1FAA58402034FFBF1CC72040901600E021AA584040CAFD5F8CC620406C1400201DAA5840009E0020ADC52040947C008013AA5840209603A0FEC42040E87200200EAA58406063FF3FC5C42040FC3F01C003AA584060EFFEBF84C420406C7A00C0EEA95840800004C039C420405081FFBFE7A9584080DCFDBF24C4204010D5FFFFA0A85840C004044003C2204040CF00609AA8584020FB03E0FDC12040C0A4FF1F86A85840C062FE9FFDC12040F46400407FA85840606702E000C220408CCDFF5FE0A7584020E001A0B5C22040B06F0000D7A75840604A02C0F0C2204090330000D2A7584080DCFDBF24C320405890FFFFC8A75840E039036092C320409C6E00A0C4A7584040D40180CBC32040003A0040C0A75840A0D8FCFF05C4204074A0FF9FBCA75840C012FF1F43C42040E44600C05CA75840A075FF9F5DCE2040CC7F00A02AA75840600D044096D52040D40B00202AA75840E047FE3FD2D520406465FFFF29A75840404E03800FD62040609FFF3F2AA75840802801804FD62040DC1A00602BA75840C008000093D6204010BE006034A758402094FEDF8AD7204018E4FF3F62A75840E02700E087DB2040CC7F00A06AA75840407C01C028DC2040A8D7FE7F70A75840E0CDFC3F8EDC2040347700A077A758402095FE3FFDDC20405C3F00207CA75840C00702803DDD2040807B002081A7584000EDFC1F83DD20403C20004087A758404032FCDFC5DD2040ECD2FF3F9CA758408042FE5F96DE20403C030020B7A75840E0F901A06EDF204068CBFF9FBBA75840E0F901A06EDF204068CBFF9FDBA75840405101C049DF204008E3FFDFEFA7584060C3FE5F33DF204060BCFF5FFAA75840402E002036DF20407CAFFFDFFDA7584000DF014043DF204084BEFF1FFFA7584060CD028072DF204000B7FF7FFEA75840E01C034082DF2040003A0040C0A7584020F602C06CE02040046600A0B1A7584020A5FCFF93E020405CF6FF9F7AA75840C077FF5F42DE20409C85FF3F71A7584040500160D7DD20402C4B004066A7584020A2FEBF59DD204010D5FFFF60A75840404E03800FDD204074A0FF9F5CA758406063FF3FC5DC20409C08000053A75840408FFDBFF9DB2040149BFFBF40A75840C0F4FE9FC0DA2040003A004020A75840E0250200C0D82040D8D1FFDF09A75840A07EFE5F9BD720401867000004A7584040E4FF3F62D720405C3F0020FCA658402079FC9F42D7204024590020F5A65840408DFFDF31D72040F80D00E0EEA6584000BBFB3F2ED7204054CAFF3FE9A65840C054FEBF2ED72040ACC0FFDFE3A658402062FDFF35D72040404C00A0D8A65840601AFFBF63D72040DC1A0060CBA65840804E00609DD72040001D0020B0A65840C026008015D82040049AFF5FAEA65840603601601ED82040609FFF3FAAA65840205D01E033D8204008490080A1A6584000E8000081D820406C1400209DA6584020CEFE1FABD82040E07AFF7F99A65840602D02A0E0D82040C453004099A6584060A6FE3F23D92040DC8000009DA65840E07D02C062D9204094DCFF9FA1A65840408302A09ED9204028850080A6A6584080B102C0D4D92040BCC1FF3FB6A6584080E5FC7F62DA2040640500E0BBA65840C09DFE3F90DA20402CE5FF9FD4A65840E0A201403EDB2040C89C00C0DAA65840C04CFF5F63DB2040844100E0E0A6584060F0032086DB2040ACA3FFBFF3A65840E08D0080F9DB2040D0280040FAA65840A00D012024DC2040880700A000A75840C08F036050DC20403411000006A75840404902607EDC2040581300C00AA75840409FFD5FADDC2040F80D00E00EA7584080C1FB7FDCDC20401801006012A75840C01B03E00FDD2040B4B2FFFF14A75840208BFF1F4DDD2040BCDEFF5F26A758400020FC7F2DE12040A47D00E025A75840E01602C07EE120402C7FFFFF22A75840C064FC7FC5E12040047DFF3F1EA758404030FEFFFDE1204058ADFF1F19A75840E08AFD3F30E220401C4A00E013A75840C0ABFE1F5FE22040E87200200EA7584000F9FE1F8AE220405081FFBF07A7584080060040AEE22040E85B018001A75840E0DD01E0D0E22040E4C3FFFFFAA65840C0E6FEBFF1E220407C2CFF1FDCA658406048FDFF7CE320402C680060D6A65840C0D3FBDF74E320407CAFFFDFDDA6584020E0FC9F26E0204068CBFF9FFBA658400078FC3FD0DD2040807B002001A758406029FFFFA4DD204020F3FF7F03A75840A0B5006081DD204020F3FF7F03A7584060B0026062DD204098A2FF5F01A758408083FF7F2CDD2040CC680100FEA65840803E02A006DD2040E8EFFF5FECA6584080A1FD1F92DC2040640500E0DBA65840E0E400E029DC2040F8A7FF3F9DA65840C0D20260AEDA2040A877FF5F82A65840A010FF5F5EDA20406C1400207DA65840C0D9FC5F78DA204040C9FFDF76A65840408302A09EDA204018E4FF3F62A65840A0CB018038DB20401CAAFFFF41A6584080A2FD7F04DC2040B895FFDF24A6584020A000E091DC2040F01B00C01DA658408011FDDFB3DC2040488FFF9F16A6584000730020CEDC2040C02700E007A65840809D016002DD204008C6FFBFFFA5584080CBFF9F1BDD2040B43500C0F6A55840E0E9FCFF2BDD20408092FFBFEDA55840C00101003ADD2040A84300A0E5A55840C0FBFF7F36DD2040FCF0FFBFDEA5584000B0FE9F28DD2040404C00A0D8A55840400C04E023DD2040A494FF7FD2A5584080DCFDBF24DD2040ECB5FF1FCCA5584020E6FD1F2ADD2040901C0160C5A55840800004C039DD20401C4A00E0B3A55840604EFE7F80DD20402487FE5FAEA55840809402A0C4DD20407CCCFFFFADA55840600E04A008DE2040FC560060B0A55840600901A05ADE20409093FF1FC0A558400001FE7F55DF2040047DFF3FBEA55840200D026079DF204074A0FF9FBCA55840E093FCFF6DDF20407CCCFFFFADA55840C009006005DF2040DC97FF9FA9A5584080CDFD7FE3DE204040E6FFFFA6A55840005BFC1FC0DE20409C6E00A0A4A55840C0B4FDDF9CDE2040105800C0A2A55840A039018075DE204010D5FFFFA0A5584000EAFEDF48DE204084BEFF1F9FA5584020BE006014DE20407C3200A09FA55840E0E30260D4DD204098250020A3A55840E03EFF7F94DD20405081FFBFA7A55840A0FEFBDF53DD2040F01B00C0BDA55840A03F020079DC2040B81800A0C6A5584020DB008024DC20402453FF9FD1A558400001FE7F55DC2040F8A7FF3FDDA5584060F0032086DC2040D0C2FF9FE8A55840E03EFF7F94DC204024590020F5A5584020A000E091DC2040705D00A0FEA55840A03F020079DC2040049AFF5F0EA65840404B004046DC2040CC96FF3F17A6584080DCFDBF24DC20407C3200A01FA65840E0280040FADB2040280200C044A65840404E03800FDB2040E0FDFF3F5BA65840408B01006ADA2040F08100606FA65840A0E0FB5FD1D92040A417004074A658402063FD5FA8D920400C0F0040E1A658406032FEBFE2D420406C7A00C0EEA658400032FFFF37D42040143B00A0F2A65840A0D2FB7F02D420408CCDFF5F00A75840A0DEFD7F09D320405C5C00400CA75840A03200A0FFD02040F80D00E00EA75840603E00C0E9CF2040784F00C00FA7584060BAFF9FF5CD2040646B00800DA75840007CFFDF0BCD2040C4EDFF9F07A7584040CC022000CC20405CF6FF9FFAA6584040A5FEDFB0C9204024590020F5A6584060F9FBFF17C9204044ACFFBFE6A65840A08D03A06BC82040E4BDFE7FB7A65840809A0320C8C62040CCFCFFDF88A6584040330140C7C520400400000080A6584000870180A0C52040B8FBFF7F76A658400058FEDF85C520405CD9FF7F6AA6584000BA02C067C52040D8EEFFFF39A65840E039FE5F03C52040A0CEFFBF32A6584060EB0200F5C42040D8D1FFDF29A65840A026FE9FF8C42040B81800A026A6584080270300FAC42040149BFFBF20A6584080270300FAC42040DC8000001DA658404089FC3FF6C42040A466014019A65840A0040260E6C4204030C8FF7F04A658404046042044C32040600B0160FFA5584000140160D2C22040F098FFFFFBA5584080FD008070C22040E4C3FFFFFAA5584000DF014043C220405CF6FF9FFAA558402057006030C220405CF6FF9FFAA5584080FBFBBFFCC1204084BEFF1FFFA55840802603A087C120409825002003A65840E0E7FE1F64C120408092FFBF0DA65840C00EFC7F07C12040B4B2FFFF14A65840A0ED02C0D9C02040F4E1FF7F1DA658408000FFBFAAC02040683100402DA6584000C003406BC02040543000E03AA65840C06000C035C0204094DCFF9F41A65840E05CFFFF16C0204048F5FF3F48A65840C0E70320F3BF204000D4FF9F4EA65840E0A0FC7FCABF204050E7FF5F59A65840001F030067BF204028B9FF3F63A6584080FBFBBFFCBE204040E6FFFF66A65840C0F4FE9FC0BE2040D04500606AA6584060E9FD3F81BE2040544D00006BA65840C066014039BE2040609FFF3F6AA65840A01B0100F3BD2040B009006065A65840A055FC3F84BD20400400000060A65840604804E028BD204070DAFFDF5CA6584000B8FDFFF3BC2040B4B2FFFF54A658408073FFDF78BC2040ACC0FFDF43A65840004204A07ABB2040FC6DFFFF3CA6584000B6FF1F2CBB2040B8FBFF7F36A6584080D9FF7FEABA204074EF00A021A65840C06000C035BA20402CE5FF9F14A65840404A00E0D3B9204064E8FFBF0BA65840006FFD7F92B92040B895FFDF04A65840A02803606CB92040BCC1FF3FF6A5584020E702802BB92040786C00E0DFA558408057FF1FDBB8204030ABFF5FD4A55840801DFFDFBAB82040901600E0C1A55840A0C6FE7F8AB82040208DFFDF51A55840C05DFD7F6CB72040CCFCFFDF48A5584080DAFFDF5CB72040CC96FF3F37A5584080C8FC5F52B7204008E3FFDF2FA55840E0D0FF7F57B7204048D8FF1F18A5584000E7FB9F7FB72040F80D00E00EA5584020B7FF7F9EB7204038D7FFBF05A55840C0A101E0CBB7204000B7FF7FFEA45840804403200AB82040BC440000F8A45840E01DFCBF48B8204018010060F2A458400052FD5F82B820408C92569DD1A45840C00AC1A209BA2040A0510080D4A45840E0530040D9B92040A8DDFFFFD3A4584060D2FE9F74B92040046600A0D1A45840E0E3FB7F28B920409C6E00A0C4A458402062FDFF35B82040F01B00C0BDA45840403C000005B82040F06A01C062A45840C01CFC5FD6B52040E0FDFF3F5BA45840203F0340CEB520407489000050A45840E02B0380C3B52040684E00603DA45840E013FF7FB5B52040208DFFDF31A458402080FD7FB8B520405CF6FF9F1AA4584080520300D9B520401801006012A45840C0DB0120ECB5204018670000E4A3584040DEFEBF5EB62040D47100C0DBA35840C0E5FE5F7FB62040C00A00C0D7A35840205800C0A2B620408CEAFF7FD0A35840A03200A0FFB6204048F5FF3FC8A358406057FD3FBEB720403C9DFF7FC5A35840E0C6FB5F18B82040B4CFFF1FC5A35840C04102C05DB8204094E20020C5A3584040ABFF5FB4B82040D0C2FF9FC8A35840A09F0120E7B9204048780000CAA35840000D00805CBA20406C7A00C0CEA35840C04CFF5F63BB204014B8FFDFD0A35840C0AEFC5F99BB204098420040D3A35840E0420220D0BB204044120060D8A3584040AAFFFF41BC2040D47100C0DBA35840201303E07CBC2040C8B3FF5FE7A35840E09C00C03ABD2040F4470020EFA35840A03D0420B1BD20409C080000F3A358406097FEFFE1BD2040A034006004A458400044FD7FB3BE2040347700A017A45840E0D3FDBF91BF2040A877FF5F22A45840A037FCBF01C02040D0C2FF9F28A4584000DEFCDF41C02040187EFF9F30A45840003C032077C02040F46400403FA45840C0DAFCBFEAC02040A86000C055A45840205DFCDFA4C120403C8600E078A4584000900040DEC22040B05200E086A4584040D40180CBC32040CCFCFFDF88A4584080FFFE5F38C42040FC84FE9F89A45840602403E0A2C42040DC97FF9F89A45840A0AD0100B6C52040F06A01C082A45840802EFDFFC3C6204008C6FFBF7FA45840C074012008C72040D02800407AA458402037FDFF56C72040D03FFFDF46A458400077FCDF5DC92040B06F0000D7A35840A038012003CC20405004008049A3584060BFFBBFF7CD20400849008041A35840A018FEBF29CE2040B086FF9F23A35840405F01A018CF2040047DFF3F1EA35840E01DFCBF48CF2040ED63AB431DA358402483132053CF2040CB7FC7CA27A358400EEBCFB855D22040EE8C4D3747A35840FBF1629802D82040498189EB71A358402835CA179DDD204055926ABDA7A35840F85CD7A91FE3204053A6D977E8A35840C0F8F2D884E82040BC0CEDDA33A4584038CA864CC7ED20404901279C89A4584018BF4BCEE1F22040C1BEBE66E9A45840A74E764FCFF7204058DCF3DB52A558402910BCED8AFC204073A56B93C5A5584073842DF80F0121409911981B41A65840CB36DFF359052140DA2767CB88A6584080A0DE6B8C0721404DF827FAC4A65840C4885DA064092140D31280AC50A7584061B0E6FB2B0D2140C5543CA8E3A758400CA96647AC102140821BB95B7DA858408014310AE21321402AACA32E1DA95840E04A7515CA16214034709182C2A958401D1E69876119214050599DB36CAA5840391E27CEA51B2140CFCE0A191BAB58406F783DAA941D2140C080ED05CDAB58400EDBEA302C1F2140B477D5C981AC5840471A07CE6A20214087B37EB138AD584014A795454F2121409BA78307F1AD5840984000B5D8212140FCDF1115AAAE58405CA1F893062221405D18A02263AF5840984000B5D8212140700CA5781BB0584014A795454F21214043484E60D2B05840471A07CE6A202140373F362487B158400EDBEA302C1F214028F1181139B258406E783DAA941D2140A7668676E7B25840391E27CEA51B2140C34F92A791B358401D1E698761192140CD1380FB36B45840E14A7515CA16214076A46ACED6B458408014310AE2132140326BE78170B558400CA96647AC10214025ADA37D03B6584061B0E6FB2B0D2140AAC7FB2F8FB65840C4885DA0640921401E98BC5ECBB6584080A0DE6B8C072140 +1 7 01030000000100000068000000E00F2635149E584000827AAF311E0C40D705FD20139E58408A222B0FDD1D0C40EDDC0525C39D584078AF2D2421090C405DFFE50D699D5840F937295D0FF50B402DBAE434059D5840B2603399BBE10B407748F0FC979C58401DDF1AFB38CF0B40EAB33BD2219C58403DA379D699BD0B407F93D329A39B5840816C949DEFAC0B40CB122A811C9B584037A619D04A9D0B40DAB49A5D8E9A5840278AD0EABA8E0B407A4DE64BF9995840599B48584E810B4063B3A7DF5D995840A98C986212750B40C6B4C1B2BC985840C59B3A26136A0B4048DEC664169858401D4F13865B600B4063AA5B9A6B975840665FAE20F5570B4093B493FCBC965840C263BC46E8500B40A8904A380B965840A29BDAF23B4B0B407CEA78FD569558408EECACC2F5460B40579686FEA094584044DD50F119440B40CE3C9AEFE9935840D0032F53AB420B408D4FE78532935840D0032F53AB420B4004F6FA767B92584044DD50F119440B40DFA10878C59158408EECACC2F5460B40B3FB363D11915840A29BDAF23B4B0B40C8D7ED785F905840C263BC46E8500B40F9E125DBB08F5840665FAE20F5570B4013AEBA10068F58401D4F13865B600B4096D7BFC25F8E5840C59B3A26136A0B40F8D8D995BE8D5840A98C986212750B40E13E9B29238D5840599B48584E810B4081D7E6178E8C5840288AD0EABA8E0B40907957F4FF8B584037A619D04A9D0B40DCF8AD4B798B5840816C949DEFAC0B4071D845A3FA8A58403DA379D699BD0B40E4439178848A58401DDF1AFB38CF0B402ED29C40178A5840B3603399BBE10B40FF8C9B67B3895840F937295D0FF50B406EAF7B505989584078AF2D2421090C4085868454098958408A222B0FDD1D0C407C7C5B400889584000827AAF311E0C4020D5FDC2C3885840D3966D962E330C40A513E1E0888858404DB7F29D00490C401CDA94E85888584074284D8A3D5F0C40BAB9B20934885840B9990656CF750C406EC0D7681A88584058796BA79F8C0C40B5D67F1F0C885840A0C4AAE697A30C40761DEC3B098858406A193454A1BA0C40AB6614C111885840EAEC3D1FA5D10C40E6D8A3A62588584042A15D7C8CE80C40A0C200D944885840DD1E1BBC40FF0C405B975F396F885840FB966961AB150D406805E19DA4885840B22CEF37B62B0D405708BBD1E48858404B6E056A4B410D4003D16C952F89584036CC5C9655560D406F51FD9E8489584047932DE5BF6A0D400F30449AE38958402B5EE11C767E0D409BDB3C294C8A5840FC7420B664910D405D6E63E4BD8A5840CD292FEF78A30D4017061B5B388B58408FEC87DEA0B40D40F82B1D14BB8B58400E909F84CBC40D40F6DEF18D458C5840E80BC2DCE8D30D400ACA6F3FD78C5840F6EAF6ECE9E10D408F2744986F8D5840618BDDD4C0EE0D400BCB81010E8E58407E5A72DB60FA0D406BC336DEB18E5840524BAF7BBE040E40E400088C5A8F584077E9FA6FCF0D0E409463D26307905840F6995ABC8A150E40319250BAB7905840DCD65DB7E81B0E4066F3C4E06A9158404A79B911E3200E400120A7252092584073748BDC74240E40942054D5D692584002BC408E9A260E402EC6C03A8E935840E96F180652270E40C76B2DA04594584002BC408E9A260E405A6CDA4FFC94584073748BDC74240E40F598BC94B19558404A79B911E3200E402BFA30BB64965840DCD65DB7E81B0E40C728AF1115975840F4995ABC8A150E40778B79E9C197584077E9FA6FCF0D0E40F0C84A976A985840504BAF7BBE040E4050C1FF730E995840805A72DB60FA0D40CC643DDDAC995840638BDDD4C0EE0D4052C21136459A5840F4EAF6ECE9E10D4066AD8FE7D69A5840E80BC2DCE8D30D4063606461619B58400E909F84CBC40D404486661AE49B58408EEC87DEA0B40D40FE1D1E915E9C5840CD292FEF78A30D40C0B0444CD09C5840FC7420B664910D404C5C3DDB389D58402B5EE11C767E0D40EC3A84D6979D584047932DE5BF6A0D4058BB14E0EC9D584037CC5C9655560D400484C6A3379E58404B6E056A4B410D40F386A0D7779E5840B22CEF37B62B0D4001F5213CAD9E5840FB966961AB150D40BBC9809CD79E5840DD1E1BBC40FF0C4076B3DDCEF69E584042A15D7C8CE80C40B0256DB40A9F5840EAEC3D1FA5D10C40E56E9539139F58406A193454A1BA0C40A7B50156109F5840A0C4AAE697A30C40EECBA90C029F584058796BA79F8C0C40A1D2CE6BE89E5840B9990656CF750C4040B2EC8CC39E584074284D8A3D5F0C40B678A094939E58404EB7F29D00490C403BB783B2589E5840D3966D962E330C40E00F2635149E584000827AAF311E0C40 +1 8 0103000000010000006A0000007045DA4700805B40BDCC7990E1A5D93FD9AE32B3E17F5B40B31D1733C48CD93F4F399AE74C7F5B40C66C32E95A21D93F4EA7A9C3B17E5B40F58994CC76BFD83F334E06E1107E5B40C36DBFCF7867D83F45D605DF6A7D5B401D0F8017B819D83F47741062C07C5B403032A2A481D6D73F201DFE12127C5B40CDDDAE07189ED73FAE546F9E607B5B40D107011FB370D73F553D22B4AC7A5B40B3CC71DF7F4ED73F7E914406F7795B400608D327A037D73F7133C34840795B40676F649F2A2CD73F3F02983089785B40676F649F2A2CD73F32A41673D2775B400608D327A037D73F5BF838C51C775B40B3CC71DF7F4ED73F02E1EBDA68765B40D107011FB370D73F90185D66B7755B40CDDDAE07189ED73F69C14A1709755B403032A2A481D6D73F6B5F559A5E745B401C0F8017B819D83F7DE75498B8735B40C36DBFCF7867D83F628EB1B517735B40F58994CC76BFD83F61FCC0917C725B40C66C32E95A21D93FD78628C6E7715B40B31D1733C48CD93FF90B45E659715B4020F90C4A4801DA3F2A07997ED3705B40C00F82C9737EDA3F5D6E411455705B401436CDBACA03DB3F1AE27124DF6F5B40DD96EF0FC990DB3F7AB2F823726F5B4028385926E324DC3FB833CB7E0E6F5B40CAF72E5186BFDC3FB0D49A97B46E5B40E119896A1960DD3F036173C7646E5B40F9A11A6BFD05DE3F78CF625D1F6E5B403B5BAB078EB0DE3FD7F32A9EE46D5B4023B4C853225FDF3FA161FDC3B46D5B40B83686B48608E03F13C341FE8F6D5B40F6C92989CF62E03F9597EDD7896D5B400008EABDC678E03F83DD6671766D5B40ED2C1E3D12BEE03F9971BE36686D5B40C5131F6EF419E13F4E1B645C656D5B408753D61B1B76E13FAD4A2FE56D6D5B40A954FA012BD2E13F6363B0C8816D5B408944B0F2C82DE23F3E0639F3A06D5B40F693D8309A88E23FB37CEF45CB6D5B40E163ECC944E2E23F7733ED96006E5B4005ED12EF6F3AE33F682668B1406E5B40A7B1164DC490E33F2415E7558B6E5B40A862E362ECE4E33F0C4D803AE06E5B40C9D036D69436E43FE8CA220B3F6F5B40D90631C66C85E43F3C6CE969A76F5B4007C1711B26D1E43F4BDE77EF18705B40A5DD73D57519E53F28EF602B93705B40401ADA54145EE53FDDDB95A415715B40B57363A2BD9EE53FB32DDED99F715B4082CB40B231DBE53F37B0574231725B400E0889A33413E63F5B01FE4DC9725B405BBA8CFB8E46E63FA235396667735B407368CEDC0D75E63F0B0373EE0A745B402FEB6839839EE63F03DFB144B3745B4068D4B100C6C2E63F947539C25F755B407C8BEA47B2E1E63FABDA2FBC0F765B4056B4D76D29FBE63FAFD14684C2765B40D6841B39120FE73F4583686977775B403DDF34F1581DE73FDDF566B82D785B402F5A0A72EF25E73FD89AADBCE4785B4054E6ED39CD28E73FD43FF4C09B795B402F5A0A72EF25E73F6CB2F20F527A5B403DDF34F1581DE73F016414F5067B5B40D6841B39120FE73F055B2BBDB97B5B4056B4D76D29FBE63F1CC021B7697C5B407C8BEA47B2E1E63FAD56A934167D5B4068D4B100C6C2E63FA532E88ABE7D5B402EEB6839839EE63F0E002213627E5B407468CEDC0D75E63F55345D2B007F5B405CBA8CFB8E46E63F7A850337987F5B400F0889A33413E63F7045DA4700805B401B45BA881EEBE53FFD077D9F29805B4082CB40B231DBE53FD459C5D4B3805B40B57363A2BD9EE53F8846FA4D36815B40401ADA54145EE53F6657E389B0815B40A5DD73D57519E53F74C9710F22825B4007C1711B26D1E43FC86A386E8A825B40D90631C66C85E43FA4E8DA3EE9825B40C9D036D69436E43F8C2074233E835B40A862E362ECE4E33F480FF3C788835B40A6B1164DC490E33F39026EE2C8835B4006ED12EF6F3AE33FFDB86B33FE835B40E163ECC944E2E23F722F228628845B40F693D8309A88E23F4ED2AAB047845B408A44B0F2C82DE23F03EB2B945B845B40AA54FA012BD2E13F621AF71C64845B408753D61B1B76E13F17C49C4261845B40C5131F6EF419E13F2D58F40753845B40EE2C1E3D12BEE03F1B9E6DA13F845B400008EABDC678E03F9D72197B39845B40F6C92989CF62E03F0FD45DB514845B40B93686B48608E03FD94130DBE4835B4025B4C853225FDF3F3866F81BAA835B403B5BAB078EB0DE3FADD4E7B164835B40FBA11A6BFD05DE3F0061C0E114835B40E119896A1960DD3FF80190FABA825B40CAF72E5186BFDC3F3683625557825B4029385926E324DC3F9653E954EA815B40DD96EF0FC990DB3F53C7196574815B401536CDBACA03DB3F872EC2FAF5805B40C00F82C9737EDA3FB72916936F805B4022F90C4A4801DA3F7045DA4700805B40BDCC7990E1A5D93F +1 9 0103000000010000006A0000007045DA4700805B403DC2F2C2EF94DD3F1D7DB110F37F5B40F32ABB2A557DDD3F7455746B8F7F5B40C00AB703B2E2DC3FC531EB6A227F5B40529044F1974EDC3FA67C0B7BAC7E5B40E54A30A099C1DB3F0B9AA3102E7E5B40D338FBB2423CDB3FA73BE7A8A77D5B407605953717BFDA3FE065F3C8197D5B4011BF9724934ADA3FC49F4AFD847C5B40387F86DE29DFD93F6FD049D9E97B5B40349588C5457DD93F815396F6487B5B407A5E12CC4725D93FF3D485F4A27A5B405225E41687D7D83FDD8A8077F8795B401E21BEA65094D83F58695E284A795B40E6181F0CE75BD83F8FF1BFB398785B40E1375925822ED83F9A4163C9E4775B403F673EE74E0CD83F500E761B2F775B40E9029B306FF5D73F7E33E55D78765B40EC09ABA8F9E9D73F2389AA45C1755B40EC09ABA8F9E9D73F51AE19880A755B40E9029B306FF5D73F077B2CDA54745B403F673EE74E0CD83F12CBCFEFA0735B40E2375925822ED83F4A53317BEF725B40E6181F0CE75BD83FC4310F2C41725B401E21BEA65094D83FAEE709AF96715B405125E41687D7D83F2069F9ACF0705B407A5E12CC4725D93F32EC45CA4F705B40349588C5457DD93FDD1C45A6B46F5B40387F86DE29DFD93FC1569CDA1F6F5B4010BF9724934ADA3FFA80A8FA916E5B407405953717BFDA3F9622EC920B6E5B40D338FBB2423CDB3FFB3F84288D6D5B40E44A30A099C1DB3FDC8AA438176D5B40529044F1974EDC3F2D671B38AA6C5B40BE0AB703B2E2DC3F843FDE92466C5B40F32ABB2A557DDD3F769B9EABEC6B5B40DD267740E81DDE3F706168DB9C6B5B40B21CAB3DCCC3DE3FD9A44971576B5B4059E82AD75C6EDF3F975704B21C6B5B4066684790780EE03F652CCAD7EC6A5B409B06BD196E67E03FF39221C7E56A5B400008EABDC678E03F78ED0212C86A5B4053CD68EDB6C1E03F67801D85AE6A5B40BAFB9CA0F91CE13FF9C56B4AA06A5B40148E17D1DB78E13FAB7909709D6A5B404EBB837E02D5E13FF829CEF8A56A5B40AE6298641231E23F74574ADCB96A5B40353C7A55B08CE23FD8BDCF06D96A5B40A259089481E7E23F12BF8459036B5B404799B92D2C41E33F6EDD82AA386B5B404617B2535799E33F232700C5786B5B404C6CB8B2ABEFE33F8A6A8369C36B5B40D5A2B3C9D343E43FC9FF224E186C5B40703B5C3E7C95E43F3BEACD1E776C5B40965CCC2F54E4E43F84099F7DDF6C5B40F1619D860D30E53F6B083A03516D5B40EA6643425D78E53FCEAD313FCB6D5B40201F5AC3FBBCE53FB92A77B84D6E5B40D3529912A5FDE53FB7F7D1EDD76E5B404EA12A24193AE63FDFCA5F56696F5B4066BE1D171C72E63F22281C6201705B400635BC7076A5E63F9C056F7A9F705B40FCCD8153F5D3E63FCCF6C10243715B40A20883B16AFDE63FC54A1B59EB715B40BE9A0F7AAD21E73FC683BED697725B4053A463C29940E73FE088D1D047735B40FF2B3FE9105AE73F01EE0599FA735B40D18640B5F96DE73F2EAA457EAF745B403E82E36D407CE73F9B9062CD65755B407F780CEFD684E73F51DEC7D11C765B40E1CA0BB7B487E73F062C2DD6D3765B407F780CEFD684E73F73124A258A775B403E82E36D407CE73FA0CE890A3F785B40D18640B5F96DE73FC133BED2F1785B40FF2B3FE9105AE73FDB38D1CCA1795B4053A463C29940E73FDC71744A4E7A5B40BE9A0F7AAD21E73FD6C5CDA0F67A5B40A30883B16AFDE63F05B720299A7B5B40FCCD8153F5D3E63F7F947341387C5B400735BC7076A5E63FC2F12F4DD07C5B4066BE1D171C72E63FEAC4BDB5617D5B404EA12A24193AE63FE89118EBEB7D5B40D3529912A5FDE53FD30E5E646E7E5B40201F5AC3FBBCE53F36B455A0E87E5B40EA6643425D78E53F1DB3F0255A7F5B40F3619D860D30E53F66D2C184C27F5B40965CCC2F54E4E43F7045DA4700805B40843D5377F8B0E43FD8BC6C5521805B40703B5C3E7C95E43F17520C3A76805B40D6A2B3C9D343E43F7E958FDEC0805B404A6CB8B2ABEFE33F33DF0CF900815B404717B2535799E33F8FFD0A4A36815B404799B92D2C41E33FC9FEBF9C60815B40A259089481E7E23F2D6545C77F815B40363C7A55B08CE23FA992C1AA93815B40AE6298641231E23FF64286339C815B404EBB837E02D5E13FA8F6235999815B40158E17D1DB78E13F3A3C721E8B815B40BBFB9CA0F91CE13F29CF8C9171815B4053CD68EDB6C1E03FAE296EDC53815B400008EABDC678E03F3C90C5CB4C815B409C06BD196E67E03F0A658BF11C815B4067684790780EE03FC8174632E2805B405BE82AD75C6EDF3F315B27C89C805B40B21CAB3DCCC3DE3F2B21F1F74C805B40DD267740E81DDE3F7045DA4700805B403DC2F2C2EF94DD3F +1 10 010300000001000000680000009FEECE9BDF1B5A4000BF4228E7F011C099D9A50DD81B5A400E132A1D01F711C07864AA79BE1B5A40DEA0C362690212C074C7AEA3991B5A40A95DD678B20D12C0EAA00EB0691B5A4060159A31D11812C0EFB329CE2E1B5A40652B1D89BA2312C094773538E91A5A40676F2FB0632E12C0BAE20333991A5A409F291917C23812C014ABBF0D3F1A5A40EAB81378CB4212C0CB389E21DB195A40475979E1754C12C0719A87D16D195A4082EFA1BFB75512C019CFB489F7185A40550763E6875E12C036C644BF78185A401E8C299ADD6612C0587EC8EFF1175A401B28A498B06E12C0F6B4C6A063175A40C5A3F520F97512C006A2375FCE165A402B1567FBAF7C12C06B42F9BE32165A40A12D9280CE8212C0DCBB3C5A91155A401D7BFC9F4E8812C00E6CEDD0EA145A403E001DE62A8D12C0773B12C83F145A404A1BC6815E9112C006D229E990135A404C4FEE48E59412C0705182E1DE125A408623D3BCBB9712C05A3D8D612A125A4059F3710DDF9912C0CA3D301C74115A40C132531C4D9B12C05F6B13C6BC105A40C859A57E049C12C097D7ED1405105A40C859A57E049C12C02C05D1BE4D0F5A40C132531C4D9B12C09C057479970E5A4059F3710DDF9912C086F17EF9E20D5A408623D3BCBB9712C0F070D7F1300D5A404C4FEE48E59412C07F07EF12820C5A404A1BC6815E9112C0E8D6130AD70B5A403E001DE62A8D12C01A87C480300B5A401D7BFC9F4E8812C08B00081C8F0A5A40A12D9280CE8212C0F0A0C97BF3095A402B1567FBAF7C12C0008E3A3A5E095A40C5A3F520F97512C09EC438EBCF085A401B28A498B06E12C0C07CBC1B49085A401E8C299ADD6612C0DD734C51CA075A40560763E6875E12C085A8790954075A4082EFA1BFB75512C02B0A63B9E6065A40475979E1754C12C0E29741CD82065A40EAB81378CB4212C03C60FDA728065A409F291917C23812C062CBCBA2D8055A40676F2FB0632E12C0078FD70C93055A40652B1D89BA2312C00CA2F22A58055A4060159A31D11812C0827B523728055A40A95DD678B20D12C07EDE566103055A40DEA0C362690212C05D695BCDE9045A400E132A1D01F711C05754323FE2045A4000BF4228E7F011C085149494DB045A4020B994F484EB11C0C7C1F4C4D8045A4073AD1F4900E011C04EF22361E1045A40B28833837ED411C048AE7860F5045A40E61739080BC911C02C9E03AF14055A40DC8E502FB1BD11C0984AA32D3F055A40E05F17367CB211C0C56C23B274055A4006D2873577A711C0AF2E6707B5055A405D53FD16AD9C11C0802F9EEDFF055A40C45E6889289211C02716841A55065A406C9CBCF6F38711C0D071AA39B4065A40E0A8A379197E11C0059ECCEC1C075A40E2A97ED3A27411C060562DCC8E075A40E288C062996B11C0909CFD6609085A40FE52AA19066311C0348ACC438C085A4062DC7275F15A11C0B29FFFE016095A40B45AE275635311C0CB1853B5A8095A4063376A95634C11C021C86130410A5A409FDFC0C1F84511C080F333BBDF0A5A40D7D50855294011C036A4D4B8830B5A40E9BE890FFB3A11C03FD8EC862C0C5A40A4950112733611C0CFFB637ED90C5A405E9993D8953211C0500D05F4890D5A4043F15936672F11C0C2C927393D0E5A40035E9E51EA2C11C0443B5D9CF20E5A4092AFBDA0212B11C093001F6AA90F5A40A70CB9E70E2A11C07BA180ED60105A40116E7736B32911C06342E27018115A40A70CB9E70E2A11C0B207A43ECF115A4092AFBDA0212B11C03479D9A184125A40035E9E51EA2C11C0A635FCE637135A4043F15936672F11C027479D5CE8135A405E9993D8953211C0B76A145495145A40A4950112733611C0C09E2C223E155A40E9BE890FFB3A11C0764FCD1FE2155A40D7D50855294011C0D57A9FAA80165A409FDFC0C1F84511C02B2AAE2519175A4062376A95634C11C044A301FAAA175A40B45AE275635311C0C2B8349735185A4062DC7275F15A11C066A60374B8185A40FE52AA19066311C096ECD30E33195A40E288C062996B11C0F1A434EEA4195A40E1A97ED3A27411C026D156A10D1A5A40E0A8A379197E11C0CF2C7DC06C1A5A406C9CBCF6F38711C0761363EDC11A5A40C45E6889289211C047149AD30C1B5A405C53FD16AD9C11C031D6DD284D1B5A4006D2873577A711C05EF85DAD821B5A40E05F17367CB211C0CAA4FD2BAD1B5A40DC8E502FB1BD11C0AE94887ACC1B5A40E61739080BC911C0A850DD79E01B5A40B18833837ED411C02F810C16E91B5A4073AD1F4900E011C0712E6D46E61B5A4020B994F484EB11C09FEECE9BDF1B5A4000BF4228E7F011C0 +1 11 010300000001000000680000007045DA4700E059406F5394D093720AC0628A7D5FF2DF5940C4E7E0021A5C0AC01261D08DEFDF59401CE71D8310450AC0D2CCE223F8DF5940E3647AC80C2E0AC07045DA4700E05940BFEC464BB5240AC0A7F71A190CE05940BAC4039E25170AC05B749C592BE059408E6E4DB271000AC06D485CC655E05940B9B0FB8007EA09C0430440358BE059405DC5813CFDD309C0A1C94771CBE05940F9F928B868BE09C06D15C33A16E1594032AE76525FA909C0991790476BE15940447E06E0F59409C062596543CAE159400E79ED96408109C05A6825D032E25940B0B8BAFA526E09C0A7323C86A4E25940831729C93F5C09C0ADB705F51EE359404B0B95E7184B09C019A73DA3A1E35940AFED4851EF3A09C0AD7F770F2CE45940BA24B106D32B09C0F5B69EB0BDE459407BB888FDD21D09C0136B7EF655E5594083ED0D12FD1009C0B118504AF4E559403F774DF95D0509C04BC8500F98E65940F9C1913401FB08C0911F5CA340E75940B4B40206F1F108C03EBE8C5FEDE75940611E816636EA08C04348E1989DE8594041CBC7FCD8E308C0E67AE5A050E959405DFEDA15DFDE08C056A65EC605EA594079C1CD9E4DDB08C003E2FA55BCEA5940932DE31F28D908C0654F029B73EB5940177610B970D808C0C8BC09E02AEC5940942DE31F28D908C075F8A56FE1EC59407AC1CD9E4DDB08C0E5231F9596ED59405DFEDA15DFDE08C08856239D49EE594040CBC7FCD8E308C08DE077D6F9EE5940611E816636EA08C03A7FA892A6EF5940B5B40206F1F108C080D6B3264FF05940F7C1913401FB08C01A86B4EBF2F0594040774DF95D0509C0B833863F91F1594083ED0D12FD1009C0D6E7658529F259407BB888FDD21D09C01D1F8D26BBF25940BA24B106D32B09C0B2F7C69245F35940AFED4851EF3A09C01EE7FE40C8F359404A0B95E7184B09C0246CC8AF42F45940831729C93F5C09C07136DF65B4F45940B0B8BAFA526E09C069459FF21CF559400D79ED96408109C0328774EE7BF55940447E06E0F59409C05D8941FBD0F5594031AE76525FA909C02AD5BCC41BF65940F8F928B868BE09C0889AC4005CF659405DC5813CFDD309C05E56A86F91F65940B9B0FB8007EA09C0702A68DCBBF659408E6E4DB271000AC024A7E91CDBF65940B9C4039E25170AC0F9D12112EFF65940E2647AC80C2E0AC0B93D34A8F7F659401CE71D8310450AC0691487D6F4F65940C4E7E0021A5C0AC0A6FDCB9FE6F65940DA13CE7612730AC02CD7FD11CDF659400CD0A01EE3890AC036405346A8F65940D11C4E6175A00AC07703266178F65940D16667E3B2B60AC07576CF913DF65940FA14419D85CC0AC0F0EF7912F8F5594070DEC6F0D7E10AC0F881E727A8F559405B40E8BE94F60AC0DA2E2E214EF55940B5CF877CA70A0BC0CFDA6957EAF45940BE9BD846FC1D0BC06446642D7DF45940F56315F77F300BC0C667330F07F45940F6057E3520420BC0BA82CE7188F35940CB3D888BCB520BC063689AD201F35940FE9B317571620BC00A51EDB673F25940A36F617102710BC082CA8AABDEF15940D14F4A11707E0BC0F43D184443F15940EDECBB06AD8A0BC059978A1AA2F0594082DF5631AD950BC0FB9F8DCEFBEF5940F84595AA659F0BC0E6A3E50451EF5940E82B9CD0CCA70BC0D2FFCB66A2EE594059EBCA4FDAAE0BC0A03A47A1F0ED59403CF9FE2A87B40BC0D8517F643CED5940F1D883C2CDB80BC0AFE40E6386EC59408946A8D9A9BB0BC04CEC5151CFEB59405301F39A18BD0BC07FB2B2E417EB59405301F39A18BD0BC01CBAF5D260EA59408946A8D9A9BB0BC0F34C85D1AAE95940F1D883C2CDB80BC02B64BD94F6E859403DF9FE2A87B40BC0F99E38CF44E8594059EBCA4FDAAE0BC0E5FA1E3196E75940E82B9CD0CCA70BC0D0FE7667EBE65940F84595AA659F0BC072077A1B45E6594082DF5631AD950BC0D760ECF1A3E55940EDECBB06AD8A0BC049D4798A08E55940D14F4A11707E0BC0C14D177F73E45940A36F617102710BC068366A63E5E35940FF9B317571620BC0111C36C45EE35940CB3D888BCB520BC00437D126E0E25940F6057E3520420BC06758A0086AE25940F56315F77F300BC0FCC39ADEFCE15940BF9BD846FC1D0BC0F16FD61499E15940B5CF877CA70A0BC0D31C1D0E3FE159405B40E8BE94F60AC0DBAE8A23EFE059406FDEC6F0D7E10AC0562835A4A9E05940FA14419D85CC0AC0539BDED46EE05940D16667E3B2B60AC0955EB1EF3EE05940D21C4E6175A00AC09EC706241AE059400CD0A01EE3890AC025A1389600E05940DA13CE7612730AC07045DA4700E059406F5394D093720AC0 +1 12 01030000000100000068000000E57B462234D759403EE4A9BBCC8CFF3F81AF673EEBD7594004B745D25D8BFF3FA4921FA5A1D85940ABAC9B811287FF3FF9BEB8A156D959407C9ACA0AEF7FFF3F2DEFE48009DA594090895580FA75FF3FE3CF6E91B9DA5940E30E20BF3E69FF3F94B9E92466DB5940B430A664C859FF3F65A45E900EDC59403E9779C2A647FF3FB4A9F52CB2DC5940748110CFEB32FF3FAF6A9B5850DD5940D4A1F513AC1BFF3F7FB7A176E8DD59403CA36A99FE01FF3F5ED75AF079DE5940449F91CFFCE5FE3F5BD7AE3504DF59404D443475C2C7FE3FB64BAABD86DF594090C7417C6DA7FE3F7045DA4700E0594092DE788F5385FE3F50F6050701E059406D031FEC1D85FE3FD5CBA69872E05940534AE6C1F560FE3F1DD91502DBE05940BC81B6CE183BFE3F3B92EFDB39E1594059023394AC13FE3F811D4AC88EE159405584571FD8EAFD3FB6351273D9E15940ED02C5E1C3C0FD3F23485E9219E25940B403AE899995FD3F007DB7E64EE259404F078AD88369FD3F9A62583B79E25940AA1BB978AE3CFD3F21FE606698E259403D9141D2450FFD3F6B0F0049ACE259404FB4D2DE76E1FC3FB75F91CFB4E25940E92038FD6EB3FC3F61FEB0F1B1E2594064D169C45B85FC3F5C5943B2A3E25940D76665D66A57FC3F4D2A721F8AE25940B45CFDB2C929FC3F213C9E5265E25940C9DFCA8AA5FCFB3F0518467035E2594074DB6E122BD0FB3F72B3E1A7FAE15940C07E4E5686A4FB3FEF43B333B5E1594092FEF68EE279FB3FB8678D5865E1594096B253F66950FB3F1BDD8E650BE15940CAD8E09D4528FB3F5A0CD4B3A7E059406A5004469D01FB3F2FB21EA63AE059400A79B43697DCFA3F7045DA4700E059400F961B9F27CBFA3F730274A8C4DF5940C217941958B9FA3FFAA3B12F46DF5940D8B2A8D50298FA3F8FEF19B9BFDE5940583ECF6CB878FA3FA3E4D7C931DE5940A13A11DB975BFA3F5B4E7BEE9CDD59406A8BFAF7BD40FA3F4E9B6DBA01DD5940A7520E5A4528FA3F86F15FC760DC5940C402773C4612FA3FEE0EB3B4BADB594008A70B67D6FEF93F698CD92610DB59404C13C21809EEF93F671FB5C661DA5940B946A3F4EEDFF93FA87AEF40B0D95940F0D254F195D4F93F45734F45FCD85940E48B474B09CCF93F9C120B8646D85940D927997951C6F93FF24017B78FD75940D6CBB32574C3F93FD8B6758DD8D65940D6CBB32574C3F93F2EE581BE21D65940D927997951C6F93F85843DFF6BD55940E48B474B09CCF93F227D9D03B8D45940F1D254F195D4F93F63D8D77D06D45940B946A3F4EEDFF93F606BB31D58D359404C13C21809EEF93FDBE8D98FADD2594007A70B67D6FEF93F44062D7D07D25940C402773C4612FA3F7C5C1F8A66D15940A7520E5A4528FA3F6EA91156CBD059406A8BFAF7BD40FA3F2613B57A36D05940A13A11DB975BFA3F3A08738BA8CF5940583ECF6CB878FA3FCF53DB1422CF5940D8B2A8D50298FA3F57F5189CA3CE5940C217941958B9FA3F9B456E9E2DCE59400A79B43697DCFA3F70EBB890C0CD59406A5004469D01FB3FAF1AFEDE5CCD5940CAD8E09D4528FB3F1190FFEB02CD594096B253F66950FB3FDAB3D910B3CC594092FEF68EE279FB3F5744AB9C6DCC5940C07E4E5686A4FB3FC5DF46D432CC594074DB6E122BD0FB3FA8BBEEF102CC5940C9DFCA8AA5FCFB3F7DCD1A25DECB5940B45CFDB2C929FC3F6D9E4992C4CB5940D76665D66A57FC3F69F9DB52B6CB594064D169C45B85FC3F1398FB74B3CB5940E92038FD6EB3FC3F5FE88CFBBBCB594051B4D2DE76E1FC3FA9F92BDECFCB59403D9141D2450FFD3F30953409EFCB5940AA1BB978AE3CFD3FCA7AD55D19CC59404F078AD88369FD3FA6AF2EB24ECC5940B403AE899995FD3F14C27AD18ECC5940ED02C5E1C3C0FD3F48DA427CD9CC59405584571FD8EAFD3F8F659D682ECD594059023394AC13FE3FAD1E77428DCD5940BC81B6CE183BFE3FF52BE6ABF5CD5940514AE6C1F560FE3F7A01873D67CE59406D031FEC1D85FE3F13ACE286E1CE594092C7417C6DA7FE3F6E20DE0E64CF59404D443475C2C7FE3F6B203254EECF5940449F91CFFCE5FE3F4A40EBCD7FD059403EA36A99FE01FF3F1B8DF1EB17D15940D2A1F513AC1BFF3F164E9717B6D15940748110CFEB32FF3F65532EB459D259403C9779C2A647FF3F363EA31F02D35940B430A664C859FF3FE7271EB3AED35940E30E20BF3E69FF3F9D08A8C35ED4594092895580FA75FF3FD138D4A211D559407C9ACA0AEF7FFF3F26656D9FC6D55940ABAC9B811287FF3F484825067DD6594004B745D25D8BFF3FE57B462234D759403EE4A9BBCC8CFF3F +1 13 0103000000010000006A0000007045DA4700C05940B61312771DD81340D1C2FDB2D6BF5940171813293FD613403A7595F83ABF5940AC323F5A21D013408201FE7A99BE594035118EE4A1CA1340667C13DAF2BD5940E4DFFC38C6C51340A8F5C4BA47BD5940BD21672693C113404C7F71C698BC59406305C7D40CBE13402ADC40AAE6BB59407D9D1AC136BB1340EA79781632BB5940A802F2B913B91340115ECDBD7BBA59404FBFA9DCA5B713409CB0B354C4B95940A73C5493EEB61340EE90AC900CB95940A73C5493EEB6134079E3922755B859404FBFA9DCA5B71340A1C7E7CE9EB75940A802F2B913B9134060651F3BEAB659407D9D1AC136BB13403EC2EE1E38B659406405C7D40CBE1340E34B9B2A89B55940BD21672693C1134025C54C0BDEB45940E3DFFC38C6C513400840626A37B4594035118EE4A1CA134050CCCAEC95B35940AC323F5A21D01340B97E6232FAB25940171813293FD61340D07054D564B2594011D34A43F5DC13408D548269D6B15940929060043DE41340B331F27B4FB15940998F97370FEC1340CADC4292D0B05940A3C01A1F64F413407AB1272A5AB059404611A37B33FD1340BC12ECB8ECAF594061E19D9474061440C3E5CC3D93AF59400041BDD7180F1440372CFFAA88AF5940B0A0CB401E101440FD6788632EAF5940B91C4EEF261A1440A103053CDEAE59403B981DB184241440CC26EF8398AE59400D5FDB422D2F14404DD46E805DAE59401A2CF816163A14403205156C2DAE59404A632460344514408833A17608AE5940CFD6FF1B7D5014405A90D1C4EEAD5940F48EFE1DE55B144014163E70E0AD59408BD5771A61671440F89D3E87DDAD59404B9AD4B1E57214406514DC0CE6AD59408224D27B677E144000DDCCF8F9AD594010F0CD12DB8914409B6C7C3719AE594089880F1F35951440CC131EAA43AE5940C23405626AA0144002EACA2679AE5940CC4F68C16FAB1440DCBDAA78B9AE59400B423E523AB614408EE4276004AF5940BC2DAB63BFC0144038B72D9359AF5940CB8D8B89F4CA14404E8271BDB8AF59400B3CCAA6CFD41440D5A0C58021B05940CC9468F746DE1440B272767593B0594064B92E1A51E714405FD3B02A0EB15940FB47FA19E5EF1440C7ACF12691B15940D33EA276FAF7144075387EE81BB259409729672D89FF14409678E4E5ADB259402E29E7C089061540B169838E46B359401FE18D40F50C154003661A4BE5B359404AD8784FC5121540FD2B5F7E89B459402465C92AF41715406FF2998532B559408ECF5DAF7C1C1540ABEE46B9DFB5594047EAEB5E5A20154014ABBC6D90B65940B404786489231540BB87D6F343B759403BB92398062615400EBAA299F9B75940B4C45082CF271540191D13ABB0B8594050C0145EE2281540C520B07268B959409B47FB1A3E29154071244D3A20BA594050C0145EE22815407C87BD4BD7BA5940B5C45082CF271540CFB989F18CBB59403BB92398062615407696A37740BC5940B304786489231540DF52192CF1BC594047EAEB5E5A2015401B4FC65F9EBD59408FCF5DAF7C1C15408D15016747BE59402565C92AF417154087DB459AEBBE59404AD8784FC5121540D9D7DC568ABF594020E18D40F50C15407045DA4700C0594013A2B383FF071540F4C87BFF22C059402E29E7C0890615401609E2FCB4C059409729672D89FF1440C3946EBE3FC15940D33EA276FAF714402B6EAFBAC2C15940FB47FA19E5EF1440D8CEE96F3DC2594064B92E1A51E71440B5A09A64AFC25940CC9468F746DE14403CBFEE2718C359400A3CCAA6CFD41440528A325277C35940CB8D8B89F4CA1440FC5C3885CCC35940BC2DAB63BFC01440AE83B56C17C459400B423E523AB61440885795BE57C45940CC4F68C16FAB1440BE2D423B8DC45940C23405626AA01440F0D4E3ADB7C4594089880F1F359514408A6493ECD6C459400FF0CD12DB891440252D84D8EAC459408224D27B677E144092A3215EF3C459404B9AD4B1E5721440762B2275F0C459408CD5771A6167144030B18E20E2C45940F48EFE1DE55B1440020EBF6EC8C45940CFD6FF1B7D501440583C4B79A3C459404A632460344514403D6DF16473C45940192CF816163A1440BE1A716138C459400D5FDB422D2F1440E93D5BA9F2C359403C981DB1842414408DD9D781A2C35940B91C4EEF261A14405315613A48C35940B0A0CB401E101440C75B93A73DC359400041BDD7180F1440CE2E742CE4C2594060E19D9474061440109038BB76C259404611A37B33FD1340C0641D5300C25940A4C01A1F64F41340D70F6E6981C15940998F97370FEC1340FDECDD7BFAC05940929060043DE41340BAD00B106CC0594010D34A43F5DC13407045DA4700C05940B61312771DD81340 +1 14 0103000000010000006A0000007045DA4700A059404FC581B47F750740D2F06857D69F59405CC4E60B0C73074096A7CAA12B9F5940BBF17593A56A0740CCC732197D9E59404A7146A9986307401AF86A6ACB9D594034072848EC5D07404AE65945179D594057EAE70DA65907405F5C555C619C5940E426C335CA56074080F07163AA9B5940229837945B5507403CFDD00FF39A5940229837945B5507405D91ED163C9A5940E426C335CA5607407207E92D8699594057EAE70DA6590740A2F5D708D298594035072848EC5D0740F025105A209859404A7146A998630740264678D171975940BBF17593A56A0740EAFCD91BC79659405CC4E60B0C730740300F37E2209659400D54ADC1C37C0740393E22C97F955940EC49FB15C3870740BF821C70E4945940D2D5A225FF930740A345F7704F94594082D1DAD36BA107409E313C5FC19359401D2339D6FBAF07400A349BC73A9359402C92D7C1A0BF0740C33C5F2FBC925940CF1B96194BD00740B346EA13469259402EB96C5DEAE107405C2B39EAD8915940F57EBD1A6DF4074045BC6F1E75915940DFFF96FDC007084010956D131B91594035EED5E2D21B08408104EDEF1191594000827AAF311E084082106C22CB9059408D1B14EB8E300840F1C1A59A85905940522A528EE04508402FCB07C14A9059402D8B48B0B25B08407E5DEDCF1A90594023BA4BB5EF71084030AAE5F6F58F5940161CAF97818808404D7D845ADC8F5940B35E91FD519F08404CB23D14CE8F594076D2FC4E4AB608402FA84B32CB8F59407DE145CC53CD084097CFA0B7D38F5940D88591A457E408408662E49BE78F594041786D0C3FFB0840654A7ACB0690594070BB6354F3110940182E962731905940842A73FF5D2809409C965986669059402CC856D9683E0940F90DFDB2A69059406EBA850CFE5309402711046EF19059400B22D63708690940FBA27B6D469159402256AD83727D09406243435DA5915940D97AA8B628910940F80360DF0D92594097EDA84917A40940C568598C7F925940AD99307B2BB609400FBA9FF3F9925940CCF3FA6153C70940C262FB9B7C935940C21AC0FE7DD70940B2ED040407945940C86D104D9BE60940462BA5A298945940EECC37539CF40940C5009DE73095594047AD183173010A40CB5A143CCF9559400730F02D130D0A40F5B42F0373965940F27FF5C470170A4043A2AB9A1B97594048DBC7B081200A4066BB7D5BC8975940F2DF9FF53C280A40AC527A9A78985940AEEA39EA9A2E0A405A48FEA82B9959404BA0703F95330A40FC559CD5E09959409906800627370A409923CD6C979A594013EAEAB54C390A40DE76A1B94E9B594080ADFD2C043A0A4023CA7506069C594013EAEAB54C390A40C097A69DBC9C59409906800627370A4062A544CA719D59404BA0703F95330A40109BC8D8249E5940AEEA39EA9A2E0A405632C517D59E5940F2DF9FF53C280A40794B97D8819F594049DBC7B081200A407045DA4700A059407C1A4920B5190A40C73813702AA05940F27FF5C470170A40F1922E37CEA059400730F02D130D0A40F7ECA58B6CA1594048AD183173010A4076C29DD004A25940EECC37539CF409400A003E6F96A25940C86D104D9BE60940FA8A47D720A35940C21AC0FE7DD70940AD33A37FA3A35940CCF3FA6153C70940F784E9E61DA45940AD99307B2BB60940C4E9E2938FA4594097EDA84917A409405AAAFF15F8A45940DA7AA8B628910940C14AC70557A559402256AD83727D094095DC3E05ACA559400B22D63708690940C3DF45C0F6A559406EBA850CFE5309402057E9EC36A659402CC856D9683E0940A4BFAC4B6CA65940832A73FF5D28094057A3C8A796A6594070BB6354F3110940368B5ED7B5A6594040786D0C3FFB0840251EA2BBC9A65940D88591A457E408408D45F740D2A659407DE145CC53CD0840703B055FCFA6594076D2FC4E4AB608406F70BE18C1A65940B35E91FD519F08408C435D7CA7A65940161CAF97818808403E9055A382A6594024BA4BB5EF7108408D223BB252A659402D8B48B0B25B0840CB2B9DD817A65940522A528EE04508403ADDD650D2A559408D1B14EB8E3008403BE955838BA5594000827AAF311E0840AC58D55F82A5594035EED5E2D21B08407731D35428A55940DFFF96FDC007084060C20989C4A45940F57EBD1A6DF4074009A7585F57A459402EB96C5DEAE10740F9B0E343E1A35940D01B96194BD00740B2B9A7AB62A359402C92D7C1A0BF07401EBC0614DCA259401D2339D6FBAF074019A84B024EA2594081D1DAD36BA10740FD6A2603B9A15940D2D5A225FF93074083AF20AA1DA15940EC49FB15C38707408CDE0B917CA059400D54ADC1C37C07407045DA4700A059404FC581B47F750740 +1 15 01060000000300000001030000000100000084000000F40DC99CE07059400004F55E633CF83F60E5B6E3BA705940D26F01EA9B33F83FC82311F72C705940D671284A7B16F83F5D28671E986F594025638F59A1FBF73F9F0E20EDFC6E59408F10E7AE28E3F73F8CB5EAFC5B6E59406537858529CDF73F6BA025EDB56D59409C2969A5B9B9F73FCC3941620B6D59408564AD4DECA8F73F7D141D055D6C594005687B21D29AF73F51CB6082AB6B594014A29417798FF73FB324D189F76A59405DB4806CEC86F73FEA22A2CD416A594043BD6E973481F73FB1ACC6018B69594008A2D441577EF73F837C3EDBD368594008A2D441577EF73F4906630F1D68594043BD6E973481F73F80043453676759405DB4806CEC86F73FE25DA45AB366594015A29417798FF73FB614E8D70166594005687B21D29AF73F67EFC37A536559408564AD4DECA8F73FC888DFEFA86459409C2969A5B9B9F73FA7731AE0026459406537858529CDF73F941AE5EF616359408F10E7AE28E3F73FD6009EBEC662594025638F59A1FBF73F6B05F4E531625940D671284A7B16F83FD3434EF9A3615940D16F01EA9B33F83F3F1B3C407E6159400004F55E633CF83FFD283A851D6159407B1B5F61E652F83F884CE00E9F6059409E9607B43B74F83FD797801329605940BD45EDDF7A97F83F7045DA4700605940A38B28CA54A5F83F5E3DF607BC5F59407C68DAFD80BCF83FC5FA4358585F5940B821FD6329E3F83F72182967FE5E594093CB31CA4D0BF93F6190BF8DAE5E594060BAE76FC634F93F3BBD231B695E594058FB7A436A5FF93F4AE925542E5E5940F42ADC0A0F8BF93F250B0673FE5D5940C9335E8D89B7F93FFBF339A7D95D594046A380BEADE4F93F21293E15C05D5940B1468CE94E12FA3F179871D6B15D59404BF1D6DD3F40FA3F6749FCF8AE5D594051A5821B536EFA3F2A2CC17FB75D59406BDD8B005B9CFA3FE7085B62CB5D59403263FAF529CAFA3F8F9F248DEA5D594095FA079D92F7FA3FD5B45956EC5D59405DA321E076F9FA3FB05200E0065E594000131A002BF9FA3F08E3FFDF0F5E594000E214C00EF8FA3FA8DDFFFF735E59400089208034F1FA3F70DAFFDFBC5E5940003B0720CCEDFA3FD8D1FFDF695F594000E10A607ED2FA3FB4CFFF1F855F59400093FE1FA6CBFA3FD47100C0BB5F59400076DF5F74B9FA3F7045DA4700605940009D3C580BABFA3FF01B00C01D60594000EA0B00D9A4FA3F94DCFF9F81605940000B10007A8CFA3F442F00808860594000AFE61FCE8AFA3F6482FF1FFA605940007DE71F9678FA3FE07AFF7FB9615940008FF17F4C5DFA3F141E0080C261594000E7F5FF285CFA3FD85400A0CB61594000971840025BFA3FB05200E00662594000FBE57F3154FA3FF08100608F625940008D0340F949FA3F8CCDFF5FA063594000B400A09C36FA3F143B00A0B2635940000D05207935FA3F40C9FFDFF66359400066E73F3433FA3F543000E07A6459400017F03FED30FA3FF46400409F6459400017F03FED30FA3FBC610020E86459400017F03FED30FA3F7883FF7F0C6559400017F03FED30FA3FFCF0FFBF1E6559400017F03FED30FA3F141E0080C26559400066E73F3433FA3FBCDEFF5F6666594000B400A09C36FA3F7CAFFFDF7D66594000150B60F037FA3F704000804E675940002B02808946FA3F7C2CFF1F5C67594000D2FDFFAC47FA3F0C7500E0726759400022F5FFF349FA3FF098FFFF7B67594000BF0880154BFA3FB43500C01668594000DEE87F935FFA3F9033000032685940007109C03F65FA3F208DFFDF516859400069DFDF176CFA3F94F9FFBF7168594000B7EB1FF072FA3F046600A09168594000AD00007C77FA3FC8ADFEDFA36859400090E9BFBD79FA3F70BDFFBFCC6859400024F0BF497EFA3F10F2FF1F1169594000CA1AC01E85FA3FAC26008035695940005FEF1F8788FA3FF08100608F6A5940006BE7BFFDA4FA3FC0A4FF1FA66A59400012E33F21A6FA3FA0CEFFBF926B594000FFDD7F87A9FA3FC4700060C96B594000FFDD7F87A9FA3F68CBFF9FDB6B594000FFDD7F87A9FA3FC8360020096C594000FFDD7F87A9FA3F506A00203B6C59400057E2FF63A8FA3F5CF6FF9FBA6C5940004DF7DFEFACFA3F10F2FF1FF16C594000FD0C0039AFFA3F10F2FF1F116D594000421C007EB1FA3F5C5C00402C6D5940009B0320C8B3FA3F14B8FFDF306D59400043FF9FEBB4FA3FD8EEFFFF396D594000E00A000CB6FA3F00B7FF7F5E6D5940002F020053B8FA3FCC79FF1FC76D594000CC126002BEFA3F200AFF1FF06D5940007EFEFF28BFFA3FC8190000196E5940007EFEFF28BFFA3F508700404B6E5940007EFEFF28BFFA3F6C1400207D6E5940007EFEFF28BFFA3F10D5FFFF006F59400038148077B9FA3FD8D1FFDF696F5940009B0320C8B3FA3FF098FFFF7B6F594000E91780A1B2FA3FCC96FF3F976F594000AFF89F5FB0FA3F786C00E01F705940007216A0B4A2FA3F903300003270594000D7E0BF71A0FA3F348EFF3F44705940001CE39F269EFA3F84BEFF1FBF70594000A5E1BF398EFA3F8884FFDFFE70594000C2DEBFD782FA3F748900001071594000CCEB3F6D7FFA3F58ADFF1FF97159400005F3FFC650FA3FA47D00E0E57259400051F75FB91EFA3FA0CEFFBF9273594000BCF31F2DF9F93F7CCCFFFFCD73594000E3FD9F60EAF93F506A00203B7459400044F2DFCDCCF93F189F58BE67745940F8A5737487C0F93F0E1EFF6960745940C9335E8D89B7F93FE93FDF8830745940F32ADC0A0F8BF93FF86BE1C1F573594058FB7A436A5FF93FD298454FB073594060BAE76FC634F93FC110DC756073594093CB31CA4D0BF93F6E2EC18406735940B821FD6329E3F83FD5EB0ED5A27259407C68DAFD80BCF83F5C9184C935725940BD45EDDF7A97F83FABDC24CEBF7159409E9607B43B74F83F3600CB57417159407B1B5F61E652F83FF40DC99CE07059400004F55E633CF83F0103000000010000000A00000050040080096959400064F71FC44EFB3F243C0000056959400064F71FC44EFB3F243C0000056959400047F47FCF56FB3F207600400569594000EEEFFFF257FB3F88A1FFFF0E6959400001FABF1B59FB3F143B00A01269594000EDF67F2C59FB3F3C0300201769594000EEEFFFF257FB3F383D00601769594000A0EBDF1B51FB3F143B00A01269594000F8EF5FF84FFB3F50040080096959400064F71FC44EFB3F01030000000100000072000000D2D57D98985E5940100422E6B283FB3F9F75E9FCD45E594018790439BCA5FB3F8A1CDEE7295F5940B9E96CA890CEFB3F7CF817C0885F59401F9D9CDCFCF5FB3F144BAE27F15F59409056BCC8D91BFC3F7045DA4700605940543D70A6AA20FC3F79FB3EB76260594058AE4FEB0140FC3F6AE154FEDC605940AC8060735162FC3F8F13D7835F615940A061FE63A682FC3F29CB80C6E96159408DD2EEB5E0A0FC3FC066613D7B62594016BE6C77E2BCFC3F3D0D6458136359406BAAD7E98FD6FC3F0D6BDE80B16359400A1C359DCFEDFC3FEFFB251A556459405CCB68898A02FD3F4C4D2B82FD645940B4930A25AC14FD3FEF9E1A12AA6559409F5FC3792224FD3F8E42011F5A66594083CB1C36DE30FD3FBC1577FA0C67594066C0B1BCD23AFD3F946D4BF3C1675940B7E4B030F641FD3F25C8345678685940EA6BA47F4146FD3F9A94826E2F695940088A7568B047FD3F0E61D086E6695940EA6BA47F4146FD3F9FBBB9E99C6A5940B7E4B030F641FD3F77138EE2516B594066C0B1BCD23AFD3FA5E603BE046C594083CB1C36DE30FD3F448AEACAB46C59409F5FC3792224FD3FE7DBD95A616D5940B4930A25AC14FD3F442DDFC2096E59405CCB68898A02FD3F26BE265CAD6E59400C1C359DCFEDFC3FF61BA1844B6F59406BAAD7E98FD6FC3FDC25FF9A756F5940EE89DE0C75CFFC3F149BFFBF606F594000F4FA7F14AEFC3F347700A0576F594000730E80B3A2FC3FD40B00202A6F594000CAFDFF196CFC3F609FFF3F0A6F59400054EAFF0D53FC3FE44600C0DC6E594000A2E19FE238FC3F98A2FF5FC16E594000B4FB9F0D32FC3F08490080A16E59400021F59F812DFC3FD4A5FF7F986E594000DA1740602CFC3F2C4B0040866E594000DA1740602CFC3FA8DDFFFF736E594000E6E5FF412BFC3FF8A7FF3F5D6E5940002BE0BFF528FC3F843BFF5F3D6E5940003EE57F8F25FC3F508700400B6E594000F0E05FB81EFC3F001D0020F06D594000591920E017FC3F90B0FF3FD06D594000D1E3FFA70DFC3F3C200040A76D5940009FE9FFFEFFFB3F8CEAFF7F906D59400015F07F08F8FB3F602200008C6D594000D81FA07CF3FB3F9CEBFFDF826D59400032F59FA7ECFB3F187EFF9F706D5940009EEE9F1BE8FB3F705D00A03E6D59400063EF3FDEDDFB3F8884FFDFFE6C594000CE05A0E4D5FB3F488FFF9FF66C59400008E1FFFED4FB3FBC610020E86C59400028E8BF9FD3FB3FC4700060C96B594000D7F55F75B9FB3FE04C0140C06B59400038148077B9FB3F3C030020B76B5940007FF1DF98BAFB3FE8720020AE6B59400026ED5FBCBBFB3F50040080A96B594000C300E0DDBCFB3F383D0060976B594000C300E0DDBCFB3FF098FFFF7B6B59400026ED5FBCBBFB3FD8D1FFDF696B594000EAFF1F9EBAFB3F509EFFDF176B594000AFF89F5FB0FB3FF80D00E00E6B594000FD14203AAFFB3F38D7FFBF056B594000420FE0EDACFB3FD0450060EA6A594000C20580FAA4FB3FF4470020CF6A5940007DF99F279EFB3F2C4B0040866A59400056EABFF28BFB3FF8A7FF3F5D6A594000231720FA83FB3F141E0080C2695940006709603A6DFB3F345A00806769594000D4FD5F1F64FB3FD8EEFFFF3969594000D4FD5F1F64FB3F5C5C00402C695940009710200163FB3F28B9FF3F23695940008FF67FDB61FB3F6482FF1FFA6859400054EFFF9C57FB3F846FFE1FFA68594000A10B807756FB3FA86000C0F5685940005DFC7F3254FB3F30DFFE1FF1685940000E0580EB51FB3F34F4FFDFD56859400021FD1FF549FB3F1C2D00C0C368594000830E206745FB3F6CF7FFFFAC68594000480CA0B73FFB3F543000E09A68594000F814A0703DFB3F302E00209668594000F814A0703DFB3FFC8AFF1F8D685940005119204D3CFB3F18010060F2675940000122C0931DFB3F7CCCFFFFCD67594000141A609D15FB3F68CBFF9FBB6759400082F1FFEF0FFB3F143B00A0B2675940002902C0A40DFB3FE883FE3F976759400033EDDF1809FB3F740600408E67594000ED0F80F707FB3FD8D1FFDF69675940004E21806903FB3F149BFFBF6067594000EFE0FF4502FB3F509EFFDF1767594000EFE0FF4502FB3FD0450060EA665940004E21806903FB3F74A0FF9FDC66594000F61C008D04FB3F48D8FF1FD86659400046F29FB205FB3FF4470020CF66594000B11A20D806FB3F404C00A07866594000EC1CA0870CFB3FBCDEFF5F66665940008A20E0A60DFB3F30ABFF5F346659400082F1FFEF0FFB3F4C2100A099655940006FDF1FC616FB3FFCA1FEBF7965594000641160E417FB3FA86000C0D5645940005CE27F2D1AFB3FA034006004645940000122C0931DFB3F68CBFF9FDB6359400051F75FB91EFB3FB895FFDFC463594000F9F2DFDC1FFB3FF8A7FF3F5D62594000BD0D203233FB3F3C8600E038625940000C0D407A35FB3F64E8FFBFAB61594000480CA0B73FFB3F30C8FF7F0460594000D4FD5F1F64FB3F7045DA47006059400070E09E7F64FB3F44120060B85E594000DD1760B781FB3FD2D57D98985E5940100422E6B283FB3F +1 16 010300000001000000F3000000B81E85EB516859401A92F85DCABF0640CCF671250969594005A6A2E712BF06407655D1A9BF695940CE4A6A3AEDBC064099E9C9C3746A5940C138F7765BB90640C7FEE9BF276B5940E707B32661B40640678AD9ECD76B59409BB3463803AE0640EF1F0A9C846C59403DA7B5FA47A606407E1C64222D6D594009351B17379D06402A5DF0D8D06D594013B81089D9920640BED67D1D6F6E5940ABF8C39539870640F1694253076F59408AB9C6C1627A0640225476E3986F59401999A1C5616C0640DAA1E93D23705940A6AD3581445D0640CB0E93D9A57059406472F9ED194D06408AC51735207159409ABD1E10F23B064089784BD791715940BD8CB0E6DD2906409B55A84FFA7159407878B85AEF160640A85DBE3659725940F6977C2D39030640B8B2992EAE725940E67EE7E5CEEE0540D5771FE3F8725940FAD32CBDC4D9054091E7600A3973594074BDBD8A2FC40540EF4FE4646E735940BB0DA1AF24AE054070ACE3BD9873594029B24301BA97054091A180EBB7735940706AD6B30581054058A7EDCECB735940E43D4E441E6A0540953B8C54D4735940757D1D621A53054037FFFF73D17359406E65BBD8103C0540FCAC3630C373594028A010791825054010E36497A97359402B03DF02480E0540CDC3F7C2847359405AE23A0EB6F70440287D7BD7547359406D412CF578E104400CD176041A735940BF018EBDA6CB0440B9C33B84D4725940F9EB400355B60440E49FAD9B84725940C11FC8E298A104400120EB7B84725940D095C9CD91A104407CAFFFDF7D725940809901E038A20440FCF0FFBF5E7259408018FFFF7E9F0440E07AFF7F59725940805AEF9FBD9E04409C080000537259408016FA3F0B9E0440CC7F00A04A72594080F3FD9F869D0440901600E041725940805309A02F9D04404C2100A03972594000E90D80839D0440A0CEFFBF32725940003DFABF209E0440DC9D00202D72594000B70980BC9E04405890FFFF28725940000602C0589F0440B086FF9F237259400045000098A00440B05200E0067259408089F37F2DA804401C2D00C0037259408054F13F2CA904408884FFDFFE715940803709E091AD04408CCDFF5F0072594000F505806CAE0440141E00800272594000ED0D004DAF0440243C00000572594080AB028026B00440B81800A006725940809E0EE004B10440B81800A00672594000081180EAB10440901C016005725940808A0A60C0B20440003A00400072594080CE0CE002B40440684E0060FD71594000BC01E084B404408C500020E271594080D8092096B80440DC800000DD71594000C9F43F51B90440B43500C0D6715940009DF2FFE2B90440F01B00C05D715940800A0B00EBC3044030ABFF5FF470594000ACF55F96CB0440049AFF5FEE705940002DFF1FFCCB04401CAAFFFFE17059400004F3FFC6CC0440D47100C0DB7059400085FCBF2CCD044088A1FFFFCE70594000F00460DBCD044018E4FF3FC2705940007AF83F7BCE0440B0ECFF3FB570594080D8EF7F03CF04402CCE0000A87059400036EFDF8CCF0440A4B1FF9F62705940008D0EE0DED1044084DBFF3F4F705940808EF35F69D20440406900C04870594080B5F3DF7ED20440E4A6FFDFEA6F59408045F6FF79D30440D4A5FF7FB86F5940004BF8BFD2D20440704000808E6F5940807AFEDFD1D10440B0090060856F59400049F3FF5ED10440AC260080156F594080FCF93F52CB0440FC8AFF1F0D6F5940808AFFBFDACA044034110000066F59400067032056CA044080F8FF5FFF6E59400023FBFFBCC90440CCDFFFBFF86E5940005E046018C90440A84300A0E56E5940804DFE1F0EC70440404C00A0D86E5940801FF21F9DC50440302E0020D66E594000D9060024C50440A8DDFFFFD36E594080B00E409DC40440A494FF7FD26E5940004BFA9FEFC3044050870040CB6E594080C80E8071C20440B81800A0C66E5940806EF81F91C1044054360160BE6E594000650680E3C00440A0510080B46E5940806AF57F55C00440D0450060AA6E594000DF0E60D3BF04407C3200A09F6E59408011F23FCEBF04406C1400203D6E59400033F69FE1BF04408CCDFF5F206E594080410AE05EC004408CEAFF7FD06D594000B8070012C2044074230060BE6D5940008C05C0A3C2044098420040B36D594000210400A1C304409476FFFFAF6D594000F501C032C4044068310040AD6D594000C30B80C3C40440A8DDFFFF936D594080230880BFCA0440E0FDFF3F1B6D594000A90AE042D30440A4170040146D5940005B0440C1D30440880700A0006D594080E504A063D50440E4C3FFFFFA6C5940804BF09FD1D60440E4C3FFFFFA6C594000150BC062D80440E49501C0016D59408061F6BF17DB0440740600400E6D594080CAFF3FA9DD0440640500E01B6D59408003FE3F3AE00440E06300E02C6D59400089F4BF82E204409093FF1F406D59408011F73F5DE40440807B0020816D59408011032098EA044054CAFF3F896D59408081FFBF47EB044084DBFF3FAF6D59408013F9DF5EEE0440BCC1FF3FB66D594080ACFBFFECEE0440F8A7FF3FBD6D5940803A0E0078EF0440348EFF3FC46D5940802DF7FFFAEF0440846FFE1FDA6D5940802A00A06CF10440C4EDFF9FE76D594080DAFB1F23F2044070400080EE6D59400082F79F46F3044070400080EE6D59400056F55FD8F30440CCFCFFDFE86D59400063FE9FFDF3044058790060DC6D59400063FE9FFDF3044050870040CB6D59400009FBFF03F30440B895FFDFC46D594080BD0000A2F2044084DBFF3FAF6D5940804001C023F104409C6E00A0A46D594080EEF77F66F00440E889FFBF9A6D594000520760A0EF04405890FFFF886D59408050FA7F2BEE044074890000506D5940008008C02BE90440609FFF3F4A6D594000220960A2E80440280200C0446D5940801308E00BE80440F82A00001F6D5940800211A0AEE30440485B00E0196D59408070F5BF03E304405890FFFF086D59408000F23FA8E00440B086FF9F036D594080BDEF1FD8DF04401052FF3FFF6C59400021FFFF11DF044068CBFF9FFB6C594000B1FA3F61DE04402C680060F66C594080E2066029DD0440A8DDFFFFF36C594080980A408FDC044098BFFF7FF16C59400069FD3FE4DB044084DBFF3FEF6C5940005C02C016DB04406CF7FFFFEC6C5940807C01C028DA0440E0E0FF1FEB6C5940803105401CD90440E4A6FFDFEA6C594080DE03C0EDD70440FC8AFF1FED6C594080220500DBD604408CEAFF7FF06C594080CD05A0E4D50440A0510080F46C5940005A0BC0FAD404403C8600E0F86C594000F0104016D40440608800A0FD6C59408047F4DF41D304400C920000036D5940002BF23F87D204405890FFFF086D5940006CEF9FE3D104408CEAFF7F106D59400049F3FF5ED1044058ADFF1F196D59408011FC3FECD00440945F0060236D59408000FDDF8DD00440683100402D6D5940803F0F2009D004403C030020376D5940005901606ACF0440807B0020416D5940802EF07FC1CE0440DC1A00604B6D594000F4F25F13CE0440CC13FF7F556D594080D2FCBF57CD04407C3200A05F6D59408005F9FF8FCC0440F4E1FF7F7D6D594000BA05E0D9C90440B81800A0866D5940002507A0DCC80440EC3800E08D6D5940801FFEFFD7C70440208DFFDF916D594080AD0920B7C604402453FF9F916D5940004DFF5F63C50440E87200208E6D5940801B024048C40440C8360020896D594000E40260D4C30440684E00607D6D594000050260E6C2044044120060786D5940802AEFBFA2C20440609FFF3F6A6D594000CCF77F1AC2044010D5FFFF606D5940001B1140F5C10440CCF6FE5FE56C5940804310E027C00440C8B3FF5FC76C59400033F69FE1BF04405CD9FF7FAA6C594000560D204EC00440F098FFFF7B6C594080EB03404AC10440786C00E03F6C5940008A07E0DBC60440CC7F00A00A6C594000FDF8FFFCCC0440243C0000056C5940808503A0D8CE0440C0A4FF1F066C5940802D11A08DD10440A0B70020066C5940002E09808CD50440CC96FF3FF76B59408054FAFFF4DB0440B06F0000D76B59408058EFDFD8E10440280200C0C46A594080840A2012F00440581300C00A6A59400041F99F22F90440A4FAFF1F046A594080A0F6DF73F90440947C0080F3695940005E0E6036FA0440EC66FE1FE769594000460F60B7FA0440FC730080E069594080180DC0D6FA0440A4B1FF9FC2695940800708C021FB0440DC97FF9FA969594000340A0090FA044028B9FF3FC368594000CE09601E04054014B8FFDF506859408028FDBF150905405081FFBF2768594080191040BB0A054050040080C96759408001EF7FA80D05405004008089675940800011E0C90E05403C6F01404C67594000650A401D0F0540E4C3FFFFFA66594080140660610F05409C6E00A0C466594000D2EF3F55100540A0D400409666594000F307A0A412054090B0FF3F906659408091F69F32130540F098FFFFFB65594080FFF3BF52210540C4530040B9655940000D01C0B1270540AC26008035655940806AF07FC63305406482FF1FFA645940002FFADF51380540A01D01C0D7635940805AFCBF4D3F0540A4FAFF1F8463594080D90CA07A4005407CCCFFFF6D63594080910960193F0540E889FFBFBA62594000360CA0914805404C2100A0D9615940006BF61F1D57054008C6FFBF1F605940807D1020487005407045DA4700605940806AC56C0672054018E4FF3F625F5940000101A0C77A0540742300601E5F5940803C0320777C05400C8CFF7FDF5E594080340F60917F05402C4B0040A65E5940007A074028840540B4B2FFFF745E5940801904608088054054CAFF3FE95D5940005C02C0169B0540CC7F00A08A5D594080620DA0AAA905402C4B0040865D594000820D809FAA0540844100E0605D59408035F51F71B60540ECC122E7535D5940BD34B9EBB9BA0540DE55A9CC6A5D594074BDBD8A2FC405409AC5EAF3AA5D5940FAD32CBDC4D90540B78A70A8F55D5940E67EE7E5CEEE0540C7DF4BA04A5E5940F6977C2D39030640D4E76187A95E59407778B85AEF160640E6C4BEFF115F5940BC8CB0E6DD290640E577F2A1835F59409ABD1E10F23B0640A42E77FDFD5F59406472F9ED194D06407045DA470060594087AACF82624D0640959B209980605940A7AD3581445D06404DE993F30A6159401999A1C5616C06407ED3C7839C6159408CB9C6C1627A0640B2668CB934625940AAF8C3953987064045E019FED262594012B81089D9920640F120A6B4766359400A351B17379D0640801D003B1F6459403CA7B5FA47A6064008B330EACB6459409BB3463803AE0640A83E20177C655940E707B32661B40640D65340132F665940C138F7765BB90640F9E7382DE4665940CD4A6A3AEDBC0640A34698B19A67594005A6A2E712BF0640B81E85EB516859401A92F85DCABF0640 +1 17 0103000000010000006A0000004769F96A797259400008EABDC678E03FC4BA821FA67259400EDABD19923CE03FA65E8270DB7259406F22C4F6CDC8DF3F08B848C30573594088E0E3CE7815DF3F3781EEED2473594075EA5959D65FDE3F313B99D1387359406D280F7B9AA8DD3FB6AB995A41735940353B2CAE7AF0DC3F624C7F803E735940A7055F4E2D38DC3FD89720463073594060D841E46880DB3F272D98B91673594081FC9670E3C9DA3F40CC36F4F1725940E7810BB85115DA3FAE396A1AC2725940D9AC33906663D93F6022995B87725940AC27722ED2B4D83F7323F4F141725940E7367979410AD83F66243C22F2715940FE9B125E5D64D73FB63C7E3B98715940D096D727CAC3D63F5D68C59634715940D0927EDE2629D63F7258C296C77059405D865FA80C95D53F2DB769A751705940F5E5CA320E08D53FE33F893DD36F59406A4DC920B782D43FA71454D64C6F5940CAADD3808B05D43FE0C3E6F6BE6E594043EC0C4A0791D33F7C78C32B2A6E5940E8747EE19D25D33F16D746088F6D59409C58D1A7B9C3D23F9A111626EE6C5940CC2AF58FBB6BD23FF0C18624486C594080FE1CBFFA1DD23F162201A89D6B5940E9A57136C4DAD13FF13D5D59EF6A5940BDBBCE865AA2D13FFBBF3BE53D6A59408F0CD68EF574D13F5AFE5AFB8969594007B99A43C252D13FBEF1E84DD46859404FEB1A84E23BD13F55C2D2901D6859408244B4F76C30D13FBA991279666759408244B4F76C30D13F516AFCBBAF6659404FEB1A84E23BD13FB55D8A0EFA65594007B99A43C252D13F149CA92446655940900CD68EF574D13F1D1E88B094645940BCBBCE865AA2D13FF939E461E6635940E9A57136C4DAD13F1F9A5EE53B63594080FE1CBFFA1DD23F754ACFE395625940CB2AF58FBB6BD23FF9849E01F56159409C58D1A7B9C3D23F93E321DE59615940E8747EE19D25D33F2F98FE12C560594042EC0C4A0791D33F6747913337605940C9ADD3808B05D43F7045DA4700605940F571EF64B138D43F2B1C5CCCB05F59406A4DC920B782D43FE2A47B62325F5940F5E5CA320E08D53F9D032373BC5E59405D865FA80C95D53FB2F31F734F5E5940CF927EDE2629D63F581F67CEEB5D5940D096D727CAC3D63FA937A9E7915D5940FE9B125E5D64D73F9C38F117425D5940E6367979410AD83FAF394CAEFC5C5940AB27722ED2B4D83F61227BEFC15C5940D7AC33906663D93FCE8FAE15925C5940E6810BB85115DA3FE82E4D506D5C594081FC9670E3C9DA3F36C4C4C3535C59405ED841E46880DB3FAD0F6689455C5940A6055F4E2D38DC3F58B04BAF425C5940353B2CAE7AF0DC3FDE204C384B5C59406C280F7B9AA8DD3FD8DAF61B5F5C594074EA5959D65FDE3F06A49C467E5C594088E0E3CE7815DF3F69FD6299A85C59406D22C4F6CDC8DF3F4BA162EADD5C59400EDABD19923CE03FC8F2EB9E0A5D59400008EABDC678E03F60F2D0041E5D5940A28B6F6FE692E03F4A3334A9685D59405588887B0EE7E03F3553A28DBD5D594076ABF1E3B638E13FC0110A5E1C5E594079B4FAC78E87E13F223286BC845E59400FFD761048D3E13F9D6BBA41F65E5940491918BD971BE23F7CBB397D705F594014B8B92E3660E23FB6B3F5F5F25F5940E41D556EDFA0E23F7045DA47006059407E678DF6B2A6E23F2D58B62A7D605940AEDF557053DDE23F33149A920E61594044170D545615E33FA4499D9DA66159404D1F049FB048E33F99F328B544625940FAF9F3732F77E33F57CFA73CE862594029D42AC5A4A0E33FDF77219290635940019E2D82E7C4E33F4BDADA0E3D645940936968C0D3E3E33FE763FB07ED6459401427C5DE4AFDE33F414636CF9F655940E06405A43311E43F1A2977B35466594072ECC0577A1FE43FF79E91010B675940B266F0D51028E43F07AEF204C2675940FA95F09CEE2AE43F18BD530879685940B266F0D51028E43FF5326E562F69594072ECC0577A1FE43FCE15AF3AE4695940E06405A43311E43F28F8E901976A59401427C5DE4AFDE33FC4810AFB466B5940936968C0D3E3E33F30E4C377F36B5940019E2D82E7C4E33FB78C3DCD9B6C59402AD42AC5A4A0E33F7668BC543F6D5940FAF9F3732F77E33F6B12486CDD6D59404E1F049FB048E33FDC474B77756E594044170D545615E33FE2032FDF066F5940AEDF557053DDE23F59A8EF13916F5940E51D556EDFA0E23F92A0AB8C1370594014B8B92E3660E23F72F02AC88D705940491918BD971BE23FEC295F4DFF7059400FFD761048D3E13F4E4ADBAB677159407AB4FAC78E87E13FD908437CC671594076ABF1E3B638E13FC528B1601B7259405688887B0EE7E03FAF69140566725940A28B6F6FE692E03F4769F96A797259400008EABDC678E03F +1 18 0103000000010000006A000000B65D5C64A26F59400008EABDC678E03F0482D416B96F5940E68A67FFE665E03F5C5A39FB0D7059401D670C9A3E14E03F1142979F5870594003A258212D80DF3F230C04BA98705940830EA57A84D3DE3FDED6050BCE705940CB73C6412E23DE3F34D9D15DF8705940E2BAD61CD96FDD3F0C9B80881771594039D846A936BADC3F4B65376C2B715940FE73F5CBFA02DC3FCCC046F5337159407EE104FFDA4ADB3F76E63D1B3171594002C4209E8D92DA3F7E0DF3E022715940CE45E531C9DAD93FBF8F805409715940009E1ABB4324D93F46E7368FE4705940B7C977FEB16FD83F0B9383B5B4705940E0D49ED1C6BDD73FB6FBCCF679705940C1D7036A320FD73F597C438D3470594061E86DAEA164D63F7CBDA7BDE46F594040ADBE8BBDBED53F909B06D78A6F59408402AA4D2A1ED53F1CDC6A32276F5940F63F03FC8683D43FF9FE8432BA6E5940162841BD6CEFD33FE0824943446E59406163D43E6E62D33FBFFD85D9C56D59401FACE62317DDD23FA8726D723F6D5940067B127BEB5FD23F8C571C93B16C59403E239C3B67EBD13F7EC514C81C6C594075E2ADCAFD7FD13FB655B3A4816B5940777E1089191ED13F0E359DC2E06A5940DEA6D1691BC6D03F50FE27C13A6A5940407C40925A78D03FEBF1BB4490695940E4619F032435D03F3B2731F6E1685940E945C19D74F9CF3F9A58288230685940F10573A5AA9ECF3FC8EF5F987C67594065279A08445ACF3F01FC05EBC66659403E584B85842CCF3F2FBE072E106659404A4C526A9915CF3FEF795F16596559404A4C526A9915CF3F1C3C6159A26459403E584B85842CCF3F564807ACEC63594065279A08445ACF3F84DF3EC238635940F10573A5AA9ECF3FE310364E87625940E745C19D74F9CF3F3346ABFFD8615940E4619F032435D03FCE393F832E615940407C40925A78D03F0F03CA8188605940DDA6D1691BC6D03F7045DA4700605940E2D6BA329D10D13F68E2B39FE75F5940777E1089191ED13FA072527C4C5F594075E2ADCAFD7FD13F91E04AB1B75E59403E239C3B67EBD13F76C5F9D1295E5940037B127BEB5FD23F5F3AE16AA35D59401FACE62317DDD23F3EB51D01255D59405F63D43E6E62D33F2539E211AF5C5940162841BD6CEFD33F025CFC11425C5940F63F03FC8683D43F8E9C606DDE5B59408402AA4D2A1ED53FA27ABF86845B594040ADBE8BBDBED53FC5BB23B7345B59405FE86DAEA164D63F673C9A4DEF5A5940C0D7036A320FD73F13A5E38EB45A5940DFD49ED1C6BDD73FD75030B5845A5940B6C977FEB16FD83F5FA8E6EF5F5A5940009E1ABB4324D93FA02A7463465A5940CC45E531C9DAD93FA8512929385A594002C4209E8D92DA3F5277204F355A59407EE104FFDA4ADB3FD3D22FD83D5A5940FE73F5CBFA02DC3F129DE6BB515A594037D846A936BADC3FEA5E95E6705A5940E2BAD61CD96FDD3F406161399B5A5940CA73C6412E23DE3FFB2B638AD05A5940810EA57A84D3DE3F0DF6CFA4105B594003A258212D80DF3FC2DD2D495B5B59401C670C9A3E14E03F19B6922DB05B5940E68A67FFE665E03F67DA0AE0C65B59400008EABDC678E03F4D2BEDFD0E5C594026AA1AE0BEB4E03F7DF6575C775C5940F15F07257800E13F86CE76E1E85C5940465BEECDC748E13F48BADC1C635D5940CB02BB3B668DE13F695F7B95E55D594057B076770FCEE13FA0DF1ACA6F5E5940482B9D75830AE23F0ECED931015F5940F19B8F558642E23F1BBEB43C995F59404507E79CE075E23F7045DA4700605940B8507AD72E94E23FC1E5145437605940A6766B6E5FA4E23FFB4565DBDA605940083F79BCD4CDE23FACC5AD3083615940CC60A27617F2E23F39A533AD2F62594027B25EB20311E33FCFAB1EA6DF6259404B6CA2CE7A2AE33F726A226D926359404BC23792633EE33FF9EC2A5147645940825BBD44AA4CE33F7A2D0C9FFD64594077DC30C24055E33F0F9C33A2B46559401029F2881E58E33FA40A5BA56B66594077DC30C24055E33F254B3CF321675940825BBD44AA4CE33FACCD44D7D66759404BC23792633EE33F4F8C489E896859404B6CA2CE7A2AE33FE59233973969594026B25EB20311E33F7272B913E6695940CC60A27617F2E23F23F201698E6A5940083F79BCD4CDE23F5D5252F0316B5940A5766B6E5FA4E23F037AB207D06B59404507E79CE075E23F106A8D12686C5940F29B8F558642E23F7D584C7AF96C5940482B9D75830AE23FB5D8EBAE836D594058B076770FCEE13FD67D8A27066E5940CC02BB3B668DE13F9869F062806E5940465BEECDC748E13FA1410FE8F16E5940F15F07257800E13FD10C7A465A6F594026AA1AE0BEB4E03FB65D5C64A26F59400008EABDC678E03F +1 19 0103000000010000006600000000000000005059405A665E73030C0F40474D806BB7505940EA4D5CFB4B0B0F4042153A216E5159401CD7284926090F40EF521B6C2352594002E0877D94050F40254D7A98D65259402C6211229A000F4036F9C7F4865359403732AE253CFA0E40414240D2335459404EFCB1D780F20E40A9849785DC5459402E6497E16FE90E404E90A46780555940808F643F12DF0E409E8706D61E565940F7B7BF3672D30E40F5F5C533B756594016B3BC4C9BC60E409C7CF0E94857594001A36D3A9AB80E40807B2E68D35759403E4042E07CA90E40A2205225565859403A5B423852990E405D50DF9FD0585940595831472A880E40C9DF8B5E42595940CC7CA90C16760E4058A2B7F0AA595940B9E83E7227630E4044D4DBEE095A5940F50FBB39714F0E401A74F1FA5E5A59406E6682EA063B0E40DA26CEC0A95A5940F5C736BEFC250E40DA4B77F6E95A594068E1A98C67100E404EEF6A5C1F5B5940BA8D33B75CFA0D40C054DEBD495B5940B5B08013F2E30D40F5DCF1F0685B5940E59BEFD53DCD0D40A515DAD67C5B5940F7788F7B56B60D40F1CBFD5B855B59409D8CD8B3529F0D40B5050978825B59405A68334A49880D409DD0F42D745B5940674E650F51710D4003E1038C5A5B59401222F8C2805A0D40BA05B4AB355B5940B03FB4FCEE430D40B980A4B1055B59408A824216B22D0D40AC5F71CDCA5A594041940C15E0170D401CFA8339855A59406D6071948E020D408BC5D83A355A5940EB3163B0D2ED0C4034BABA20DB595940FA9384F0C0D90C40788D7444775959403C92D8336DC60C409F10F8080A595940645F1A9DEAB30C40770C7CDA93585940BCC2CF7F4BA20C404EFB102E155859400AF4284EA1910C40E30B2D818E575940BBC0BF87FC810C4015E02F590057594029F846A96C730C406982DE426B565940972F3A1D00660C40A914D8D1CF555940A9F09C2DC4590C405BC103A02E555940AD59D7F6C44E0C40D47FF84C885459407E1DBD5B0D450C401C415F7DDD5359402EAACAFAA63C0C40E02050DA2E535940090DA3249A350C40C33AAB107D52594000EED7D3ED2F0C4063C86DD0C8515940EBB703A6A72B0C402D3004CC125159405CB63CD6CB280C40E2AF99B75B505940489EE6385D270C401E506648A44F5940489EE6385D270C40D3CFFB33ED4E59405CB63CD6CB280C409D37922F374E5940EBB703A6A72B0C403DC554EF824D594001EED7D3ED2F0C4020DFAF25D14C5940090DA3249A350C40E4BEA082224C59402EAACAFAA63C0C402B8007B3774B59407E1DBD5B0D450C40A53EFC5FD14A5940AD59D7F6C44E0C4057EB272E304A5940A9F09C2DC4590C40977D21BD94495940972F3A1D00660C40EB1FD0A6FF4859402AF846A96C730C401DF4D27E71485940BAC0BF87FC810C40B204EFD1EA4759400AF4284EA1910C4089F383256C475940BAC2CF7F4BA20C4061EF07F7F5465940645F1A9DEAB30C4088728BBB884659403E92D8336DC60C40CC4545DF24465940FA9384F0C0D90C40753A27C5CA455940EB3163B0D2ED0C40E4057CC67A4559406D6071948E020D4054A08E323545594041940C15E0170D40477F5B4EFA4459408A824216B22D0D4046FA4B54CA445940B03FB4FCEE430D40FD1EFC73A54459401222F8C2805A0D40632F0BD28B445940694E650F51710D404AFAF6877D4459405A68334A49880D400F3402A47A4459409D8CD8B3529F0D405BEA252983445940F7788F7B56B60D400B230E0F97445940E59BEFD53DCD0D4040AB2142B6445940B5B08013F2E30D40B21095A3E0445940BA8D33B75CFA0D4026B488091645594066E1A98C67100E4026D9313F56455940F6C736BEFC250E40E68B0E05A14559406E6682EA063B0E40BC2B2411F6455940F50FBB39714F0E40A85D480F55465940B9E83E7227630E40372074A1BD465940CA7CA90C16760E40A3AF20602F475940595831472A880E405EDFADDAA94759403A5B423852990E408084D1972C4859403E4042E07CA90E4064830F16B748594001A36D3A9AB80E400B0A3ACC4849594014B3BC4C9BC60E406278F929E1495940F7B7BF3672D30E40B26F5B987F4A5940808F643F12DF0E40577B687A234B5940306497E16FE90E40BFBDBF2DCC4B59404EFCB1D780F20E40CA06380B794C59403732AE253CFA0E40DBB28567294D59402C6211229A000F4011ADE493DC4D594002E0877D94050F40BEEAC5DE914E59401CD7284926090F40B9B27F94484F5940EA4D5CFB4B0B0F4000000000005059405A665E73030C0F40 +1 20 0103000000010000006800000005067E1F835A59400004F55E633CF03FB2F95C4F645A59408FAB79E2C01FF03FB1D8338D295A5940F5BE56E037E8EF3FB4F2F51FE4595940ABFA07DBEF92EF3F830F6B4C945959406D97492AFE3FEF3F1CF2A6613A595940CFD1A5F2B4EFEE3F23FBBAB8D6585940263B9AB663A2EE3FB0E95DB4695859407725DF075758EE3FCF118AC0F3575940BDB1983BD811EE3F8B69125275575940517CBC212DCFED3F39D62EE6EE56594046B7F3BF9790ED3F852C00026156594085113E105656ED3FD65D0C32CC555940231496C3A120ED3F6355B309315559407DA9D308B0EFEC3F8B0E9D22905459401C510558B1C3EC3FBA74211CEA5359404D197442D19CEC3F45A3AA9A3F5359409EDA8147367BEC3F312112479152594078648CAE015FEC3FEEB9F9CDDF515940DB53FB654F48EC3F4C9820DF2B515940593099E73537EC3FA04CB52C765059405C2E5322C62BEC3F5A6AA56ABF4F5940DC9D75690B26EC3F4C6BEB4D084F5940DC9D75690B26EC3F0689DB8B514E59405C2E5322C62BEC3F5A3D70D99B4D5940593099E73537EC3FB81B97EAE74C5940DC53FB654F48EC3F75B47E71364C594078648CAE015FEC3F6132E61D884B59409EDA8147367BEC3FEC606F9CDD4A59404D197442D19CEC3F1BC7F395374A59401C510558B1C3EC3F4380DDAE964959407DA9D308B0EFEC3FD0778486FB485940231496C3A120ED3F21A990B66648594086113E105656ED3F6DFF61D2D847594044B7F3BF9790ED3F1B6C7E6652475940517CBC212DCFED3FD7C306F8D3465940BDB1983BD811EE3FF6EB32045E4659407725DF075758EE3F83DAD5FFF0455940283B9AB663A2EE3F8AE3E9568D455940CFD1A5F2B4EFEE3F23C6256C334559406D97492AFE3FEF3FF2E29A98E3445940A9FA07DBEF92EF3FF5FC5C2B9E445940F5BE56E037E8EF3FF4DB3369634459408FAB79E2C01FF03FA1CF1299444459400004F55E633CF03F3B1E578C33445940E64A418B3B4CF03F437D34C40E445940357F94DD5F79F03F96BE4035F54359407786632401A7F03FC47FD3F8E64359405295C12EF2D4F03F9A010E1DE44359401BDDA47C0503F13FE20BCDA4EC435940BF57F56B0D31F13F15F8A587004459405AC4BD65DC5EF13F6CE6EEB11F445940DF1B520B458CF13FB115D2044A44594090CD3E631AB9F13F434B6C567F4459405C47D40530E5F13FB92DF671BF44594069AE22495A10F23FE569F8170A455940F7373A6C6E3AF23F286F8AFE5E4559403A4985C14263F23F86859BD1BD455940226612D8AE8AF23FD2F54533264659407C01A5A38BB0F23FFFF12BBC97465940477465A3B3D4F23F38E2DDFB11475940F3BC090703F7F23FC3B1497994475940F32752D25717F33FE4AD32B31E48594065A9B6FE9135F33F4D80B120B04859404378239B9351F33F7FC6BB31484959405566A5E9406BF33FFABFB24FE6495940867AE87A8082F33FD985F8DD894A5940217B6D473B97F33FEF378B3A324B594015595DC65CA9F33F8385A5BEDE4B594064CAE301D3B8F33F45F263BF8E4C5940B2D5FCA88EC5F33F6D336E8E414D59408799A31E83CFF33FB2FCA37AF64D594029345386A6D6F33F5691CCD0AC4E5940085BCDCDF1DAF33FD36A48DC634F594091E31CB460DCF33F5044C4E71A505940085BCDCDF1DAF33FF4D8EC3DD150594029345386A6D6F33F39A2222A865159408799A31E83CFF33F61E32CF938525940B2D5FCA88EC5F33F2350EBF9E852594063CAE301D3B8F33FB79D057E9553594015595DC65CA9F33FCD4F98DA3D545940217B6D473B97F33FAC15DE68E1545940877AE87A8082F33F270FD5867F5559405566A5E9406BF33F5955DF97175659404378239B9351F33FC2275E05A956594065A9B6FE9135F33FE323473F33575940F32752D25717F33F6EF3B2BCB5575940F2BC090703F7F23FA7E364FC2F585940477465A3B3D4F23FD4DF4A85A15859407D01A5A38BB0F23F2050F5E609595940236612D8AE8AF23F7E6606BA685959403A4985C14263F23FC16B98A0BD595940F7373A6C6E3AF23FEDA79A46085A594069AE22495A10F23F638A2462485A59405C47D40530E5F13FF5BFBEB37D5A594090CD3E631AB9F13F3AEFA106A85A5940DF1B520B458CF13F91DDEA30C75A59405AC4BD65DC5EF13FC4C9C313DB5A5940BF57F56B0D31F13F0BD4829BE35A59401BDDA47C0503F13FE155BDBFE05A59405295C12EF2D4F03F0F175083D25A59407786632401A7F03F63585CF4B85A5940357F94DD5F79F03F6BB7392C945A5940E64A418B3B4CF03F05067E1F835A59400004F55E633CF03F +1 21 0103000000010000006A000000E8CF29FFE05359400041BDD7180F1040AD3D6570F253594070E5C48EC0081040A2D14457065459402C00A5CA99FA0F404B4B66DC0E5459400B0FE00B96E30F406CDE77F70B545940E3D49DAC8CCC0F40AABD75ABFD5359400733987D94B50F40B3ADA606E4535940C50E483EC49E0F40746C8D22BF5359405E4F5F8632880F409FFCCE238F535940E71F6BAFF5710F40A1F00D3A545359405A8FB4BE235C0F40E0DABA9F0E5359401069754FD2460F40DA13DA99BE5259401BC9767D16320F400011BF776452594021852ED0041E0F405691BD9200525940F1027026B10A0F40E0EED04D93515940FA81C4A22EF80E4042ED39151D515940D0357E988FE60E406667135E9E50594083D69879E5D50E40AC46DEA517505940DC7E79C540C60E409E360572894F5940DAD39EF8B0B70E402590584FF44E5940F782517D44AA0E40CEFE82D1584E59405827649D089E0E4083697792B74D5940829A107509930E40CAAED831114D5940619BFFE651890E4094CA5B54664C5940EC908691EB800E40CB0025A3B74B5940F1FB25C5DE790E40E6AC20CB054B59405FF0517C32740E40D75A587C514A594077A98C54EC6F0E4033D144699B495940F6FFD988106D0E4079B71D46E44859404A3691EDA16B0E40A78527C82C4859404B3691EDA16B0E40ED6B00A575475940F6FFD988106D0E4049E2EC91BF46594077A98C54EC6F0E403A9024430B4659405FF0517C32740E40553C206B59455940F1FB25C5DE790E408C72E9B9AA445940EC908691EB800E40568E6CDCFF4359405F9BFFE651890E409DD3CD7B59435940829A107509930E40523EC23CB84259405827649D089E0E40FBACECBE1C425940F782517D44AA0E408206409C87415940DCD39EF8B0B70E4074F66668F9405940DC7E79C540C60E40BAD531B07240594083D69879E5D50E407045DA4700405940B3066B3AF1E40E40DE4F0BF9F33F5940CE357E988FE60E40404E74C07D3F5940FA81C4A22EF80E40CAAB877B103F5940F3027026B10A0F40202C8696AC3E594021852ED0041E0F4046296B74523E59401BC9767D16320F4040628A6E023E59401069754FD2460F407F4C37D4BC3D59405A8FB4BE235C0F40814076EA813D5940E71F6BAFF5710F40ACD0B7EB513D59405E4F5F8632880F406D8F9E072D3D5940C50E483EC49E0F40767FCF62133D59400933987D94B50F40B45ECD16053D5940E3D49DAC8CCC0F40D5F1DE31023D59400B0FE00B96E30F407E6B00B70A3D59402C00A5CA99FA0F4073FFDF9D1E3D594070E5C48EC0081040386D1B0F303D59400041BDD7180F1040F0A6E5D23D3D5940391C8BAA1A14104057104637683D594002D21AFA4F1F104071A920A19D3D594016753F63552A10404EA8A8DBDD3D594014A31AFB1F35104015EC58A7283E5940C223F610A53F10400A8432BA7D3E59405AB2DC38DA4910409FA005C0DC3E59408E0DEF55B5531040C0A7C45A453F5940A6086BA42C5D1040841BE122B73F594071A35AC3366610407045DA4700405940545D5AC6556B104046F8B1A7314059407284E1BDCA6E1040D525E36FB44059408D901F14E0761040FA8DEDF93E41594003C49FC36E7E1040C96197BCD0415940E4E24A4F6F851040FD0E7C27694259404511D6C6DA8B1040815E9BA307435940AEE6A5CDAA911040122EEF93AB435940201A1FA1D99610401530075654445940EC755E1E629B10405316AA4201455940435A52C73F9F104009877BAEB145594095B330C76EA210405236A6EA64465940EAEC44F6EBA410400C7A89451A475940800E12DDB4A6104066AA690BD1475940B5E4C5B6C7A71040909E2287884859403EBBFA7223A81040BA92DB0240495940B5E4C5B6C7A7104014C3BBC8F6495940800E12DDB4A61040CE069F23AC4A5940EAEC44F6EBA4104017B6C95F5F4B594095B330C76EA21040CD269BCB0F4C5940435A52C73F9F10400B0D3EB8BC4C5940EC755E1E629B10400E0F567A654D5940201A1FA1D99610409FDEA96A094E5940AEE6A5CDAA911040232EC9E6A74E59404511D6C6DA8B104057DBAD51404F5940E5E24A4F6F85104026AF5714D24F594003C49FC36E7E10404B17629E5C5059408D901F14E0761040DA449366DF5059407284E1BDCA6E10409C2164EB5951594071A35AC336661040609580B3CB515940A6086BA42C5D1040819C3F4E345259408E0DEF55B553104016B91254935259405AB2DC38DA4910400B51EC66E8525940C223F610A53F1040D2949C323353594013A31AFB1F351040AF93246D7353594016753F63552A1040C92CFFD6A853594002D21AFA4F1F104030965F3BD3535940391C8BAA1A141040E8CF29FFE05359400041BDD7180F1040 +1 22 0103000000010000006E0000007061DD7877475940DB3985BD69720F40AA1325E72E485940586E7045B2710F40AE1DA39FE5485940BE5405938C6F0F402F5C42ED9A495940BC1F09C7FA6B0F40F000561C4E4A59403FFC156B00670F40EDF94B7BFE4A594051C6176EA2600F40033D5D5BAB4B59406AF8651FE7580F40AA473B11544C594007BB7F28D64F0F403125BAF5F74C59405F5B708578450F4099517666964D594028C7E37BD8390F407CD375C62E4E594072FBF390012D0F4050ECC37EC04E5940959AB97D001F0F4002C406FF4A4F59404319AB22E30F0F40057B0EBECD4F59407F22D779B8FF0E40D5145D3A4850594006F5078890EE0E4039B4A6FAB9505940A191DE4C7CDC0E4066AA498E22515940C696F5B18DC90E408CE2BD8D81515940A39B1C79D7B50E402B3CFB9AD65159403FC0BD296DA10E40996FD661215259404CFA7FFD628C0E40E921549861525940CE6539CCCD760E4031D7F1FE96525940F98B45F7C2600E40597CE460C1525940972A5454584A0E40174C4C94E0525940F588C517A4330E406CDD5D7AF452594096D5A9BEBC1C0E40A83480FFFC525940146079F8B8050E4002BB5F1BFA525940C7C59B90AFEE0D40C90BF6D0EB5259408154D457B7D70D403091862ED252594085FDA90DE7C00D40E7F58F4DAD5259403934E14955AA0D40727BB2527D52594092FE1D6618940D405E508B6D42525940A752C467467E0D40F40B85D8FC5159400A9C2CEAF4680D40E67F9DD8AC51594007EE410939540D40A91A21BD52515940B9FC9E4C27400D406F1F5CDFEE5059407D773E93D32C0D40A80242A28150594049CCD2FF501A0D4053430B720B505940C4B3D8E5B1080D40E822CAC38C4F5940E42C77B707F80C402FA6F614064F5940B8C23EF462E80C402152F2EA774E5940AC19D918D3D90C409C1F84D2E24D59405DD4B88F66CC0C407A284D5F474D594040E2D8A22AC00C407D97362BA64C5940153F996E2BB50C40256BD9D5FF4B59401C10C6D573AB0C4056A1E003554B594061E6D4760DA30C404767665EA64A5940B2BB62A2009C0C40DAED4C92F44959403004FC5254960C40DC86934F40495940A4E936260E920C400AB3A8488A485940E1782657328F0C40F6CBB931D347594057362CBAC38D0C40EBF600C01B47594057362CBAC38D0C40D70F12A964465940E1782657328F0C40043C27A2AE455940A4E936260E920C4006D56D5FFA4459403104FC5254960C40995B549348445940B2BB62A2009C0C407C7E7565F243594033520D3A7B9F0C40683100408D4359408096FE9F6FBD0C40702901E081435940003DF3DF74C00C40807B00206143594080740C607BC60C40EC5500007E4259408055F87F4AE70C40BCC1FF3F1642594000B8EF3F9CF50C40F098FFFF3B41594000360AC074130D4028B9FF3F03415940009C0740741A0D406C91FF5FBB40594000CC0A4001220D407045DA470040594000F963D91A350D405CD9FF7FCA3F594080A90860983A0D40FC560060B03E59408021F9BF2D580D408CEAFF7F903E59408068F0BFE15A0D40048300C0813E59400013F83F975B0D40A8DDFFFF733E594000E7F5FF285C0D40F8A7FF3F5D3E594000E7F5FF285C0D403C8600E0383E594000FDF61FE05B0D40A0340060243E594000981080735B0D4038BAFF9F153E59408068F0BFE15A0D40754F6B1F113D5940EA042E7EEA600D40ECB63519F23C5940099C2CEAF4680D4083722F84AC3C5940A752C467467E0D406E47089F713C594090FE1D6618940D40FACC2AA4413C59403934E14955AA0D40B13134C31C3C594085FDA90DE7C00D4018B7C420033C59408154D457B7D70D40DE075BD6F43B5940C7C59B90AFEE0D40398E3AF2F13B5940146079F8B8050E4075E55C77FA3B594096D5A9BEBC1C0E40CA766E5D0E3C5940F588C517A4330E408846D6902D3C5940972A5454584A0E40AFEBC8F2573C5940F98B45F7C2600E40F8A066598D3C5940CE6539CCCD760E404753E48FCD3C59404CFA7FFD628C0E40B586BF56183D59403FC0BD296DA10E4054E0FC636D3D5940A39B1C79D7B50E407B187163CC3D5940C696F5B18DC90E40A70E14F7343E5940A091DE4C7CDC0E400BAE5DB7A63E594006F5078890EE0E40DB47AC33213F59408122D779B8FF0E40DFFEB3F2A33F59404319AB22E30F0F407045DA4700405940CF29B5AAF6190F4090D6F6722E405940959AB97D001F0F4064EF442BC040594072FBF390012D0F404771448B5841594026C7E37BD8390F40AF9D00FCF64159405D5B708578450F40367B7FE09A42594007BB7F28D64F0F40DE855D96434359406AF8651FE7580F40F3C86E76F043594053C6176EA2600F40F0C164D5A04459403FFC156B00670F40B166780454455940BC1F09C7FA6B0F4033A5175209465940BE5405938C6F0F4036AF950AC0465940586E7045B2710F407061DD7877475940DB3985BD69720F40 +1 23 01060000000400000001030000000100000016000000901600E0012559404059FE3FF8AB1140684E0060FD245940405AF99FDBAB1140D47100C0FB24594080630120E2AB114024D6FF5FF3245940C06103401AAC1140243C0000E52459408005F9FF8FAC1140B87E0040D8245940C0EF07804DAD1140A494FF7FD2245940005104E0BBAD11403C200040C72459408064FA9FA8AE11408C500020C224594040CA02601BAF1140880700A0C02459404046FD3F98AF1140105800C0C2245940801300C00AB011402C4B0040C62459404017FA9F7DB0114054CAFF3FC9245940001D05209FB011408092FFBFCD2459408014F93FD1B01140C8190000D924594000A7FFDFEAB0114020F3FF7FE324594040D3FA3FADB0114030C8FF7FE4245940404702A099B01140D8D1FFDFE9245940C088FB9F2EB0114040E6FFFF06255940402301A013AD1140D40B00200A25594080350100ACAC1140406900C008255940004A030046AC1140901600E0012559404059FE3FF8AB11400103000000010000000F000000049AFF5F6E2B594040D301205972114020760040652B594040D3012059721140D0AB00005C2B5940406707E08F72114020760040652B5940004C01E00D74114040C9FFDF762B5940C01302A027751140880700A0802B594080F8FD7FC2751140487800008A2B5940401D0020107611408CCDFF5FA02B5940401D002010761140F4F8FE1FAA2B5940406AF75F7275114050040080A92B5940C02400C030751140243C0000A52B5940C009FE7FE87411408CCDFF5FA02B5940C009FE7FE8741140740600408E2B594000620200C5731140383D0060772B5940C00E018096721140049AFF5F6E2B594040D301205972114001030000000100000031000000D94F8B9655255940ECCFE2F8CE4E12400CAA1602CD2559403BF460F394521240E31CCCDC7526594078F2B5731D5712401A5EBBE22227594066A06D1FFB5A1240446E6C68D3275940A09EAA212A5E1240D68BEDBE86285940FB15A852A76012406AEE80343C295940481DDC3A7062124034504D15F3295940F2186B1583631240C8F5DF3E372A59406371AD24A563124030ABFF5F342A5940409AFF5F8E631240C8190000192A594080C601A0FC621240C4EDFF9FA7295940C04603E0EE601240347700A09729594000460740B6601240C8B3FF5F8729594000460740B6601240602200008C295940C052F9FFBA5C124090B0FF3FD029594040A007C03D4A1240B878FFBFD4295940C03BFA5FAE471240B878FFBFD4295940007EFDBFD3461240B878FFBFD429594040AAFFFF41461240B09DFE3FD029594080EC026067451240D85400A0CB295940803100408D441240E07AFF7FB9295940008A04C06943124044FB00C0AB295940C00700A020431240B4B2FFFF94295940C00700A0204312406CFD008090295940008A04C069431240ECB5FF1F6C295940008A04C069431240982500204329594080BBFAFFD8421240F82A0000DF285940008F00E06B411240ACC0FFDFA32859404085FE9F49401240046600A09128594040160120B73F1240E0FDFF3F1B285940005104E0BB3B12400400000000285940C0A7F99F063A12409093FF1FE0275940C002F89FE3381240B06F0000D7275940C01507609B381240C4700060C9275940002FFADF51381240E8720020AE275940405BFC1FC0371240B4CFFF1F85275940405BFC1FC03712400C7500E07227594000AAFB3F083812405C3F00205C275940C018F9BF9A381240048300C081265940405BFEFFDC3E1240CC79FF1FC7255940C04701604444124064E8FFBFAB255940C01805A0D5441240105800C0A2255940803100408D4412402C7FFFFF6225594000CF01A08F421240882400C05025594000CF01A08F421240705D00A03E255940007403A0B2431240D8EEFFFF3925594080D60140B0451240D8EEFFFF39255940007EFDBFD3461240D94F8B9655255940ECCFE2F8CE4E12400103000000010000006F010000997048275B3459400041BDD7180F12407399E6EBB6345940E996B270950512404564210A0C35594032ECF94860FB11400D2AA3DF563559408A933434DBF01140CA7C6522973559404C366A9E10E61140249CDD92CC355940872660380BDB11400CF83AFCF6355940174300EDD5CF11403E0A9B3416365940BFB786D67BC411403D55321D2A365940B958813308B9114043626AA232365940DE84AB5B86AD11407AA2F4BB2F3659409C9AB1B401A21140F422D26C213659403822E7A685961140CA0D50C30736594080DAFA911D8B1140F0FDF8D8E2355940B0D5B3C1D47F114044367BD2B2355940D6C8C262B674114033D783DF77355940709AB277CD69114023398F3A323559406A1C03CE245F1140709CAE28E234594023B278F3C65411401C6A43F9873459406F6DAB2BBE4A1140764AB00524345940A8EDDF6514411140F16000B1B6335940B1013433D3371140CA0585674033594031BA28BD032F1140745F6A9EC1325940613C93BCAE261140654643D33A3259405A3FFD70DC1E11407AE68C8BAC31594097AF7D9894171140EB982A54173159402C7D1168DE1011405879DAC07B3059409D1A7C84C00A1140933FA36BDA2F59408BACB6FB4005114002EE3BF4332F5940B65DF43E65001140A0EA6DFF882E5940C2B7411D32FC1040E41D7236DA2D59401049C4BEABF810408DB74946282D5940E6419FA0D5F51040A63D13DF732C59405D118191B2F31040808D5CB3BD2B59403D64DDAE44F2104061797277062B5940D63ED6628DF11040E4AFAEE04E2A5940D73ED6628DF11040C59BC4A4972959403D64DDAE44F21040D3DB7EFE5C295940F1BFA57ABAF21040FC560060B02959400040FFDF06FA1040CC79FF1FC7295940C0CEFFBF72FB104094DCFF9FE1295940C0BFFF7F31FD104008E3FFDF0F2A5940C051F99F48001140ACC0FFDF232A5940005FFDDFDE011140C8360020292A594000D1FF7F57021140F0FEFF9F2D2A59408085FB7FD7021140B8FBFF7F562A5940804DFBFF9B091140DC8000005D2A594000DE06E05F0B1140901600E0612A5940C0D4F95FCA0E1140B003FFDF612A594080A8F71F5C0F1140B4CFFF1F852B5940C0FD07601C071140F0FEFF9F0D2C594040EFFAFF4AFE1040886D0040122C5940401BFD3FB9FD1040485B00E0192C594080A104003EFD1040E0FDFF3F1B2C5940404AF9FF27FD1040506A00203B2C5940808CFC5F4DFC1040880D0120642C5940C0B8FE9FBBFB104094F9FFBF712C5940C0B8FE9FBBFB104004000000802C594000EC01C09FFC104030C8FF7F042C5940C0580660F9051140EC3800E0ED2B5940C01D06C066071140FC730080E02B594040310360FF071140B06F0000D72B594000F600E04F081140B895FFDFC42B5940007901A0D108114068CBFF9FBB2B5940C0EFFB9F12091140BC440000B82B594080A0FDBF1F091140143B00A0B22B5940007901A0D10811408CCDFF5FA02B5940408F02C088081140383D0060972B5940408F02C088081140ECD2FF3F3C2B59404040FCBF940A11407CCCFFFF2D2B594000DE06E05F0B11405CD9FF7FCA2A5940C09B056057131140048300C0812A5940C05E000051191140404C00A0782A594080ED00E0BC1A1140347700A0772A594040B706604A1D1140DC8000007D2A5940000902E0AF201140048300C0812A59400073F93F2222114048D8FF1FD82A59400067FE1FC7311140B81800A0E62A5940C05205E0F5321140646B0080ED2A59400052FB7F65331140BC440000F82A5940C0C9FCBFC4331140E80C00801C2B5940809DFA7F56341140581300C04A2B594000F4FB1FDC341140003A0040C02B5940800903807737114090330000522C594080C7FCFFDF3A1140BCC1FF3F562C5940008AF8DF2E3B1140243C0000452C594040C20300503C11408092FFBFED2B5940003B03C0043C114088B8FE9FDB2B5940C0B8FE9FBB3B114088A1FFFFCE2B594000180720803B1140B0090060C52B594000D9FEDF223B1140383D0060B72B5940C0E906004A3A1140B0090060A52B5940C0A9054026391140243C0000A52B594080980360E3381140740600408E2B5940800903807737114038D7FFBF852B5940003DFC9F3D371140149BFFBF602B5940C03505C0E5361140487800004A2B59408009038077371140844100E0402B594080F30160C0371140C00A00C0372B594080DD004009381140E80C00801C2B594000560000BE3911404C8D01C00E2B594000FB0100E13A11409476FFFFEF2A5940405BFC1FC03D11407C1500806F2A5940007906A0604B114034110000662A5940C0B4FBFF7F4C1140901600E0612A594080C50320A74D1140B003FFDF612A5940C0A8F9FF78561140E8EFFF5F6C2A5940405EFA5FFA5711406C1400207D2A5940007207A0075911400C2C0060F12A5940C093FA1F515C1140807B0020012B594080E804C0BA5C1140A0510080142B594080C50320A75D114000D4FF9F2E2B5940001F01204A5F1140B06F0000372B5940803203C071601140C00A00C0372B594000CF01A08F6011400CC0FE3F3C2B59404044FD7FB3621140ECD2FF3F3C2B5940C090FAFFF9641140404C00A0382B5940C0BCFC3F6866114098420040332B594080A4F97FAF66114000D4FF9F2E2B59408075FDBF40671140D40B00202A2B59404033FA5F1B681140D40B00202A2B5940000907E03E691140E4C3FFFF3A2B594000ACF91FD06B1140E0E0FF1F4B2B5940405702404D6D1140201000A0932B59400084034066721140143B00A0B22B5940003600C056741140C8360020C92B5940804902607E75114094F9FFBFD12B59400096FFDFC4751140A494FF7FD22B594080E2FC5F0B76114050040080C92B5940C0B502409E76114010D5FFFFA02B594000B6F83F80771140A0CEFFBF922B594000B6F83F80771140740600408E2B594000B6F83F807711404C0A01008D2B594000B6F83F80771140D0AB00007C2B5940406CFC1FE67611404063FF3F652B594080F8FD7FC2751140347700A0572B5940C009FE7FE874114090330000522B5940003600C056741140509EFFDF172B594000CE03203A6D1140487800000A2B594080BDFDDF2F6B1140609FFF3F0A2B5940401DF93F6468114038D7FFBF052B594080A4F97FAF661140D0450060EA2A594080E1FEDFB56011400C0F0040E12A594000350240015F1140201000A0D32A594080A10220215E1140901600E0612A59400084F75F2B5A1140E0E0FF1F4B2A594040B0F99F9959114028850080462A594040B0F99F9959114030ABFF5F342A5940409AF87FE25911408075FF9F1D2A594000860620BD5A1140DC1A00600B2A5940C012F83F975B114058ADFF1FF929594080E804C0BA5C1140A877FF5FE2295940004B0360B85E1140B878FFBFD429594080F7FFFF6C601140FCF0FFBF9E29594040840020F46C1140403501008C29594040D3012059721140602200008C295940C03FFDFFE97211408CEAFF7F902959404056FBFF2E771140303401A099295940C018F9BF9A781140705D00A09E29594000DBFCBFEA781140FC560060B0295940C0A7F99F067A114090330000B2295940405D01E0337B1140B4B2FFFF94295940005104E0BB7B1140186700008429594080FFFE5F38771140DCB4FFBF79295940004C01E00D74114098BFFF7F71295940008000A02A711140187EFF9F7029594080040080C96F1140B086FF9F832959408046FA1F266A11404CBBFFFFA72959404035FA1F00611140DC800000DD29594080B90120A05B11400CF800A0F42959400084F75F2B5A11401CAAFFFF212A5940800FF81F40541140544D00002B2A59408096076038521140544D00002B2A594040EDFEFF9F511140C8190000192A5940C0E5FE5F7F501140A0510080142A5940C0E5FE5F7F501140345A0080072A5940804C0280D54C114040E6FFFF862A59404099FAFF8C44114040E6FFFF862A5940403EFCFFAF431140048300C0812A5940800AFA1F21431140CCDFFFBF782A5940800207A0904211407C1500806F2A5940C062FE9FFD41114034110000662A5940C0A7FB7F23411140DC8000005D2A5940C0550320303F1140F8A7FF3F5D2A5940C0B8FE9FBB3B1140901600E0612A594080980360E3381140544D00006B2A59408009038077371140048300C0812A5940007808200B3611402C4B0040862A5940007808200B361140805E0000912A5940C01104C05F351140A8DDFFFF932A594080A0FDBF1F351140D4A5FF7F982A59408087F95F9F341140D4A5FF7F982A5940C0C9FCBFC43311401C4A00E0932A594080F8F87F333311400CA9FF9F4F2A5940C090FCDF162C114078E9FF1F3E2A5940406B08A0AE2A1140608800A01D2A594000C905201B281140A0510080142A594000F5076089271140B05200E0062A5940000E030041271140B06F0000F729594080DAFDFF3F2711402CE5FF9FF4295940000E0300412711402C7FFFFFE22959404097080000271140001D0020F029594080F907E0521711402CE5FF9FF4295940C07EF95F0C151140F01B00C0FD29594000F0F87FA013114018E4FF3F022A594000B206800E13114040E6FFFF062A5940C059FFDFBF121140946501E0062A594080C00700A50F114058ADFF1FF92959400001FC9F380E1140105800C0A229594080E7064065071140D8EEFFFF7929594080AAFCDFCF0411409033000032295940C06C00E01F0111406CF7FFFFCC285940400EFBDF3FFD10404CBBFFFFC72859404086F71F10FD1040280200C0E4275940C013FD9F98FA10406CF7FFFFCC275940C07AFBBF5FFA10408CEAFF7F9027594000FFF8BFE1F91040602200008C275940C0B600C0F3F91040F80D00E0EE265940007403A0B2031140201000A0D3265940802706206C01114098A2FF5FC1265940C051F99F48001140DC800000BD265940C051F99F48001140CCDFFFBFB8265940C051F99F48001140F47BFFDF8B265940803FFE3F3F011140F82A00007F265940809C010090011140608800A01D26594000E006A044041140B878FFBFD4255940802C04208B061140AC8C0020C725594080E7064065071140141E0080C2255940000802803D0A1140141E0080C225594080CAFF3FA90B1140CC79FF1FC725594000A00300040C1140D85400A0CB25594040840700A00C1140EC550000DE2559400017FDBFEF11114040E6FFFFE6255940805406E02F16114040E6FFFFE6255940402804A0C11611401CAAFFFFE12559404021FCDF9F171140B4B2FFFFD4255940804A07C07F1811406CF7FFFF8C25594040C6FFBFDF1F11406CF7FFFF8C2559404004FCBF8F2311406CF7FFFF8C25594080DAFDFF3F2911406CF7FFFF8C255940801503A0612A1140141E008082255940805A0080872B1140CC620080DA245940C05D00A0DE3C114084BEFF1FBF2459408071FF1F943E11407CCCFFFFAD245940C0550320303F11403C8600E0982459408056FFBF683F1140A03400602424594000EAFEDF48401140B06F000017245940400000000040114054CAFF3FE92359408047FF7F273D114068E2FE3FE8235940806001008B3C11409093FF1FE0235940003B03C0043C11407883FF7FCC235940C0EB04E0113C11401CAAFFFFC1235940C0A1FAFF1F3C114074A0FF9FBC23594080D503C05A3C114060BCFF5FBA235940C0AD0100B63C1140D47100C0BB235940802C02406E3F1140C4700060C9235940C0FD07601C4711407C150080CF235940C0E9FA1F0F48114098420040D3235940C0B9F71F82481140A0510080D423594040FA0560A8481140280200C0E4235940003404C0AB491140208DFFDF5124594000ABFB9F7A541140A4FAFF1F6424594080E1F7FF095711402C7FFFFF62255940400AF83F047C1140AC26008075255940402F03A01A7F1140C819000079255940802CF83F50801140DCB4FFBF79255940C0CD0640AC801140FCA1FEBF7925594040110500B5811140E4A6FFDF6A255940C044F91FEC821140E4C3FFFF5A25594040C1FCBF31841140485B00E059255940C047016044841140485B00E059255940801BFF1FD6841140602200008C25594040D201C0E695114064E8FFBFAB255940C008FE1F769811401C44FF5FB025594080F2FCFFBE981140281F00E0B4255940C0B9FEFF2D9A1140281F00E0B4255940C02A0820E09B1140D8EEFFFF992559400078FC3FD0A51140602200008C255940C0A1FFFFAEA61140105800C082255940C050FB1FF3A61140B88FFE5F81255940808BFEDFF7A611404CBBFFFF6725594080CBF8BF6FA81140C4EDFF9F472559400018FD1F62AA1140C02700E04725594080E5068080AA114064E8FFBF4B255940004A030046AC114088A1FFFF6E255940403EFCFFAFB111409CEBFFDF8225594000620200C5B31140602200008C2559400067032056B41140EC550000DE25594000EC01C09FB811404CBBFFFFE725594000EC01C09FB81140842A0140F425594080DD004009B81140608800A0FD255940003BFEBF75B711400C0F004001265940C0040260E6B611407CAFFFDFFD2559408083F89F80B21140F01B00C0FD25594080D007A058B21140D0C2FF9F082659408065FFFFA9B11140DC1A00600B265940801505807EB1114040E6FFFF262659408022F91FA0B01140E0E0FF1F4B265940005808E0A3B01140D4A5FF7F98265940C04107C0ECB01140E8EFFF5FCC265940806B0080ADB21140C8B3FF5FE7265940804804E028B41140A0CEFFBFF2265940404E01A0F2B41140947C0080F3265940409C042002B51140BC440000F8265940800EFF9F79B511401CAAFFFF01275940C01A03809DB61140F44700202F275940C0170040D4BF1140149BFFBF6027594080CAFF3FA9CB1140740600408E27594080C6FC9F6DE0114088A1FFFF8E2759400042F8BF3FE3114088A1FFFF8E275940C090FCDF16EC1140F80D00E08E275940C0C6056036F11140DC97FF9F892759408009038077F71140740600408E275940006DFF9FCA0E1240543FDF7E8F2759400041BDD7180F1240C4530040992759400016FD5F7D11124050040080A927594000ABFB9F7A1412403C030020B727594080F7F81FC1161240003A0040C02759404021FCDF9F171240D882FEDFC4275940400D076008191240C4700060C9275940409AF87FE219124018010060F2275940001F01204A1F124040C9FFDFF6275940C0230240DB1F1240B43500C016285940C0EBF8FFD6231240488FFF9F36285940807C08A0D42512407057FF1F3B285940807C08A0D4251240302E0020962859400018FB3F45231240B00900602529594080DAFDFF3F271240742300605E295940801007805F2C1240441801E0FB29594040160120B73F1240C0A4FF1F062A5940404001C023411240784F00C02F2A594080CAFA3F1A491240784F00C02F2A5940C03603403B4C1240508700402B2A5940C0F10560154D12406C630120222A5940801B0600824E1240608800A01D2A594040540400134F1240C8190000192A5940002802C0A44F1240B05200E0062A5940403907C059511240F01B00C0FD295940000D0580EB51124058ADFF1FF9295940C0E002407D521240A877FF5FE229594000A6FF7F785612407CAFFFDFDD295940804DFBFF9B57124054E7FF5FD92959404055FB9FBC5A124054E7FF5FD9295940C0F7FAFFDD5D124068AEFF7FEB29594040B706604A5F1240FC07FF5F0B2A5940001A0500486112408C500020222A5940C0080500226212403C8600E0382A594080F203E06A621240E0E0FF1F4B2A594080C601A0FC6212407D0CADA4592A5940CAEDC954B6631240A39410ACAA2A5940719FE9D1DE63124011D9D342622B5940F2186B1583631240DB3AA023192C5940481DDC3A70621240709D3399CE2C5940FB15A852A760124001BBB4EF812D5940A09EAA212A5E12402BCB6575322E594066A06D1FFB5A1240630C557BDF2E594078F2B5731D571240397F0A56882F59403CF460F3945212408125325E2C3059401246961C664D1240DA1A42F1CA3059403272511296471240B8E01B726331594084FE47972A411240D23CA849F53159406F0431082A3A12408B0E6DE77F325940520974559B3212404A861CC2023359409E6A45FC852A124077301D587D335940064237FFF1211240284D0A30EF335940953046DEE7181240B1F52BD95734594060026A8E700F1240997048275B3459400041BDD7180F1240 +1 24 010600000003000000010300000001000000C0000000B39B12F4C92759400041BDD7180F144097C3B9F5822759402000C4DF770B14403C120196ED26594099990CC7C1041440D318D4D85126594057B09FF9A3FE13406F876058B02559408856388524F91340CF1685B4092559400724D0DA48F41340528633925E245940135C3FC915F0134075A1CD9AAF235940914D7D788FEC1340B5EC7D7BFD225940B5958565B9E91340F49D8CE448225940AD4AE65E96E71340FD87B18892215940A76EFA8128E61340A1A3631CDB2059404362D33871E5134036E32655232059404362D33871E513407045DA4700205940AF54683F94E51340DAFED8E86B1F5940A76EFA8128E61340E3E8FD8CB51E5940AD4AE65E96E71340229A0CF6001E5940B5958565B9E9134062E5BCD64E1D5940914D7D788FEC1340850057DF9F1C5940135C3FC915F01340097005BDF41B59400724D0DA48F4134068FF29194E1B59408856388524F91340046EB698AC1A594057B09FF9A3FE13409B7489DB101A594099990CC7C104144041C3D07B7B1959402000C4DF770B144024EB777D341959400041BDD7180F14402DD490642E19594069BC8CA0680F1440C02700E027195940805B07C0A5111440345A00802719594080B803A04A121440149BFFBF20195940C092FFBF6D1414408CEAFF7F10195940C0B502409E1814400C8CFF7FDF18594000E3FDFFD2241440886D0040121859400055F9BF9F4314407CCCFFFFCD175940800A0100CD4C144050040080A917594080E3F7BFEE4F1440B4CFFF1F851759404077F9BFEB511440509EFFDF17175940408808C0BE581440F80D00E00E175940C0D3FBDF74591440609FFF3F0A175940C01903202B5A1440609FFF3F0A175940C06F0500065B14406C7A00C00E17594080E1FEDFB5601440D40B00202A17594080ED07C068641440ECD2FF3F3C175940003A0540AF66144008C6FFBF5F175940C057060087691440F098FFFF7B17594080AFFDFF606B1440CC96FF3F971759400042FDBFCE6B14408CCDFF5FA0175940403F0340CE6B144064E8FFBFAB1759404083FBBFF26B14407CCCFFFFCD17594040FA0560A86C144068CBFF9FDB17594000E7FEBFF16C1440280200C0E417594000CE03203A6D144030C8FF7F04185940007305205D6E1440CCFCFFDF4818594080A90180EC701440600B01607F185940C0B3FB9F0D741440FC8AFF1F8D1859404071F83FE87414406CF7FFFFAC185940002FFADF5178144020F3FF7FE3185940C08CFE3F6A8314402C4B0040E6185940008AFFBFDA8614408CCDFF5F00195940C0D7F79F048B14404CBBFFFF07195940C03603403B8C14407883FF7F0C195940C0070780CC8C14406482FF1F1A19594080C50320A78D14402C7FFFFF62195940C0A0018059911440DCB4FFBF79195940C0A0018059911440105800C0A2195940C0E002407D92144064E8FFBFAB195940C0CA0120C69214401C44FF5FB0195940801901400E931440508700400B1A5940C0C3004050991440A0510080141A59400084F75F2B9A14406CF7FFFF8C1A5940003F0160B1A4144048D8FF1FF81A5940400AF83F04AC14407CCCFFFF0D1B59408063FA3F36AE14407CCCFFFF0D1B5940403206E0E3B11440609FFF3F0A1B594080E9024010B2144078E9FF1FFE1A5940800207A090B21440F82A0000DF1A5940808DFADFA2B31440201000A0D31A594080E7FA5F2ABB1440201000A0D31A594040AAFFFF41C61440A47D00E0E51A59400046F97F5ECD1440F80D00E0EE1A5940409405C036D01440947C0080F31A5940403907C059D11440BC440000F81A5940000D0580EBD11440F80D00E00E1B5940804301E07AD41440A4170040141B5940C04AF8BFD2D41440509EFFDF171B59404081FBFF0DD5144010D5FFFF201B5940809903C055D51440D40B00202A1B594080F203E06AE21440D40B00202A1B5940406B03A01FE4144010D5FFFF201B594000C905201BE81440E80C00801C1B5940805A008087E91440609FFF3F0A1B59408009FC9FCBED14400C0F0040E11A5940C01F04A02EF7144074A0FF9FDC1A5940C0DA06C008F814402C7FFFFF621A594080C5FE1F180B1540F8A7FF3F5D1A594080BE04204E0B1540488FFF9F361A59400069FA1F720C1540784F00C02F1A59404088FCDF830C1540508700400B1A594080850260830C154018E4FF3F021A594080C00200160D1540D02E01C0FD1959400017FDBFEF0D154058ADFF1FF9195940405C08606D1015405CD9FF7F4A1A5940005A01C0DC1515401CAAFFFF811A5940C04608E07D1915405081FFBF871A5940405AF99FDB1915404C2100A0991A5940C026FE9FF81A154098420040B31A594080460600611C1540EC210140E11A5940406104806F1E1540F4E1FF7F3D1B5940409FFB7F9021154010D5FFFF401B5940C0BAFC7F832115405C3F00205C1B59404044FD7FB3221540683100406D1B5940C08CFE3F6A231540880700A0801B59408060FCFFFB231540A0CEFFBF921B594040E406200E241540105800C0A21B594040930560C423154064E8FFBFAB1B594040930560C4231540FC560060B01B5940C0B3FB9F0D24154098BFFF7FB11B5940804301E07A241540F4470020AF1B59404071F83FE8241540D4A5FF7F981B5940807C08A0D425154024420180881B594040AAFFFF412615403488FEBF601B5940406607801D261540CCDFFFBF581B5940004705C00B2615409C080000531B594000D5FB3FE7251540C4EDFF9F471B594000EBFC5F9E2515403C9DFF7F251B59408087F95F9F24154038BAFF9F151B594040D7FDDFE8231540807B0020011B5940005DF81F6B22154004000000001B5940805EFE1F3422154058ADFF1FF91A5940C074FF3FEB211540947C0080F31A594040110500B521154098BFFF7FF11A594040110500B521154040E6FFFFE61A594000AFFE3FB621154048D8FF1FD81A59404089FA5FD9211540B8FBFF7FD61A5940C02207E0F7211540B88FFE5FC11A5940C001FA1F8E231540ACA3FFBFB31A594000A401A0B024154084DBFF3FAF1A594040AB046043251540105800C0A21A5940C079FD3F0A27154094DCFF9FA11A5940000E0300412715401C4A00E0931A5940007305205D2E1540543000E07A1A594080D0FBBF1D36154094F9FFBF711A5940C002F89FE338154084A77CB3621A59403D26572CAB3B154058F1D6F6FB1A594092A93FD1474115403DC6262DA01B59402390D3AC76461540E7DB8537491C594045CAA731FF4A1540B10C6D6EF61C59409AED6FE1DC4E1540B66A2F26A71D594071522EE70B521540EEC4A4AF5A1E5940F4EA021B8954154097ABD758101F594015224E05525615405045B66DC71F5940619B24E1645715407045DA47002059401B3DA741815715406C43C5387F205940485D119EC05715408741D40337215940619B24E16457154040DBB218EE21594015224E0552561540E9C1E5C1A3225940F4EA021B89541540211C5B4B5723594071522EE70B521540277A1D030824594099ED6FE1DC4E1540F0AA043AB524594045CAA731FF4A15409AC063445E2559402390D3AC764615407F95B37A0226594092A93FD1474115401A27393AA1265940C24710C2773B15402A20A7E539275940F3BD26420C3515409BF0B9E5CB2759408B5368AE0B2E15406FD7CDA956285940F94A6CF77C2615402A496EA8D92859408B73979A671E15404724DE5F54295940612EAC9AD31515408D2C9856C6295940C358D677C90C1540444EC71B2F2A59402D113B27520315409432B6478E2A5940C1C6140A77F9144082B7357CE32A5940A27464E441EF144008E7F9642E2B59409B5341D3BCE41440BF12EDB76E2B594072ABD042F2D9144067C37835A42B5940DBC2EFE3ECCE1440F635C4A8CE2B594028389AA1B7C31440612AE8E7ED2B5940B04D17965DB814400BD317D4012C594033EBF8FFE9AC1440A7BEBE590A2C59401D41F73668A114405CA19370072C5940E31AB4A0E3951440F3EC9F1BF92B59409D0371A5678A144018333C69DF2B5940936AC3A4FF7E14408E570173BA2B5940DCF451EAB673144060A3AE5D8A2B59401A1EA6A298681440EDD404594F2B5940CA331DD0AF5D14406D53969F092B5940D39503400753144033B78C76B92A59400CFBE47FA94814402EE2632D5F2A594082421CD3A03E14406DEE9A1DFB295940CC19AD28F734144025425BAA8D29594083767111B62B14403A22164017295940FB8CA4B6E622144088251954982859400491D4D0911A1440D0F31964112859406F27449FBF121440B39B12F4C92759400041BDD7180F1440010300000001000000100000006CF7FFFFAC185940C0FBF89F8A0615406CF7FFFFAC1859400012FABF41061540F46400409F185940C042054042061540302E002096185940C0FD07601C071540F82A0000DF18594000F0F87FA013154020F3FF7FE318594000DAF75FE9131540E42900A0EC18594000DAF75FE91315404CBBFFFF071959404006FA9F5713154010F2FF1F11195940401CFBBF0E1315406482FF1F1A1959408048FDFF7C121540FCF0FFBF1E1959404077F9BFEB111540FCF0FFBF1E1959400041FD5F5C0F15406482FF1F1A1959400001FC9F380E15407883FF7F0C195940409EFDFF3A0C154048F5FF3FC8185940C0E5F77FD30615406CF7FFFFAC185940C0FBF89F8A0615400103000000010000000F000000C087FFFFB5185940C00502C058111540C087FFFFB518594080B9FC1F111115406CF7FFFFAC18594040B9044012111540D488FF5FA81859404061F89F34121540C89C00C09A18594040CD05A0E41715405CF6FF9FBA1859400018FB3F4523154048F5FF3FC8185940805704206A241540A86000C0D5185940805704206A241540BC610020E818594080CBFDBFFE201540BC610020E8185940C00D012024201540F82A0000DF185940C068F83F55151540CC620080DA18594000F0F87FA013154070BDFFBFCC1859404061F89F3412154048F5FF3FC81859404077F9BFEB111540C087FFFFB5185940C00502C058111540 +1 25 01060000000900000001030000000100000012000000B0ECFF3F757D5D40C03002C037EF164020AAFFFF617D5D40C04603E0EEEE1640A894FF7F527D5D40C055032030EF1640882400C0507D5D40C03002C037EF1640A0B1FF9F427D5D4000030160ACEF16406082FF1F3A7D5D4080BF02A0A3F01640D8EEFFFF397D5D40C04107C0ECF0164098250020437D5D4080E9024010F21640882400C0507D5D4080A40560EAF21640702300605E7D5D40408E044033F31640A0EBFFDF827D5D40408E044033F31640602200008C7D5D4080A40560EAF21640502100A0997D5D4080E9024010F2164078E9FF1F9E7D5D40801505807EF1164078E9FF1F9E7D5D40C02B06A035F11640502100A0997D5D4080BF02A0A3F01640602200008C7D5D40009D05C0C9EF1640B0ECFF3F757D5D40C03002C037EF164001030000000100000012000000509EFFDF977C5D4080E50180F1B11640A84300A0857C5D4080A4FB5FCCB1164088EAFF7F707C5D40C0B2F81F29B2164080DBFF3F4F7C5D4040C5064019B316407074FF3F4B7C5D40006BFADF56B31640E097FF9F497C5D40C0250020A3B3164068E8FFBF4B7C5D4040200540F6B31640F84700204F7C5D40408AFABF4BB4164030ABFF5F547C5D408042FE5F96B41640D04500608A7C5D40C0CC06E039B6164010F2FF1F917C5D40800901A05AB61640509EFFDF977C5D4080A400605BB61640D02800409A7C5D4080A400605BB6164010D5FFFFA07C5D4080FEF9FF36B61640A84300A0A57C5D40800A06005CB51640A84300A0A57C5D40C01EF85F81B4164000F1FFBF9E7C5D4040FDF7BFA7B21640509EFFDF977C5D4080E50180F1B1164001030000000100000031010000794254B18E7A5D400041BDD7180F1640BE2F0D555E7A5D406DC7A314ED0C1640F5FB3077C2795D4039A13E57CF0616408BB3F7D420795D40817D8FF14F011640C35B5F0E7A785D402A97675474FC15409E9079C8CE775D40B48F7A4E41F81540D074C8AC1F775D409B509F07BBF41540094A97686D765D4095C0B5FCE4F11540A6554FACB8755D40585035FCC1EF15401AB9C92A02755D4008BE672354EE1540A3E89E984A745D4008C552DC9CED1540A36C74AB92735D4008C552DC9CED15402C9C4919DB725D4008BE672354EE1540A0FFC39724725D40585035FCC1EF15403D0B7CDB6F715D4096C0B5FCE4F1154076E04A97BD705D409B509F07BBF41540A8C4997B0E705D40B48F7A4E41F8154083F9B335636F5D402A97675474FC1540BBA11B6FBC6E5D40817D8FF14F0116405159E2CC1A6E5D4039A13E57CF061640882506EF7E6D5D406DC7A314ED0C1640CD12BF924E6D5D400041BDD7180F1640695CD36FE96C5D403C682F1CA3131640DF1E4CE35A6C5D4063E68DC9EA1A164071FB95D6D36B5D40C7D636E8BC22164075496ECF546B5D406AEA8ABA112B164093C6A54BDE6A5D40CF807901E13316401BF9A3C0706A5D40E362A604223D1640AFD2F29A0C6A5D4019A8079BCB4616409206D33DB2695D407150F333D4501640D57ED90262695D4007AF93E0315B1640B35296391C695D40E85AB95DDA6516406E974527E1685D40F8FD001EC3701640755B8A06B1685D403B064354E17B1640BC1134078C685D4083F242FE2987164043A90E4E72685D40D9B393EF919216404482BDF463685D40BD65A6DC0D9E1640FD67A10961685D400574F86592A9164054ABC98F69685D405523562314B51640A36FF07E7D685D40CD5A26AF87C01640ED2F82C39C685D40DA81B4B1E1CB1640D477B03EC7685D4042446DEC16D7164051C08FC6FC685D40E81803451CE21640645540263D695D40227D6FD0E6EC1640B41E221E88695D40C6F6C5DD6BF716404D1A1364DD695D40491ACE00A1011740DD4DB8A33C6A5D4049085A1C7C0B17403CE8D07EA56A5D40BF164F6CF3141740CA42938D176B5D40B7A4568FFD1D1740E967135F926B5D40016D2D90912617406FBAB279156C5D404E0B88EEA62E17402B50985BA06C5D40A2CF83A7353617400789317B326D5D40076E9B3D363D17405163BA47CB6D5D40A89217C0A14317404606CD296A6E5D409FE5F3D171491740C1F3F7830E6F5D40E59331B0A04E1740B44B5AB3B76F5D40EF049137295317401285451065705D40B6FCACE906571740B3437A08DC705D4085F57D222B5917400800000000715D40402606C0F9521740A877FF5F02715D40803C07E0B0521740E01A00600B715D40803C07E0B05217407889000010715D40406AFE3F1E531740F0E1FF7F1D715D40C0EC044084581740E0FDFF3F1B715D4080AA01E05E591740B810AE5715715D40B36CA738335A174088FAE3EE15715D40A90872F1355A174016A4E39FC9715D40ACB9ED26B35C17402A5224717F725D4039D671127C5E17405BBB68AE36735D40335C08EF8E5F1740A3AA09A2EE735D408BDB35ACEA5F1740EB99AA95A6745D40335C08EF8E5F17401C03EFD25D755D403AD671127C5E174030B12FA413765D40ACB9ED26B35C1740BE5A2F55C7765D40A90872F1355A174034D0CD3378775D40B6FCACE9065717409209B99025785D40F00491372953174085611BC0CE785D40E59331B0A04E1740004F461A73795D409FE5F3D171491740F5F158FC117A5D40A89217C0A14317403FCCE1C8AA7A5D40076E9B3D363D17401B057BE83C7B5D40A2CF83A735361740D79A60CAC77B5D404E0B88EEA62E1740A141FBCBCC7B5D40884D77E9572E174050040080697B5D40C03BF87F91301740A0340060447B5D40C03A06E076311740500A0100ED7A5D40C0F2F9DF4C331740D865FEBFD47A5D40404FFA1FB933174048120060B87A5D40007D04400D34174098DCFF9FA17A5D40007D04400D341740A8DDFFFF937A5D40C0FFFB3FC6331740C0A4FF1F867A5D40401303E07C331740D0DFFFBF587A5D40C0CEF8DFC6311740D87100C07B795D40005C06805029174058ADFF1FF9795D4080EE0540BE1F1740F01B00C0FD795D4000DE06E05F1F1740A877FF5F027A5D40C0CDFF5F001F1740B05200E0067A5D4040AC02E0981E174018840020347A5D404010FEBF96191740B06F0000377A5D4040C8F89F18191740B8FBFF7F367A5D40C0B502409E18174030ABFF5F347A5D40402F05803718174098BFFF7F317A5D40403206E0E3171740686B00802D7A5D408023FE7FA117174050BBFFFF277A5D400063F99F6E171740A06E00A0047A5D40C05601A085161740D0280040DA795D4040E802E09D1517409076FFFFAF795D40407FF81FB7141740F873008060795D40801DFDFF9D121740C0EDFF9FC7785D4040CBFBDFE10D174030C8FF7F44795D4080E7F87F0D06174030ABFF5F54795D40807C08A0D40517402868006076795D40807C08A0D4051740E8A6FFDF8A795D4080E7F87F0D061740F82A0000DF795D4080FE030055071740D8370080FB795D40C04D04C064081740003A0040007A5D40C0A6FB1FB1081740508700400B7A5D40804FF9DF6309174058ADFF1F197A5D4000EBFC5F9E091740287FFFFF227A5D40C0CC01E0AA091740E097FF9F297A5D408039F8BFAC09174070F7FFFF2C7A5D40406A0340AD091740B83500C0367A5D4080CFFB5FAB091740786C00E03F7A5D40C01B08E09E091740805E0000517A5D40809407A053091740C8530040597A5D40C065FAFF1A09174010D5FFFF607A5D40C056FABFD9081740D079FF1F677A5D40002800E087081740086600A0717A5D40804C00A0B8071740907C0080537A5D40401A04C0F2021740F06400403F7A5D404084F93F4801174028250160187A5D40C061F93FFCFE1640A894FF7F127A5D40C0B90300BDFE1640C8360020097A5D4000D6FB9F59FE1640280200C0047A5D4080F1FC9F4CFE164028680060F6795D408017018029FE1640105800C082795D4000D204E058F91640B8B2FFFF94795D4000750800B4F81640D09401608F7A5D4080BA0680A1F2164080DBFF3FAF7A5D40408E044033F3164090330000327B5D4040AFFB1F44F41640008300C0617C5D4080EE0540BEEF1640086600A0717C5D40003D088078EF1640208DFFDF917C5D4080FBF9DFDFEE1640E0FDFF3F9B7C5D408011FBFF96EE16408884FFDF9E7C5D4040AE00C060EE164078B50060A17C5D4040C401E017EE164020F3FF7FA37C5D40409F0760CBED16405081FFBFA77C5D40C082FA1F2BED164050BBFFFFC77C5D400091F9BFA4E61640D8D1FFDFC97C5D40403B07803EE61640C0EDFF9FC77C5D4040EA00C065E5164028B9FF3FC37C5D400086FA3F82E41640705D00A0BE7C5D40004B01809BE31640E889FFBFBA7C5D40C04D02E047E31640B8FBFF7FB67C5D400098F8BFFDE2164098080000937C5D40C0B407E0BAE0164070F7FFFF8C7C5D408061FC5F6EE01640304B0040867C5D4040C6F8DF33E01640488FFF9F767C5D4040EF01E0F6DF1640D0C2FF9F687C5D40004FFF1F48E0164040C9FFDF167C5D4080330320E4E0164020590020B57B5D40C0C80340FEE0164050040080A97B5D408091FD7FDEE01640082C0060917B5D4040650040FFDF164090330000327B5D40003607A002DC1640F8470020EF7A5D400035024001D9164080AFFFDFFD7A5D40C01CFA7FB9D8164030C8FF7F247B5D40808D06C0DDD91640783200A05F7B5D40008FFBDFDCDA1640184A00E0937B5D40C04CF87FB7DB16407883FF7FEC7B5D4080220500DBDC16406082FF1FFA7B5D4080220500DBDC1640B0090060057C5D4080B0FB7FB6DC1640E82900A00C7C5D40802FF99FFCDB1640F0FEFF9F0D7C5D4000E104209ADB16408092FFBF0D7C5D4040A3F73F20DB16408092FFBF0D7C5D40C07301C095DA164060A500C00D7C5D40C0CCFAFFFED91640F0FEFF9F0D7C5D40C0E5FC7F62D9164040C9FFDFF67B5D40001E08A083CC1640D8370080FB7B5D40006DF8BF1EC51640506A00203B7C5D408020FB3FD8C21640C8FCFFDF487C5D4080DBFD5FB2C31640B8FBFF7F567C5D40407B066045C41640A8FAFF1F647C5D404099FAFF8CC41640402F0080887C5D40006DF8BF1EC51640E87801A0917C5D40006DF8BF1EC5164038200040E77C5D400086FA3F82C41640E07AFF7FF97C5D4080CA044038C4164010D5FFFF007D5D408060FCFFFBC31640A0CEFFBF127D5D4040B0008045C31640B0ECFF3F157D5D404058F9DFF6C2164058ADFF1F197D5D40005605004DC21640E07AFF7F197D5D40403BFB9F03C2164030E5FF9F147D5D4000C106C04FC1164088EAFF7F707C5D40C074FA3F5CB91640886D0040127C5D404076F95F79B51640704000804E7B5D400067FE1FC7B11640800FFFFFAB7A5D400075030025B0164060220000AC7A5D40C0C407806EAF1640600500E0BB7A5D40001B0560BAAB164068E8FFBFCB7A5D4000D5FD1F04AB1640280200C0E47A5D40405D08C0DFAA164018E4FF3F027B5D4000D5FD1F04AB164008E90060137B5D40007801405FAC164028D6FF5F137B5D4040FA0560A8AC164010D5FFFF207B5D4000A201E0CBAD1640C00A00C0377B5D40C03002C037AF1640A877FF5FC27B5D40C0A3F8DFE7B1164078C6FE7FCA7B5D40C049FF3F0CB21640D888FF5FE87B5D4040FEFA3F8CB21640E8550000FE7B5D40C0EE0720DBB21640B09DFE3F107C5D404071F83FE8B2164040C9FFDF167C5D40C03FFDFFE9B21640388C01601C7C5D40C0F8FF5FDFB21640F08100602F7C5D40801B060082B21640F0E1FF7F3D7C5D40002BF91F33B21640B086FF9F437C5D404016FA3F0BB2164028080140487C5D4040DA0200CFB116407883FF7F4C7C5D40C030FBDF8BB11640805E0000517C5D40401501C044B1164058F6FF9F5A7C5D4080CB04A0AAB01640F06400405F7C5D400013010060B0164098250020637C5D40C0F706E018B01640A8FAFF1F647C5D40407B0160B6AF1640A8FAFF1F647C5D408032F9BF53AF164018670000647C5D400087FA9FF4AE1640A0EBFFDF627C5D4000AFFC5F99AE164088BEFF1F5F7C5D408090F81FDDAD1640789AFE1F597C5D40800D0620B3AC164070BDFFBF4C7C5D408049FD5FEFAA1640D0C2FF9F687C5D4080ED07C068A41640683100406D7C5D40C001FA1F8EA31640A8C0FFDFA37C5D40C012FD3F269E164070F7FFFFAC7C5D4040A60160959D164070BDFFBFCC7C5D40407C01C0289C164038F4FFDFD57C5D40407C01C0289C1640D0620080DA7C5D40406600A0719C1640F82A0000DF7C5D40003AFE5F039D164020F3FF7FE37C5D4040A60160959D1640D0620080DA7C5D40C012FD3F269E164048F5FF3FC87C5D40C012FD3F269E1640C087FFFFB57C5D40001F01204A9F1640F06400409F7C5D404089FA5FD9A1164080F8FF5F7F7C5D40406607801DA61640583000E07A7C5D40000E030041A71640583000E07A7C5D40C0DC04A0D0A9164080F8FF5F7F7C5D4040FF0180AAAA164018670000847C5D408055046085AB164038F4FFDFD57C5D40C0B3FB9F0DB41640B8FBFF7FF67C5D40008706802FB71640E0C3FFFFFA7C5D4000FA03808BB7164018E4FF3F027D5D40C05A0440C1B71640581300C00A7D5D4000DAF75FE9B7164018840020147D5D40C05BF8BFF8B71640E07AFF7FB97D5D40009F0580AEB81640485B00E0197E5D40802CFD3FDFB61640F8470020EF7E5D4080AE04809AB81640B0ECFF3FF57E5D40C0FF0540E4B816406891FF5FFB7E5D4080B1FBDF28B9164008490080017F5D4000E8FE1F64B9164010F2FF1F117F5D4040EF01E0F6B91640F0E1FF7F1D7F5D404063026037BA1640283C0000257F5D408056FDDF4BBA164070BDFFBF2C7F5D40C093FA1F51BA164030E5FF9F347F5D40C093FA1F51BA1640D87100C03B7F5D40801BFA1F47BA1640189BFFBF407F5D4080C6038019BA1640A0340060447F5D40808D06C0DDB91640B0090060457F5D4080FFF77F8CB91640280200C0447F5D40807F08C02BB9164068CBFF9F1B7F5D40806201C06FB1164028D6FF5F137F5D4080EEFE5F12B0164038D7FFBF057F5D40C04603E0EEAE164030E5FF9FF47E5D40401BFD3FB9AD1640A085FF3FF17E5D4000B8020083AD164020F3FF7FE37E5D4040E40440F1AC164070DAFFDFBC7E5D40C02A0820E0AB164080DBFF3FAF7E5D408055046085AB1640D0A5FF7F987E5D40406607801DA61640C0440000F87E5D40C0B8FE9FBBAB164010D5FFFF207F5D40007305205DAE1640603F00203C7F5D4080A90180ECB0164080F8FF5F3F7F5D40002FFCBF6EB1164050BBFFFF477F5D40005CFA9F15B31640AC53E9ED727F5D400A11CE1886B51640F2A949B4737F5D405523562314B5164049ED713A7C7F5D400574F86592A9164002D3554F797F5D40BD65A6DC0D9E164003AC04F66A7F5D40D9B393EF919216408A43DF3C517F5D4083F242FE29871640D1F9883D2C7F5D403B064354E17B1640D8BDCD1CFC7E5D40F8FD001EC370164093027D0AC17E5D40E85AB95DDA65164071D639417B7E5D4007AF93E0315B1640B44E40062B7E5D407150F333D4501640978220A9D07D5D4019A8079BCB4616402B5C6F836C7D5D40E262A604223D1640B38E6DF8FE7C5D40CF807901E1331640D10BA574887C5D406AEA8ABA112B1640D5597D6D097C5D40C7D636E8BC2216406736C760827B5D4063E68DC9EA1A1640DDF83FD4F37A5D403B682F1CA3131640794254B18E7A5D400041BDD7180F16400103000000010000001F000000784F00C04F7A5D4000EAFCFF2BF91640008300C0417A5D4000EAFCFF2BF91640F0E1FF7F3D7A5D404087FE5F2EF9164098910140387A5D4080D501E03DF91640E82900A02C7A5D40C0E107A07EF91640B8DEFF5F267A5D4040650720ABF91640003A0040207A5D408032FEBFE2F9164058F6FF9F1A7A5D4080CA044038FA1640A86000C0157A5D40801F07C0A0FA164090330000127A5D40408801E012FB1640103B00A0127A5D40C0E107A07EFB1640D8B4FFBF197A5D40C01DFFDFBAFB1640488FFF9F967A5D4000E605402BFE1640E8720020AE7A5D400063FB7F8BFE1640B89B0060C87A5D40004DFA5FD4FE164020C1FE9FCE7A5D4000C3F8BFDCFE1640A8260080D57A5D40C0F30340DDFE1640E0800000DD7A5D40C027F9FFDBFE1640A06E00A0E47A5D4000430220D0FE1640107500E0F27A5D400070000077FE1640D0620080FA7A5D4040DEFCDF41FE1640D8860180207B5D4000D5FD1F04FD1640C02700E0277B5D400078FA5FB3FC1640780600402E7B5D4000AA072043FC1640A8DDFFFF337B5D40C06B0260CAFB164048120060387B5D4040ADFB5F5FFB1640E85500001E7B5D40C018F9BF9AFA164038D7FFBF657A5D400060FB5F34F91640F06400405F7A5D40C01A08802CF91640C0440000587A5D4000EAFCFF2BF91640784F00C04F7A5D4000EAFCFF2BF916400103000000010000002400000098420040537C5D40C0950200370B1740801500804F7C5D40405EFC3F170B1740602200004C7C5D40C0FBFD9F190B1740D87100C03B7C5D40805BFBDF6A0B1740F0B5FF1F2C7C5D40004C08C0B90B1740587900601C7C5D40007808200B0C1740A86000C0157C5D404003FE3F3A0C1740E8BB00A00F7C5D4000EEFF9F670C1740D80B00200A7C5D4000110320980C1740A87D00E0057C5D4080220500DB0C174090CDFF5FE07B5D4080C00020F90F1740603F0020DC7B5D408085026083101740D0620080DA7B5D4000B103002A111740C8190000D97B5D404007FAFFC9111740D87100C0DB7B5D404000F91F5412174090CDFF5FE07B5D40808F0680C2121740B81800A0E67B5D40404F08E010131740F8C4FF5FED7B5D40006BFADF56131740281F00E0F47B5D400069FCFF8E131740E87200202E7C5D40C00607205A141740B83500C0367C5D40400BF89F761417408884FFDF3E7C5D40C09CF7FF71141740B0CFFF1F457C5D40C076FAFF40141740E8A6FFDF4A7C5D40408200600F141740187EFF9F507C5D40802100A0D913174048F5FF3F687C5D40400AFD3F93121740F88AFF1F6D7C5D40409EFDFF3A12174010B8FFDF707C5D40C003FDFFE411174090F9FFBF717C5D408042FC7F79111740A0CEFFBF727C5D4040D704C0940F1740103B00A0727C5D40C00CFCBF220F1740082C0060717C5D40C0AD0100B60E174068AEFF7F6B7C5D40005901606A0D1740F82A00005F7C5D40C0720840CF0B1740D8EEFFFF597C5D40C0D8F7FF760B174098420040537C5D40C0950200370B17400103000000010000000E00000098080000537B5D40C047016044041740E0E0FF1F4B7B5D4040D7FDDFE803174090CDFF5F407B5D40005B0800FB0317404080FF5F157B5D40C0F4FE9FC0041740F80D00E00E7B5D4040970800000517407883FF7F0C7B5D4040C5FF5F6D05174010B8FFDF107B5D40C0EBF8FFD605174020760040257B5D4000F7036034061740408600E0387B5D40C0A8F9FF7806174008C6FFBF3F7B5D404094FEDF8A061740704000804E7B5D4040AAFFFF41061740307700A0577B5D4080D60140B0051740307700A0577B5D40803100408D04174098080000537B5D40C04701604404174001030000000100000017000000E86CFF9F6A795D4080E804C0BA0617404812006058795D4080E804C0BA06174070F7FFFF4C795D4040BC02804C071740A8FAFF1F44795D4000DEFAFF240917401867000044795D40806702E0000A1740304B004046795D40408107E0480B1740585C00404C795D40C094F89FA60D1740882400C050795D40801B0600820E1740485B00E059795D4080C00700A50F1740385A008067795D40407E04A07F101740087DFF3F7E795D40002306A0A2111740C8B3FF5F87795D40000D0580EB111740581300C08A7A5D40801906409D17174080DBFF3FAF7A5D40801906409D171740201000A0D37A5D40C0FEFBDF53171740A87D00E0E57A5D4000BC00A02F161740A87D00E0E57A5D408059020032141740100F0040E17A5D40C0850440A0131740201000A0D37A5D40C0CA0120C612174098A2FF5FC17A5D40000D0580EB111740B878FFBFD4795D40000DFE9F3F081740D85400A0CB795D40C025F93FF7071740E86CFF9F6A795D4080E804C0BA0617400103000000010000004C0000001E37B1E5317C5D4040CEF91F1C2817405DEDFFE44A7C5D40016D2D90912617407C1280B6C57C5D40B7A4568FFD1D17400A6D42C5377D5D40C0164F6CF314174069075BA0A07D5D4049085A1C7C0B1740F93A00E0FF7D5D40491ACE00A1011740AC8967A44F7E5D408E8E0E9814F816402070FFBF417E5D4000F302A015F81640907C0080337E5D4080F1FC9F4CF81640A0510080147E5D40C01A08802CF91640E01A00600B7E5D408004076075F91640A085FF3FF17D5D408091F87F4FFA164068AEFF7FEB7D5D40408DF8FF85FA164098420040B37D5D40406B01C002FD1640385A0080A77D5D40C056FC9FF6FD1640882400C0907D5D4040850580F5FF1640585C00408C7D5D400059034087001740683100404D7D5D40006D04A05909174020F3FF7F237D5D40C0B104A0F10F1740F8730080207D5D40C060F9DF89101740807B0020217D5D408033FC3F38111740583601601E7D5D400055F9BF9F11174030CE0000087D5D40C0560880311417408884FFDFFE7C5D404017FF9F0C151740F82A0000DF7C5D40C0FEFBDF53171740583000E09A7C5D40C0F403A04F1B1740F88AFF1F8D7C5D400095FC5FE01B174090F9FFBF717C5D404086FEFFBB1C1740B0D500A0687C5D40003AFE5F031D1740E0C3FFFF5A7C5D400024FD3F4C1D1740A0340060447C5D40803E07A0951D1740B83500C0167C5D40803E07A0951D174010ECFE9F0D7C5D40803E07A0951D1740D85400A00B7C5D40407DFF3F7E1D1740B05200E0067C5D4040B706604A1D174098A2FF5F017C5D40409EFB1F1E1D174080470160E47B5D40C02FFD5F361C174028B9FF3FA37B5D40C08F0180331C1740D8EEFFFF797B5D40C0E906004A1C1740086600A0317B5D40C052F9FFBA1C1740F07BFFDF2B7B5D4040FCFC5FC41C1740509EFFDF177B5D40C0D2FB7F021D1740B0A3FFBFB37A5D404044F87F24201740B895FFDFA47A5D4000F605E0DE201740006EFFFF9C7A5D40004B086047211740184A00E0937A5D40C01E0620D9211740008300C0817A5D40406B03A01F241740008300C0817A5D400013FF1F43251740201000A0937A5D40403F08405D26174070DAFFDF9C7A5D40C0010420AC261740C0A4FF1FA67A5D400065FE5FE226174070400080AE7A5D40004D06400F27174038F4FFDFB57A5D40C0B0F85F4427174078A0FF9FBC7A5D4000C2FA3F8727174098250020C37A5D40C081FC9FD52717405890FFFFC87A5D4000B704802D28174018C7FF1FD27A5D4000F4FB1FDC28174048120060D87A5D404020F95FBB291740D87100C0DB7A5D40C02908C06D2B1740D8370080DB7A5D40C0260080152C1740A8260080D57A5D4000900040DE2D1740F8470020CF7A5D4040B1F71FEF2E1740F8470020CF7A5D400056F91F1230174048D8FF1FD87A5D40005808E0A3301740100F0040E17A5D40005808E0A3301740F8A7FF3FFD7A5D40C0C400A0C22F17408092FFBF4D7B5D4040970120542D1740087DFF3F5E7B5D40808AF79FD92C1740D8B4FFBFD97B5D40C0F701E089291740003A0040E07B5D400048034061291740B0ECFF3FF57B5D4040730200EB281740587900601C7C5D4080E1F9DF2628174090500020227C5D4040CEF91F1C2817405081FFBF277C5D4040CEF91F1C281740686B00802D7C5D4040CEF91F1C2817401E37B1E5317C5D4040CEF91F1C2817400103000000010000001A000000304B0040867A5D40000E030041271740008300C0817A5D40000E030041271740B8DEFF5F667A5D40803000E01A281740F8A7FF3F5D7A5D40801AFFBF63281740E0E0FF1F4B7A5D40805A00808729174028850080467A5D40802B04C0182A174028850080467A5D400018FD1F622A1740D0DFFFBF787A5D4040E0FC9F262F174070DAFFDF7C7A5D40802BFFBF892F1740A84300A0857A5D4040EFFAFF4A301740508700408B7A5D40C03BF87F9130174090330000927A5D40800A0100CD301740D8B4FFBF997A5D40C068FF1F0131174008490080A17A5D40C0B8F99F2C31174008A9FF9FAF7A5D400007FD1F3C31174028680060B67A5D4080A5F9DF2131174068CBFF9FBB7A5D4000550780F730174098A2FF5FC17A5D400040F8FF5A30174030110000C67A5D40C01FFDBF822D1740C0700060C97A5D4040C1FE9F4E2B1740D079FF1FC77A5D40402EFE3F192A174098A2FF5FC17A5D40805A00808729174048120060B87A5D40808602C0F5281740705D00A09E7A5D40007E07C0F12717404080FF5F957A5D404003F93FAB271740304B0040867A5D40000E030041271740 +1 26 0106000000020000000103000000010000007A000000BA490C022B575D40E386BA8119B51740D582ADFCE2575D403F4983C4BDB41740B4A0EA409A585D40DCE2CFE7AAB3174040E3141950595D405ED01CFCE1B11740158AE7D0035A5D4071CE61C664AF1740970C3BB6B45A5D4027774FBE35AC17404531B619625B5D405735DB0B58A817402B527C4F0B5C5D40E9041C84CFA31740621FD8AFAF5C5D4096287BA5A09E17403F34E2974E5D5D4093AA3C93D098174053D8226AE75D5D40F4256410659217402E4A2E8F795E5D40E7FAFA79648B174029570786DD5E5D403CA23E1CF4851740A877FF5FE25D5D40C0BAFC7F837117409093FF1FE05D5D4080250840A46F1740F01B00C0FD5D5D4040C2FC1FA4701740B8DEFF5F665E5D4080F8F87F33731740581300C08A5E5D40C0B3FB9F0D741740C0A4FF1FA65E5D408087F95F9F74174048D8FF1FD85E5D404089082031751740E04600C0DC5E5D404089082031751740D0450060EA5E5D404089082031751740E80C00801C5F5D40408908203175174010D5FFFF205F5D404071F83FE874174098420040335F5D404071F83FE8741740E8D2FF3F3C5F5D40408908203175174098080000535F5D400062070054761740D8D1FFDF695F5D40C0F007E0BF771740A8E30080775F5D408004076075791740880700A0805F5D40407D06202A7B1740880700A0805F5D40A9F0B3382E7C17404D10B195875F5D4028CD3662C07B17407045DA4700605D40047370A35273174091DBB26B02605D4019404E612C7317405C589D7E74605D40D981543E226A1740CE99805DDD605D402CB391EEAA6017405D728FA03C615D407DFD60D3CF561740ECFC85E991615D40EF28E1B09A4C1740F69F06E4DC615D40D8B243A4154217404230ED451D625D40A210C4194B37174086E297CF52625D405E2051C2452C1740DFC5254C7D625D403813F38810211740898BAA919C625D40A35EF887B6151740066B5781B0625D403077F4FD420A1740C1FC9807B9625D4002439B42C1FE164060F0291CB6625D40305284BB3CF31640E68E1AC2A7625D40A300E1D0C0E71640ED03CD078E625D4080AF30E258DC16407374E60669625D40ED42FE3A10D116405EF434E438625D40D205B207F2C5164042778ACFFD615D4017FE824A09BB16401FE38C03B8615D40EE9A92D060B01640FE767BC567615D403D7A3D2703A616400CC1E9640D615D401DCCAB91FA9B16407F6A703BA9605D40BC9AABFE50921640E62854AC3B605D403AF2DEFE0F8916407045DA4700605D4034F4FBF4A5841640582E2324C55F5D40239047BB40801640CC7A4918465F5D409F6439ECEB771640E9799C06BF5E5D4033C9BCD019701640A360DE74305E5D40E1E16826D268164010C639F09A5D5D40CE26BE211C62164036FAB50CFF5C5D409F950867FE5B16406DA4A4645D5C5D406084D0037F56164055390998B65B5D40F682E068A351164069DEFA4B0B5B5D40F825E664704D16400656012A5C5A5D4060FDB21FEA49164021946DDFA9595D40735C221614471640509FAE1CF5585D40CAF6A716F1441640D465A3943E585D40DFB08A3E83431640D930EAFB86575D40F459CEF7CB4216409B622E08CF565D40F459CEF7CB4216409F2D756F17565D40DFB08A3E8343164023F469E760555D40CAF6A716F144164053FFAA24AC545D40735C2216144716406E3D17DAF9535D4060FDB21FEA4916400BB51DB84A535D40F825E664704D16401E5A0F6C9F525D40F682E068A351164007EF739FF8515D405F84D0037F5616403E9962F756515D409F950867FE5B164064CDDE13BB505D40CE26BE211C621640D0323A8F25505D40E2E16826D26816408B197CFD964F5D4033C9BCD019701640A818CFEB0F4F5D409F6439ECEB7716401C65F5DF904E5D40239047BB408016408E6AC4571A4E5D403AF2DEFE0F891640F428A8C8AC4D5D40BC9AABFE5092164067D22E9F484D5D401DCCAB91FA9B1640761C9D3EEE4C5D403D7A3D2703A6164054B08B009E4C5D40EE9A92D060B01640321C8E34584C5D4017FE824A09BB1640169FE31F1D4C5D40D205B207F2C51640011F32FDEC4B5D40EE42FE3A10D11640878F4BFCC74B5D4080AF30E258DC16408E04FE41AE4B5D40A300E1D0C0E7164013A3EEE79F4B5D40305284BB3CF31640B3967FFC9C4B5D4002439B42C1FE16406E28C182A54B5D403077F4FD420A1740EB076E72B94B5D40A35EF887B615174095CDF2B7D84B5D403813F38810211740EEB08034034C5D405E2051C2452C174032632BBE384C5D40A210C4194B3717407EF31120794C5D40D8B243A4154217408896921AC44C5D40EE28E1B09A4C174017218963194D5D407DFD60D3CF561740A6F997A6784D5D402CB391EEAA601740183B7B85E14D5D40D881543E226A1740E3B76598534E5D4019404E612C7317402783676ECE4E5D4028CD3662C07B17406E9BDD8D514F5D403BAEBDC0D58317404649EA74DC4F5D40E7FAFA79648B174021BBF5996E505D40F325641065921740355F366C07515D4093AA3C93D098174012744054A6515D4096287BA5A09E174048419CB44A525D40E9041C84CFA317402E6262EAF3525D405635DB0B58A81740DD86DD4DA1535D4028774FBE35AC17405F09313352545D4071CE61C664AF174034B003EB05555D405ED01CFCE1B11740BFF22DC3BB555D40DBE2CFE7AAB317409E106B0773565D403F4983C4BDB41740BA490C022B575D40E386BA8119B5174001030000000100000012000000644731C8135F5D401CBAB3F8E382174072016134745F5D4083B22F3DF27C1740189BFFBF605F5D4080D80420077A17405064FF9F575F5D4080980360E3781740704000804E5F5D40C0F007E0BF771740E8D2FF3F3C5F5D40004C06E09C761740404C00A0785E5D404071F83FE8741740801500806F5E5D40407307007A751740801500806F5E5D40007808200B761740A8DDFFFF735E5D40C03505C0E5761740304B0040865E5D408004076075791740581300C08A5E5D4080D80420077A1740D0A5FF7F985E5D4040670500737B174048120060B85E5D40C0B30280B97D174060D9FF7FCA5E5D40405BFEFFDC7E1740F80D00E0EE5E5D40C0A7FB7F23811740609FFF3F0A5F5D408036FC5F8F821740644731C8135F5D401CBAB3F8E3821740 +1 27 0103000000010000006A000000601A63A4353E5D400041BDD7180F16401E4AF952773E5D4006DE00B0730E1640F8FF24EA2A3F5D405BD78D7BF60B1640AD2661AFDB3F5D40C6D80C75C70816407045DA4700405D4009539B6CF6071640210277F388405D401C8A64C4E90416400F7EAB0A32415D4015679A3E610016406BC5694DD6415D40C8C4036232FB1540B6AEE91875425D4062ADCE5162F515407A56D1CF0D435D40900EE7D0F6EE15406346D1DA9F435D40135B3D3CF6E71540058D3AA92A445D40295E748467E01540D9308EB1AD445D40DF97FC2652D81540C96F057228455D40450BA426BECF15401F4412719A455D40A2F1A103B4C61540C8AFD73D03465D40F24826B33CBD15404757997062465D402EAE759661B31540A2FE21ABB7465D40CC659A712CA91540ED83209902475D40A0DDB361A79E1540DCFC7AF042475D405852EED2DC931540C3A6977178475D4030982C76D7881540A6629BE7A2475D409C516E36A27D1540B5819D28C2475D40901FFD2D487215402AB1D015D6475D40178D6B9BD466154062DFA09BDE475D4064A370D6525B154032FFC5B1DB475D401434AB44CE4F154050994B5BCD475D40C5FC574E52441540E0268DA6B3475D40E0D10553EA381540533B27AD8E475D409AFD529EA12D15408E8EDD935E475D4010F5BE5C832215407903768A23475D40CA6E9B909A1715409DD188CBDD465D40B8C42707F20C15403303469C8D465D402F60E14D940215405A83304C33465D40E2B713A88BF814402602CF34CF455D40A927B204E2EE14404DFE52B961455D405E9F85F4A0E51440684D3646EB445D4012D2B7A0D1DC14402185CF506C445D40DC31C5C17CD414404DB0DD56E5435D40B19DDE96AACC14403DC30BDE56435D407A38C3DD62C51440AA4A6C73C1425D404D691ACBACBE1440A8D7EDAA25425D403B8655038FB8144092B2C81E84415D40632520940FB314409664E66EDD405D40578365EE33AE14400FAE434032405D4064DBF0E000AA14407045DA4700405D409D53F740FFA81440A0844D3C833F5D40F1F5AD937AA614402AB93910D13E5D4006948E83A4A314408AE95B6C1C3E5D4005BD187F81A11440A6657703663D5D404A4EA1A313A0144065B20E8AAE3C5D40A383355B5C9F14401A57B1B5F63B5D40A383355B5C9F1440D9A3483C3F3B5D404A4EA1A313A01440F51F64D3883A5D4005BD187F81A114405550862FD4395D4007948E83A4A31440DF84720322395D40F2F5AD937AA61440705B7CFF72385D4064DBF0E000AA1440E9A4D9D0C7375D40578365EE33AE1440ED56F72021375D40632520940FB31440D731D2947F365D403B8655038FB81440D5BE53CCE3355D404D691ACBACBE14404246B4614E355D407B38C3DD62C514403259E2E8BF345D40B19DDE96AACC14405D84F0EE38345D40DC31C5C17CD4144017BC89F9B9335D4011D2B7A0D1DC1440320B6D8643335D405E9F85F4A0E514405907F10AD6325D40AA27B204E2EE144025868FF371325D40E2B713A88BF814404C067AA317325D402F60E14D94021540E2373774C7315D40B8C42707F20C154006064AB581315D40CA6E9B909A171540F07AE2AB46315D4010F5BE5C832215402BCE989216315D409AFD529EA12D15409FE23299F1305D40E0D10553EA3815402F7074E4D7305D40C5FC574E524415404C0AFA8DC9305D401434AB44CE4F15401C2A1FA4C6305D4064A370D6525B15405558EF29CF305D40178D6B9BD4661540CA872217E3305D40911FFD2D48721540D9A6245802315D409C516E36A27D1540BC6228CE2C315D4030982C76D7881540A30C454F62315D405852EED2DC93154092859FA6A2315D40A0DDB361A79E1540DD0A9E94ED315D40CB659A712CA9154038B226CF42325D402EAE759661B31540B759E801A2325D40F24826B33CBD154060C5ADCE0A335D40A1F1A103B4C61540B699BACD7C335D40450BA426BECF1540A6D8318EF7335D40DF97FC2652D815407A7C85967A345D40295E748467E015401CC3EE6405355D40135B3D3CF6E7154005B3EE6F97355D40900EE7D0F6EE1540C85AD62630365D4062ADCE5162F51540144456F2CE365D40C8C4036232FB1540708B143573375D4014679A3E610016405E07494C1C385D401C8A64C4E9041640D2E25E90C9385D40C6D80C75C708164087099B557A395D405BD78D7BF60B164061BFC6EC2D3A5D4006DE00B0730E16401FEF5C9B6F3A5D400041BDD7180F164013B8DDA3E33A5D408815C19A3C101640FEFFBEC69A3B5D409E4EDF764F111640BF04E09F523C5D409D3DE433AB111640810901790A3D5D409E4EDF764F1116406C51E29BC13D5D408815C19A3C101640601A63A4353E5D400041BDD7180F1640 +1 28 0103000000010000006A000000302B8860854B5C4000FC0AA19CC3FFBFE5146E297C4B5C40D79F02962FC7FFBF15E9091D0F4B5C40C6255DBF36ECFFBF1F736E1F994A5C4071F0997FBB0700C02F8774A51A4A5C405CCFFDB4661800C058645E2C94495C40A2DAA07C0C2800C07CAE5B3906495C40368834569D3600C03586055971485C40183FB3D30A4400C01541D31ED6475C4029D3AEA7475000C0654C882435475C40CC1981B2475B00C02ECB9B098F465C40AD75510E006500C0E8869A72E4455C40CF60E319676D00C0C9CF830836455C40AE302382747400C0EEEE217884445C405B90674A217A00C01FD15E71D0435C40BF755FD3677E00C0B09495A61A435C4094A9A5E0438100C043A8E1CB63425C407852F49CB28200C0BD2A6C96AC415C407852F49CB28200C0503EB8BBF5405C4094A9A5E0438100C0E101EFF03F405C40BF755FD3677E00C07045DA4700405C40D7606FC7E47C00C012E42BEA8B3F5C405B90674A217A00C03703CA59DA3E5C40AE302382747400C0184CB3EF2B3E5C40CF60E319676D00C0D207B258813D5C40AD75510E006500C09B86C53DDB3C5C40CC1981B2475B00C0EB917A433A3C5C4029D3AEA7475000C0CB4C48099F3B5C40183FB3D30A4400C08424F2280A3B5C40368834569D3600C0A86EEF357C3A5C40A2DAA07C0C2800C0D14BD9BCF5395C405CCFFDB4661800C0E25FDF4277395C4071F0997FBB0700C0EBE9434501395C40C6255DBF36ECFFBF1BBEDF3894385C40D89F02962FC7FFBFD0A7C5018B385C4000FC0AA19CC3FFBF619CAE8930385C40D7C92A3186A0FFBFE555649AD6375C40A15BFCDC6078FFBFA62A0BC486375C40B593C95DE74EFFBF4BBFAB5541375C406787A7C84224FFBF9601FF9306375C402F99B85A9DF8FEBF1B482AB9D6365C40757A524F22CCFEBF67EF85F4B1365C40393C29B5FD9EFEBFB3AC6E6A98365C40E4E5A9425C71FEBF34C321348A365C40A8D4AF296B43FEBFB03EA45F87365C4085C4C0EA5715FEBFC54AB5EF8F365C40E7D8FB2750E7FDBF9AB3CBDBA3365C407442E87781B9FDBF7F911E10C3365C40B63C5038198CFDBF0816B96DED365C40400E5561445FFDBF626699CA22375C40A67AE8582F33FDBF7863DAF162375C409CAED7C60508FDBF1636E8A3AD375C40E71B9369F2DDFCBF616ABF9602385C405AFEDDEB1EB5FCBFC95B367661385C40285C8FBBB38DFCBF5EA850E4C9385C40A4438DE1D767FCBFBB589C793B395C4085D129DBB043FCBF166098C5B5395C40A32B08756221FCBFEE0E244F383A5C409418AEA70E01FCBF060AF794C23A5C405834E575D5E2FBBFFA4E210E543B5C4024F80CCDD4C6FBBFD7C7922AEC3B5C404AE07C6728ADFBBFC4E7A9538A3C5C4098F113B1E995FBBFBDC3C8EC2D3D5C40B2B810AE2F81FBBF3314F053D63D5C40119D4AE40E6FFBBF63865FE2823E5C401107E346995FFBBF23BF3AED323F5C40B1688224DE52FBBF3A6C32C6E53F5C40ECB43218EA48FBBF7045DA4700405C40896C8577DE47FBBF4DBD30BC9A405C40C73BE6FCC641FBBF2E9A081C51415C400F3EB7E37B3DFBBF80E9263108425C408BD3E90C0D3CFBBFD3384546BF425C400F3EB7E37B3DFBBFB3151DA675435C40C83BE6FCC641FBBFC6661B9C2A445C40ECB43218EA48FBBFDC131375DD445C40B1688224DE52FBBF9D4CEE7F8D455C401007E346995FFBBFCDBE5D0E3A465C40119D4AE40E6FFBBF430F8575E2465C40B2B810AE2F81FBBF3DEBA30E86475C4098F113B1E995FBBF290BBB3724485C404AE07C6728ADFBBF06842C54BC485C4022F80CCDD4C6FBBFFAC856CD4D495C405834E575D5E2FBBF13C42913D8495C409418AEA70E01FCBFEA72B59C5A4A5C40A22B08756221FCBF457AB1E8D44A5C4085D129DBB043FCBFA22AFD7D464B5C40A4438DE1D767FCBF377717ECAE4B5C40285C8FBBB38DFCBF9F688ECB0D4C5C405AFEDDEB1EB5FCBFEB9C65BE624C5C40E51B9369F2DDFCBF886F7370AD4C5C409EAED7C60508FDBF9E6CB497ED4C5C40A87AE8582F33FDBFF8BC94F4224D5C40400E5561445FFDBF81412F524D4D5C40B63C5038198CFDBF661F82866C4D5C407442E87781B9FDBF3B889872804D5C40E7D8FB2750E7FDBF5094A902894D5C4085C4C0EA5715FEBFCC0F2C2E864D5C40A6D4AF296B43FEBF4D26DFF7774D5C40E2E5A9425C71FEBF99E3C76D5E4D5C40393C29B5FD9EFEBFE68A23A9394D5C40757A524F22CCFEBF6AD14ECE094D5C402F99B85A9DF8FEBFB513A20CCF4C5C406787A7C84224FFBF5AA8429E894C5C40B593C95DE74EFFBF1B7DE9C7394C5C40A15BFCDC6078FFBF9F369FD8DF4B5C40D7C92A3186A0FFBF302B8860854B5C4000FC0AA19CC3FFBF +1 29 01030000000100000068000000D7F0F44A59285C4095CDDFCDC97800C064FB986910295C40B9692A38817900C08DDCEFD2C6295C403C6071C1A67B00C0F1BF5FD27B2A5C406AE29D49387F00C056CAB4B42E2B5C406C5B3548328400C0CB5CD2C8DE2B5C402437D8CF8F8A00C0E94762608B2C5C4083B721934A9200C0794281D0332D5C40630CE4EA5A9B00C043F96772D72D5C40C286BADDB7A500C0641110A4752E5C40026EEA2857B100C03B7AD4C80D2F5C4024B4894A2DBE00C0CF700C4A9F2F5C40CD83E18C2DCC00C09C9BA09729305C407F6B01134ADB00C09AAB9928AC305C4072BC76E673EB00C089F5A77B26315C4025931A069BFC00C09C7DA31798315C4089ECE775AE0E01C0D7F6038C00325C40A225CA4F9C2101C05F3F50715F325C40514E53D5513501C06EEB8469B4325C4059D24882BB4901C028797120FF325C40F927F41FC55E01C0EBD00B4C3F335C40FC7024D9597401C0F4BEB9AC74335C40DF4BCD4E648A01C04F1D900D9F335C407C762EADCEA001C0DB6E8744BE335C40505B6FC182B701C0A3B6A532D2335C40002F990F6ACE01C07B611DC4DA335C40A5E4D9E86DE501C0412261F0D7335C40F8F5F98177FC01C045AC2CBAC9335C409AC8EE09701302C02D42822FB0335C404C5B73C0402A02C0AC1A9D698B335C40ECDD900CD34002C074A7D88C5B335C40B8E40093105702C0BDD58CC820335C400C0B534CE36C02C0A76ADF56DB325C409C13C09A358202C09AA88A7C8B325C405CE3945FF29602C04975998831325C403D19201005AB02C0B04219D4CD315C40A47E0DCA59BE02C04807C2C160315C403D1B1B67DDD002C0F89B94BDEA305C407C5914907DE202C0A2DF6F3C6C305C407E5F00CF28F302C0AF099DBBE52F5C40588971A0CE0203C0949D53C0572F5C404DCDE4835F1103C0157A35D7C22E5C405FBF200BCD1E03C0C086C393272E5C40EEE684E8092B03C0B289CB8F862D5C4028263BFC093603C0B2B6CF6AE02C5C404F0E3E60C23F03C0128F68C9352C5C40CD242873294803C066AFA154872B5C40FB51C1E1364F03C09F2D52B9D52A5C40C6F240AFE35403C0562E71A7212A5C400D503C3C2A5903C0155C67D16B295C4037913A4C065C03C0ADEE5DEBB4285C40BB98E809755D03C002F38BAAFD275C40BB98E809755D03C09A8582C446275C4037913A4C065C03C059B378EE90265C400D503C3C2A5903C00FB497DCDC255C40C6F240AFE35403C0493248412B255C40FB51C1E1364F03C09C5281CC7C245C40CD242873294803C0FC2A1A2BD2235C404F0E3E60C23F03C0FD571E062C235C4028263BFC093603C0EF5A26028B225C40EEE684E8092B03C09967B4BEEF215C405FBF200BCD1E03C01B4496D55A215C404ECDE4835F1103C000D84CDACC205C40588971A0CE0203C00C027A5946205C407E5F00CF28F302C07045DA4700205C406A8C7A3FEDE902C0B74555D8C71F5C407B5914907DE202C066DA27D4511F5C403D1B1B67DDD002C0FE9ED0C1E41E5C40A47E0DCA59BE02C0656C500D811E5C403D19201005AB02C015395F19271E5C405CE3945FF29602C007770A3FD71D5C409B13C09A358202C0F10B5DCD911D5C400C0B534CE36C02C03A3A1109571D5C40B9E40093105702C003C74C2C271D5C40EDDD900CD34002C0819F6766021D5C404C5B73C0402A02C06A35BDDBE81C5C409BC8EE09701302C06EBF88A5DA1C5C40F8F5F98177FC01C03480CCD1D71C5C40A5E4D9E86DE501C00C2B4463E01C5C40022F990F6ACE01C0D4726251F41C5C40505B6FC182B701C05FC45988131D5C407C762EADCEA001C0BB2230E93D1D5C40DF4BCD4E648A01C0C410DE49731D5C40FC7024D9597401C086687875B31D5C40F927F41FC55E01C040F6642CFE1D5C405AD24882BB4901C04FA29924531E5C40514E53D5513501C0D7EAE509B21E5C40A325CA4F9C2101C01264467E1A1F5C4089ECE775AE0E01C026EC411A8C1F5C4025931A069BFC00C07045DA4700205C40D81A8A8950EC00C01536506D06205C4072BC76E673EB00C0124649FE88205C407F6B01134ADB00C0E070DD4B13215C40CD83E18C2DCC00C0746715CDA4215C4024B4894A2DBE00C04BD0D9F13C225C40026EEA2857B100C06CE88123DB225C40C286BADDB7A500C0369F68C57E235C40630CE4EA5A9B00C0C599873527245C4083B721934A9200C0E48417CDD3245C402537D8CF8F8A00C0591735E183255C406C5B3548328400C0BE218AC336265C406AE29D49387F00C02205FAC2EB265C403C6071C1A67B00C04BE6502CA2275C40B9692A38817900C0D7F0F44A59285C4095CDDFCDC97800C0 +1 30 010300000001000000680000007045DA4700205C40B46BD8CAC12204C0BBA2957A58205C4066AF4568CB2304C0750BBC880D215C40A8D029E95C2704C078E9A579C0215C4079E6B0DD562C04C02B572B9C70225C406E829C58B43204C00BD6EA411D235C409AADB10C6F3A04C0CBC4F5BFC5235C40A263F4527F4304C0AF6F796F69245C40330B3932DC4D04C09B1464AE07255C409E7504687B5904C0153805E09F255C406BA5B172516604C02BADA86D31265C40B05AD39C517404C052B72BC7BB265C40332CC6096E8304C013B38B633E275C4023C866C3979304C09EB86DC1B8275C40D6D6DEC8BEA404C063B29E672A285C4099E6791ED2B604C0DA688BE592285C407BC372DEBFC904C0E00CB0D3F1285C40B0A7A84A75DD04C004D2FED346295C406CCC2ADFDEF104C011333D9291295C408A0E8865E80605C0198457C4D1295C401398CF087D1C05C0D57EAA2A072A5C40DECF2E6A873205C0FA7F4290312A5C40143418B6F14805C0113610CB502A5C40D93BDEB9A55F05C0FB8C12BC642A5C4007E8ACF98C7605C086AB754F6D2A5C40C84BCCC6908D05C067E3A67C6A2A5C40AE0B16569AA405C0BD7E5D465C2A5C40579D87D692BB05C0246198BA422A5C4042F3DA8763D205C06A7D90F21D2A5C4036370FD1F5E805C0112CA012EE295C405E43CB5633FF05C0A0791F4AB3295C40F9AC8311061506C0E28E35D36D295C40F26C4E63582A06C0E05F9FF21D295C404F814E2D153F06C01FD96BF7C3285C401247A2E4275306C034CDAD3A60285C4018C5BFA67C6606C0DBEE231FF3275C4065AF2A4D007906C0DE2DD7107D275C40238C7080A08A06C0B9D6AF84FE265C40231A58CA4B9B06C01EDE01F877265C408BE131A7F1AA06C091CA10F0E9255C40CDB2389682B906C0B3B68BF954255C40F6C4F128F0C606C0D7EE01A8B9245C4077217D112DD306C01DB4509518245C40E318C82F2DDE06C039B60A6172235C405B97949DE5E706C06ADBD9AFC7225C40985849B94CF006C01BF4DB2A19225C40C230802F5AF706C0C7FCFA7E67215C40BCDF480307FD06C04995415CB3205C40D82E18954D0107C07045DA4700205C40AEF7E74B1E0407C0B8582C75FD1F5C40BD6B5CA8290407C00AC4F87D461F5C40F8ACB167980507C0695CF22B8F1E5C40F8ACB167980507C0BCC7BE34D81D5C40BD6B5CA8290407C02A8BA94D221D5C40D82E18954D0107C0AC23F02A6E1C5C40BCDF480307FD06C0582C0F7FBC1B5C40C230802F5AF706C0094511FA0D1B5C40985849B94CF006C03A6AE048631A5C405A97949DE5E706C0566C9A14BD195C40E418C82F2DDE06C09C31E9011C195C4077217D112DD306C0C0695FB080185C40F6C4F128F0C606C0E255DAB9EB175C40CEB2389682B906C05642E9B15D175C408BE131A7F1AA06C0BB493B25D7165C40231A58CA4B9B06C095F2139958165C40228C7080A08A06C09831C78AE2155C4065AF2A4D007906C03F533D6F75155C4019C5BFA67C6606C054477FB211155C401247A2E4275306C094C04BB7B7145C404F814E2D153F06C09191B5D667145C40F16C4E63582A06C0D3A6CB5F22145C40F9AC8311061506C062F44A97E7135C405F43CB5633FF05C009A35AB7B7135C4036370FD1F5E805C04FBF52EF92135C4042F3DA8763D205C0B6A18D6379135C40599D87D692BB05C00D3D442D6B135C40AE0B16569AA405C0ED74755A68135C40C84BCCC6908D05C07893D8ED70135C4007E8ACF98C7605C062EADADE84135C40D93BDEB9A55F05C079A0A819A4135C40143418B6F14805C09EA1407FCE135C40DECF2E6A873205C05A9C93E503145C401398CF087D1C05C062EDAD1744145C408A0E8865E80605C06F4EECD58E145C406BCC2ADFDEF104C094133BD6E3145C40B0A7A84A75DD04C09AB75FC442155C407CC372DEBFC904C0106E4C42AB155C4099E6791ED2B604C0D5677DE81C165C40D6D6DEC8BEA404C0606D5F4697165C4023C866C3979304C02169BFE219175C40332CC6096E8304C04873423CA4175C40B05AD39C517404C05EE8E5C935185C406CA5B172516604C0D90B87FBCD185C409E7504687B5904C0C5B0713A6C195C40330B3932DC4D04C0A85BF5E90F1A5C40A263F4527F4304C0684A0068B81A5C409AADB10C6F3A04C048C9BF0D651B5C406F829C58B43204C0FB364530151C5C4079E6B0DD562C04C0FE142F21C81C5C40A8D029E95C2704C0B87D552F7D1D5C4065AF4568CB2304C0675679A7331E5C40B90F67E3A52104C03A90F5D4EA1E5C40BA33967AEE2004C00DCA7102A21F5C40B90F67E3A52104C07045DA4700205C40B46BD8CAC12204C0 +1 31 01030000000100000068000000030E550BC8FA5B40007E8550CEE103C0B2292A7819FB5B406D84A5B58EF003C052441E6478FB5B40465DE127440404C01B516C62CDFB5B403ED736C2AD1804C0D83CDC1E18FC5B40B58D244EB72D04C011B05C4F58FC5B40F7A6AAF64B4304C00A974CB48DFC5B405FF8E85C565904C06A5ABA18B8FC5B40760F46ADC06F04C077899852D7FC5B40933B0AB5748604C0D1C0E742EBFC5B40783F59F85B9D04C05DA3D5D5F3FC5B400EF374C85FB404C0A6C5D002F1FC5B409DD3325A69CB04C0EA7691CCE2FC5B405C4C8EDC61E204C0F55D1741C9FC5B40C662418F32F904C0EEEA9B79A4FC5B402E694DD9C40F05C03DAA799A74FC5B40D1675D5F022605C0BE8F08D339FC5B408E0FEC19D53B05C04C5C6F5DF4FB5B400C43176B275105C0954A6A7EA4FB5B40A7910B34E46505C0C73A07854AFB5B406164F2E9F67905C02A9F57CAE6FA5B404F164EAA4B8D05C0E87518B179FA5B4066C0AF4ECF9F05C05FA650A503FA5B401E23B37F6FB105C0C921E61B85F95B407CCE2DC71AC205C091302A92FEF85B404E727FA1C0D105C06C5E5D8D70F85B40A01BF28D51E005C0D07F2B9ADBF75B4082111A1EBFED05C07453204C40F75B4067002604FCF905C0F449153D9FF65B4079301120FC0406C0C004990CF9F55B40A09EAA8BB40E06C0E123515F4EF55B403BF964A51B1706C0DEFF56DE9FF45B4054B6E419291E06C06BF28F36EEF35B4026BA42ECD52306C0F3D401183AF35B40304EFB7C1C2806C0F95F243584F25B40827C828FF82A06C09B193042CDF15B40413C784E672C06C0E5836BF415F15B40413C784E672C06C0863D77015FF05B40827C828FF82A06C08CC8991EA9EF5B40304EFB7C1C2806C014AB0B00F5EE5B4026BA42ECD52306C0A19D445843EE5B4054B6E419291E06C09E794AD794ED5B403BF964A51B1706C0BF98022AEAEC5B40A09EAA8BB40E06C08C5386F943EC5B407A301120FC0406C00B4A7BEAA2EB5B4067002604FCF905C0AF1D709C07EB5B4082111A1EBFED05C0133F3EA972EA5B40A11BF28D51E005C0EE6C71A4E4E95B404E727FA1C0D105C0B67BB51A5EE95B407CCE2DC71AC205C020F74A91DFE85B401E23B37F6FB105C09727838569E85B4066C0AF4ECF9F05C056FE436CFCE75B4050164EAA4B8D05C0B86294B198E75B406164F2E9F67905C0EA5231B83EE75B40A7910B34E46505C033412CD9EEE65B400D43176B275105C0C10D9363A9E65B408F0FEC19D53B05C043F3219C6EE65B40D2675D5F022605C092B2FFBC3EE65B402F694DD9C40F05C08A3F84F519E65B40C662418F32F904C095260A6A00E65B405C4C8EDC61E204C0D9D7CA33F2E55B409DD3325A69CB04C022FAC560EFE55B400EF374C85FB404C0AEDCB3F3F7E55B40783F59F85B9D04C0081403E40BE65B40933B0AB5748604C01643E11D2BE65B40760F46ADC06F04C076064F8255E65B405FF8E85C565904C06EED3EE78AE65B40F8A6AAF64B4304C0A860BF17CBE65B40B58D244EB72D04C0644C2FD415E75B403DD736C2AD1804C02D597DD26AE75B40465DE127440404C0CE7371BEC9E75B406D84A5B58EF003C07C8F462B1BE85B40007E8550CEE103C01A5B003A32E85B40245985EFA0DD03C079E0A8DDA3E85B4029ADB3938DCB03C0797EDA381EE95B407E45088866BA03C0C2DF64D2A0E95B408B664AC83CAA03C0D4E7EF282BEA5B4083376355209B03C0DEC67BB3BCEA5B407C918825208D03C0029AE8E154EB5B405AD86F154A8003C00012851DF3EB5B40A87797DAAA7403C07A93A3C996EC5B40B489B4F64D6A03C0933D35443FED5B405C0F51AC3D6103C0023E6AE6EBED5B4048F2A5F4825903C075D456059CEE5B4098D3BA76255303C09A629CF24EEF5B402568D47F2B4E03C016E215FD03F05B40E2D838FD994A03C082168771BAF05B402A4D5077744803C0C0CE4D9B71F15B402E6B260EBD4703C0FD8614C528F25B402A4D5077744803C069BB8539DFF25B40E3D838FD994A03C0E53AFF4394F35B402568D47F2B4E03C00AC9443147F45B4098D3BA76255303C07D5F3150F7F45B4048F2A5F4825903C0EC5F66F2A3F55B405D0F51AC3D6103C0040AF86C4CF65B40B489B4F64D6A03C07F8B1619F0F65B40A87797DAAA7403C07E03B3548EF75B405AD86F154A8003C0A2D61F8326F85B407B918825208D03C0ABB5AB0DB8F85B4083376355209B03C0BDBD366442F95B408B664AC83CAA03C0061FC1FDC4F95B407E45088866BA03C007BDF2583FFA5B4029ADB3938DCB03C065429BFCB0FA5B40245985EFA0DD03C0030E550BC8FA5B40007E8550CEE103C0 +1 32 01060000000A0000000103000000010000001600000046DD4F06A6665D40721B1F5236081A40AF0A9A1AC4665D40ABF94F2007021A40C017CCA3EE665D40C065B5F3D1F619407EA855A303675D40C4A1A6CE34EF1940D0450060EA665D4080A90180ECF0194090500020A2665D40805D05A06DF51940583000E09A665D404027FF3FC0F51940580DFF3F87665D4040B4FC3FD5F61940A87D00E065665D40403506003BF91940702300605E665D40409F0760CBF91940C044000058665D40C0DFFBFF5EFA1940C070006049665D4080280680DEFB1940105800C042665D4040E5FF9FD4FC194020AAFFFF41665D40C0F9FFBF51FD19409050002042665D408009FC9FCBFD194020F3FF7F43665D40C040FB7F3FFE1940D0C2FF9F48665D40000FFE5F24FF1940D85400A04B665D404052FF3F9FFF194088A1FFFF4E665D4080D2058020001A40A86000C095665D4000C4FFFFFA061A40705D00A09E665D40806C05E0AE071A4046DD4F06A6665D40721B1F5236081A400103000000010000001C000000BDA3C684985B5D40BF61D1CDDA8A1A40962B26BF505C5D403AE150107F8A1A407D9AEE52945C5D407168F2D8198A1A40404C00A0F85B5D408067FDDF71831A40E889FFBFDA5B5D4040F805A0C3811A40B878FFBFD45B5D4000FB060070811A4038BAFF9FB55B5D4040F1FFBFBE7F1A40F8470020AF5B5D40000AFB5F767F1A40907C0080935B5D4000710740957E1A40585C00408C5B5D40007305205D7E1A40E0FDFF3F7B5B5D40000B07A0237E1A40086600A0715B5D4080D80420077E1A40D888FF5F685B5D40800202A0017E1A40307700A0575B5D40C0FF0720017E1A40F80D00E04E5B5D40406FF87F037E1A40389DFF7F455B5D408082F83F0E7E1A40A8260080F55A5D404086FEFFBB7E1A40F0E1FF7FBD5A5D4000A7F8FF3E7F1A4098250020C3595D40406C0120757B1A40882400C050595D408031FE5F707D1A40705D00A03E595D4000760860267F1A40585C00402C595D40C0ECF85F49801A40C799C7D1B1585D401E7650ACEA841A4061D69EB9BE585D400C591F0E26851A40F07A95AF72595D404A0CB245A3871A40055DB0C6285A5D4085A8C4326C891A40E51B674AE05A5D403AE150107F8A1A40BDA3C684985B5D40BF61D1CDDA8A1A400103000000010000000D00000000570060B05D5D40C01805A0D5441A4038200040A75D5D40C01805A0D5441A40502100A0995D5D4080D60140B0451A40502100A0995D5D4080160300D4461A4078E9FF1F9E5D5D4080D10520AE471A4068E8FFBFAB5D5D40408F02C088481A40281F00E0B45D5D40007901A0D1481A40C8680100BE5D5D40408F02C088481A4008C6FFBFBF5D5D400093F79F6C481A40181E0080C25D5D4040A503E03F481A40181E0080C25D5D4080160300D4461A40E07AFF7FB95D5D4080D60140B0451A4000570060B05D5D40C01805A0D5441A40010300000001000000B50200007045DA4700605D40CEDC771D7D2619406DEA25137A5F5D4032448201342319401E66C1B7CA5E5D403DC7D2CAAD1F1940F4D6D132185E5D4064760FCDD71C1940A132FB34635D5D40BFD18BD6B41A194070C65171AC5C5D401E32760447191940028AA99CF45B5D4015D9C1C08F18194079BDE36C3C5B5D4015D9C1C08F1819400B813B98845A5D401E32760447191940DA1492D4CD595D40BFD18BD6B41A19408770BBD618595D4064760FCDD71C19405DE1CB5166585D403DC7D2CAAD1F19400E5D67F6B6575D403244820134231940833513720B575D4071046AF466271940E1D78A6E64565D40778FE67B422C1940053E1891C2555D403F747EC9C131194073B6F07926555D40189DA06CDF37194063A296C390545D4070BD0258953E19400DC5400202545D40119A9AE7DC45194025BA47C37A535D4041542CE7AE4D19407F239A8CFB525D40FB4D679903561940AD1738DC84525D40EFB08ABFD25E19406354B62717525D40C21C8AA1136819404CB0C9DBB2515D40C681AA16BD711940C83FDB5B58515D40F5B68E8EC57B19403B98A50108515D40CAEAAA1A238619408994DB1CC2505D40FCA71578CB90194065F4D8F286505D40C4C9AC19B49B1940BC265DBE56505D40B3668432D2A61940958550AF31505D40716E95C01AB21940D83F94EA17505D40B075A19782BD1940EB22DD8909505D40F3F4406CFEC81940C36B999B06505D404F1211DF82D4194005BCE1220F505D4072ECF58704E019400A45751723505D402E46660178EB1940BB2FC16542505D40C065B5F3D1F61940CC3CF3EE6C505D40ABF94F2007021A40288E1789A2505D40CEDEE06C0C0D1A405A6BC3C5AE505D400041BDD7180F1A40D87E41FFE2505D406EB852EED6171A400D64BF112E515D40AF6AA3F35B221A40AB06597683515D40A3B27E10912C1A40069A98D8E2515D40064D95276C361A40F4E91DDA4B525D40AE5CA674E33F1A40B06FFB12BE525D40EA0C3196ED481A4042F31C1239535D405BC0C49681511A40F954B75DBC535D40637CE6F596591A405611C17347545D4052A982B025611A408C0873CAD9545D4034B1E24826681A405CFD145456555D40E8DC8FD85F6D1A40485B00E059555D40C035FEDF396D1A40A0EBFFDF82555D40C035FEDF396D1A4008E3FFDF0F565D404093006035711A40C819000019565D400016FD5F7D711A40C087FFFF75565D4040B70160BB761A40C1E7956577565D40EE5CC6D093771A40AD78FC87B6565D40BCCC62C490791A407786E39EB6565D408B11416191791A4060D9FF7FCA565D4040C7FF1F52781A40F8470020CF565D40C01F04A02E771A4048D8FF1FD8565D404076F95F79751A40100F0040E1565D404071F83FE8741A40A87D00E0E5565D40403D0380E9741A40C0440000F8565D40004705C00B761A4078A0FF9FFC565D40005D06E0C2751A40A068FF1F01575D404089082031751A4038D7FFBF05575D40C0B3FB9F0D741A4038D7FFBF05575D4080F9FDDF34721A4018EA00C005575D400051FDFF0F721A4078A0FF9FFC565D4080040080C96F1A40D0450060EA565D408009FC9FCB6D1A4030110000C6565D4040FF0180AA6A1A40404C00A078565D40409503408C651A40386001006B565D4080D706A0B1641A40F8A7FF3F5D565D4080ED07C068641A4008A9FF9F4F565D4080ED07C068641A4020AAFFFF41565D4080ED07C068641A4030ABFF5F34565D4080D706A0B1641A40508700402B565D4080C10580FA641A40B81800A026565D4040AB046043651A40B05200E006565D40409503408C651A40001D0020F0555D40409503408C651A4040E6FFFFE6555D4040AB046043651A4050E7FF5FD9555D4080ED07C068641A4050E7FF5FD9555D4000750800B4621A4080AFFFDFDD555D404089FA5FD9611A40E01A00600B565D4080ED00E0BC5A1A4008E3FFDF0F565D40C01903202B5A1A409050002022565D4000B704802D581A4008A9FF9F4F565D40801405200C551A40D0DFFFBF58565D40C0DB06207B541A4078A0FF9FDC565D400041F85FCD4C1A40100F0040E1565D40005CFA9F154D1A4080F8FF5FDF565D404021FCDF9F4D1A4098A2FF5FC1565D40804DF91F7F501A40B0A3FFBFB3565D40403907C059511A40C0A4FF1FA6565D4000F7036034521A40E08000009D565D4000C807A0C5521A40581300C08A565D40C0850440A0531A40008300C081565D4000DAF75FE9531A40404C00A078565D4000F2074032541A40901600E061565D40C068F83F55551A4008A9FF9F4F565D40803E05C078561A400898018046565D40C0FD07601C571A40784F00C02F565D40008B0240BF581A409050002022565D4080030200745A1A40608800A01D565D4080D7FFBF055B1A4018E4FF3F02565D404049F99FB5601A40A051008014565D40C0EBF8FFD6631A40B81800A026565D400018FB3F45631A4020AAFFFF41565D4000750800B4621A40D0DFFFBF58565D40005DF81F6B621A40901600E061565D40005DF81F6B621A40581300C08A565D4080ED07C068641A40B0A3FFBFB3565D4000240420F8661A4030110000C6565D40C0B2040064681A40201000A0D3565D40805A008087691A40C00A00C037575D4080F3FA7F146E1A4040C9FFDFF6575D404082FB5F806F1A40F8470020CF585D400040FFDF066A1A40180701E0D5585D404044FF5FD0691A40705D00A03E595D40006400E08C651A40485B00E059595D400033F87FFE601A40502100A099595D40C068F83F55551A40502100A099595D40C0DB06207B541A40B8B2FFFF94595D4000DAF75FE9531A40A811FFBF50595D40C0470840F04D1A409825002043595D404072FBBFCC4C1A40A826008035595D4080B4FE1FF24B1A40C80201602C595D4080CAFF3FA94B1A4028B9FF3F23595D4080AFFDFF604B1A4010F2FF1F11595D4080AFFDFF604B1A4050BBFFFF07595D4080CAFF3FA94B1A408884FFDFFE585D4080B4FE1FF24B1A40A86000C0F5585D40409EFDFF3A4C1A40D83D0100DF585D40C02FF85FA74D1A40082C0060D1585D400001FC9F384E1A4048F5FF3FC8585D400001FC9F384E1A4088BEFF1FBF585D400001FC9F384E1A40A8C0FFDFA3585D400001FC9F384E1A40E889FFBF9A585D40C0470840F04D1A40086600A091585D400046F97F5E4D1A401867000084585D404088FCDF834C1A4080F8FF5F7F585D4080B4FE1FF24B1A4080F8FF5F7F585D40007E02C062491A401867000084585D40C0580660F9451A409093FF1F80585D40800EFDBF5C401A40681400205D585D4040C604C06E381A40E889FFBF5A585D40807B0520F0371A4048D8FF1F58585D4080E401207F371A4010F2FF1F31585D40809AFC3F1C321A408092FFBF2D585D400078FC3FD0311A40E097FF9F29585D40808B03E086311A4090CDFF5F20585D4080A40560EA301A40E0C3FFFF1A585D40004AFC1F9A301A40B8B2FFFF14585D400088F81F4A301A40F0FEFF9F0D585D400056F91F12301A400800000000585D4000C90700382F1A40985F006003585D4040B1F71FEF2E1A40E01A00600B585D4080580480DC2E1A40B06F000017585D4080C4F7DFF92E1A4040E6FFFF26585D40C08A0060A22F1A40E0E0FF1F2B585D4040BC0780DB2F1A40F84700202F585D4000CCF77F1A301A40C00A00C037585D40C04107C0EC301A40B086FF9F63585D40404D03209D351A40406900C068585D40000306603B361A40E06300E06C585D40806DF77FC9361A4090B0FF3F70585D4000DCFC1F5D371A409033000072585D4040D9FBBFB0371A4038F4FFDF75585D400032F81F8C381A40D8B4FFBF99585D40C08A07404E411A4070DAFFDF9C585D408083F89F80421A4008C6FFBF9F585D40807006A0CD431A40F06400409F585D40408EF85FF8461A4088BEFF1F9F585D40C1B607A09F471A40702300609E585D4040E3F8FF43481A40A0170040D4585D4040CA0760AA4B1A4038BAFF9F35595D40402704404F4A1A40B0ECFF3FB5595D4040D0FCFF72461A406082FF1FBA595D40002F08A0A9451A40E8550000BE595D4000B3FAFF45451A4020AAFFFFC1595D40402C0560E0441A4060D9FF7FCA595D40801F02C011441A4080DBFF3FCF595D4080420860B4431A40A0B1FF9FE2595D40406E01E059421A40C0EDFF9FE7595D40400C020007421A40E06300E0EC595D40804C00A0B8411A4018010060F2595D40C0D505A077411A40985F0060235A5D40007EFBDFB63F1A4040ACFFBF465A5D4000110320983E1A40009AFF5F6E5A5D40C01FFDBF823D1A4030E5FF9F745A5D40C0640360713D1A40600500E07B5A5D404096FE9F6F3D1A4020F3FF7F835A5D404034F8DF703D1A40508700408B5A5D4080090380773D1A40105800C0C25A5D40002AFEBF4F3E1A40801B0100F35A5D40C096FD5F1A3F1A40608800A0FD5A5D404086F71F103F1A40C0EDFF9F075B5D40C0DE0460B53E1A40383D0060175B5D40403F0340CE3D1A40684E00601D5B5D40C00AFEDF5A3D1A40406900C0485B5D40C09E01C074381A40D8D1FFDF495B5D408012F97FEC371A40487800004A5B5D40005C068050371A40682BFFBF495B5D4080410300B3361A40385A0080475B5D408076FD1FB3351A40609FFF3F2A5B5D40004F0420D7321A40B0A3FFBF135B5D4000F803C0A6301A40F08100600F5B5D4080FB07A037301A40504D00000B5B5D4040F6FBDFC02F1A40C8B3FF5F075B5D40400403A03B2F1A40105800C0025B5D40C0D6FE1F3E2E1A40380BFF7F025B5D4040C6FFBFDF2D1A4018670000045B5D40C01402009A2D1A40C0A4FF1F065B5D40804005805D2D1A4070BDFFBF0C5B5D40003CFC3FCB2C1A40082C0060115B5D40005F02E06D2C1A40C896FF3F175B5D40C07FFEBF0D2C1A40F01B00C01D5B5D40406FFF5FAF2B1A40283C0000255B5D4000E8FE1F642B1A40107500E0325B5D40404FFCFFD52A1A40485B00E0395B5D40C002FF7F8F2A1A40008300C0415B5D40803508E0572A1A40F032FF5F4A5B5D40801CFD9F2B2A1A4030ABFF5F545B5D4040CEF91F1C2A1A408884FFDF5E5B5D40C0F5FC1F162A1A40C0700060695B5D40400AF83F042A1A40907C0080735B5D40400904C0CC291A40401801E07B5B5D40C0E603C080291A40008300C0815B5D40803BFD7F20291A40308EFF3F845B5D4040FA0560A8281A4018670000845B5D404041062025281A40807B0020815B5D40C05EF91FA5271A40684E00607D5B5D40C0AE016028271A40D0DFFFBF785B5D40C0ABF73FB3261A40F8960120745B5D40C0DD040043261A4088A1FFFF6E5B5D40C0940480E1251A4058CAFF3F695B5D400035074090251A40287FFFFF625B5D404006FA9F57251A40E08000005D5B5D4040ECFE9F2D251A40C087FFFF555B5D40808A038014251A40009AFF5F4E5B5D404001056001251A40B05200E0465B5D40C0150080EF241A40702300603E5B5D40807504A0EC241A4028B9FF3F235B5D40C0A9F95FEB241A40583000E01A5B5D4040E2FA7FEE241A40107500E0125B5D4080F5FA3FF9241A40E8A6FFDF0A5B5D404052FA3F10251A4050E7FF5FF95A5D408039FF9F58251A40001D0020F05A5D40C0FCFDFF8B251A40D079FF1FE75A5D40805DFEBFC1251A40705D00A0DE5A5D400088FFFFF5251A40C8D0FF7FD75A5D404057006030261A4010F2FF1FD15A5D4080C8066070261A40581300C0CA5A5D4080DC02C0B3261A40308EFF3FC45A5D40408EF85FF8261A40684E0060BD5A5D40003800803B271A40C087FFFFB55A5D40C04608E07D271A408092FFBFAD5A5D40C0E405E0B8271A40280200C0A45A5D40806A0700E7271A40E0FDFF3F9B5A5D40C02F0440E2271A4098420040935A5D4000200180BC271A40686B00808D5A5D40C05C022089271A40406900C0885A5D4000FC016053271A4068CBFF9F5B5A5D40C0E801A048251A4048D8FF1F585A5D40400200C0E4241A40401801E07B5A5D40C0D6FE1F3E241A4088EAFF7F105B5D40C0AA05A098231A40B83500C0165B5D4080A1FD1F92231A40E08000001D5B5D404004FCBF8F231A40A8C0FFDF235B5D40409AFF5F8E231A40E01A00602B5B5D40809705E08D231A4098420040335B5D400069FCFF8E231A40E0C3FFFF3A5B5D40007002E093231A40B87E0040785B5D40C0580660F9231A40108CFF7F7F5B5D404076FE5F08241A40B81800A0865B5D4040E1010028241A40E80C00809C5B5D40401BFB5F9C241A40F0E1FF7F9D5B5D4000AB0780B5281A40600500E09B5B5D4040D8FB5F3E291A40E097FF9F895B5D400018FD1F622A1A4028850080865B5D40409608A08D2A1A4000B7FF7F7E5B5D40C03A01E0E72A1A40C087FFFF755B5D400060FB5F342B1A408092FFBF6D5B5D40408C06606B2B1A4090330000525B5D4040B8FFDF102C1A40280200C0445B5D4000CAF99F522C1A40E85500003E5B5D40C0DFFBFF5E2C1A40383D0060375B5D400024FF1F692C1A40082C0060315B5D4080200220842C1A40585C00402C5B5D40803FF93FB02C1A4018E4FF3F225B5D40402F03A01A2D1A4080AFFFDF1D5B5D4000F9FC3F6D2D1A40E0C3FFFF1A5B5D4000010380E42D1A40D06200801A5B5D4040970300712E1A40684E00601D5B5D40C037FCBF012F1A40008300C0215B5D40C09E03A0912F1A40A0510080545B5D4040FB05C01A351A40D0DFFFBF585B5D4040BF00C086351A4040ACFFBF665B5D40801305C099361A4068E8FFBF6B5B5D400000FE1FE3361A40F8130160725B5D4080EE00402F371A40E0C3FFFF7A5B5D4040DBF99F78371A4030C8FF7F845B5D4000C5F87FC1371A40780600408E5B5D40C0DF02E00A381A4010D5FFFFA05B5D40C078FDDF97381A40D0450060AA5B5D40809EFADFC8381A40A84300A0255C5D40402C0740FD391A4088EAFF7F505C5D4040DA07005E3A1A40D02800407A5D5D40401A04C0F23C1A4020AAFFFF815D5D40C0E70140D63C1A40C02700E0875D5D40403B00A0923C1A40602200008C5D5D4040A3F91F3D3C1A40F08100608F5D5D40C008F91FE73B1A40A0510080945D5D40804D0500BA3A1A40C8190000995D5D4000A5FF1F06381A40502100A0995D5D40803CFDDF92371A40502100A0995D5D400068FC9F1C371A40502100A0995D5D4040C406E0A6361A40502100A0995D5D40405204402E361A40502100A0995D5D40804C00A0B8351A40502100A0995D5D40407A010044351A40502100A0995D5D40C080F73FD4341A4050E7FF5F995D5D40409207E06E341A40B8B2FFFF745D5D40800CF8FFE8261A40184A00E0735D5D40C07B02007E261A40F08100606F5D5D4000A0F71FC9251A40901600E0615D5D40C0E20700F1231A4000F1FFBF1E5D5D4080CBF8BF6F1E1A40D00B0020EA5C5D4000DAFE3F950F1A40106BD48AE95C5D400041BDD7180F1A40C0700060E95C5D40C0B70520F50E1A4048780000EA5C5D4040330640560E1A40F03800E0ED5C5D40805103A0660D1A40A0170040F45C5D4080D007A0580C1A40D8EEFFFF395D5D40008A04C069031A40F0E1FF7F3D5D5D4000A005E020031A4028850080465D5D400018FB3F45031A40485B00E0595D5D404094FEDF8A061A40485B00E0595D5D400052FB7F65071A400040F74D305D5D400041BDD7180F1A4090CDFF5F205D5D40C049FF3F0C121A40585C00404C5D5D400084F75F2B1A1A40B06F0000775D5D40C07DFEFF28231A40B8B2FFFF945D5D4000800780D62A1A4060BCFF5F9A5D5D40801505807E2F1A408884FFDF9E5D5D404030FC1FE12F1A40A84300A0A55D5D4000CCF77F1A301A40F88AFF1FAD5D5D40002006804B301A4080F8FF5FBF5D5D40404702A099301A40C8360020C95D5D40003EFA1F93301A40304B0040865E5D40C035FEDF392D1A40F092FE7F985E5D40C090FCDF162C1A40E08000009D5E5D4000BDFE1F852B1A4098DCFF9FA15E5D400002FCFFAA2A1A40985F0060A35E5D40C02903C0DE261A40804100E0A05E5D4040D3FA3FAD241A40E08000009D5E5D40C0FF0540E4221A40082C0060915E5D400081F91FF11D1A40581300C08A5E5D40C0440500271B1A40E097FF9F895E5D4000DDFA9FB21A1A40A87D00E0855E5D40402704404F1A1A40404C00A0785E5D4080DAFDFF3F191A40184A00E0735E5D4040F80780E0181A402087FE5F6E5E5D4040B9FF3F83181A4018330140675E5D4000BC00A02F181A40705D00A05E5E5D408097FEFFE1171A40E8A6FFDF4A5E5D400033046039171A409093FF1F405E5D40C0150080EF161A40C8190000195E5D40007808200B161A40580DFF3F075E5D4080360660AD151A4078E9FF1FFE5D5D40C0D7FC9F93151A4040C9FFDFF65D5D40C003F8FF55151A40103B00A0D25D5D40801F02C011141A40E8EFFF5FCC5D5D40C0D5FEBFCB131A40C8B3FF5FC75D5D40C0B004407F131A40A0EBFFDFC25D5D400054F95F2D131A4000F1FFBFBE5D5D40C0D4F95FCA121A40E06300E0AC5D5D40808FFF9F16111A40F07BFFDFAB5D5D40C013FD9F98101A4068E8FFBFAB5D5D40C05C072018101A4068E8FFBFAB5D5D400041BDD7180F1A4068E8FFBFAB5D5D40C0CCFAFFFE0D1A407883FF7FAC5D5D40C0E5FC7F620D1A4030C8FF7FC45D5D40C0EBF8FFD6091A405081FFBFC75D5D40005901606A091A40585C0040CC5D5D4080F0FE1FF7081A4008C6FFBFDF5D5D40C0D7FC9F93071A40D079FF1FE75D5D40C09807201D071A4008E3FFDF0F5E5D40801BFF1FD6041A40B81800A0265E5D40007403A0B2031A4028850080465E5D40809D016002FE194020AAFFFF415E5D40800EFF9F79F5194030ABFF5F345E5D40C01FFF9F9FF41940B81800A0265E5D40C01FFF9F9FF41940608800A01D5E5D40C01FFF9F9FF4194008E3FFDF0F5E5D40C01FFF9F9FF4194068AEFF7FEB5D5D40C02400C030F51940A877FF5FE25D5D40800EFF9F79F51940B878FFBFD45D5D40800EFF9F79F51940E8550000BE5D5D40800EFF9F79F5194000570060B05D5D40C02400C030F5194038200040A75D5D40C01FFF9F9FF4194030E5FF9FF45D5D40C03FFDFFE9F21940C8190000195E5D40C03FFDFFE9F21940408600E0385E5D40C0DFFDDF7BF3194020AAFFFF415E5D4000620200C5F31940A0170040545E5D40C009FE7FE8F41940304B0040865E5D40809D016002FE1940681400207D5E5D40C06E05A093FE194098250020E35D5D4040A706C0960C1A40D0620080DA5D5D40C07EFE5F9B0D1A40488FFF9FD65D5D40003E01003F0E1A40C8C2E200D25D5D400041BDD7180F1A4010F2FF1FD15D5D40C0ABFC3F420F1A4090B0FF3FD05D5D4080BBF81FBC0F1A40707A00C0CE5D5D40C0B004407F111A4090330000D25D5D40801F02C011121A40B06F0000D75D5D400045000098121A4080AFFFDFDD5D5D4000A4FABF04131A4030110000E65D5D40C00006A056131A40009AFF5FEE5D5D408064F8BF8B131A40901600E0015E5D40401FF9FF48131A4020760040055E5D4040B7FA7F0F131A40E097FF9F095E5D400098F8BFFD121A40704000800E5E5D40808D08A0FA121A40D04500602A5E5D40804C00A0B8131A4000A000E0315E5D4080AA06E0ED131A4008C0FE3F5C5E5D4000DEFAFF24151A4098250020635E5D40403CF91F59151A4040C9FFDF765E5D40809500201A161A4090C7FEDF7C5E5D40406BFCBF73161A40287FFFFF825E5D4000540040D9161A40707A00C08E5E5D4080DD0720B5171A40B0A3FFBF935E5D40C095F8FF18181A40307700A0975E5D400030012070181A40108CFF7F9F5E5D40C074FA3F5C191A40A8FAFF1FA45E5D4080ADF83FED191A40D84EFF1FA85E5D40C0A3FDDF761A1A40483E00C0A95E5D40C0FB0480C51A1A40F88AFF1FAD5E5D4040E1FCFF981B1A40609FFF3FCA5E5D408021078085231A40201000A0D35E5D40C06508C072261A40383D0060D75E5D404034FABF8D281A40C896FF3FD75E5D4000C4FFFFFA281A40107500E0D25E5D4040500660662A1A4010D5FFFFC05E5D40C04BFFFFF02C1A40E0800000BD5E5D4000AEFEDF432D1A40D0DFFFBFB85E5D400033FD7F8D2D1A4030ABFF5FB45E5D4040BAFDBFD82D1A40F0810060AF5E5D4000EB01602D2E1A40E097FF9FA95E5D408065FD1F8D2E1A40F08100608F5E5D400056F91F12301A40506A00205B5E5D40809903C055331A40D0DFFFBF585E5D40407803207C331A4088EAFF7F505E5D4080A4074007341A40E0E0FF1F4B5E5D40401E008082341A4028850080465E5D4000F2FD3F14351A40A0B1FF9F425E5D400014FF7FB5351A40804100E0405E5D4000E7F9BF62361A4008000000405E5D40007207A007371A40A0EBFFDF425E5D40C0DF02E00A381A40086600A0915E5D40008E0080F9401A40502100A0995E5D40C0F4FCBFA3411A408884FFDF9E5E5D40801F02C011421A4028850080A65E5D4080E105C061421A4080DBFF3FAF5E5D4040CEFE1FAB421A406039FF9FB85E5D4000B8FDFFF3421A40105800C0C25E5D4080EE00402F431A40F88AFF1FCD5E5D40C01BFCFF63431A40B06F0000D75E5D40C0D505A077431A40F82A0000DF5E5D400027FB7F86431A40A84300A0E55E5D4080640680E3421A4060D9FF7FEA5E5D40C001FF1F1D421A4078CCFFFFED5E5D40C07604005F411A40F8470020EF5E5D40C0BAFE5FA0401A40F0810060EF5E5D4000970620E33F1A4080DBFF3FEF5E5D400085FCBF2C3F1A40F8470020EF5E5D40C0AFFFDF7D3E1A40106FFF5FEF5E5D40407DF85FD23D1A40187EFF9FF05E5D4040AAFD1F253D1A4028D6FF5FF35E5D40C04FFBBF803C1A40B8FBFF7FF65E5D4000900040DE3B1A40502100A0F95E5D40C0F9FFBF513B1A40608800A0FD5E5D40C03505C0E53A1A40A0EBFFDF025F5D40C013FD9F983A1A4020760040655F5D40C02400C030351A40D8D1FFDF695F5D40C02400C030351A40187EFF9F705F5D40C08008209E351A40009AFF5F6E5F5D4080E2FC5F0B361A4008490080615F5D40C0C9F7BF35371A4078A0FF9F5C5F5D40008BF83FA1371A40488FFF9F565F5D4040C2F71F15381A40C8360020095F5D40C0B1FDBF45401A4048F5FF3FC85E5D40C04D02E047471A4020AAFFFFC15E5D40C058FF7F4D481A4008490080C15E5D4000E507C0D5481A40008300C0C15E5D40000804605A491A40105800C0C25E5D40005A01C0DC491A4050BBFFFFC75E5D400005FA3FE54A1A4018E4FF3F225F5D400065FE5FE2541A40B0ECFF3F755F5D4080FC0020FE5C1A40E8720020AE5F5D40006EF81F91611A407045DA4700605D40401EE1988B681A409825002043605D40C0310720396E1A40F06400407F605D404076F95F79751A401867000084605D400062070054761A40086600A091605D4040C7FF1F52781A405AF9EE6D95605D40D1B53CBAB7781A40BBA9531A1F615D40E47802E361741A40B951716B8E615D4003B57DD950701A4050E7FF5FD9615D40004B086047611A40E04600C0BC625D4080CF00603A461A40F8730080C0625D408073F8FFCC451A4050BBFFFFC7625D4000F505806C441A40D00B0020CA625D4040A0FBDF02441A4060D9FF7FCA625D404089014085431A40C87F00A0CA625D40808D08A0FA421A4060D9FF7FCA625D4040BA02C067421A40D8D1FFDFC9625D404007FAFFC9411A40280200C0C4625D404094FEDF8A401A40A0510080B4625D4040850580F53F1A40F80D00E0AE625D40C094FF7F523F1A40F80D00E0AE625D4000670820E53E1A40886D0040B2625D4000D80740793D1A40A8FAFF1FC4625D40C06203A08C3A1A4098250020E3625D40802AF87F6B391A4010D5FFFF20635D400080FB9F9B381A40189BFFBF40635D4040C7FF1F52381A40704000804E635D4040C7FF1F52381A40307700A057635D4040C7FF1F52381A40A0CEFFBF92635D4080B300A09C361A40283C0000A5635D40C049FF3F0C361A4040030020B7635D40C02400C030351A40D87100C0BB635D40C03A01E0E7341A40D87100C0BB635D40005102009F341A40103B00A0B2635D4080A40560EA321A402016012057635D40C0E6FCDFD4251A40907C008053635D400005F85FC8251A4050BBFFFF47635D408009088006261A40E80C00803C635D4080A400605B261A40208DFFDF31635D40C047FA7F98261A40B81800A006635D400043FB3F24271A40600500E0FB625D4080FFFE5F38271A40A085FF3FF1625D40C09AFE1F39271A40B05200E0E6625D400074FE9F23271A4068140020DD625D40C0150080EF261A40A0170040D4625D40C00BFC5FB0261A407883FF7FCC625D40C0CDFA5F71261A40D079FF1FC7625D40805EFE1F34261A40A0B1FF9FC2625D400034FDDFFF251A4080DBFF3F6F625D40005500A04B211A40E87200206E625D4000F9F73FDE201A4098BFFF7F71625D40C0FAFD3FA7201A400849008081625D40C0FAFD3FA7201A40E04600C09C625D40808A038014211A40307700A0B7625D4040AE00C060221A40100F0040C1625D400007FD1F3C231A4038D7FFBFC5625D40808DFADFA2231A4050040080C9625D40C0C4F9BF16241A4070F7FFFFCC625D40006FFD7F92241A40805E0000D1625D40C074012008251A40583000E0DA625D4000E204800C261A40402F0080E8625D40801808C047261A4078A0FF9FFC625D4000B8070012261A40A017004074635D4040C90200A9241A40F098FFFF7B635D408015F99F43251A40F098FFFF7B635D4080BAFA9F66261A40880700A080635D40808BFEDFF7261A4028B9FF3F83635D4000D9FEDF22271A40E097FF9F89635D40805FFC9F89271A40A0CEFFBF92635D4080E4FA3FD3271A40283C0000A5635D4080E4FA3FD3271A40003A0040C0635D408075FDBF40271A40C0700060C9635D4080F5FA3FF9261A4000F1FFBF3E645D40809DFA7F56201A40E04600C09C645D404057FB5FA11B1A409842004013655D4080FD008070171A40683100408D655D4080710100B1131A40A085FF3FF1655D40801901400E111A40087CAF081B665D400041BDD7180F1A4000F1FFBF1E665D400011FC3FEC0E1A40B05200E066665D40C04A04A00D0B1A40E82900A06C665D40808800A0BD0A1A40805E000071665D400018FD1F620A1A40A894FF7F72665D40C070F79F200A1A409842004073665D40002F01C0FD091A4090F9FFBF71665D40C09201A08A091A40F8C4FF5F6D665D40801FFBDF65081A40786C00E05F665D4000B003A0B7061A40681400205D665D40407A03E060061A40302E002056665D40409503408C051A4010B8FFDF50665D40C054FCDF11051A4068E8FFBF4B665D40402A028089041A4040ACFFBF46665D40C09BF79FFF031A409050002042665D40C04608E07D031A40D85400A02B665D40C027F9FFDB001A40D0C2FF9F28665D408073FDFF5B001A40D079FF1F27665D40806900C0C8FF1940D079FF1F27665D40C0D5F9BF3CFF19405081FFBF27665D4040E10600B7FE1940508700402B665D4080CC022000FC1940602200002C665D40C0BFF89F85FB194010F2FF1F31665D40002DFCFF89FA1940A051008034665D4080C1076017FA1940D0A5FF7F38665D4040650720ABF9194078A0FF9F3C665D40003800803BF91940D03FFFDF46665D4000FD06C054F81940E85500005E665D4000A1036076F61940A034006064665D4040B1F9FF0BF6194098BFFF7F91665D40C093F83F34F3194070DAFFDFBC665D4080BF02A0A3F019408DA6A59A0D675D405AB996B697EB1940710218F20D675D402E46660178EB19407FBAB91F11675D404C8BB11DA5E91940C87F00A00A675D40C0EDFDBF4AE919400800000000675D40C02BFABFFAE81940B878FFBFF4665D400090FE5FC1E81940C8360020E9665D40C0D305E092E81940D87100C0BB665D408037FADFE4E71940304B0040A6665D400006068092E71940801500806F665D4040F3F89FF7E61940D0DFFFBF58665D4040F3F89FF7E61940F0E1FF7F3D665D408075FDBF40E71940A82C010019665D40401DF93F64E81940E01A00600B665D40001F0800F6E8194030E5FF9FF4655D40C07FF7DF61EA1940001D0020F0655D40003FFA7F05EB194060D9FF7FCA655D40005CFA9F15ED1940D87100C0BB655D40C0B30280B9ED1940105800C0A2655D40005D0400A6EE1940502100A099655D40C04603E0EEEE1940B8B2FFFF94655D40C04603E0EEEE1940C8B3FF5F87655D40C04603E0EEEE1940E88F00407E655D40C04603E0EEEE19402093006055655D400089064014EE1940C0EDFF9F47655D4000CE03203AED19406082FF1F1A655D4000C2FA3F87E919406082FF1F1A655D40000907E03EE91940C02700E027655D408075FDBF40E71940105800C0A2655D4080E1F7FF09D7194068E8FFBFAB655D40C0A8F9FF78D61940480CFFDFB4655D40C0A8F9FF78D61940D079FF1FC7655D4080F7F81FC1D61940D85400A0CB655D4080E1F7FF09D7194080AFFFDFDD655D4040230880BFD81940A877FF5FE2655D4040C6FABF50D91940A877FF5FE2655D40409AF87FE2D91940A877FF5FE2655D40009C074074DA1940A877FF5FE2655D40C06F050006DB194050E7FF5FD9655D40C014070029DC1940281F00E0B4655D40406104806FDE1940A0EBFFDF82655D40409FFB7F90E11940C0EDFF9F47655D4040F3F89FF7E619409825002043655D40805FFC9F89E71940D8EEFFFF39655D40001F0800F6E81940D8EEFFFF39655D4000ACF91FD0E91940705D00A03E655D40C07FF7DF61EA1940C0EDFF9F47655D40808106A0F3EA1940385A008067655D4040FA0560A8EC1940D8B4FFBF79655D4000CE03203AED1940602200008C655D4000B8020083ED194078E9FF1F9E655D4000B8020083ED194038200040A7655D4040E40440F1EC194068E8FFBFAB655D40801007805FEC1940281F00E0B4655D40808106A0F3EA1940E8550000BE655D4000C2FA3F87E91940B878FFBFD4655D404033FA5F1BE8194080AFFFDFDD655D404049FB7FD2E7194010D5FFFF20675D40406104806FDE19403A9FDF1223675D40406104806FDE1940B7DBF36D2A675D404F1211DF82D419409024B07F27675D40F3F4406CFEC81940A307F91E19675D40B175A19782BD1940E6C13C5AFF665D40716E95C01AB21940BF20304BDA665D40B3668432D2A619401653B416AA665D40C3C9AC19B49B1940F2B2B1EC6E665D40FCA71578CB90194040AFE70729665D40CAEAAA1A23861940B307B2ADD8655D40F5B68E8EC57B19402F97C32D7E655D40C681AA16BD71194018F3D6E119655D40C21C8AA113681940CE2F552DAC645D40EFB08ABFD25E1940FC23F37C35645D40FB4D679903561940568D4546B6635D4041542CE7AE4D19406E824C072F635D40119A9AE7DC45194018A5F645A0625D406FBD0258953E194008919C8F0A625D40189DA06CDF371940760975786E615D403F747EC9C13119409A6F029BCC605D40778FE67B422C1940F7117A9725605D4071046AF4662719407045DA4700605D40CEDC771D7D2619400103000000010000001D0000005098FE5F145E5D408080FA5F46461A40F80D00E00E5E5D4000E006A044461A40487800000A5E5D408044FC3F5E461A40B0090060055E5D404069FEDFAB461A40C09EFE9F025E5D400033046039471A4018E4FF3F025E5D40003E08E0EA471A40985F0060035E5D40808EFF3FA4481A40B895FFDF045E5D40C03505C0E5481A40D079FF1F075E5D40004EFABF46491A40D85400A00B5E5D40C0460100D2491A4098420040135E5D4080E0F79F974A1A40408600E0185E5D4040E20660294B1A40F06400401F5E5D400067FE1FC74B1A40202D00C0235E5D4040120560274C1A40C87F00A02A5E5D4000CDF7DF8C4C1A40201000A0335E5D40407A08E0EF4C1A40783200A03F5E5D404064FBDFFD4C1A40B8610020485E5D40407004C0B04C1A40E8A6FFDF4A5E5D40002BF91F334C1A4088A1FFFF4E5E5D40C05B02C016491A40F8C4FF5F4D5E5D4040A3FE1FCC481A40581300C04A5E5D4040D702E077481A4028850080465E5D40407705A026481A40B88FFE5F415E5D4080AB0140D1471A40F84700202F5E5D40C0400080CE461A405081FFBF275E5D40408F02C088461A4068D100201F5E5D40005605004D461A406891FF5F1B5E5D404020FE5F4A461A405098FE5F145E5D408080FA5F46461A400103000000010000000C000000B878FFBFB4595D4080A8F71F5C4F1A40B878FFBFB4595D4000BCFEBF124F1A40105800C0A2595D40C0BEF83F134F1A40105800C0A2595D40804DF91F7F501A40181E0080C2595D40C0EF00A0A1531A4000570060D0595D40C0DB06207B541A4050E7FF5FD9595D40C0DB06207B541A4050E7FF5FD9595D400088F81F4A521A4050E7FF5FD9595D40404F08E010511A40D85400A0CB595D40409405C036501A400843FFFFBD595D4080C00700A54F1A40B878FFBFB4595D4080A8F71F5C4F1A400103000000010000000D00000058FC00209E595D40004B086047611A40602200008C595D4080C6FC9F6D601A4070BDFFBF8C595D4080B2FB3F9B631A4068E8FFBFAB595D404093006035711A40281F00E0B4595D40407DFF3F7E711A40E8550000BE595D40407DFF3F7E711A4090B0FF3FD0595D4040D8FD3F5B701A40A877FF5FE2595D4080AEFD9FEE6E1A40001D0020F0595D4000A7FDFFCD6B1A40001D0020F0595D4000F5076089671A40B878FFBFD4595D4080C10580FA641A4048FB00C0AB595D400073F93F22621A4058FC00209E595D40004B086047611A400103000000010000000A000000A86000C0D5585D404098FC7F376F1A4070F7FFFFAC585D404098FC7F376F1A40A8C0FFDFA3585D404093006035711A40A8C0FFDFA3585D40C024FBBFA1721A40C087FFFFB5585D4080F8F87F33731A4038F4FFDFD5585D4080F8F87F33731A40E82900A0EC585D400067FE1FC7711A40E82900A0EC585D40807B0340D3701A40E82900A0EC585D4080EEFE5F12701A40A86000C0D5585D404098FC7F376F1A400103000000010000000A000000C0440000F8565D40C03505C0E5761A4040E6FFFFE6565D40007808200B761A40D0A5FF7FD8565D4000C9FB1FFD761A40C0C1FF3FD6565D40402E0520C5771A40C0C1FF3FD6565D40C0CCFFFF8D781A40087DFF3FDE565D40C02BFABFFA781A4060D9FF7FEA565D40400200C0E4781A40907C0080F3565D4040C7FF1F52781A40A0570100F8565D4080F30160C0771A40C0440000F8565D40C03505C0E5761A400103000000010000000F00000064EA9AEE16575D40FCB88F0D257C1A404D5E0FD127575D40B2B8AFB3987C1A40784F00C02F575D4080CAFF3FA97B1A4030ABFF5F34575D40C013FD9F987A1A40A826008035575D40002AFEBF4F7A1A40C00A00C037575D4000560000BE791A40408600E038575D40C03703A0AD781A40408600E038575D40C01F04A02E771A40C00A00C037575D4040F3F89FF7761A4010B8FFDF30575D40004C06E09C761A4000D4FF9F2E575D40807B0340D3761A40508700402B575D408009038077771A40D02800401A575D4040C8F89F187B1A40C819000019575D40C0CEFFBF727B1A4064EA9AEE16575D40FCB88F0D257C1A40 \ No newline at end of file diff --git a/src/test/scala/org/globalforestwatch/TestEnvironment.scala b/src/test/scala/org/globalforestwatch/TestEnvironment.scala new file mode 100644 index 00000000..7c87a10c --- /dev/null +++ b/src/test/scala/org/globalforestwatch/TestEnvironment.scala @@ -0,0 +1,30 @@ +package org.globalforestwatch + + +import java.nio.file.{Files, Path} +import com.typesafe.scalalogging.Logger +import geotrellis.raster.testkit.RasterMatchers +import org.apache.spark.sql._ +import org.apache.spark.SparkContext +import org.globalforestwatch.summarystats.SummarySparkSession +import org.scalactic.Tolerance +import org.scalatest._ +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.LoggerFactory + +trait TestEnvironment extends AnyFunSpec + with Matchers with Inspectors with Tolerance with RasterMatchers { + @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) + + lazy val scratchDir: Path = { + val outputDir = Files.createTempDirectory("gfw-scratch-") + outputDir.toFile.deleteOnExit() + outputDir + } + + def sparkMaster = "local[*, 2]" + + implicit lazy val spark: SparkSession = SummarySparkSession(s"Test Session") + implicit def sc: SparkContext = spark.sparkContext +} \ No newline at end of file diff --git a/src/test/scala/org/globalforestwatch/layers/GladAlertsSuite.scala b/src/test/scala/org/globalforestwatch/layers/GladAlertsSuite.scala index f8535d2b..256ff708 100644 --- a/src/test/scala/org/globalforestwatch/layers/GladAlertsSuite.scala +++ b/src/test/scala/org/globalforestwatch/layers/GladAlertsSuite.scala @@ -15,32 +15,32 @@ class GladAlertsSuits extends AnyFunSuite { test("Unconfirmed date 1") { assert( - glad.lookup(20001) == Some(("2015-01-01", false)) + glad.lookup(20001) == Some((LocalDate.of(2015, 1, 1), false)) ) } test("Unconfirmed date 2") { assert( - glad.lookup(20366) === Some(("2016-01-01", false)) + glad.lookup(20366) === Some((LocalDate.of(2016,1,1), false)) ) } test("Unconfirmed date 3") { assert( - glad.lookup(20732) === Some(("2017-01-01", false)) + glad.lookup(20732) === Some((LocalDate.of(2017,1,1), false)) ) } test("Confirmed date 1") { assert( - glad.lookup(30001) === Some(("2015-01-01", true)) + glad.lookup(30001) === Some((LocalDate.of(2015,1,1), true)) ) } test("Confirmed date 2") { assert( - glad.lookup(30366) === Some(("2016-01-01", true)) + glad.lookup(30366) === Some((LocalDate.of(2016,1,1), true)) ) } test("Confirmed date 3") { assert( - glad.lookup(30732) === Some(("2017-01-01", true)) + glad.lookup(30732) === Some((LocalDate.of(2017,1,1), true)) ) } } diff --git a/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala b/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala new file mode 100644 index 00000000..676497eb --- /dev/null +++ b/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticAnalysisSpec.scala @@ -0,0 +1,82 @@ +package org.globalforestwatch.summarystats.forest_change_diagnostic + +import cats.data.{ NonEmptyList, Validated } +import com.github.mrpowers.spark.fast.tests.DataFrameComparer +import geotrellis.vector._ +import org.apache.sedona.core.spatialRDD.SpatialRDD +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.{ DataFrame, SaveMode } +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.types.IntegerType +import org.globalforestwatch.features.{ ValidatedFeatureRDD, FeatureFilter } +import org.globalforestwatch.features.GfwProFeatureId +import org.globalforestwatch.summarystats.{ JobError, ValidatedLocation, Location } +import org.globalforestwatch.TestEnvironment + +class ForestChangeDiagnosticAnalysisSpec extends TestEnvironment with DataFrameComparer { + def palm32InputTsvPath = getClass.getResource("/palm-oil-32.tsv").toString() + def palm32ExpectedOutputPath = getClass.getResource("/palm-32-fcd-output").toString() + + def FCD(features: RDD[ValidatedLocation[Geometry]]) = { + val fireAlertsRdd = { + // I guess this is how they do things in Java? + val spatialRDD = new SpatialRDD[Geometry] + spatialRDD.rawSpatialRDD = spark.sparkContext.emptyRDD[Geometry].toJavaRDD() + spatialRDD + } + + ForestChangeDiagnosticAnalysis( + features, + intermediateResultsRDD = None, + fireAlerts = fireAlertsRdd, + saveIntermidateResults = identity, + kwargs = Map.empty) + } + + /** Function to update expected results when this test becomes invalid */ + def saveExpectedFcdResult(fcd: DataFrame): Unit = { + fcd.repartition(1) + .write + .mode(SaveMode.Overwrite) + .options(ForestChangeDiagnosticExport.csvOptions) + .csv(path = palm32ExpectedOutputPath) + } + + def readExpectedFcdResult = { + spark.read + .options(ForestChangeDiagnosticExport.csvOptions) + .csv(palm32ExpectedOutputPath) + .withColumn("status_code", col("status_code").cast(IntegerType)) + } + + it("matches recorded output for palm oil mills location 31") { + val featureLoc31RDD = ValidatedFeatureRDD( + NonEmptyList.one(palm32InputTsvPath), + "gfwpro", + FeatureFilter.empty, + splitFeatures = true + ).filter { + case Validated.Valid(Location(GfwProFeatureId(_, 31, _, _), _)) => true + case _ => false + } + val fcd = FCD(featureLoc31RDD) + val fcdDF = ForestChangeDiagnosticDF.getFeatureDataFrame(fcd, spark) + // saveExpectedFcdResult(fcdDF) + val expectedDF = readExpectedFcdResult + + assertSmallDataFrameEquality(fcdDF, expectedDF) + } + + it("reports inValid geometries") { + // match it to tile of location 31 so we fetch less tiles + val x = 114 + val y = -1 + val selfIntersectingPolygon = Polygon(LineString(Point(x+0,y+0), Point(x+1,y+0), Point(x+0,y+1), Point(x+1,y+1), Point(x+0,y+0))) + val featureRDD = spark.sparkContext.parallelize(List( + Validated.valid[Location[JobError], Location[Geometry]](Location(GfwProFeatureId("1", 1, 0, 0), selfIntersectingPolygon)) + )) + val fcd = FCD(featureRDD) + val res = fcd.collect() + res.head.isInvalid shouldBe true + } +} diff --git a/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataSpec.scala b/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataSpec.scala new file mode 100644 index 00000000..b598af9f --- /dev/null +++ b/src/test/scala/org/globalforestwatch/summarystats/forest_change_diagnostic/ForestChangeDiagnosticDataSpec.scala @@ -0,0 +1,33 @@ +package org.globalforestwatch.summarystats.forest_change_diagnostic + +import org.globalforestwatch.TestEnvironment +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.apache.spark.sql.types.StringType +import org.globalforestwatch.summarystats.SummaryDF + + +case class Boop( + a: ForestChangeDiagnosticDataDouble, + b: ForestChangeDiagnosticDataBoolean, + c: ForestChangeDiagnosticDataLossYearly, + d: ForestChangeDiagnosticDataValueYearly, + e: ForestChangeDiagnosticDataDoubleCategory) + +class ForestChangeDiagnosticDataSpec extends TestEnvironment with SummaryDF { + + it ("implicit scope contains correct encoder") { + val enc = implicitly[ExpressionEncoder[ForestChangeDiagnosticData]] + + // If this test fails likely ExpressionEncoder.apply is being called to derive the encoder through reflection + // the likely cause is that encoder defined through TypedExpressionEncoder are not present in implicit scope + withClue(enc.schema.treeString) { + forAll(enc.schema.fields) { field => field.dataType shouldBe StringType } + } + } + + it("creates empty DataFrame") { + import spark.implicits._ + val data = List(ForestChangeDiagnosticData.empty) + val df = data.toDF + } +} diff --git a/src/test/scala/org/globalforestwatch/util/GeodesySuits.scala b/src/test/scala/org/globalforestwatch/util/GeodesySuite.scala similarity index 94% rename from src/test/scala/org/globalforestwatch/util/GeodesySuits.scala rename to src/test/scala/org/globalforestwatch/util/GeodesySuite.scala index af0c9360..cc2ae3b7 100644 --- a/src/test/scala/org/globalforestwatch/util/GeodesySuits.scala +++ b/src/test/scala/org/globalforestwatch/util/GeodesySuite.scala @@ -4,7 +4,7 @@ import geotrellis.raster.CellSize import org.globalforestwatch.util.Geodesy.pixelArea import org.scalatest.funsuite.AnyFunSuite -class GeodesySuits extends AnyFunSuite { +class GeodesySuite extends AnyFunSuite { val cellSize = CellSize(0.00025, 0.00025) diff --git a/src/test/scala/org/globalforestwatch/util/GeometryFixerSuite.scala b/src/test/scala/org/globalforestwatch/util/GeometryFixerSuite.scala new file mode 100644 index 00000000..81f80cfa --- /dev/null +++ b/src/test/scala/org/globalforestwatch/util/GeometryFixerSuite.scala @@ -0,0 +1,89 @@ +package org.globalforestwatch.util + +import geotrellis.vector.{LineString, Polygon} +import org.globalforestwatch.util.GeotrellisGeometryValidator.makeValidGeom +import org.scalatest.funsuite.AnyFunSuite + +class GeometryFixerSuite extends AnyFunSuite { + + test("A 'bowtie' polygon") { + val poly: Polygon = Polygon((0.0, 0.0), (0.0, 10.0), (10.0, 0.0), (10.0, 10.0), (0.0, 0.0)) + val validGeom = makeValidGeom(poly) + info(poly.toText()) + info(validGeom.toText()) + + assert(!poly.isValid) + assert(validGeom.isValid) + } + + test("Inner ring with one edge sharing part of an edge of the outer ring") { + val outerRing = LineString((0.0, 0.0), (10.0, 0.0), (10.0, 10.0), (0.0, 10.0), (0.0, 0.0)) + val innerRing = LineString((5.0, 2.0), (5.0, 7.0), (10.0, 7.0), (10.0, 2.0), (5.0, 2.0)) + val poly: Polygon = Polygon(outerRing, innerRing) + val validGeom = makeValidGeom(poly) + + assert(!poly.isValid) + assert(validGeom.isValid) + assert(validGeom.getGeometryType === poly.getGeometryType) + } + + test("Dangling edge") { + val poly: Polygon = Polygon((0.0, 0.0), (10.0, 0.0), (15.0, 5.0), (10.0, 0.0), (10.0, 10.0), (0.0, 10.0), (0.0, 0.0)) + val validGeom = makeValidGeom(poly) + + assert(!poly.isValid) + assert(validGeom.isValid) + assert(validGeom.getGeometryType === poly.getGeometryType) + } + + + test("Two adjacent inner rings") { + val outerRing = LineString((0.0, 0.0), (10.0, 0.0), (10.0, 10.0), (0.0, 10.0), (0.0, 0.0)) + val innerRing1 = LineString((1.0, 1.0), (1.0, 8.0), (3.0, 8.0), (3.0, 1.0), (1.0, 1.0)) + val innerRing2 = LineString((3.0, 1.0), (3.0, 8.0), (5.0, 8.0), (5.0, 1.0), (3.0, 1.0)) + + val poly: Polygon = Polygon(outerRing, innerRing1, innerRing2) + val validGeom = makeValidGeom(poly) + + assert(!poly.isValid) + assert(validGeom.isValid) + assert(validGeom.getGeometryType === poly.getGeometryType) + } + + test("Non overlapping Holes") { + val outerRing = LineString((10.0, 90.0), (90.0, 90.0), (90.0, 10.0), (10.0, 10.0), (10.0, 90.0)) + val innerRing1 = LineString((80.0, 70.0), (30.0, 70.0), (30.0, 20.0), (30.0, 70.0), (80.0, 70.0)) + val innerRing2 = LineString((70.0, 80.0), (70.0, 30.0), (20.0, 30.0), (70.0, 30.0), (70.0, 80.0)) + + val poly: Polygon = Polygon(outerRing, innerRing1, innerRing2) + val validGeom = makeValidGeom(poly) + + assert(!poly.isValid) + assert(validGeom.isValid) + assert(validGeom.getGeometryType === poly.getGeometryType) + } + + test("Polygon with an inner ring inside another inner ring") { + val outerRing = LineString((0.0, 0.0), (10.0, 0.0), (10.0, 10.0), (0.0, 10.0), (0.0, 0.0)) + val innerRing1 = LineString((2.0, 8.0), (5.0, 8.0), (5.0, 2.0), (2.0, 2.0), (2.0, 8.0)) + val innerRing2 = LineString((3.0, 3.0), (4.0, 3.0), (3.0, 4.0), (3.0, 3.0)) + + val poly: Polygon = Polygon(outerRing, innerRing1, innerRing2) + val validGeom = makeValidGeom(poly) + + assert(!poly.isValid) + assert(validGeom.isValid) + assert(validGeom.getGeometryType === poly.getGeometryType) + } + + test("Positive and negative overlap") { + val poly: Polygon = Polygon((10.0, 90.0), (50.0, 90.0), (50.0, 30.0), (70.0, 30.0), (70.0, 50.0), (30.0, 50.0), (30.0, 70.0), + (90.0, 70.0), (90.0, 10.0), (10.0, 10.0), (10.0, 90.0)) + val validGeom = makeValidGeom(poly) + + assert(!poly.isValid) + assert(validGeom.isValid) + assert(validGeom.getGeometryType === poly.getGeometryType) + } + +} diff --git a/src/test/scala/org/globalforestwatch/util/GridRDDSuite.scala b/src/test/scala/org/globalforestwatch/util/GridRDDSuite.scala new file mode 100644 index 00000000..89ed681b --- /dev/null +++ b/src/test/scala/org/globalforestwatch/util/GridRDDSuite.scala @@ -0,0 +1,30 @@ +package org.globalforestwatch.util + +import org.locationtech.jts.geom.Envelope +import org.apache.spark.sql.SparkSession +import org.globalforestwatch.summarystats.SummarySparkSession +import org.scalatest.funsuite.AnyFunSuite + +class GridRDDSuite extends AnyFunSuite { + + val spark: SparkSession = SummarySparkSession("Test") + + test("Create single grid cell") { + val envelope = new Envelope(1.0, 2.0, 0.0, 1.0) + val gridRDD = GridRDD(envelope, spark) + assert(gridRDD.countWithoutDuplicates() === 1) + val geom = gridRDD.rawSpatialRDD.take(1).get(0) + assert(geom.getArea === 1.0) + assert(geom.getGeometryType === "Polygon") + // assert(geom.getSRID === 4326) + assert(geom.getCentroid.getX === 1.5) + assert(geom.getCentroid.getY === 0.5) + } + + test("Create multiple grid cells") { + val envelope = new Envelope(0.0, 2.0, 0.0, 2.0) + val gridRDD = GridRDD(envelope, spark) + assert(gridRDD.countWithoutDuplicates() === 4) + + } +} diff --git a/src/test/scala/org/globalforestwatch/util/SpatialJoinRDDSuite.scala b/src/test/scala/org/globalforestwatch/util/SpatialJoinRDDSuite.scala new file mode 100644 index 00000000..4f37bb14 --- /dev/null +++ b/src/test/scala/org/globalforestwatch/util/SpatialJoinRDDSuite.scala @@ -0,0 +1,12 @@ +package org.globalforestwatch.util + +import org.scalatest.funsuite.AnyFunSuite + +class SpatialJoinRDDSuite extends AnyFunSuite { + + + // + // test("Geodesic Area Lat 0.0") { + // assert(pixelArea(0.0, cellSize) === 769.3170049535535) + // } +} diff --git a/version.sbt b/version.sbt deleted file mode 100644 index 1fdc9d83..00000000 --- a/version.sbt +++ /dev/null @@ -1 +0,0 @@ -version in ThisBuild := "1.5.5"