From e72ee2f9e758e1c24d60ea87a40e9121065c0464 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Tue, 8 Mar 2022 07:59:55 -0800 Subject: [PATCH 01/21] Use Planted Forests dataset instead of Plantations, add integrated alerts layers to raster config --- src/main/resources/raster-catalog-default.json | 10 +++++++++- src/main/resources/raster-catalog-pro.json | 2 +- .../annualupdate_minimal/AnnualUpdateMinimalDF.scala | 2 +- .../AnnualUpdateMinimalDataGroup.scala | 2 +- .../AnnualUpdateMinimalGridSources.scala | 6 +++--- .../AnnualUpdateMinimalSummary.scala | 4 ++-- .../annualupdate_minimal/AnnualUpdateMinimalTile.scala | 2 +- .../summarystats/firealerts/FireAlertsDF.scala | 2 +- .../summarystats/firealerts/FireAlertsDataGroup.scala | 2 +- .../firealerts/FireAlertsGridSources.scala | 6 +++--- .../summarystats/firealerts/FireAlertsSummary.scala | 4 ++-- .../summarystats/firealerts/FireAlertsTile.scala | 2 +- .../summarystats/gladalerts/GladAlertsDF.scala | 8 ++++---- .../summarystats/gladalerts/GladAlertsDataGroup.scala | 2 +- .../gladalerts/GladAlertsGridSources.scala | 6 +++--- .../summarystats/gladalerts/GladAlertsSummary.scala | 4 ++-- .../summarystats/gladalerts/GladAlertsTile.scala | 2 +- .../integrated_alerts/IntegratedAlertsDF.scala | 2 +- .../integrated_alerts/IntegratedAlertsDataGroup.scala | 2 +- .../integrated_alerts/IntegratedAlertsExport.scala | 3 --- .../IntegratedAlertsGridSources.scala | 6 +++--- .../integrated_alerts/IntegratedAlertsSummary.scala | 4 ++-- .../integrated_alerts/IntegratedAlertsTile.scala | 2 +- 23 files changed, 45 insertions(+), 40 deletions(-) diff --git a/src/main/resources/raster-catalog-default.json b/src/main/resources/raster-catalog-default.json index 0125098b..e990c303 100644 --- a/src/main/resources/raster-catalog-default.json +++ b/src/main/resources/raster-catalog-default.json @@ -246,7 +246,7 @@ }, { "name":"gfw_planted_forests", - "source_uri":"s3://gfw-data-lake/gfw_planted_forests/v20201209/raster/epsg-4326/{grid_size}/{row_count}/species_simp/geotiff/{tile_id}.tif" + "source_uri":"s3://gfw-data-lake/gfw_planted_forests/v20201209/raster/epsg-4326/{grid_size}/{row_count}/type/geotiff/{tile_id}.tif" }, { "name":"per_forest_concessions", @@ -268,6 +268,14 @@ "name":"umd_glad_landsat_alerts", "source_uri":"s3://gfw-data-lake/umd_glad_landsat_alerts/latest/raster/epsg-4326/{grid_size}/{row_count}/date_conf/geotiff/{tile_id}.tif" }, + { + "name":"umd_glad_sentinel2_alerts", + "source_uri":"s3://gfw-data-lake/umd_glad_sentinel2_alerts/latest/raster/epsg-4326/{grid_size}/{row_count}/date_conf/geotiff/{tile_id}.tif" + }, + { + "name":"wur_radd_alerts", + "source_uri":"s3://gfw-data-lake/wur_radd_alerts/latest/raster/epsg-4326/{grid_size}/{row_count}/date_conf/geotiff/{tile_id}.tif" + }, { "name":"gfw_plantations", "source_uri":"s3://gfw-data-lake/gfw_plantations/v2014/raster/epsg-4326/{grid_size}/{row_count}/type/geotiff/{tile_id}.tif" diff --git a/src/main/resources/raster-catalog-pro.json b/src/main/resources/raster-catalog-pro.json index 3d2079b1..6e8d0f0f 100644 --- a/src/main/resources/raster-catalog-pro.json +++ b/src/main/resources/raster-catalog-pro.json @@ -242,7 +242,7 @@ }, { "name":"gfw_planted_forests", - "source_uri":"s3://gfw-data-lake/gfw_planted_forests/v20201209/raster/epsg-4326/{grid_size}/{row_count}/species_simp/gdal-geotiff/{tile_id}.tif" + "source_uri":"s3://gfw-data-lake/gfw_planted_forests/v20201209/raster/epsg-4326/{grid_size}/{row_count}/type/gdal-geotiff/{tile_id}.tif" }, { "name":"per_forest_concessions", diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala index 6c717267..0a7860ea 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala @@ -44,7 +44,7 @@ object AnnualUpdateMinimalDF { $"data_group.globalLandCover" as "esa_land_cover_2015__class", $"data_group.primaryForest" as "is__umd_regional_primary_forest_2001", $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_sites", - $"data_group.plantations" as "gfw_planted_forests__type", + $"data_group.plantedForests" as "gfw_planted_forests__type", $"data_group.mangroves1996" as "is__gmw_global_mangrove_extent_1996", $"data_group.mangroves2016" as "is__gmw_global_mangrove_extent_2016", $"data_group.intactForestLandscapes" as "ifl_intact_forest_landscapes__year", diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala index 932c878b..96d3668e 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala @@ -11,7 +11,7 @@ case class AnnualUpdateMinimalDataGroup(lossYear: Integer, // biodiversityIntactness: Boolean, wdpa: String, aze: Boolean, - plantations: String, + plantedForests: String, // riverBasins: String, // ecozones: String, // urbanWatersheds: Boolean, diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala index 0adda57c..06a4443e 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala @@ -21,7 +21,7 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val primaryForest: PrimaryForest = PrimaryForest(gridTile, kwargs) val protectedAreas: ProtectedAreas = ProtectedAreas(gridTile, kwargs) val aze: Aze = Aze(gridTile, kwargs) - val plantations: Plantations = Plantations(gridTile, kwargs) + val plantedForests: PlantedForests = PlantedForests(gridTile, kwargs) val mangroves1996: Mangroves1996 = Mangroves1996(gridTile, kwargs) val mangroves2016: Mangroves2016 = Mangroves2016(gridTile, kwargs) val intactForestLandscapes: IntactForestLandscapes = IntactForestLandscapes(gridTile, kwargs) @@ -66,7 +66,7 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val primaryForestTile = primaryForest.fetchWindow(windowKey, windowLayout) val wdpaTile = protectedAreas.fetchWindow(windowKey, windowLayout) val azeTile = aze.fetchWindow(windowKey, windowLayout) - val plantationsTile = plantations.fetchWindow(windowKey, windowLayout) + val plantedForestsTile = plantedForests.fetchWindow(windowKey, windowLayout) val mangroves1996Tile = mangroves1996.fetchWindow(windowKey, windowLayout) val mangroves2016Tile = mangroves2016.fetchWindow(windowKey, windowLayout) val intactForestLandscapesTile = intactForestLandscapes.fetchWindow(windowKey, windowLayout) @@ -98,7 +98,7 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String primaryForestTile, wdpaTile, azeTile, - plantationsTile, + plantedForestsTile, mangroves1996Tile, mangroves2016Tile, intactForestLandscapesTile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala index 7ee59610..8120cd98 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala @@ -58,7 +58,7 @@ object AnnualUpdateMinimalSummary { // raster.tile.biodiversityIntactness.getData(col, row) val wdpa: String = raster.tile.wdpa.getData(col, row) val aze: Boolean = raster.tile.aze.getData(col, row) - val plantations: String = raster.tile.plantations.getData(col, row) + val plantedForests: String = raster.tile.plantedForests.getData(col, row) // val riverBasins: String = raster.tile.riverBasins.getData(col, row) // val ecozones: String = raster.tile.ecozones.getData(col, row) // val urbanWatersheds: Boolean = @@ -151,7 +151,7 @@ object AnnualUpdateMinimalSummary { primaryForest, wdpa, aze, - plantations, + plantedForests, mangroves1996, mangroves2016, intactForestLandscapes, diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala index df3ef50a..db142186 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala @@ -24,7 +24,7 @@ case class AnnualUpdateMinimalTile( // biodiversityIntactness: BiodiversityIntactness#OptionalDTile, wdpa: ProtectedAreas#OptionalITile, aze: Aze#OptionalITile, - plantations: Plantations#OptionalITile, + plantedForests: PlantedForests#OptionalITile, // riverBasins: RiverBasins#OptionalITile, // ecozones: Ecozones#OptionalITile, // urbanWatersheds: UrbanWatersheds#OptionalITile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala index 0c0da61c..6a9e52c6 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala @@ -43,7 +43,7 @@ object FireAlertsDF { $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_sites", $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas", $"data_group.landmark" as "is__landmark_indigenous_and_community_lands", - $"data_group.plantations" as "gfw_planted_forests__type", + $"data_group.plantedForests" as "gfw_planted_forests__type", $"data_group.mining" as "is__gfw_mining_concessions", $"data_group.logging" as "is__gfw_managed_forests", $"data_group.rspo" as "rspo_oil_palm__certification_status", diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDataGroup.scala index 2cefced5..c1c7c827 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDataGroup.scala @@ -7,7 +7,7 @@ case class FireAlertsDataGroup(threshold: Integer, aze: Boolean, keyBiodiversityAreas: Boolean, landmark: Boolean, - plantations: String, + plantedForests: String, mining: Boolean, logging: Boolean, rspo: String, diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsGridSources.scala index 53720697..4e93227b 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsGridSources.scala @@ -13,7 +13,7 @@ case class FireAlertsGridSources(gridTile: GridTile, kwargs: Map[String, Any]) e val keyBiodiversityAreas: KeyBiodiversityAreas = KeyBiodiversityAreas(gridTile, kwargs) // not found val aze: Aze = Aze(gridTile, kwargs) val landmark: Landmark = Landmark(gridTile, kwargs) // not found - val plantations: Plantations = Plantations(gridTile, kwargs) + val plantedForests: PlantedForests = PlantedForests(gridTile, kwargs) val mining: Mining = Mining(gridTile, kwargs) val oilPalm: OilPalm = OilPalm(gridTile, kwargs) val peatlands: Peatlands = Peatlands(gridTile, kwargs) @@ -37,7 +37,7 @@ case class FireAlertsGridSources(gridTile: GridTile, kwargs: Map[String, Any]) e val azeTile = aze.fetchWindow(windowKey, windowLayout) val keyBiodiversityAreasTile = keyBiodiversityAreas.fetchWindow(windowKey, windowLayout) val landmarkTile = landmark.fetchWindow(windowKey, windowLayout) - val plantationsTile = plantations.fetchWindow(windowKey, windowLayout) + val plantedForestsTile = plantedForests.fetchWindow(windowKey, windowLayout) val miningTile = mining.fetchWindow(windowKey, windowLayout) val loggingTile = logging.fetchWindow(windowKey, windowLayout) val rspoTile = rspo.fetchWindow(windowKey, windowLayout) @@ -64,7 +64,7 @@ case class FireAlertsGridSources(gridTile: GridTile, kwargs: Map[String, Any]) e azeTile, keyBiodiversityAreasTile, landmarkTile, - plantationsTile, + plantedForestsTile, miningTile, loggingTile, rspoTile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsSummary.scala index 6b01aa74..391adf67 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsSummary.scala @@ -42,7 +42,7 @@ object FireAlertsSummary { val keyBiodiversityAreas: Boolean = raster.tile.keyBiodiversityAreas.getData(col, row) val landmark: Boolean = raster.tile.landmark.getData(col, row) - val plantations: String = raster.tile.plantations.getData(col, row) + val plantedForests: String = raster.tile.plantedForests.getData(col, row) val mining: Boolean = raster.tile.mining.getData(col, row) val logging: Boolean = raster.tile.logging.getData(col, row) val rspo: String = raster.tile.rspo.getData(col, row) @@ -80,7 +80,7 @@ object FireAlertsSummary { aze, keyBiodiversityAreas, landmark, - plantations, + plantedForests, mining, logging, rspo, diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsTile.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsTile.scala index 6b0ca7e7..1efb3fa2 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsTile.scala @@ -16,7 +16,7 @@ case class FireAlertsTile( aze: Aze#OptionalITile, keyBiodiversityAreas: KeyBiodiversityAreas#OptionalITile, landmark: Landmark#OptionalITile, - plantations: Plantations#OptionalITile, + plantedForests: PlantedForests#OptionalITile, mining: Mining#OptionalITile, logging: Logging#OptionalITile, rspo: RSPO#OptionalITile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala index 7683578a..b68fec02 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala @@ -38,13 +38,13 @@ object GladAlertsDF { def defaultCols = List( - $"data_group.alertDate" as "alert__date", $"data_group.isConfirmed" as "is__confirmed_alert", + $"data_group.alertDate" as "umd_glad_landsat_alerts__date", $"data_group.primaryForest" as "is__umd_regional_primary_forest_2001", $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_sites", $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas", $"data_group.landmark" as "is__landmark_indigenous_and_community_lands", - $"data_group.plantations" as "gfw_planted_forests__type", + $"data_group.plantedForests" as "gfw_planted_forests__type", $"data_group.mining" as "is__gfw_mining_concessions", $"data_group.logging" as "is__gfw_managed_forests", $"data_group.rspo" as "rspo_oil_palm__certification_status", @@ -85,7 +85,7 @@ object GladAlertsDF { val spark = df.sparkSession import spark.implicits._ - val gladCols = List("alert__date", "is__confirmed_alert", "umd_glad_landsat_alerts__confidence") + val gladCols = List("umd_glad_landsat_alerts__date", "is__confirmed_alert", "umd_glad_landsat_alerts__confidence") val cols = if (!wdpa) @@ -93,7 +93,7 @@ object GladAlertsDF { else groupByCols ::: gladCols ::: contextualLayers - df.filter($"alert__date".isNotNull) + df.filter($"umd_glad_landsat_alerts__date".isNotNull) .groupBy(cols.head, cols.tail: _*) .agg( sum("alert__count") as "alert__count", diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDataGroup.scala index 70c33276..0f944c49 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDataGroup.scala @@ -11,7 +11,7 @@ case class GladAlertsDataGroup(alertDate: String, aze: Boolean, keyBiodiversityAreas: Boolean, landmark: Boolean, - plantations: String, + plantedForests: String, mining: Boolean, logging: Boolean, rspo: String, diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGridSources.scala index f90b7f26..e438f091 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsGridSources.scala @@ -19,7 +19,7 @@ case class GladAlertsGridSources(gridTile: GridTile, kwargs: Map[String, Any]) e val aze: Aze = Aze(gridTile, kwargs) val keyBiodiversityAreas: KeyBiodiversityAreas = KeyBiodiversityAreas(gridTile, kwargs) val landmark: Landmark = Landmark(gridTile, kwargs) - val plantations: Plantations = Plantations(gridTile, kwargs) + val plantedForests: PlantedForests = PlantedForests(gridTile, kwargs) val mining: Mining = Mining(gridTile, kwargs) val logging: Logging = Logging(gridTile, kwargs) val rspo: RSPO = RSPO(gridTile, kwargs) @@ -52,7 +52,7 @@ case class GladAlertsGridSources(gridTile: GridTile, kwargs: Map[String, Any]) e val azeTile = aze.fetchWindow(windowKey, windowLayout) val keyBiodiversityAreasTile = keyBiodiversityAreas.fetchWindow(windowKey, windowLayout) val landmarkTile = landmark.fetchWindow(windowKey, windowLayout) - val plantationsTile = plantations.fetchWindow(windowKey, windowLayout) + val plantedForestsTile = plantedForests.fetchWindow(windowKey, windowLayout) val miningTile = mining.fetchWindow(windowKey, windowLayout) val loggingTile = logging.fetchWindow(windowKey, windowLayout) val rspoTile = rspo.fetchWindow(windowKey, windowLayout) @@ -79,7 +79,7 @@ case class GladAlertsGridSources(gridTile: GridTile, kwargs: Map[String, Any]) e azeTile, keyBiodiversityAreasTile, landmarkTile, - plantationsTile, + plantedForestsTile, miningTile, loggingTile, rspoTile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsSummary.scala index fced60c6..2c0192d1 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsSummary.scala @@ -56,7 +56,7 @@ object GladAlertsSummary { val keyBiodiversityAreas: Boolean = raster.tile.keyBiodiversityAreas.getData(col, row) val landmark: Boolean = raster.tile.landmark.getData(col, row) - val plantations: String = raster.tile.plantations.getData(col, row) + val plantedForests: String = raster.tile.plantedForests.getData(col, row) val mining: Boolean = raster.tile.mining.getData(col, row) val logging: Boolean = raster.tile.logging.getData(col, row) val rspo: String = raster.tile.rspo.getData(col, row) @@ -127,7 +127,7 @@ object GladAlertsSummary { aze, keyBiodiversityAreas, landmark, - plantations, + plantedForests, mining, logging, rspo, diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsTile.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsTile.scala index aa685401..7e207665 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsTile.scala @@ -17,7 +17,7 @@ case class GladAlertsTile( aze: Aze#OptionalITile, keyBiodiversityAreas: KeyBiodiversityAreas#OptionalITile, landmark: Landmark#OptionalITile, - plantations: Plantations#OptionalITile, + plantedForests: PlantedForests#OptionalITile, mining: Mining#OptionalITile, logging: Logging#OptionalITile, rspo: RSPO#OptionalITile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala index 17fb8b7e..03ed63f7 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala @@ -57,7 +57,7 @@ object IntegratedAlertsDF { $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_sites", $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas", $"data_group.landmark" as "is__landmark_indigenous_and_community_lands", - $"data_group.plantations" as "gfw_planted_forests__type", + $"data_group.plantedForests" as "gfw_planted_forests__type", $"data_group.mining" as "is__gfw_mining", $"data_group.logging" as "is__gfw_managed_forests", $"data_group.rspo" as "rspo_oil_palm__certification_status", diff --git a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDataGroup.scala index 23d29d47..2ead99a9 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDataGroup.scala @@ -18,7 +18,7 @@ case class IntegratedAlertsDataGroup(gladLAlertDate: Option[String], aze: Boolean, keyBiodiversityAreas: Boolean, landmark: Boolean, - plantations: String, + plantedForests: String, mining: Boolean, logging: Boolean, rspo: String, diff --git a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsExport.scala b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsExport.scala index 66d4f5f0..50101215 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsExport.scala @@ -9,9 +9,6 @@ object IntegratedAlertsExport extends SummaryExport { outputUrl: String, kwargs: Map[String, Any]): Unit = { - val changeOnly: Boolean = - getAnyMapValue[Boolean](kwargs, "changeOnly") - summaryDF.cache() val spark = summaryDF.sparkSession diff --git a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsGridSources.scala index cbcb100e..5b219a6f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsGridSources.scala @@ -22,7 +22,7 @@ case class IntegratedAlertsGridSources(gridTile: GridTile, kwargs: Map[String, A val aze: Aze = Aze(gridTile, kwargs) val keyBiodiversityAreas: KeyBiodiversityAreas = KeyBiodiversityAreas(gridTile, kwargs) val landmark: Landmark = Landmark(gridTile, kwargs) - val plantations: Plantations = Plantations(gridTile, kwargs) + val plantedForests: PlantedForests = PlantedForests(gridTile, kwargs) val mining: Mining = Mining(gridTile, kwargs) val logging: Logging = Logging(gridTile, kwargs) val rspo: RSPO = RSPO(gridTile, kwargs) @@ -56,7 +56,7 @@ case class IntegratedAlertsGridSources(gridTile: GridTile, kwargs: Map[String, A val azeTile = aze.fetchWindow(windowKey, windowLayout) val keyBiodiversityAreasTile = keyBiodiversityAreas.fetchWindow(windowKey, windowLayout) val landmarkTile = landmark.fetchWindow(windowKey, windowLayout) - val plantationsTile = plantations.fetchWindow(windowKey, windowLayout) + val plantedForestsTile = plantedForests.fetchWindow(windowKey, windowLayout) val miningTile = mining.fetchWindow(windowKey, windowLayout) val loggingTile = logging.fetchWindow(windowKey, windowLayout) val rspoTile = rspo.fetchWindow(windowKey, windowLayout) @@ -85,7 +85,7 @@ case class IntegratedAlertsGridSources(gridTile: GridTile, kwargs: Map[String, A azeTile, keyBiodiversityAreasTile, landmarkTile, - plantationsTile, + plantedForestsTile, miningTile, loggingTile, rspoTile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsSummary.scala index fb1ecc93..b09a25ca 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsSummary.scala @@ -57,7 +57,7 @@ object IntegratedAlertsSummary { val keyBiodiversityAreas: Boolean = raster.tile.keyBiodiversityAreas.getData(col, row) val landmark: Boolean = raster.tile.landmark.getData(col, row) - val plantations: String = raster.tile.plantations.getData(col, row) + val plantedForests: String = raster.tile.plantedForests.getData(col, row) val mining: Boolean = raster.tile.mining.getData(col, row) val logging: Boolean = raster.tile.logging.getData(col, row) val rspo: String = raster.tile.rspo.getData(col, row) @@ -170,7 +170,7 @@ object IntegratedAlertsSummary { aze, keyBiodiversityAreas, landmark, - plantations, + plantedForests, mining, logging, rspo, diff --git a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsTile.scala b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsTile.scala index b25aea06..f589a8e8 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsTile.scala @@ -19,7 +19,7 @@ case class IntegratedAlertsTile( aze: Aze#OptionalITile, keyBiodiversityAreas: KeyBiodiversityAreas#OptionalITile, landmark: Landmark#OptionalITile, - plantations: Plantations#OptionalITile, + plantedForests: PlantedForests#OptionalITile, mining: Mining#OptionalITile, logging: Logging#OptionalITile, rspo: RSPO#OptionalITile, From 1ff2bc7fffbfeb0c7806c599d98717f731c29f78 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Wed, 16 Mar 2022 15:47:57 -0700 Subject: [PATCH 02/21] Temporarily include both old and new column names --- .../AnnualUpdateMinimalDF.scala | 124 ++++++++++++++++-- .../firealerts/FireAlertsDF.scala | 76 +++++++++-- .../gladalerts/GladAlertsDF.scala | 76 +++++++++-- .../IntegratedAlertsDF.scala | 77 +++++++++-- 4 files changed, 316 insertions(+), 37 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala index 6c717267..8dc0ee73 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala @@ -27,7 +27,24 @@ object AnnualUpdateMinimalDF { "is__gfw_wood_fiber", "is__gfw_resource_rights", "is__gfw_managed_forests", - "is__umd_tree_cover_gain" + "is__umd_tree_cover_gain", + + "umd_tree_cover_density__threshold", + "tsc_tree_cover_loss_drivers__type", + "is__birdlife_alliance_for_zero_extinction_site", + "gfw_plantation__type", + "is__gmw_mangroves_1996", + "is__gmw_mangroves_2016", + "ifl_intact_forest_landscape__year", + "is__gfw_tiger_landscape", + "is__landmark_land_right", + "is__gfw_land_right", + "is__birdlife_key_biodiversity_area", + "is__gfw_mining", + "is__peatland", + "is__gfw_resource_right", + "is__gfw_managed_forest", + "is__umd_tree_cover_gain_2000-2012" ) def unpackValues(cols: List[Column], @@ -75,13 +92,39 @@ object AnnualUpdateMinimalDF { $"data.totalNetFluxCo2" as "gfw_forest_carbon_net_flux__Mg_CO2e", $"data.totalGrossEmissionsCo2eCo2Only" as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", $"data.totalGrossEmissionsCo2eNonCo2" as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", - $"data.totalGrossEmissionsCo2e" as "gfw_forest_carbon_gross_emissions__Mg_CO2e" + $"data.totalGrossEmissionsCo2e" as "gfw_forest_carbon_gross_emissions__Mg_CO2e", + + $"data_group.threshold" as "umd_tree_cover_density__threshold", + $"data_group.drivers" as "tsc_tree_cover_loss_drivers__type", + $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_site", + $"data_group.plantations" as "gfw_plantation__type", + $"data_group.mangroves1996" as "is__gmw_mangroves_1996", + $"data_group.mangroves2016" as "is__gmw_mangroves_2016", + $"data_group.intactForestLandscapes" as "ifl_intact_forest_landscape__year", + $"data_group.tigerLandscapes" as "is__gfw_tiger_landscape", + $"data_group.landmark" as "is__landmark_land_right", + $"data_group.landRights" as "is__gfw_land_right", + $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_area", + $"data_group.mining" as "is__gfw_mining", + $"data_group.peatlands" as "is__peatland", + $"data_group.resourceRights" as "is__gfw_resource_right", + $"data_group.logging" as "is__gfw_managed_forest", + $"data_group.isGain" as "is__umd_tree_cover_gain_2000-2012", + $"data.totalGainArea" as "umd_tree_cover_gain_2000-2012__ha", + $"data.totalGrossCumulAbovegroundRemovalsCo2" as "gfw_gross_cumulative_aboveground_co2_removals__Mg", + $"data.totalGrossCumulBelowgroundRemovalsCo2" as "gfw_gross_cumulative_belowground_co2_removals__Mg", + $"data.totalGrossCumulAboveBelowgroundRemovalsCo2" as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", + $"data.totalNetFluxCo2" as "gfw_net_flux_co2e__Mg", + $"data.totalGrossEmissionsCo2eCo2Only" as "gfw_gross_emissions_co2e_co2_only__Mg", + $"data.totalGrossEmissionsCo2eNonCo2" as "gfw_gross_emissions_co2e_non_co2__Mg", + $"data.totalGrossEmissionsCo2e" as "gfw_gross_emissions_co2e_all_gases__Mg" ) val unpackCols = { if (!wdpa) { defaultUnpackCols ::: List( - $"data_group.wdpa" as "wdpa_protected_areas__iucn_cat" + $"data_group.wdpa" as "wdpa_protected_areas__iucn_cat", + $"data_group.wdpa" as "wdpa_protected_area__iucn_cat" ) } else defaultUnpackCols } @@ -97,7 +140,8 @@ object AnnualUpdateMinimalDF { val cols = if (!wdpa) groupByCols ::: contextualLayers ::: List( - "wdpa_protected_areas__iucn_cat" + "wdpa_protected_areas__iucn_cat", + "wdpa_protected_area__iucn_cat" ) else groupByCols ::: contextualLayers @@ -118,6 +162,15 @@ object AnnualUpdateMinimalDF { sum("gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", sum("gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e", + + sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", + sum("gfw_gross_cumulative_aboveground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_co2_removals__Mg", + sum("gfw_gross_cumulative_belowground_co2_removals__Mg") as "gfw_gross_cumulative_belowground_co2_removals__Mg", + sum("gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", + sum("gfw_net_flux_co2e__Mg") as "gfw_net_flux_co2e__Mg", + sum("gfw_gross_emissions_co2e_co2_only__Mg") as "gfw_gross_emissions_co2e_co2_only__Mg", + sum("gfw_gross_emissions_co2e_non_co2__Mg") as "gfw_gross_emissions_co2e_non_co2__Mg", + sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg", ) } @@ -127,7 +180,8 @@ object AnnualUpdateMinimalDF { val cols = if (!wdpa) groupByCols ::: contextualLayers ::: List( - "wdpa_protected_areas__iucn_cat" + "wdpa_protected_areas__iucn_cat", + "wdpa_protected_area__iucn_cat" ) else groupByCols ::: contextualLayers @@ -148,6 +202,15 @@ object AnnualUpdateMinimalDF { sum("gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", sum("gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e", + + sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", + sum("gfw_gross_cumulative_aboveground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_co2_removals__Mg", + sum("gfw_gross_cumulative_belowground_co2_removals__Mg") as "gfw_gross_cumulative_belowground_co2_removals__Mg", + sum("gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", + sum("gfw_net_flux_co2e__Mg") as "gfw_net_flux_co2e__Mg", + sum("gfw_gross_emissions_co2e_co2_only__Mg") as "gfw_gross_emissions_co2e_co2_only__Mg", + sum("gfw_gross_emissions_co2e_non_co2__Mg") as "gfw_gross_emissions_co2e_non_co2__Mg", + sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg", ) } @@ -157,7 +220,8 @@ object AnnualUpdateMinimalDF { val cols = if (!wdpa) groupByCols ::: List("umd_tree_cover_loss__year") ::: contextualLayers ::: List( - "wdpa_protected_areas__iucn_cat" + "wdpa_protected_areas__iucn_cat", + "wdpa_protected_area__iucn_cat" ) else groupByCols ::: List("umd_tree_cover_loss__year") ::: contextualLayers @@ -169,6 +233,9 @@ object AnnualUpdateMinimalDF { sum("gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", sum("gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e", + sum("gfw_gross_emissions_co2e_co2_only__Mg") as "gfw_gross_emissions_co2e_co2_only__Mg", + sum("gfw_gross_emissions_co2e_non_co2__Mg") as "gfw_gross_emissions_co2e_non_co2__Mg", + sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg" ) } @@ -201,14 +268,34 @@ object AnnualUpdateMinimalDF { max($"is__gfw_wood_fiber") as "is__gfw_wood_fiber", max($"is__gfw_resource_rights") as "is__gfw_resource_rights", max($"is__gfw_managed_forests") as "is__gfw_managed_forests", - max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain" + max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain", + + max(length($"tsc_tree_cover_loss_drivers__type")).cast("boolean") as "tsc_tree_cover_loss_drivers__type", + max($"is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", + max(length($"gfw_plantation__type")) + .cast("boolean") as "gfw_plantation__type", + max($"is__gmw_mangroves_1996") as "is__gmw_mangroves_1996", + max($"is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", + max(length($"ifl_intact_forest_landscape__year")) + .cast("boolean") as "ifl_intact_forest_landscape__year", + max($"is__gfw_tiger_landscape") as "is__gfw_tiger_landscape", + max($"is__landmark_land_right") as "is__landmark_land_right", + max($"is__gfw_land_right") as "is__gfw_land_right", + max($"is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", + max($"is__gfw_mining") as "is__gfw_mining", + max($"is__peatland") as "is__peatland", + max($"is__gfw_resource_right") as "is__gfw_resource_right", + max($"is__gfw_managed_forest") as "is__gfw_managed_forest", + max($"is__umd_tree_cover_gain_2000-2012") as "is__umd_tree_cover_gain_2000-2012" ) val aggCols = if (!wdpa) defaultAggCols ::: List( max(length($"wdpa_protected_areas__iucn_cat")) - .cast("boolean") as "wdpa_protected_areas__iucn_cat" + .cast("boolean") as "wdpa_protected_areas__iucn_cat", + max(length($"wdpa_protected_area__iucn_cat")) + .cast("boolean") as "wdpa_protected_area__iucn_cat", ) else defaultAggCols @@ -243,12 +330,29 @@ object AnnualUpdateMinimalDF { max($"is__gfw_wood_fiber") as "is__gfw_wood_fiber", max($"is__gfw_resource_rights") as "is__gfw_resource_rights", max($"is__gfw_managed_forests") as "is__gfw_managed_forests", - max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain" + max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain", + + max($"tsc_tree_cover_loss_drivers__type") as "tsc_tree_cover_loss_drivers__type", + max($"is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", + max($"gfw_plantation__type") as "gfw_plantation__type", + max($"is__gmw_mangroves_1996") as "is__gmw_mangroves_1996", + max($"is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", + max($"ifl_intact_forest_landscape__year") as "ifl_intact_forest_landscape__year", + max($"is__gfw_tiger_landscape") as "is__gfw_tiger_landscape", + max($"is__landmark_land_right") as "is__landmark_land_right", + max($"is__gfw_land_right") as "is__gfw_land_right", + max($"is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", + max($"is__gfw_mining") as "is__gfw_mining", + max($"is__peatland") as "is__peatland", + max($"is__gfw_resource_right") as "is__gfw_resource_right", + max($"is__gfw_managed_forest") as "is__gfw_managed_forest", + max($"is__umd_tree_cover_gain_2000-2012") as "is__umd_tree_cover_gain_2000-2012" ) val aggCols = if (!wdpa) defaultAggCols ::: List( - max($"wdpa_protected_areas__iucn_cat") as "wdpa_protected_areas__iucn_cat" + max($"wdpa_protected_areas__iucn_cat") as "wdpa_protected_areas__iucn_cat", + max($"wdpa_protected_area__iucn_cat") as "wdpa_protected_area__iucn_cat" ) else defaultAggCols diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala index 0c0da61c..9997673e 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala @@ -25,7 +25,21 @@ object FireAlertsDF { "is__gfw_oil_gas", "is__gmw_global_mangrove_extent_2016", "is__ifl_intact_forest_landscapes_2016", - "ibge_bra_biomes__name" + "ibge_bra_biomes__name", + + "umd_tree_cover_density__threshold", + "is__birdlife_alliance_for_zero_extinction_site", + "is__birdlife_key_biodiversity_area", + "is__landmark_land_right", + "gfw_plantation__type", + "is__gfw_mining", + "is__gfw_managed_forest", + "is__peatland", + "idn_forest_area__type", + "per_forest_concession__type", + "is__gmw_mangroves_2016", + "is__ifl_intact_forest_landscape_2016", + "bra_biome__name" ) def unpackValues(unpackCols: List[Column], @@ -57,11 +71,25 @@ object FireAlertsDF { $"data_group.mangroves2016" as "is__gmw_global_mangrove_extent_2016", $"data_group.intactForestLandscapes2016" as "is__ifl_intact_forest_landscapes_2016", $"data_group.braBiomes" as "ibge_bra_biomes__name", + + $"data_group.threshold" as "umd_tree_cover_density__threshold", + $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_site", + $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_area", + $"data_group.landmark" as "is__landmark_land_right", + $"data_group.plantations" as "gfw_plantation__type", + $"data_group.mining" as "is__gfw_mining", + $"data_group.logging" as "is__gfw_managed_forest", + $"data_group.peatlands" as "is__peatland", + $"data_group.indonesiaForestArea" as "idn_forest_area__type", + $"data_group.peruForestConcessions" as "per_forest_concession__type", + $"data_group.mangroves2016" as "is__gmw_mangroves_2016", + $"data_group.intactForestLandscapes2016" as "is__ifl_intact_forest_landscape_2016", + $"data_group.braBiomes" as "bra_biome__name" ) val cols = if (!wdpa) - unpackCols ::: ($"data_group.protectedAreas" as "wdpa_protected_areas__iucn_cat") :: defaultCols + unpackCols ::: ($"data_group.protectedAreas" as "wdpa_protected_areas__iucn_cat") :: ($"data_group.protectedAreas" as "wdpa_protected_area__iucn_cat") :: defaultCols else unpackCols ::: defaultCols df.select(cols: _*) @@ -79,7 +107,7 @@ object FireAlertsDF { val cols = if (!wdpa) - groupByCols ::: fireCols ::: "wdpa_protected_areas__iucn_cat" :: contextualLayers + groupByCols ::: fireCols ::: "wdpa_protected_areas__iucn_cat" :: "wdpa_protected_area__iucn_cat" :: contextualLayers else groupByCols ::: fireCols ::: contextualLayers @@ -130,7 +158,7 @@ object FireAlertsDF { val aggCols = List(col(aggCol)) val contextLayers: List[String] = - if (!wdpa) "wdpa_protected_areas__iucn_cat" :: contextualLayers + if (!wdpa) "wdpa_protected_areas__iucn_cat" :: "wdpa_protected_area__iucn_cat" :: contextualLayers else contextualLayers val selectCols: List[Column] = cols.foldRight(Nil: List[Column])( @@ -175,13 +203,31 @@ object FireAlertsDF { max("is__gfw_oil_gas") as "is__gfw_oil_gas", max("is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", max("is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", - max(length($"ibge_bra_biomes__name")).cast("boolean") as "ibge_bra_biomes__name" + max(length($"ibge_bra_biomes__name")).cast("boolean") as "ibge_bra_biomes__name", + + max("is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", + max("is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", + max("is__landmark_land_right") as "is__landmark_land_right", + max(length($"gfw_plantation__type")) + .cast("boolean") as "gfw_plantation__type", + max("is__gfw_mining") as "is__gfw_mining", + max("is__gfw_managed_forest") as "is__gfw_managed_forest", + max("is__peatland") as "is__peatland", + max(length($"idn_forest_area__type")) + .cast("boolean") as "idn_forest_area__type", + max(length($"per_forest_concession__type")) + .cast("boolean") as "per_forest_concession__type", + max("is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", + max("is__ifl_intact_forest_landscape_2016") as "is__ifl_intact_forest_landscape_2016", + max(length($"bra_biome__name")).cast("boolean") as "bra_biome__name" ) val aggCols = if (!wdpa) (max(length($"wdpa_protected_areas__iucn_cat")) - .cast("boolean") as "wdpa_protected_areas__iucn_cat") :: defaultAggCols + .cast("boolean") as "wdpa_protected_areas__iucn_cat") :: + (max(length($"wdpa_protected_area__iucn_cat")) + .cast("boolean") as "wdpa_protected_area__iucn_cat") :: defaultAggCols else defaultAggCols df.groupBy(groupByCols.head, groupByCols.tail: _*) @@ -209,12 +255,26 @@ object FireAlertsDF { max("is__gfw_oil_gas") as "is__gfw_oil_gas", max("is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", max("is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", - max("ibge_bra_biomes__name") as "ibge_bra_biomes__name" + max("ibge_bra_biomes__name") as "ibge_bra_biomes__name", + + max("is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", + max("is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", + max("is__landmark_land_right") as "is__landmark_land_right", + max("gfw_plantation__type") as "gfw_plantation__type", + max("is__gfw_mining") as "is__gfw_mining", + max("is__gfw_managed_forest") as "is__gfw_managed_forest", + max("is__peatland") as "is__peatland", + max("idn_forest_area__type") as "idn_forest_area__type", + max("per_forest_concession__type") as "per_forest_concession__type", + max("is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", + max("is__ifl_intact_forest_landscape_2016") as "is__ifl_intact_forest_landscape_2016", + max("bra_biome__name") as "bra_biome__name" ) val aggCols = if (!wdpa) - (max("wdpa_protected_areas__iucn_cat") as "wdpa_protected_areas__iucn_cat") :: defaultAggCols + (max("wdpa_protected_areas__iucn_cat") as "wdpa_protected_areas__iucn_cat") :: + (max("wdpa_protected_area__iucn_cat") as "wdpa_protected_area__iucn_cat") :: defaultAggCols else defaultAggCols df.groupBy(groupByCols.head, groupByCols.tail: _*) diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala index 7683578a..743791ae 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala @@ -24,7 +24,20 @@ object GladAlertsDF { "is__gfw_oil_gas", "is__gmw_global_mangrove_extent_2016", "is__ifl_intact_forest_landscapes_2016", - "ibge_bra_biomes__name" + "ibge_bra_biomes__name", + + "is__birdlife_alliance_for_zero_extinction_site", + "is__birdlife_key_biodiversity_area", + "is__landmark_land_right", + "gfw_plantation__type", + "is__gfw_mining", + "is__gfw_managed_forest", + "is__peatland", + "idn_forest_area__type", + "per_forest_concession__type", + "is__gmw_mangroves_2016", + "is__ifl_intact_forest_landscape_2016", + "bra_biome__name" ) def unpackValues(unpackCols: List[Column], @@ -61,12 +74,25 @@ object GladAlertsDF { $"data.totalAlerts" as "alert__count", $"data.alertArea" as "alert_area__ha", $"data.co2Emissions" as "whrc_aboveground_co2_emissions__Mg", - $"data.totalArea" as "area__ha" + $"data.totalArea" as "area__ha", + + $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_site", + $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_area", + $"data_group.landmark" as "is__landmark_land_right", + $"data_group.plantations" as "gfw_plantation__type", + $"data_group.mining" as "is__gfw_mining", + $"data_group.logging" as "is__gfw_managed_forest", + $"data_group.peatlands" as "is__peatland", + $"data_group.indonesiaForestArea" as "idn_forest_area__type", + $"data_group.peruForestConcessions" as "per_forest_concession__type", + $"data_group.mangroves2016" as "is__gmw_mangroves_2016", + $"data_group.intactForestLandscapes2016" as "is__ifl_intact_forest_landscape_2016", + $"data_group.braBiomes" as "bra_biome__name" ) val cols = if (!wdpa) - unpackCols ::: ($"data_group.protectedAreas" as "wdpa_protected_areas__iucn_cat") :: defaultCols + unpackCols ::: ($"data_group.protectedAreas" as "wdpa_protected_areas__iucn_cat") :: ($"data_group.protectedAreas" as "wdpa_protected_area__iucn_cat") :: defaultCols else unpackCols ::: defaultCols val unpackedDf = df.filter($"data_group.tile.z" === minZoom) @@ -89,7 +115,7 @@ object GladAlertsDF { val cols = if (!wdpa) - groupByCols ::: gladCols ::: "wdpa_protected_areas__iucn_cat" :: contextualLayers + groupByCols ::: gladCols ::: "wdpa_protected_areas__iucn_cat" :: "wdpa_protected_area__iucn_cat" :: contextualLayers else groupByCols ::: gladCols ::: contextualLayers @@ -107,7 +133,7 @@ object GladAlertsDF { val cols = if (!wdpa) - groupByCols ::: "wdpa_protected_areas__iucn_cat" :: contextualLayers + groupByCols ::: "wdpa_protected_areas__iucn_cat" :: "wdpa_protected_area__iucn_cat" :: contextualLayers else groupByCols ::: contextualLayers @@ -143,13 +169,31 @@ object GladAlertsDF { max("is__gfw_oil_gas") as "is__gfw_oil_gas", max("is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", max("is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", - max(length($"ibge_bra_biomes__name")).cast("boolean") as "ibge_bra_biomes__name" + max(length($"ibge_bra_biomes__name")).cast("boolean") as "ibge_bra_biomes__name", + + max("is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", + max("is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", + max("is__landmark_land_right") as "is__landmark_land_right", + max(length($"gfw_plantation__type")) + .cast("boolean") as "gfw_plantation__type", + max("is__gfw_mining") as "is__gfw_mining", + max("is__gfw_managed_forest") as "is__gfw_managed_forest", + max("is__peatland") as "is__peatland", + max(length($"idn_forest_area__type")) + .cast("boolean") as "idn_forest_area__type", + max(length($"per_forest_concession__type")) + .cast("boolean") as "per_forest_concession__type", + max("is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", + max("is__ifl_intact_forest_landscape_2016") as "is__ifl_intact_forest_landscape_2016", + max(length($"bra_biome__name")).cast("boolean") as "bra_biome__name" ) val aggCols = if (!wdpa) (max(length($"wdpa_protected_areas__iucn_cat")) - .cast("boolean") as "wdpa_protected_areas__iucn_cat") :: defaultAggCols + .cast("boolean") as "wdpa_protected_areas__iucn_cat") :: + (max(length($"wdpa_protected_area__iucn_cat")) + .cast("boolean") as "wdpa_protected_area__iucn_cat") :: defaultAggCols else defaultAggCols df.groupBy(groupByCols.head, groupByCols.tail: _*) @@ -177,12 +221,26 @@ object GladAlertsDF { max("is__gfw_oil_gas") as "is__gfw_oil_gas", max("is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", max("is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", - max("ibge_bra_biomes__name") as "ibge_bra_biomes__name" + max("ibge_bra_biomes__name") as "ibge_bra_biomes__name", + + max("is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", + max("is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", + max("is__landmark_land_right") as "is__landmark_land_right", + max("gfw_plantation__type") as "gfw_plantation__type", + max("is__gfw_mining") as "is__gfw_mining", + max("is__gfw_managed_forest") as "is__gfw_managed_forest", + max("is__peatland") as "is__peatland", + max("idn_forest_area__type") as "idn_forest_area__type", + max("per_forest_concession__type") as "per_forest_concession__type", + max("is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", + max("is__ifl_intact_forest_landscape_2016") as "is__ifl_intact_forest_landscape_2016", + max("bra_biome__name") as "bra_biome__name" ) val aggCols = if (!wdpa) - (max("wdpa_protected_areas__iucn_cat") as "wdpa_protected_areas__iucn_cat") :: defaultAggCols + (max("wdpa_protected_areas__iucn_cat") as "wdpa_protected_areas__iucn_cat") :: + (max("wdpa_protected_area__iucn_cat") as "wdpa_protected_area__iucn_cat") :: defaultAggCols else defaultAggCols df.groupBy(groupByCols.head, groupByCols.tail: _*) diff --git a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala index 17fb8b7e..fb10600f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala @@ -32,7 +32,20 @@ object IntegratedAlertsDF { "is__gfw_oil_gas", "is__gmw_global_mangrove_extent_2016", "is__ifl_intact_forest_landscapes_2016", - "ibge_bra_biomes__name" + "ibge_bra_biomes__name", + + "is__birdlife_alliance_for_zero_extinction_site", + "is__birdlife_key_biodiversity_area", + "is__landmark_land_right", + "gfw_plantation__type", + "is__gfw_mining", + "is__gfw_managed_forest", + "is__peatland", + "idn_forest_area__type", + "per_forest_concession__type", + "is__gmw_mangroves_2016", + "is__ifl_intact_forest_landscape_2016", + "bra_biome__name" ) def unpackValues(unpackCols: List[Column], @@ -74,12 +87,25 @@ object IntegratedAlertsDF { $"data.totalAlerts" as "alert__count", $"data.alertArea" as "alert_area__ha", $"data.co2Emissions" as "whrc_aboveground_co2_emissions__Mg", - $"data.totalArea" as "area__ha" + $"data.totalArea" as "area__ha", + + $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_site", + $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_area", + $"data_group.landmark" as "is__landmark_land_right", + $"data_group.plantations" as "gfw_plantation__type", + $"data_group.mining" as "is__gfw_mining", + $"data_group.logging" as "is__gfw_managed_forest", + $"data_group.peatlands" as "is__peatland", + $"data_group.indonesiaForestArea" as "idn_forest_area__type", + $"data_group.peruForestConcessions" as "per_forest_concession__type", + $"data_group.mangroves2016" as "is__gmw_mangroves_2016", + $"data_group.intactForestLandscapes2016" as "is__ifl_intact_forest_landscape_2016", + $"data_group.braBiomes" as "bra_biome__name" ) val cols = if (!wdpa) - unpackCols ::: ($"data_group.protectedAreas" as "wdpa_protected_areas__iucn_cat") :: defaultCols + unpackCols ::: ($"data_group.protectedAreas" as "wdpa_protected_areas__iucn_cat") :: ($"data_group.protectedAreas" as "wdpa_protected_area__iucn_cat") :: defaultCols else unpackCols ::: defaultCols df.select(cols: _*) @@ -93,7 +119,7 @@ object IntegratedAlertsDF { val cols = if (!wdpa) - groupByCols ::: "wdpa_protected_areas__iucn_cat" :: contextualLayers + groupByCols ::: "wdpa_protected_areas__iucn_cat" :: "wdpa_protected_area__iucn_cat" :: contextualLayers else groupByCols ::: contextualLayers @@ -111,7 +137,7 @@ object IntegratedAlertsDF { val cols = if (!wdpa) - groupByCols ::: "wdpa_protected_areas__iucn_cat" :: contextualLayers + groupByCols ::: "wdpa_protected_areas__iucn_cat" :: "wdpa_protected_area__iucn_cat" :: contextualLayers else groupByCols ::: contextualLayers @@ -147,13 +173,31 @@ object IntegratedAlertsDF { max("is__gfw_oil_gas") as "is__gfw_oil_gas", max("is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", max("is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", - max(length($"ibge_bra_biomes__name")).cast("boolean") as "ibge_bra_biomes__name" + max(length($"ibge_bra_biomes__name")).cast("boolean") as "ibge_bra_biomes__name", + + max("is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", + max("is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", + max("is__landmark_land_right") as "is__landmark_land_right", + max(length($"gfw_plantation__type")) + .cast("boolean") as "gfw_plantation__type", + max("is__gfw_mining") as "is__gfw_mining", + max("is__gfw_managed_forest") as "is__gfw_managed_forest", + max("is__peatland") as "is__peatland", + max(length($"idn_forest_area__type")) + .cast("boolean") as "idn_forest_area__type", + max(length($"per_forest_concession__type")) + .cast("boolean") as "per_forest_concession__type", + max("is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", + max("is__ifl_intact_forest_landscape_2016") as "is__ifl_intact_forest_landscape_2016", + max(length($"bra_biome__name")).cast("boolean") as "bra_biome__name" ) val aggCols = if (!wdpa) (max(length($"wdpa_protected_areas__iucn_cat")) - .cast("boolean") as "wdpa_protected_areas__iucn_cat") :: defaultAggCols + .cast("boolean") as "wdpa_protected_areas__iucn_cat") :: + (max(length($"wdpa_protected_area__iucn_cat")) + .cast("boolean") as "wdpa_protected_area__iucn_cat") :: defaultAggCols else defaultAggCols df.groupBy(groupByCols.head, groupByCols.tail: _*) @@ -181,16 +225,29 @@ object IntegratedAlertsDF { max("is__gfw_oil_gas") as "is__gfw_oil_gas", max("is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", max("is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", - max("ibge_bra_biomes__name") as "ibge_bra_biomes__name" + max("ibge_bra_biomes__name") as "ibge_bra_biomes__name", + + max("is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", + max("is__birdlife_key_biodiversity_area") as "is__birdlife_key_biodiversity_area", + max("is__landmark_land_right") as "is__landmark_land_right", + max("gfw_plantation__type") as "gfw_plantation__type", + max("is__gfw_mining") as "is__gfw_mining", + max("is__gfw_managed_forest") as "is__gfw_managed_forest", + max("is__peatland") as "is__peatland", + max("idn_forest_area__type") as "idn_forest_area__type", + max("per_forest_concession__type") as "per_forest_concession__type", + max("is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", + max("is__ifl_intact_forest_landscape_2016") as "is__ifl_intact_forest_landscape_2016", + max("bra_biome__name") as "bra_biome__name" ) val aggCols = if (!wdpa) - (max("wdpa_protected_areas__iucn_cat") as "wdpa_protected_areas__iucn_cat") :: defaultAggCols + (max("wdpa_protected_areas__iucn_cat") as "wdpa_protected_areas__iucn_cat") :: + (max("wdpa_protected_area__iucn_cat") as "wdpa_protected_area__iucn_cat") :: defaultAggCols else defaultAggCols df.groupBy(groupByCols.head, groupByCols.tail: _*) .agg(aggCols.head, aggCols.tail: _*) } - } From 9039f62ddef73333d1810b0918a3203869f108c7 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Thu, 17 Mar 2022 10:01:43 -0700 Subject: [PATCH 03/21] No longer using plantations --- .../firealerts/FireAlertsDF.scala | 2 +- .../gladalerts/GladAlertsDF.scala | 2 +- .../IntegratedAlertsDF.scala | 20 +++++++++---------- .../IntegratedAlertsExport.scala | 1 - 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala index 4dc595b2..dbcbbcd7 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsDF.scala @@ -76,7 +76,7 @@ object FireAlertsDF { $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_site", $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_area", $"data_group.landmark" as "is__landmark_land_right", - $"data_group.plantations" as "gfw_plantation__type", + $"data_group.plantedForests" as "gfw_plantation__type", $"data_group.mining" as "is__gfw_mining", $"data_group.logging" as "is__gfw_managed_forest", $"data_group.peatlands" as "is__peatland", diff --git a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala index 8f5fb269..324e6def 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/gladalerts/GladAlertsDF.scala @@ -79,7 +79,7 @@ object GladAlertsDF { $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_site", $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_area", $"data_group.landmark" as "is__landmark_land_right", - $"data_group.plantations" as "gfw_plantation__type", + $"data_group.plantedForests" as "gfw_plantation__type", $"data_group.mining" as "is__gfw_mining", $"data_group.logging" as "is__gfw_managed_forest", $"data_group.peatlands" as "is__peatland", diff --git a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala index c4ba7a58..db58facd 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsDF.scala @@ -20,14 +20,14 @@ object IntegratedAlertsDF { "is__birdlife_key_biodiversity_areas", "is__landmark_indigenous_and_community_lands", "gfw_planted_forests__type", - "is__gfw_mining", + "is__gfw_mining_concessions", "is__gfw_managed_forests", "rspo_oil_palm__certification_status", "is__gfw_wood_fiber", "is__gfw_peatlands", "is__idn_forest_moratorium", "is__gfw_oil_palm", - "idn_forest_area__type", + "idn_forest_area__class", "per_forest_concessions__type", "is__gfw_oil_gas", "is__gmw_global_mangrove_extent_2016", @@ -71,14 +71,14 @@ object IntegratedAlertsDF { $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas", $"data_group.landmark" as "is__landmark_indigenous_and_community_lands", $"data_group.plantedForests" as "gfw_planted_forests__type", - $"data_group.mining" as "is__gfw_mining", + $"data_group.mining" as "is__gfw_mining_concessions", $"data_group.logging" as "is__gfw_managed_forests", $"data_group.rspo" as "rspo_oil_palm__certification_status", $"data_group.woodFiber" as "is__gfw_wood_fiber", $"data_group.peatlands" as "is__gfw_peatlands", $"data_group.indonesiaForestMoratorium" as "is__idn_forest_moratorium", $"data_group.oilPalm" as "is__gfw_oil_palm", - $"data_group.indonesiaForestArea" as "idn_forest_area__type", + $"data_group.indonesiaForestArea" as "idn_forest_area__class", $"data_group.peruForestConcessions" as "per_forest_concessions__type", $"data_group.oilGas" as "is__gfw_oil_gas", $"data_group.mangroves2016" as "is__gmw_global_mangrove_extent_2016", @@ -92,7 +92,7 @@ object IntegratedAlertsDF { $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_site", $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_area", $"data_group.landmark" as "is__landmark_land_right", - $"data_group.plantations" as "gfw_plantation__type", + $"data_group.plantedForests" as "gfw_plantation__type", $"data_group.mining" as "is__gfw_mining", $"data_group.logging" as "is__gfw_managed_forest", $"data_group.peatlands" as "is__peatland", @@ -158,7 +158,7 @@ object IntegratedAlertsDF { max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", max(length($"gfw_planted_forests__type")) .cast("boolean") as "gfw_planted_forests__type", - max("is__gfw_mining") as "is__gfw_mining", + max("is__gfw_mining_concessions") as "is__gfw_mining_concessions", max("is__gfw_managed_forests") as "is__gfw_managed_forests", max(length($"rspo_oil_palm__certification_status")) .cast("boolean") as "rspo_oil_palm__certification_status", @@ -166,8 +166,8 @@ object IntegratedAlertsDF { max("is__gfw_peatlands") as "is__gfw_peatlands", max("is__idn_forest_moratorium") as "is__idn_forest_moratorium", max("is__gfw_oil_palm") as "is__gfw_oil_palm", - max(length($"idn_forest_area__type")) - .cast("boolean") as "idn_forest_area__type", + max(length($"idn_forest_area__class")) + .cast("boolean") as "idn_forest_area__class", max(length($"per_forest_concessions__type")) .cast("boolean") as "per_forest_concessions__type", max("is__gfw_oil_gas") as "is__gfw_oil_gas", @@ -213,14 +213,14 @@ object IntegratedAlertsDF { max("is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas", max("is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", max("gfw_planted_forests__type") as "gfw_planted_forests__type", - max("is__gfw_mining") as "is__gfw_mining", + max("is__gfw_mining_concessions") as "is__gfw_mining_concessions", max("is__gfw_managed_forests") as "is__gfw_managed_forests", max("rspo_oil_palm__certification_status") as "rspo_oil_palm__certification_status", max("is__gfw_wood_fiber") as "is__gfw_wood_fiber", max("is__gfw_peatlands") as "is__gfw_peatlands", max("is__idn_forest_moratorium") as "is__idn_forest_moratorium", max("is__gfw_oil_palm") as "is__gfw_oil_palm", - max("idn_forest_area__type") as "idn_forest_area__type", + max("idn_forest_area__class") as "idn_forest_area__class", max("per_forest_concessions__type") as "per_forest_concessions__type", max("is__gfw_oil_gas") as "is__gfw_oil_gas", max("is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", diff --git a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsExport.scala b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsExport.scala index 50101215..be62f8b6 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/integrated_alerts/IntegratedAlertsExport.scala @@ -8,7 +8,6 @@ object IntegratedAlertsExport extends SummaryExport { override protected def exportGadm(summaryDF: DataFrame, outputUrl: String, kwargs: Map[String, Any]): Unit = { - summaryDF.cache() val spark = summaryDF.sparkSession From c512c7faf2efcecbf4a9e2f2adcf3ce304602a09 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Thu, 17 Mar 2022 16:01:19 -0700 Subject: [PATCH 04/21] Add soil carbon column --- src/main/resources/raster-catalog-default.json | 4 ++++ .../org/globalforestwatch/layers/SoilCarbon.scala | 12 ++++++++++++ .../annualupdate_minimal/AnnualUpdateMinimalDF.scala | 7 +++++-- .../AnnualUpdateMinimalData.scala | 7 ++++--- .../AnnualUpdateMinimalGridSources.scala | 5 ++++- .../AnnualUpdateMinimalSummary.scala | 5 ++++- .../AnnualUpdateMinimalTile.scala | 3 ++- 7 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 src/main/scala/org/globalforestwatch/layers/SoilCarbon.scala diff --git a/src/main/resources/raster-catalog-default.json b/src/main/resources/raster-catalog-default.json index e990c303..2e21229b 100644 --- a/src/main/resources/raster-catalog-default.json +++ b/src/main/resources/raster-catalog-default.json @@ -303,6 +303,10 @@ { "name":"mapbox_river_basins", "source_uri": "s3://gfw-data-lake/mapbox_river_basins/v2018/raster/epsg-4326/{grid_size}/{row_count}/id/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_soil_carbon_stocks", + "source_uri": "s3://gfw-data-lake/gfw_soil_carbon_stocks/v20200724/raster/epsg-4326/10/40000/Mg_CO2e_ha/geotiff/{tile_id}.tif" } ] } \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/layers/SoilCarbon.scala b/src/main/scala/org/globalforestwatch/layers/SoilCarbon.scala new file mode 100644 index 00000000..d4f595d7 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/layers/SoilCarbon.scala @@ -0,0 +1,12 @@ +package org.globalforestwatch.layers + +import org.globalforestwatch.grids.GridTile + +case class SoilCarbon(gridTile: GridTile, kwargs: Map[String, Any]) + extends FloatLayer + with OptionalFLayer { + + val datasetName = "gfw_soil_carbon_stocks" + val uri: String = + uriForGrid(gridTile) +} diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala index 5d183ce6..cfe55981 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala @@ -93,11 +93,12 @@ object AnnualUpdateMinimalDF { $"data.totalGrossEmissionsCo2eCo2Only" as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", $"data.totalGrossEmissionsCo2eNonCo2" as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", $"data.totalGrossEmissionsCo2e" as "gfw_forest_carbon_gross_emissions__Mg_CO2e", + $"data.totalSoilCarbon" as "gfw_soil_carbon_stocks__Mg_CO2e", $"data_group.threshold" as "umd_tree_cover_density__threshold", $"data_group.drivers" as "tsc_tree_cover_loss_drivers__type", $"data_group.aze" as "is__birdlife_alliance_for_zero_extinction_site", - $"data_group.plantations" as "gfw_plantation__type", + $"data_group.plantedForests" as "gfw_plantation__type", $"data_group.mangroves1996" as "is__gmw_mangroves_1996", $"data_group.mangroves2016" as "is__gmw_mangroves_2016", $"data_group.intactForestLandscapes" as "ifl_intact_forest_landscape__year", @@ -117,7 +118,7 @@ object AnnualUpdateMinimalDF { $"data.totalNetFluxCo2" as "gfw_net_flux_co2e__Mg", $"data.totalGrossEmissionsCo2eCo2Only" as "gfw_gross_emissions_co2e_co2_only__Mg", $"data.totalGrossEmissionsCo2eNonCo2" as "gfw_gross_emissions_co2e_non_co2__Mg", - $"data.totalGrossEmissionsCo2e" as "gfw_gross_emissions_co2e_all_gases__Mg" + $"data.totalGrossEmissionsCo2e" as "gfw_gross_emissions_co2e_all_gases__Mg", ) val unpackCols = { @@ -162,6 +163,7 @@ object AnnualUpdateMinimalDF { sum("gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", sum("gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e", + sum("gfw_soil_carbon_stocks__Mg_CO2e") as "gfw_soil_carbon_stocks__Mg_CO2e", sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", sum("gfw_gross_cumulative_aboveground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_co2_removals__Mg", @@ -202,6 +204,7 @@ object AnnualUpdateMinimalDF { sum("gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", sum("gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e", + sum("gfw_soil_carbon_stocks__Mg_CO2e") as "gfw_soil_carbon_stocks__Mg_CO2e", sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", sum("gfw_gross_cumulative_aboveground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_co2_removals__Mg", diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalData.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalData.scala index 8c289ced..87a7ca9c 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalData.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalData.scala @@ -23,10 +23,10 @@ case class AnnualUpdateMinimalData(var treecoverLoss: Double, var totalGrossCumulBelowgroundRemovalsCo2: Double, var totalGrossCumulAboveBelowgroundRemovalsCo2: Double, var totalNetFluxCo2: Double, - var totalCo2: Double + var totalCo2: Double, + var totalSoilCarbon: Double ) { def merge(other: AnnualUpdateMinimalData): AnnualUpdateMinimalData = { - AnnualUpdateMinimalData( treecoverLoss + other.treecoverLoss, biomassLoss + other.biomassLoss, @@ -45,7 +45,8 @@ case class AnnualUpdateMinimalData(var treecoverLoss: Double, totalGrossCumulBelowgroundRemovalsCo2 + other.totalGrossCumulBelowgroundRemovalsCo2, totalGrossCumulAboveBelowgroundRemovalsCo2 + other.totalGrossCumulAboveBelowgroundRemovalsCo2, totalNetFluxCo2 + other.totalNetFluxCo2, - totalCo2 + other.totalCo2 + totalCo2 + other.totalCo2, + totalSoilCarbon + other.totalSoilCarbon ) } } diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala index 06a4443e..622af783 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala @@ -41,6 +41,7 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val netFluxCo2: NetFluxCo2e = NetFluxCo2e(gridTile, kwargs = kwargs) val grossEmissionsCo2eNonCo2: GrossEmissionsNonCo2Co2e = GrossEmissionsNonCo2Co2e(gridTile, kwargs = kwargs) val grossEmissionsCo2eCo2Only: GrossEmissionsCo2OnlyCo2e = GrossEmissionsCo2OnlyCo2e(gridTile, kwargs = kwargs) + val soilCarbon: SoilCarbon = SoilCarbon(gridTile, kwargs = kwargs) def readWindow( windowKey: SpatialKey, windowLayout: LayoutDefinition @@ -86,6 +87,7 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val netFluxCo2Tile = netFluxCo2.fetchWindow(windowKey, windowLayout) val grossEmissionsCo2eNonCo2Tile = grossEmissionsCo2eNonCo2.fetchWindow(windowKey, windowLayout) val grossEmissionsCo2eCo2OnlyTile = grossEmissionsCo2eCo2Only.fetchWindow(windowKey, windowLayout) + val soilCarbonTile = soilCarbon.fetchWindow(windowKey, windowLayout) val tile = AnnualUpdateMinimalTile( lossTile, @@ -117,7 +119,8 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String grossEmissionsCo2eCo2OnlyTile, grossCumulAbovegroundRemovalsCo2Tile, grossCumulBelowgroundRemovalsCo2Tile, - netFluxCo2Tile + netFluxCo2Tile, + soilCarbonTile, ) Raster(tile, windowKey.extent(windowLayout)) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala index 8120cd98..4cf8f820 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala @@ -124,6 +124,7 @@ object AnnualUpdateMinimalSummary { val grossCumulBelowgroundRemovalsCo2: Float = raster.tile.grossCumulBelowgroundRemovalsCo2.getData(col, row) val netFluxCo2: Float = raster.tile.netFluxCo2.getData(col, row) + val soilCarbonPerHa: Float = raster.tile.soilCarbon.getData(col, row) val netFluxCo2Pixel = netFluxCo2 * areaHa val grossCumulAbovegroundRemovalsCo2Pixel = grossCumulAbovegroundRemovalsCo2 * areaHa @@ -133,6 +134,7 @@ object AnnualUpdateMinimalSummary { val grossEmissionsCo2eCo2OnlyPixel = grossEmissionsCo2eCo2Only * areaHa val grossEmissionsCo2e = grossEmissionsCo2eNonCo2 + grossEmissionsCo2eCo2Only val grossEmissionsCo2ePixel = grossEmissionsCo2e * areaHa + val totalCarbonSoil = soilCarbonPerHa * areaHa val thresholds = List(0, 10, 15, 20, 25, 30, 50, 75) @@ -172,7 +174,7 @@ object AnnualUpdateMinimalSummary { val summary: AnnualUpdateMinimalData = stats.getOrElse( key = pKey, - default = AnnualUpdateMinimalData(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + default = AnnualUpdateMinimalData(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) ) summary.totalArea += areaHa @@ -195,6 +197,7 @@ object AnnualUpdateMinimalSummary { summary.totalGrossCumulBelowgroundRemovalsCo2 += grossCumulBelowgroundRemovalsCo2Pixel summary.totalGrossCumulAboveBelowgroundRemovalsCo2 += grossCumulAboveBelowgroundRemovalsCo2Pixel summary.totalNetFluxCo2 += netFluxCo2Pixel + summary.totalSoilCarbon += totalCarbonSoil } else if (gain) { // Adds the gain pixels that don't have any tree cover density to the flux model outputs to get // the correct flux model outputs (TCD>=threshold OR Hansen gain=TRUE) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala index db142186..d7e4c07f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala @@ -58,7 +58,8 @@ case class AnnualUpdateMinimalTile( grossEmissionsCo2eCo2Only: GrossEmissionsCo2OnlyCo2e#OptionalFTile, grossCumulAbovegroundRemovalsCo2: GrossCumulAbovegroundRemovalsCo2#OptionalFTile, grossCumulBelowgroundRemovalsCo2: GrossCumulBelowgroundRemovalsCo2#OptionalFTile, - netFluxCo2: NetFluxCo2e#OptionalFTile + netFluxCo2: NetFluxCo2e#OptionalFTile, + soilCarbon: SoilCarbon#OptionalFTile, ) extends CellGrid[Int] { def cellType: CellType = loss.cellType def cols: Int = loss.cols From ea29a21cf7b21957a0f1172a8230305ccf13d61b Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Fri, 18 Mar 2022 13:50:52 -0700 Subject: [PATCH 05/21] Fix fire alert filtering --- .../scala/org/globalforestwatch/features/FeatureRDD.scala | 6 ++++-- .../globalforestwatch/features/PolygonIntersectionDF.scala | 7 ++++--- .../summarystats/firealerts/FireAlertsCommand.scala | 5 ++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/features/FeatureRDD.scala b/src/main/scala/org/globalforestwatch/features/FeatureRDD.scala index b8ebda03..e9d50082 100644 --- a/src/main/scala/org/globalforestwatch/features/FeatureRDD.scala +++ b/src/main/scala/org/globalforestwatch/features/FeatureRDD.scala @@ -87,7 +87,8 @@ object FeatureRDD { feature2Type: String, feature2Uris: NonEmptyList[String], feature2Delimiter: String, - filters: FeatureFilter, + feature1Filters: FeatureFilter, + feature2Filters: FeatureFilter, spark: SparkSession ): RDD[geotrellis.vector.Feature[Geometry, FeatureId]] = { @@ -97,7 +98,8 @@ object FeatureRDD { feature2Uris, feature2Type, spark, - filters, + feature1Filters, + feature2Filters, feature1Delimiter, feature2Delimiter, ) diff --git a/src/main/scala/org/globalforestwatch/features/PolygonIntersectionDF.scala b/src/main/scala/org/globalforestwatch/features/PolygonIntersectionDF.scala index 8cece657..d2fd7179 100644 --- a/src/main/scala/org/globalforestwatch/features/PolygonIntersectionDF.scala +++ b/src/main/scala/org/globalforestwatch/features/PolygonIntersectionDF.scala @@ -16,15 +16,16 @@ object PolygonIntersectionDF { feature2Uris: NonEmptyList[String], feature2Type: String, spark: SparkSession, - filters: FeatureFilter, + feature1Filters: FeatureFilter, + feature2Filters: FeatureFilter, feature1Delimiter: String = "\t", feature2Delimiter: String = "\t"): DataFrame = { val feature1DF: DataFrame = - SpatialFeatureDF(feature1Uris, feature1Type, filters, "geom", spark, feature1Delimiter) + SpatialFeatureDF(feature1Uris, feature1Type, feature1Filters, "geom", spark, feature1Delimiter) val feature2DF: DataFrame = - SpatialFeatureDF(feature2Uris, feature2Type, filters, "geom", spark, feature2Delimiter) + SpatialFeatureDF(feature2Uris, feature2Type, feature2Filters, "geom", spark, feature2Delimiter) PolygonIntersectionDF(feature1DF, feature2DF, spark) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsCommand.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsCommand.scala index 3a19989b..931e6078 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsCommand.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsCommand.scala @@ -30,6 +30,7 @@ object FireAlertsCommand extends SummaryCommand { ) val featureFilter = FeatureFilter.fromOptions(default.featureType, filterOptions) + val firesFeatureFilter = FeatureFilter.fromOptions(fireAlert.alertType, filterOptions) runAnalysis { spark => val featureRDD = fireAlert.alertType match { @@ -40,6 +41,7 @@ object FireAlertsCommand extends SummaryCommand { FeatureRDD.pointInPolygonJoinAsFeature(fireAlert.alertType, fireRDD) case "burned_areas" => val burnedAreasUris = fireAlert.alertSource + FeatureRDD( fireAlert.alertType, burnedAreasUris, @@ -47,6 +49,7 @@ object FireAlertsCommand extends SummaryCommand { default.featureType, default.featureUris, "\t", + firesFeatureFilter, featureFilter, spark ) @@ -55,7 +58,7 @@ object FireAlertsCommand extends SummaryCommand { FireAlertsAnalysis( featureRDD, default.featureType, - FeatureFilter.empty, + featureFilter, spark, kwargs ) From 2a3670da014b6c9e8fc59e72472a0e04355d1708 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Fri, 18 Mar 2022 15:01:38 -0700 Subject: [PATCH 06/21] Fix VIIRS/MODIS join after JTS upgrade --- .../firealerts/FireAlertsAnalysis.scala | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala index 4d9fafea..da0e07fd 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala @@ -78,6 +78,7 @@ object FireAlertsAnalysis extends SummaryAnalysis { val featureDF = SpatialFeatureDF(featureUris, featureType, featureFilter, "geom", spark) + // Using RDD instead of DF spatial join for performance options // firePointDF // .join(featureDF) // .where("ST_Intersects(pointshape, polyshape)") @@ -101,6 +102,8 @@ object FireAlertsAnalysis extends SummaryAnalysis { fireRDD.buildIndex(IndexType.QUADTREE, buildOnSpatialPartitionedRDD) val resultPairRDD = JoinQuery.SpatialJoinQueryFlat(fireRDD, featureRDD, usingIndex, considerBoundaryIntersection) + // resultPairRDD.take(10).foreach(println) + pairRddToDf(resultPairRDD, featureType, fireAlertType, spark) } @@ -111,15 +114,15 @@ object FireAlertsAnalysis extends SummaryAnalysis { val dataType = ScalaReflection.schemaFor[FireAlertsData].dataType.asInstanceOf[StructType] val rowRdd = pairRdd.rdd.map[Row](f => { - val seq1 = f._1.toString.split("\t").toSeq - val seq2 = f._2.toString.split("\t").toSeq + val seq1 = f._1.getUserData.toString.split("\t").toSeq + val seq2 = f._2.getUserData.toString.split("\t").toSeq - val featureId = Row.fromSeq(convert(seq1(1).substring(1, seq1(1).length - 1).split(',').toList, featureIdType.toList)) - val fireId = Row.fromSeq(convert(seq2(1).substring(1, seq2(1).length - 1).split(',').toList, fireIdType.toList)) - val dataGroup = Row.fromSeq(convert(seq2(2).substring(1, seq2(2).length - 1).split(',').toList, dataGroupType.toList)) - val data = Row.fromSeq(convert(seq2(3).substring(1, seq2(3).length - 1).split(',').toList, dataType.toList)) + val featureId = Row.fromSeq(convert(seq1(0).substring(1, seq1(0).length - 1).split(',').toList, featureIdType.toList)) + val fireId = Row.fromSeq(convert(seq2(0).substring(1, seq2(0).length - 1).split(',').toList, fireIdType.toList)) + val dataGroup = Row.fromSeq(convert(seq2(1).substring(1, seq2(1).length - 1).split(',').toList, dataGroupType.toList)) + val data = Row.fromSeq(convert(seq2(2).substring(1, seq2(2).length - 1).split(',').toList, dataType.toList)) - val result = Seq(seq1(0), featureId, seq2(0), fireId, dataGroup, data) + val result = Seq(f._1.toString, featureId, f._2.toString, fireId, dataGroup, data) Row.fromSeq(result) }) From b05dea2b6a564ff685c820395a8616e4f2779e76 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Tue, 22 Mar 2022 13:51:37 -0700 Subject: [PATCH 07/21] Fix fire alerts --- src/main/resources/raster-catalog-default.json | 4 ---- .../summarystats/firealerts/FireAlertsAnalysis.scala | 5 +++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/resources/raster-catalog-default.json b/src/main/resources/raster-catalog-default.json index 2e21229b..49579700 100644 --- a/src/main/resources/raster-catalog-default.json +++ b/src/main/resources/raster-catalog-default.json @@ -172,10 +172,6 @@ "name":"gfw_belowground_carbon", "source_uri":"s3://gfw-data-lake/gfw_belowground_carbon/v20200824/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2_ha-1/geotiff/{tile_id}.tif" }, - { - "name":"umd_tree_cover_density_2000", - "source_uri":"s3://gfw-data-lake/umd_tree_cover_density_2000/v1.8/raster/epsg-4326/{grid_size}/{row_count}/threshold/geotiff/{tile_id}.tif" - }, { "name":"gfw_oil_palm", "source_uri":"s3://gfw-data-lake/gfw_oil_palm/v20191031/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" diff --git a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala index da0e07fd..de93070c 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/firealerts/FireAlertsAnalysis.scala @@ -12,8 +12,8 @@ import org.globalforestwatch.util.Util._ import cats.data.NonEmptyList import geotrellis.vector import org.apache.spark.api.java.JavaPairRDD -import org.apache.spark.sql.types.{BooleanType, DoubleType, IntegerType, StringType, StructField, StructType} -import org.globalforestwatch.summarystats.{SummaryAnalysis} +import org.apache.spark.sql.types.{BooleanType, DoubleType, FloatType, IntegerType, StringType, StructField, StructType} +import org.globalforestwatch.summarystats.SummaryAnalysis import org.apache.spark.sql.catalyst.ScalaReflection import scala.util.{Failure, Success, Try} @@ -173,6 +173,7 @@ object FireAlertsAnalysis extends SummaryAnalysis { case DoubleType => checkNullable(field, value, Try(value.toDouble)) case IntegerType => checkNullable(field, value, Try(value.toInt)) case BooleanType => checkNullable(field, value, Try(value.toBoolean)) + case FloatType => checkNullable(field, value, Try(value.toFloat)) case _ => throw new IllegalArgumentException(s"Unexpected data type ${field.dataType.toString} for field ${field.name} with value ${value}") } } From c6d825657e503be6f4be996ce45d38a74837045e Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Thu, 24 Mar 2022 14:25:57 -0700 Subject: [PATCH 08/21] Add new layers --- .../resources/raster-catalog-default.json | 52 ++++++++++++- .../layers/FaoEcozones.scala | 36 +++++++++ .../layers/ForestAgeCategory.scala | 9 +-- .../GrossCumulAbovegroundRemovalsCo2.scala | 12 +-- .../GrossCumulBelowgroundRemovalsCo2.scala | 11 +-- .../layers/GrossEmissionsCo2OnlyCo2e.scala | 8 +- .../layers/GrossEmissionsNonCo2Co2e.scala | 8 +- .../layers/IntactForestLandscapes.scala | 43 +++-------- .../layers/NetFluxCo2e.scala | 9 +-- .../AnnualUpdateMinimalDF.scala | 74 ++++++++----------- .../AnnualUpdateMinimalDataGroup.scala | 22 +----- .../AnnualUpdateMinimalDownloadDF.scala | 18 ++--- .../AnnualUpdateMinimalExport.scala | 2 +- .../AnnualUpdateMinimalGridSources.scala | 3 + .../AnnualUpdateMinimalSummary.scala | 41 +--------- .../AnnualUpdateMinimalTile.scala | 21 +----- 16 files changed, 161 insertions(+), 208 deletions(-) create mode 100644 src/main/scala/org/globalforestwatch/layers/FaoEcozones.scala diff --git a/src/main/resources/raster-catalog-default.json b/src/main/resources/raster-catalog-default.json index 49579700..c2986d6e 100644 --- a/src/main/resources/raster-catalog-default.json +++ b/src/main/resources/raster-catalog-default.json @@ -10,7 +10,7 @@ }, { "name":"wdpa_protected_areas", - "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202106/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/geotiff/{tile_id}.tif" + "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202202/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/geotiff/{tile_id}.tif" }, { "name":"gfw_oil_gas", @@ -110,7 +110,7 @@ }, { "name":"tsc_tree_cover_loss_drivers", - "source_uri":"s3://gfw-data-lake/tsc_tree_cover_loss_drivers/v2020/raster/epsg-4326/{grid_size}/{row_count}/driver/geotiff/{tile_id}.tif" + "source_uri":"s3://gfw-data-lake/tsc_tree_cover_loss_drivers/v2021/raster/epsg-4326/{grid_size}/{row_count}/driver/geotiff/{tile_id}.tif" }, { "name":"umd_tree_cover_density_2010", @@ -198,7 +198,7 @@ }, { "name":"umd_tree_cover_loss", - "source_uri":"s3://gfw-data-lake/umd_tree_cover_loss/v1.8/raster/epsg-4326/{grid_size}/{row_count}/year/geotiff/{tile_id}.tif" + "source_uri":"s3://gfw-data-lake/umd_tree_cover_loss/v1.9/raster/epsg-4326/{grid_size}/{row_count}/year/geotiff/{tile_id}.tif" }, { "name":"gfw_managed_forests", @@ -302,7 +302,51 @@ }, { "name":"gfw_soil_carbon_stocks", - "source_uri": "s3://gfw-data-lake/gfw_soil_carbon_stocks/v20200724/raster/epsg-4326/10/40000/Mg_CO2e_ha/geotiff/{tile_id}.tif" + "source_uri": "s3://gfw-data-lake/gfw_soil_carbon_stocks/v20200724/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_forest_age", + "source_uri": "s3://gfw-data-lake/gfw_forest_age/v20210621/raster/epsg-4326/{grid_size}/{row_count}/category/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_full_extent_net_flux", + "source_uri": "s3://gfw-data-lake/gfw_full_extent_net_flux/v20220316/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_full_extent_aboveground_gross_removals", + "source_uri": "s3://gfw-data-lake/gfw_full_extent_aboveground_gross_removals/v20220309/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_full_extent_belowground_gross_removals", + "source_uri": "s3://gfw-data-lake/gfw_full_extent_belowground_gross_removals/v20220309/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_full_extent_co2_gross_emissions", + "source_uri": "s3://gfw-data-lake/gfw_full_extent_co2_gross_emissions/v20220316/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"gfw_full_extent_non_co2_gross_emissions", + "source_uri": "s3://gfw-data-lake/gfw_full_extent_non_co2_gross_emissions/v20220316/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"fao_ecozones", + "source_uri": "s3://gfw-data-lake/gfw_full_extent_non_co2_gross_emissions/v20220316/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + }, + { + "name":"ifl_intact_forest_landscapes_2000", + "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2000/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" + }, + { + "name":"ifl_intact_forest_landscapes_2013", + "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2013/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" + }, + { + "name":"ifl_intact_forest_landscapes_2016", + "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2016/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" + }, + { + "name":"ifl_intact_forest_landscapes_2020", + "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2020/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" } ] } \ No newline at end of file diff --git a/src/main/scala/org/globalforestwatch/layers/FaoEcozones.scala b/src/main/scala/org/globalforestwatch/layers/FaoEcozones.scala new file mode 100644 index 00000000..8e59c2e0 --- /dev/null +++ b/src/main/scala/org/globalforestwatch/layers/FaoEcozones.scala @@ -0,0 +1,36 @@ +package org.globalforestwatch.layers + +import org.globalforestwatch.grids.GridTile + +case class FaoEcozones(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) + extends StringLayer + with OptionalILayer { + + val datasetName = "fao_ecozones" + val uri: String = uriForGrid(gridTile) + + def lookup(value: Int): String = value match { + case 1 => "Boreal coniferous forest" + case 2 => "Boreal mountain system" + case 3 => "Boreal tundra woodland" + case 4 => "No data" + case 5 => "Polar" + case 6 => "Subtropical desert" + case 7 => "Subtropical dry forest" + case 8 => "Subtropical humid forest" + case 9 => "Subtropical mountain system" + case 10 => "Subtropical steppe" + case 11 => "Temperate continental forest" + case 12 => "Temperate desert" + case 13 => "Temperate mountain system" + case 14 => "Temperate oceanic forest" + case 15 => "Temperate steppe" + case 16 => "Tropical desert" + case 17 => "Tropical dry forest" + case 18 => "Tropical moist deciduous forest" + case 19 => "Tropical mountain system" + case 20 => "Tropical rainforest" + case 21 => "Tropical shrubland" + case 22 => "Water" + } +} diff --git a/src/main/scala/org/globalforestwatch/layers/ForestAgeCategory.scala b/src/main/scala/org/globalforestwatch/layers/ForestAgeCategory.scala index 50bfebca..7369c76b 100644 --- a/src/main/scala/org/globalforestwatch/layers/ForestAgeCategory.scala +++ b/src/main/scala/org/globalforestwatch/layers/ForestAgeCategory.scala @@ -6,13 +6,8 @@ case class ForestAgeCategory(gridTile: GridTile, model: String = "standard", kwa extends StringLayer with OptionalILayer { - val datasetName = "Na" - - - val model_suffix: String = if (model == "standard") "standard" else s"$model" - val uri: String = - // s"$basePath/gfw_forest_age_category/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/category/gdal-geotiff/${gridTile.tileId}.tif" - s"s3://gfw-files/flux_1_2_1/forest_age_category/$model_suffix/${gridTile.tileId}.tif" + val datasetName = "gfw_forest_age" + val uri: String = uriForGrid(gridTile) override val externalNoDataValue = "Not applicable" diff --git a/src/main/scala/org/globalforestwatch/layers/GrossCumulAbovegroundRemovalsCo2.scala b/src/main/scala/org/globalforestwatch/layers/GrossCumulAbovegroundRemovalsCo2.scala index fe8b2d6d..11682b43 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossCumulAbovegroundRemovalsCo2.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossCumulAbovegroundRemovalsCo2.scala @@ -2,18 +2,12 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile -case class GrossCumulAbovegroundRemovalsCo2(gridTile: GridTile, - model: String = "standard", kwargs: Map[String, Any]) +case class GrossCumulAbovegroundRemovalsCo2(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_full_extent_aboveground_gross_removals" - - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - // s"$basePath/gfw_gross_cumul_aboveground_removals_co2$model_suffix/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/Mg_ha-1/geotiff/${gridTile.tileId}.tif" - s"s3://gfw-files/flux_1_2_1/gross_removals_AGCO2_all_forest_types/$model_suffix/${gridTile.tileId}.tif" - // s"s3://gfw-files/flux_1_2_0/gross_removals_AGCO2_all_forest_types/$model_suffix/${gridTile.tileId}.tif" - + uriForGrid(gridTile) } diff --git a/src/main/scala/org/globalforestwatch/layers/GrossCumulBelowgroundRemovalsCo2.scala b/src/main/scala/org/globalforestwatch/layers/GrossCumulBelowgroundRemovalsCo2.scala index f464a0a9..9f41ff97 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossCumulBelowgroundRemovalsCo2.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossCumulBelowgroundRemovalsCo2.scala @@ -2,17 +2,12 @@ package org.globalforestwatch.layers import org.globalforestwatch.grids.GridTile -case class GrossCumulBelowgroundRemovalsCo2(gridTile: GridTile, - model: String = "standard", kwargs: Map[String, Any]) +case class GrossCumulBelowgroundRemovalsCo2(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_full_extent_belowground_gross_removals" - - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - // s"$basePath/gfw_gross_cumul_belowground_removals_co2$model_suffix/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}//Mg_ha-1/geotiff/${gridTile.tileId}.tif" - s"s3://gfw-files/flux_1_2_1/gross_removals_BGCO2_all_forest_types/$model_suffix/${gridTile.tileId}.tif" - // s"s3://gfw-files/flux_1_2_0/gross_removals_BGCO2_all_forest_types/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile) } diff --git a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala index cfee7e3b..667b93ea 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala @@ -7,12 +7,8 @@ case class GrossEmissionsCo2OnlyCo2e(gridTile: GridTile, extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_full_extent_co2_gross_emissions" - - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - // s"$basePath/gfw_gross_emissions_co2e_co2_only$model_suffix/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/Mg/gdal-geotiff/${gridTile.tileId}.tif" - s"s3://gfw-files/flux_1_2_1/gross_emissions_co2_only_co2e/$model_suffix/${gridTile.tileId}.tif" - // s"s3://gfw-files/flux_1_2_0/gross_emissions_co2_only_co2e/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile) } diff --git a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2e.scala b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2e.scala index 0e9d67ec..5583fd26 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2e.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsNonCo2Co2e.scala @@ -6,12 +6,8 @@ case class GrossEmissionsNonCo2Co2e(gridTile: GridTile, model: String = "standar extends FloatLayer with OptionalFLayer { - val datasetName = "Na" + val datasetName = "gfw_full_extent_non_co2_gross_emissions" - - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - // s"$basePath/gfw_gross_emissions_co2e_non_co2$model_suffix/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/Mg/gdal-geotiff/${gridTile.tileId}.tif" - s"s3://gfw-files/flux_1_2_1/gross_emissions_non_co2_co2e/$model_suffix/${gridTile.tileId}.tif" - // s"s3://gfw-files/flux_1_2_0/gross_emissions_non_co2_co2e/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile) } diff --git a/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala b/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala index 0207d071..3cdf66d4 100644 --- a/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala +++ b/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala @@ -19,48 +19,29 @@ case class IntactForestLandscapes(gridTile: GridTile, kwargs: Map[String, Any]) } case class IntactForestLandscapes2000(gridTile: GridTile, kwargs: Map[String, Any]) - extends BooleanLayer - with OptionalILayer { - val datasetName = "ifl_intact_forest_landscapes" + extends BooleanLayer { + val datasetName = "ifl_intact_forest_landscapes_2000" val uri: String = uriForGrid(gridTile) - - override def lookup(value: Int): Boolean = { - value match { - case 0 => false - case _ => true - } - } } case class IntactForestLandscapes2013(gridTile: GridTile, kwargs: Map[String, Any]) - extends BooleanLayer - with OptionalILayer { - val datasetName = "ifl_intact_forest_landscapes" + extends BooleanLayer { + val datasetName = "ifl_intact_forest_landscapes_2013" val uri: String = uriForGrid(gridTile) - - override def lookup(value: Int): Boolean = { - value match { - case 2016 => true - case 2013 => true - case _ => false - } - } } case class IntactForestLandscapes2016(gridTile: GridTile, kwargs: Map[String, Any]) - extends BooleanLayer - with OptionalILayer { - val datasetName = "ifl_intact_forest_landscapes" + extends BooleanLayer { + val datasetName = "ifl_intact_forest_landscapes_2016" val uri: String = uriForGrid(gridTile) +} - override def lookup(value: Int): Boolean = { - value match { - case 2016 => true - case _ => false - } - } - +case class IntactForestLandscapes2020(gridTile: GridTile, kwargs: Map[String, Any]) + extends BooleanLayer { + val datasetName = "ifl_intact_forest_landscapes_2020" + val uri: String = + uriForGrid(gridTile) } diff --git a/src/main/scala/org/globalforestwatch/layers/NetFluxCo2e.scala b/src/main/scala/org/globalforestwatch/layers/NetFluxCo2e.scala index 33e187bb..c46d7f14 100644 --- a/src/main/scala/org/globalforestwatch/layers/NetFluxCo2e.scala +++ b/src/main/scala/org/globalforestwatch/layers/NetFluxCo2e.scala @@ -5,13 +5,8 @@ import org.globalforestwatch.grids.GridTile case class NetFluxCo2e(gridTile: GridTile, model: String = "standard", kwargs: Map[String, Any]) extends FloatLayer with OptionalFLayer { + val datasetName = "gfw_full_extent_net_flux" - val datasetName = "Na" - - - val model_suffix: String = if (model == "standard") "standard" else s"$model" val uri: String = - // s"$basePath/gfw_net_flux_co2e$model_suffix/v20191106/raster/epsg-4326/${gridTile.gridSize}/${gridTile.rowCount}/Mg/gdal-geotiff/${gridTile.tileId}.tif" - s"s3://gfw-files/flux_1_2_1/net_flux_all_forest_types_all_drivers/$model_suffix/${gridTile.tileId}.tif" - // s"s3://gfw-files/flux_1_2_0/net_flux_all_forest_types_all_drivers/$model_suffix/${gridTile.tileId}.tif" + uriForGrid(gridTile) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala index cfe55981..fa39daf5 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala @@ -28,6 +28,7 @@ object AnnualUpdateMinimalDF { "is__gfw_resource_rights", "is__gfw_managed_forests", "is__umd_tree_cover_gain", + "forest_age__category", "umd_tree_cover_density__threshold", "tsc_tree_cover_loss_drivers__type", @@ -77,6 +78,7 @@ object AnnualUpdateMinimalDF { $"data_group.resourceRights" as "is__gfw_resource_rights", $"data_group.logging" as "is__gfw_managed_forests", $"data_group.isGain" as "is__umd_tree_cover_gain", + $"data_group.forestage" as "forest_age__category", $"data.treecoverExtent2000" as "umd_tree_cover_extent_2000__ha", $"data.treecoverExtent2010" as "umd_tree_cover_extent_2010__ha", $"data.totalArea" as "area__ha", @@ -86,14 +88,14 @@ object AnnualUpdateMinimalDF { $"data.biomassLoss" as "whrc_aboveground_biomass_loss__Mg", $"data.co2Emissions" as "whrc_aboveground_co2_emissions__Mg", $"data.totalCo2" as "whrc_aboveground_co2_stock_2000__Mg", - $"data.totalGrossCumulAbovegroundRemovalsCo2" as "gfw_forest_carbon_gross_removals_aboveground__Mg_CO2e", - $"data.totalGrossCumulBelowgroundRemovalsCo2" as "gfw_forest_carbon_gross_removals_belowground__Mg_CO2e", - $"data.totalGrossCumulAboveBelowgroundRemovalsCo2" as "gfw_forest_carbon_gross_removals__Mg_CO2e", - $"data.totalNetFluxCo2" as "gfw_forest_carbon_net_flux__Mg_CO2e", - $"data.totalGrossEmissionsCo2eCo2Only" as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", - $"data.totalGrossEmissionsCo2eNonCo2" as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", - $"data.totalGrossEmissionsCo2e" as "gfw_forest_carbon_gross_emissions__Mg_CO2e", - $"data.totalSoilCarbon" as "gfw_soil_carbon_stocks__Mg_CO2e", + $"data.totalGrossCumulAbovegroundRemovalsCo2" as "gfw_full_extent_aboveground_gross_removals__Mg_CO2", + $"data.totalGrossCumulBelowgroundRemovalsCo2" as "gfw_full_extent_belowground_gross_removals__Mg_CO2", + $"data.totalNetFluxCo2" as "gfw_full_extent_net_flux__Mg_CO2e", + $"data.totalGrossEmissionsCo2eCo2Only" as "gfw_full_extent_gross_emissions_CO2_only__Mg_CO2", + $"data.totalGrossEmissionsCo2eNonCo2" as "gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e", + $"data.totalGrossEmissionsCo2e" as "gfw_full_extent_gross_emissions__Mg_CO2e", + $"data.totalSoilCarbon" as "gfw_soil_carbon_stocks__Mg_CO2", + $"data.totalGrossCumulAboveBelowgroundRemovalsCo2" as "gfw_full_extent_gross_removals__Mg_CO2", $"data_group.threshold" as "umd_tree_cover_density__threshold", $"data_group.drivers" as "tsc_tree_cover_loss_drivers__type", @@ -112,13 +114,9 @@ object AnnualUpdateMinimalDF { $"data_group.logging" as "is__gfw_managed_forest", $"data_group.isGain" as "is__umd_tree_cover_gain_2000-2012", $"data.totalGainArea" as "umd_tree_cover_gain_2000-2012__ha", - $"data.totalGrossCumulAbovegroundRemovalsCo2" as "gfw_gross_cumulative_aboveground_co2_removals__Mg", - $"data.totalGrossCumulBelowgroundRemovalsCo2" as "gfw_gross_cumulative_belowground_co2_removals__Mg", - $"data.totalGrossCumulAboveBelowgroundRemovalsCo2" as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", $"data.totalNetFluxCo2" as "gfw_net_flux_co2e__Mg", - $"data.totalGrossEmissionsCo2eCo2Only" as "gfw_gross_emissions_co2e_co2_only__Mg", - $"data.totalGrossEmissionsCo2eNonCo2" as "gfw_gross_emissions_co2e_non_co2__Mg", $"data.totalGrossEmissionsCo2e" as "gfw_gross_emissions_co2e_all_gases__Mg", + $"data.totalGrossCumulAboveBelowgroundRemovalsCo2" as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", ) val unpackCols = { @@ -156,22 +154,18 @@ object AnnualUpdateMinimalDF { sum("whrc_aboveground_co2_stock_2000__Mg") as "whrc_aboveground_co2_stock_2000__Mg", sum("umd_tree_cover_loss__ha") as "umd_tree_cover_loss__ha", sum("whrc_aboveground_biomass_loss__Mg") as "whrc_aboveground_biomass_loss__Mg", - sum("gfw_forest_carbon_gross_removals_aboveground__Mg_CO2e") as "gfw_forest_carbon_gross_removals_aboveground__Mg_CO2e", - sum("gfw_forest_carbon_gross_removals_belowground__Mg_CO2e") as "gfw_forest_carbon_gross_removals_belowground__Mg_CO2e", - sum("gfw_forest_carbon_gross_removals__Mg_CO2e") as "gfw_forest_carbon_gross_removals__Mg_CO2e", - sum("gfw_forest_carbon_net_flux__Mg_CO2e") as "gfw_forest_carbon_net_flux__Mg_CO2e", - sum("gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", - sum("gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", - sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e", - sum("gfw_soil_carbon_stocks__Mg_CO2e") as "gfw_soil_carbon_stocks__Mg_CO2e", + sum("gfw_full_extent_aboveground_gross_removals__Mg_CO2") as "gfw_full_extent_aboveground_gross_removals__Mg_CO2", + sum("gfw_full_extent_belowground_gross_removals__Mg_CO2") as "gfw_full_extent_belowground_gross_removals__Mg_CO2", + sum("gfw_full_extent_gross_removals__Mg_CO2") as "gfw_full_extent_gross_removals__Mg_CO2", + sum("gfw_full_extent_net_flux__Mg_CO2e") as "gfw_full_extent_net_flux__Mg_CO2e", + sum("gfw_full_extent_gross_emissions_CO2_only__Mg_CO2") as "gfw_full_extent_gross_emissions_CO2_only__Mg_CO2", + sum("gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e") as "gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e", + sum("gfw_full_extent_gross_emissions__Mg_CO2e") as "gfw_full_extent_gross_emissions__Mg_CO2e", + sum("gfw_soil_carbon_stocks__Mg_CO2") as "gfw_soil_carbon_stocks__Mg_CO2", sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", - sum("gfw_gross_cumulative_aboveground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_co2_removals__Mg", - sum("gfw_gross_cumulative_belowground_co2_removals__Mg") as "gfw_gross_cumulative_belowground_co2_removals__Mg", sum("gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", sum("gfw_net_flux_co2e__Mg") as "gfw_net_flux_co2e__Mg", - sum("gfw_gross_emissions_co2e_co2_only__Mg") as "gfw_gross_emissions_co2e_co2_only__Mg", - sum("gfw_gross_emissions_co2e_non_co2__Mg") as "gfw_gross_emissions_co2e_non_co2__Mg", sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg", ) } @@ -197,22 +191,18 @@ object AnnualUpdateMinimalDF { sum("whrc_aboveground_co2_stock_2000__Mg") as "whrc_aboveground_co2_stock_2000__Mg", sum("umd_tree_cover_loss__ha") as "umd_tree_cover_loss__ha", sum("whrc_aboveground_biomass_loss__Mg") as "whrc_aboveground_biomass_loss__Mg", - sum("gfw_forest_carbon_gross_removals_aboveground__Mg_CO2e") as "gfw_forest_carbon_gross_removals_aboveground__Mg_CO2e", - sum("gfw_forest_carbon_gross_removals_belowground__Mg_CO2e") as "gfw_forest_carbon_gross_removals_belowground__Mg_CO2e", - sum("gfw_forest_carbon_gross_removals__Mg_CO2e") as "gfw_forest_carbon_gross_removals__Mg_CO2e", - sum("gfw_forest_carbon_net_flux__Mg_CO2e") as "gfw_forest_carbon_net_flux__Mg_CO2e", - sum("gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", - sum("gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", - sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e", - sum("gfw_soil_carbon_stocks__Mg_CO2e") as "gfw_soil_carbon_stocks__Mg_CO2e", + sum("gfw_full_extent_aboveground_gross_removals__Mg_CO2") as "gfw_full_extent_aboveground_gross_removals__Mg_CO2", + sum("gfw_full_extent_belowground_gross_removals__Mg_CO2") as "gfw_full_extent_belowground_gross_removals__Mg_CO2", + sum("gfw_full_extent_gross_removals__Mg_CO2") as "gfw_full_extent_gross_removals__Mg_CO2", + sum("gfw_full_extent_net_flux__Mg_CO2e") as "gfw_full_extent_net_flux__Mg_CO2e", + sum("gfw_full_extent_gross_emissions_CO2_only__Mg_CO2") as "gfw_full_extent_gross_emissions_CO2_only__Mg_CO2", + sum("gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e") as "gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e", + sum("gfw_full_extent_gross_emissions__Mg_CO2e") as "gfw_full_extent_gross_emissions__Mg_CO2e", + sum("gfw_soil_carbon_stocks__Mg_CO2") as "gfw_soil_carbon_stocks__Mg_CO2", sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", - sum("gfw_gross_cumulative_aboveground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_co2_removals__Mg", - sum("gfw_gross_cumulative_belowground_co2_removals__Mg") as "gfw_gross_cumulative_belowground_co2_removals__Mg", sum("gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", sum("gfw_net_flux_co2e__Mg") as "gfw_net_flux_co2e__Mg", - sum("gfw_gross_emissions_co2e_co2_only__Mg") as "gfw_gross_emissions_co2e_co2_only__Mg", - sum("gfw_gross_emissions_co2e_non_co2__Mg") as "gfw_gross_emissions_co2e_non_co2__Mg", sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg", ) } @@ -233,11 +223,9 @@ object AnnualUpdateMinimalDF { sum("umd_tree_cover_loss__ha") as "umd_tree_cover_loss__ha", sum("whrc_aboveground_biomass_loss__Mg") as "whrc_aboveground_biomass_loss__Mg", sum("whrc_aboveground_co2_emissions__Mg") as "whrc_aboveground_co2_emissions__Mg", - sum("gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_co2_only__Mg_CO2e", - sum("gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e") as "gfw_forest_carbon_gross_emissions_non_co2__Mg_CO2e", - sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e", - sum("gfw_gross_emissions_co2e_co2_only__Mg") as "gfw_gross_emissions_co2e_co2_only__Mg", - sum("gfw_gross_emissions_co2e_non_co2__Mg") as "gfw_gross_emissions_co2e_non_co2__Mg", + sum("gfw_full_extent_gross_emissions_CO2_only__Mg_CO2") as "gfw_full_extent_gross_emissions_CO2_only__Mg_CO2", + sum("gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e") as "gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e", + sum("gfw_full_extent_gross_emissions__Mg_CO2e") as "gfw_full_extent_gross_emissions__Mg_CO2e", sum("gfw_gross_emissions_co2e_all_gases__Mg") as "gfw_gross_emissions_co2e_all_gases__Mg" ) } @@ -272,6 +260,7 @@ object AnnualUpdateMinimalDF { max($"is__gfw_resource_rights") as "is__gfw_resource_rights", max($"is__gfw_managed_forests") as "is__gfw_managed_forests", max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain", + max($"forest_age__category") as "forest_age__category", max(length($"tsc_tree_cover_loss_drivers__type")).cast("boolean") as "tsc_tree_cover_loss_drivers__type", max($"is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", @@ -334,6 +323,7 @@ object AnnualUpdateMinimalDF { max($"is__gfw_resource_rights") as "is__gfw_resource_rights", max($"is__gfw_managed_forests") as "is__gfw_managed_forests", max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain", + max($"forest_age__category") as "forest_age__category", max($"tsc_tree_cover_loss_drivers__type") as "tsc_tree_cover_loss_drivers__type", max($"is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala index 96d3668e..cbd7d42a 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala @@ -5,40 +5,22 @@ case class AnnualUpdateMinimalDataGroup(lossYear: Integer, drivers: String, globalLandCover: String, primaryForest: Boolean, - // idnPrimaryForest: Boolean, - // erosion: String, - // biodiversitySignificance: Boolean, - // biodiversityIntactness: Boolean, wdpa: String, aze: Boolean, plantedForests: String, - // riverBasins: String, - // ecozones: String, - // urbanWatersheds: Boolean, mangroves1996: Boolean, mangroves2016: Boolean, - // waterStress: String, intactForestLandscapes: String, - // endemicBirdAreas: Boolean, tigerLandscapes: Boolean, landmark: Boolean, landRights: Boolean, keyBiodiversityAreas: Boolean, mining: Boolean, - // rspo: String, peatlands: Boolean, oilPalm: Boolean, idnForestMoratorium: Boolean, - // idnLandCover: String, - // mexProtectedAreas: Boolean, - // mexPaymentForEcosystemServices: Boolean, - // mexForestZoning: String, - // perProductionForest: Boolean, - // perProtectedAreas: Boolean, - // perForestConcessions: String, - // braBiomes: String, woodFiber: Boolean, resourceRights: Boolean, logging: Boolean, - isGain: Boolean) -// oilGas: Boolean) + isGain: Boolean, + forestAge: String) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDownloadDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDownloadDF.scala index 336fbd75..f8056957 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDownloadDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDownloadDF.scala @@ -5,7 +5,7 @@ import org.apache.spark.sql.{Column, DataFrame, SparkSession} object AnnualUpdateMinimalDownloadDF { val treecoverLossMinYear = 2001 - val treecoverLossMaxYear = 2020 + val treecoverLossMaxYear = 2021 val fluxModelTotalYears = (treecoverLossMaxYear - treecoverLossMinYear) + 1 def sumDownload(df: DataFrame): DataFrame = { @@ -20,7 +20,7 @@ object AnnualUpdateMinimalDownloadDF { .pivot("umd_tree_cover_loss__year", yearRange) .agg( sum("umd_tree_cover_loss__ha") as "umd_tree_cover_loss__ha", - sum("gfw_forest_carbon_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e" + sum("gfw_full_extent_gross_emissions__Mg_CO2e") as "gfw_forest_carbon_gross_emissions__Mg_CO2e" ) .as("annual") .na.fill(0, Seq("adm1", "adm2")) @@ -36,9 +36,9 @@ object AnnualUpdateMinimalDownloadDF { sum("whrc_aboveground_biomass_stock_2000__Mg") / sum( "umd_tree_cover_extent_2000__ha" ) as "avg_whrc_aboveground_biomass_2000_Mg_ha-1", - sum($"gfw_forest_carbon_gross_emissions__Mg_CO2e") / fluxModelTotalYears as "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1", - sum($"gfw_forest_carbon_gross_removals__Mg_CO2e") / fluxModelTotalYears as "gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1", - sum($"gfw_forest_carbon_net_flux__Mg_CO2e") / fluxModelTotalYears as "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" + sum($"gfw_full_extent_gross_emissions__Mg_CO2e") / fluxModelTotalYears as "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1", + sum($"gfw_full_extent_gross_removals__Mg_CO2") / fluxModelTotalYears as "gfw_forest_carbon_gross_removals__Mg_CO2_yr-1", + sum($"gfw_full_extent_net_flux__Mg_CO2e") / fluxModelTotalYears as "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" ) .as("total") .na.fill(0, Seq("adm1", "adm2")) @@ -117,7 +117,7 @@ object AnnualUpdateMinimalDownloadDF { $"umd_tree_cover_extent_2000__ha" ) as "avg_whrc_aboveground_biomass_2000_Mg_ha-1", sum($"gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1") as "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1", - sum($"gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1") as "gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1", + sum($"gfw_forest_carbon_gross_removals__Mg_CO2_yr-1") as "gfw_forest_carbon_gross_removals__Mg_CO2_yr-1", sum($"gfw_forest_carbon_net_flux__Mg_CO2e_yr-1") as "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" ) ::: treecoverLossCols ::: totalGrossEmissionsCo2eAllGasesCols @@ -193,7 +193,7 @@ object AnnualUpdateMinimalDownloadDF { round($"whrc_aboveground_biomass_stock_2000__Mg") as "whrc_aboveground_biomass_stock_2000__Mg", round($"avg_whrc_aboveground_biomass_2000_Mg_ha-1") as "avg_whrc_aboveground_biomass_2000_Mg_ha-1", round($"gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1") as "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1", - round($"gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1") as "gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1", + round($"gfw_forest_carbon_gross_removals__Mg_CO2_yr-1") as "gfw_forest_carbon_gross_removals__Mg_CO2_yr-1", round($"gfw_forest_carbon_net_flux__Mg_CO2e_yr-1") as "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" ) @@ -216,7 +216,7 @@ object AnnualUpdateMinimalDownloadDF { s"${i}_gfw_forest_carbon_gross_emissions__Mg_CO2e" }).toList - val cols = "avg_whrc_aboveground_biomass_2000_Mg_ha-1" :: "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1" :: "gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1" :: "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" :: treecoverLossCols ::: totalGrossEmissionsCo2eAllGasesCols + val cols = "avg_whrc_aboveground_biomass_2000_Mg_ha-1" :: "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1" :: "gfw_forest_carbon_gross_removals__Mg_CO2_yr-1" :: "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" :: treecoverLossCols ::: totalGrossEmissionsCo2eAllGasesCols val nullColumns = df .select(cols.head, cols.tail: _*) .columns @@ -236,7 +236,7 @@ object AnnualUpdateMinimalDownloadDF { s"${i}_gfw_forest_carbon_gross_emissions__Mg_CO2e" }).toList - val cols = "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1" :: "gfw_forest_carbon_gross_removals__Mg_CO2e_yr-1" :: "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" :: totalGrossEmissionsCo2eAllGasesCols + val cols = "gfw_forest_carbon_gross_emissions__Mg_CO2e_yr-1" :: "gfw_forest_carbon_gross_removals__Mg_CO2_yr-1" :: "gfw_forest_carbon_net_flux__Mg_CO2e_yr-1" :: totalGrossEmissionsCo2eAllGasesCols val carbonColumns = df .select(cols.head, cols.tail: _*) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala index 524bf55b..f6fcbc47 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala @@ -98,7 +98,7 @@ object AnnualUpdateMinimalExport extends SummaryExport { val adm2ApiDF = df .filter($"umd_tree_cover_loss__year".isNotNull && - ($"umd_tree_cover_loss__ha" > 0 || $"gfw_forest_carbon_gross_emissions__Mg_CO2e" > 0)) + ($"umd_tree_cover_loss__ha" > 0 || $"gfw_full_extent_gross_emissions__Mg_CO2e" > 0)) .transform(AnnualUpdateMinimalDF.aggChange(List("iso", "adm1", "adm2"))) .coalesce(133) // this should result in an avg file size of 100MB diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala index 622af783..396f6a14 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala @@ -42,6 +42,7 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val grossEmissionsCo2eNonCo2: GrossEmissionsNonCo2Co2e = GrossEmissionsNonCo2Co2e(gridTile, kwargs = kwargs) val grossEmissionsCo2eCo2Only: GrossEmissionsCo2OnlyCo2e = GrossEmissionsCo2OnlyCo2e(gridTile, kwargs = kwargs) val soilCarbon: SoilCarbon = SoilCarbon(gridTile, kwargs = kwargs) + val forestAge: ForestAgeCategory = ForestAgeCategory(gridTile, kwargs = kwargs) def readWindow( windowKey: SpatialKey, windowLayout: LayoutDefinition @@ -88,6 +89,7 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val grossEmissionsCo2eNonCo2Tile = grossEmissionsCo2eNonCo2.fetchWindow(windowKey, windowLayout) val grossEmissionsCo2eCo2OnlyTile = grossEmissionsCo2eCo2Only.fetchWindow(windowKey, windowLayout) val soilCarbonTile = soilCarbon.fetchWindow(windowKey, windowLayout) + val forestAgeTile = forestAge.fetchWindow(windowKey, windowLayout) val tile = AnnualUpdateMinimalTile( lossTile, @@ -121,6 +123,7 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String grossCumulBelowgroundRemovalsCo2Tile, netFluxCo2Tile, soilCarbonTile, + forestAgeTile, ) Raster(tile, windowKey.extent(windowLayout)) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala index 4cf8f820..e6846ecd 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala @@ -43,33 +43,17 @@ object AnnualUpdateMinimalSummary { val tcd2000: Integer = raster.tile.tcd2000.getData(col, row) val tcd2010: Integer = raster.tile.tcd2010.getData(col, row) val biomass: Double = raster.tile.biomass.getData(col, row) - // val mangroveBiomass: Double = - // raster.tile.mangroveBiomass.getData(col, row) val drivers: String = raster.tile.drivers.getData(col, row) val globalLandCover: String = raster.tile.globalLandCover.getData(col, row) val primaryForest: Boolean = raster.tile.primaryForest.getData(col, row) - // val idnPrimaryForest: Boolean = - // raster.tile.idnPrimaryForest.getData(col, row) - // val erosion: String = raster.tile.erosion.getData(col, row) - // val biodiversitySignificance: Boolean = - // raster.tile.biodiversitySignificance.getData(col, row) - // val biodiversityIntactness: Boolean = - // raster.tile.biodiversityIntactness.getData(col, row) val wdpa: String = raster.tile.wdpa.getData(col, row) val aze: Boolean = raster.tile.aze.getData(col, row) val plantedForests: String = raster.tile.plantedForests.getData(col, row) - // val riverBasins: String = raster.tile.riverBasins.getData(col, row) - // val ecozones: String = raster.tile.ecozones.getData(col, row) - // val urbanWatersheds: Boolean = - // raster.tile.urbanWatersheds.getData(col, row) val mangroves1996: Boolean = raster.tile.mangroves1996.getData(col, row) val mangroves2016: Boolean = raster.tile.mangroves2016.getData(col, row) - // val waterStress: String = raster.tile.waterStress.getData(col, row) val intactForestLandscapes: String = raster.tile.intactForestLandscapes.getData(col, row) - // val endemicBirdAreas: Boolean = - // raster.tile.endemicBirdAreas.getData(col, row) val tigerLandscapes: Boolean = raster.tile.tigerLandscapes.getData(col, row) val landmark: Boolean = raster.tile.landmark.getData(col, row) @@ -77,35 +61,15 @@ object AnnualUpdateMinimalSummary { val keyBiodiversityAreas: Boolean = raster.tile.keyBiodiversityAreas.getData(col, row) val mining: Boolean = raster.tile.mining.getData(col, row) - // val rspo: String = raster.tile.rspo.getData(col, row) val peatlands: Boolean = raster.tile.peatlands.getData(col, row) val oilPalm: Boolean = raster.tile.oilPalm.getData(col, row) val idnForestMoratorium: Boolean = raster.tile.idnForestMoratorium.getData(col, row) - // val idnLandCover: String = raster.tile.idnLandCover.getData(col, row) - // val mexProtectedAreas: Boolean = - // raster.tile.mexProtectedAreas.getData(col, row) - // val mexPaymentForEcosystemServices: Boolean = - // raster.tile.mexPaymentForEcosystemServices.getData(col, row) - // val mexForestZoning: String = - // raster.tile.mexForestZoning.getData(col, row) - // val perProductionForest: Boolean = - // raster.tile.perProductionForest.getData(col, row) - // val perProtectedAreas: Boolean = - // raster.tile.perProtectedAreas.getData(col, row) - // val perForestConcessions: String = - // raster.tile.perForestConcessions.getData(col, row) - // val braBiomes: String = raster.tile.braBiomes.getData(col, row) val woodFiber: Boolean = raster.tile.woodFiber.getData(col, row) val resourceRights: Boolean = raster.tile.resourceRights.getData(col, row) val logging: Boolean = raster.tile.logging.getData(col, row) - // val oilGas: Boolean = raster.tile.oilGas.getData(col, row) - - // val cols: Int = raster.rasterExtent.cols - // val rows: Int = raster.rasterExtent.rows - // val ext = raster.rasterExtent.extent - // val cellSize = raster.cellSize + val forestAge: String = raster.tile.forestAge.getData(col, row) val lat: Double = raster.rasterExtent.gridRowToMap(row) val area: Double = Geodesy.pixelArea(lat, raster.cellSize) // uses Pixel's center coordiate. +- raster.cellSize.height/2 doesn't make much of a difference @@ -168,7 +132,8 @@ object AnnualUpdateMinimalSummary { woodFiber, resourceRights, logging, - gain + gain, + forestAge ) val summary: AnnualUpdateMinimalData = diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala index d7e4c07f..c1f1c08b 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala @@ -14,52 +14,33 @@ case class AnnualUpdateMinimalTile( tcd2000: TreeCoverDensityThreshold#ITile, tcd2010: TreeCoverDensityThreshold#ITile, biomass: BiomassPerHectar#OptionalDTile, - // mangroveBiomass: MangroveBiomass#OptionalDTile, drivers: TreeCoverLossDrivers#OptionalITile, globalLandCover: GlobalLandcover#OptionalITile, primaryForest: PrimaryForest#OptionalITile, - // idnPrimaryForest: IndonesiaPrimaryForest#OptionalITile, - // erosion: Erosion#OptionalITile, - // biodiversitySignificance: BiodiversitySignificance#OptionalDTile, - // biodiversityIntactness: BiodiversityIntactness#OptionalDTile, wdpa: ProtectedAreas#OptionalITile, aze: Aze#OptionalITile, plantedForests: PlantedForests#OptionalITile, - // riverBasins: RiverBasins#OptionalITile, - // ecozones: Ecozones#OptionalITile, - // urbanWatersheds: UrbanWatersheds#OptionalITile, mangroves1996: Mangroves1996#OptionalITile, mangroves2016: Mangroves2016#OptionalITile, - // waterStress: WaterStress#OptionalITile, intactForestLandscapes: IntactForestLandscapes#OptionalITile, - // endemicBirdAreas: EndemicBirdAreas#OptionalITile, tigerLandscapes: TigerLandscapes#OptionalITile, landmark: Landmark#OptionalITile, landRights: LandRights#OptionalITile, keyBiodiversityAreas: KeyBiodiversityAreas#OptionalITile, mining: Mining#OptionalITile, - // rspo: RSPO#OptionalITile, peatlands: Peatlands#OptionalITile, oilPalm: OilPalm#OptionalITile, idnForestMoratorium: IndonesiaForestMoratorium#OptionalITile, - // idnLandCover: IndonesiaLandCover#OptionalITile, - // mexProtectedAreas: MexicoProtectedAreas#OptionalITile, - // mexPaymentForEcosystemServices: MexicoPaymentForEcosystemServices#OptionalITile, - // mexForestZoning: MexicoForestZoning#OptionalITile, - // perProductionForest: PeruProductionForest#OptionalITile, - // perProtectedAreas: PeruProtectedAreas#OptionalITile, - // perForestConcessions: PeruForestConcessions#OptionalITile, - // braBiomes: BrazilBiomes#OptionalITile, woodFiber: WoodFiber#OptionalITile, resourceRights: ResourceRights#OptionalITile, logging: Logging#OptionalITile, - // oilGas: OilGas#OptionalITile grossEmissionsCo2eNonCo2: GrossEmissionsNonCo2Co2e#OptionalFTile, grossEmissionsCo2eCo2Only: GrossEmissionsCo2OnlyCo2e#OptionalFTile, grossCumulAbovegroundRemovalsCo2: GrossCumulAbovegroundRemovalsCo2#OptionalFTile, grossCumulBelowgroundRemovalsCo2: GrossCumulBelowgroundRemovalsCo2#OptionalFTile, netFluxCo2: NetFluxCo2e#OptionalFTile, soilCarbon: SoilCarbon#OptionalFTile, + forestAge: ForestAgeCategory#OptionalITile, ) extends CellGrid[Int] { def cellType: CellType = loss.cellType def cols: Int = loss.cols From fdc8a6453942713980ae8bbb5385b639870c95aa Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Thu, 24 Mar 2022 16:00:56 -0700 Subject: [PATCH 09/21] Finalize updates --- .../resources/raster-catalog-default.json | 4 +-- .../layers/IntactForestLandscapes.scala | 12 ++++--- .../AnnualUpdateMinimalDF.scala | 34 +++++++++++++------ .../AnnualUpdateMinimalDataGroup.scala | 9 +++-- .../AnnualUpdateMinimalGridSources.scala | 18 ++++++++-- .../AnnualUpdateMinimalSummary.scala | 21 +++++++++--- .../AnnualUpdateMinimalTile.scala | 6 +++- 7 files changed, 77 insertions(+), 27 deletions(-) diff --git a/src/main/resources/raster-catalog-default.json b/src/main/resources/raster-catalog-default.json index c2986d6e..4f55600e 100644 --- a/src/main/resources/raster-catalog-default.json +++ b/src/main/resources/raster-catalog-default.json @@ -210,7 +210,7 @@ }, { "name":"gfw_mining_concessions", - "source_uri":"s3://gfw-data-lake/gfw_mining_concessions/v202106/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" + "source_uri":"s3://gfw-data-lake/gfw_mining_concessions/v202203/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" }, { "name":"gfw_peatlands", @@ -330,7 +330,7 @@ }, { "name":"fao_ecozones", - "source_uri": "s3://gfw-data-lake/gfw_full_extent_non_co2_gross_emissions/v20220316/raster/epsg-4326/{grid_size}/{row_count}/Mg_CO2e_ha-1/geotiff/{tile_id}.tif" + "source_uri": "s3://gfw-data-lake/fao_ecozones/v2010/raster/epsg-4326/{grid_size}/{row_count}/class/geotiff/{tile_id}.tif" }, { "name":"ifl_intact_forest_landscapes_2000", diff --git a/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala b/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala index 3cdf66d4..eae421b9 100644 --- a/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala +++ b/src/main/scala/org/globalforestwatch/layers/IntactForestLandscapes.scala @@ -19,28 +19,32 @@ case class IntactForestLandscapes(gridTile: GridTile, kwargs: Map[String, Any]) } case class IntactForestLandscapes2000(gridTile: GridTile, kwargs: Map[String, Any]) - extends BooleanLayer { + extends BooleanLayer + with OptionalILayer{ val datasetName = "ifl_intact_forest_landscapes_2000" val uri: String = uriForGrid(gridTile) } case class IntactForestLandscapes2013(gridTile: GridTile, kwargs: Map[String, Any]) - extends BooleanLayer { + extends BooleanLayer + with OptionalILayer{ val datasetName = "ifl_intact_forest_landscapes_2013" val uri: String = uriForGrid(gridTile) } case class IntactForestLandscapes2016(gridTile: GridTile, kwargs: Map[String, Any]) - extends BooleanLayer { + extends BooleanLayer + with OptionalILayer{ val datasetName = "ifl_intact_forest_landscapes_2016" val uri: String = uriForGrid(gridTile) } case class IntactForestLandscapes2020(gridTile: GridTile, kwargs: Map[String, Any]) - extends BooleanLayer { + extends BooleanLayer + with OptionalILayer{ val datasetName = "ifl_intact_forest_landscapes_2020" val uri: String = uriForGrid(gridTile) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala index fa39daf5..094f3cd6 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala @@ -14,7 +14,6 @@ object AnnualUpdateMinimalDF { "gfw_planted_forests__type", "is__gmw_global_mangrove_extent_1996", "is__gmw_global_mangrove_extent_2016", - "ifl_intact_forest_landscapes__year", "is__umd_regional_primary_forest_2001", "is__wwf_tiger_conservation_landscapes", "is__landmark_indigenous_and_community_lands", @@ -29,14 +28,19 @@ object AnnualUpdateMinimalDF { "is__gfw_managed_forests", "is__umd_tree_cover_gain", "forest_age__category", + "fao_ecozones__class", + "is__ifl_intact_forest_landscapes_2000", + "is__ifl_intact_forest_landscapes_2013", + "is__ifl_intact_forest_landscapes_2016", + "is__ifl_intact_forest_landscapes_2020", + // TODO delete next data update "umd_tree_cover_density__threshold", "tsc_tree_cover_loss_drivers__type", "is__birdlife_alliance_for_zero_extinction_site", "gfw_plantation__type", "is__gmw_mangroves_1996", "is__gmw_mangroves_2016", - "ifl_intact_forest_landscape__year", "is__gfw_tiger_landscape", "is__landmark_land_right", "is__gfw_land_right", @@ -65,7 +69,6 @@ object AnnualUpdateMinimalDF { $"data_group.plantedForests" as "gfw_planted_forests__type", $"data_group.mangroves1996" as "is__gmw_global_mangrove_extent_1996", $"data_group.mangroves2016" as "is__gmw_global_mangrove_extent_2016", - $"data_group.intactForestLandscapes" as "ifl_intact_forest_landscapes__year", $"data_group.tigerLandscapes" as "is__wwf_tiger_conservation_landscapes", $"data_group.landmark" as "is__landmark_indigenous_and_community_lands", $"data_group.landRights" as "is__gfw_land_rights", @@ -78,7 +81,13 @@ object AnnualUpdateMinimalDF { $"data_group.resourceRights" as "is__gfw_resource_rights", $"data_group.logging" as "is__gfw_managed_forests", $"data_group.isGain" as "is__umd_tree_cover_gain", - $"data_group.forestage" as "forest_age__category", + $"data_group.forestAge" as "forest_age__category", + $"data_group.faoEcozones" as "fao_ecozones__class", + $"data_group.intactForestLandscapes2000" as "is__ifl_intact_forest_landscapes_2000", + $"data_group.intactForestLandscapes2013" as "is__ifl_intact_forest_landscapes_2013", + $"data_group.intactForestLandscapes2016" as "is__ifl_intact_forest_landscapes_2016", + $"data_group.intactForestLandscapes2020" as "is__ifl_intact_forest_landscapes_2020", + $"data.treecoverExtent2000" as "umd_tree_cover_extent_2000__ha", $"data.treecoverExtent2010" as "umd_tree_cover_extent_2010__ha", $"data.totalArea" as "area__ha", @@ -103,7 +112,6 @@ object AnnualUpdateMinimalDF { $"data_group.plantedForests" as "gfw_plantation__type", $"data_group.mangroves1996" as "is__gmw_mangroves_1996", $"data_group.mangroves2016" as "is__gmw_mangroves_2016", - $"data_group.intactForestLandscapes" as "ifl_intact_forest_landscape__year", $"data_group.tigerLandscapes" as "is__gfw_tiger_landscape", $"data_group.landmark" as "is__landmark_land_right", $"data_group.landRights" as "is__gfw_land_right", @@ -246,8 +254,6 @@ object AnnualUpdateMinimalDF { .cast("boolean") as "gfw_planted_forests__type", max($"is__gmw_global_mangrove_extent_1996") as "is__gmw_global_mangrove_extent_1996", max($"is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", - max(length($"ifl_intact_forest_landscapes__year")) - .cast("boolean") as "ifl_intact_forest_landscapes__year", max($"is__wwf_tiger_conservation_landscapes") as "is__wwf_tiger_conservation_landscapes", max($"is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", max($"is__gfw_land_rights") as "is__gfw_land_rights", @@ -261,6 +267,11 @@ object AnnualUpdateMinimalDF { max($"is__gfw_managed_forests") as "is__gfw_managed_forests", max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain", max($"forest_age__category") as "forest_age__category", + max($"fao_ecozones__class") as "fao_ecozones__class", + max($"is__ifl_intact_forest_landscapes_2000") as "is__ifl_intact_forest_landscapes_2000", + max($"is__ifl_intact_forest_landscapes_2013") as "is__ifl_intact_forest_landscapes_2013", + max($"is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", + max($"is__ifl_intact_forest_landscapes_2020") as "is__ifl_intact_forest_landscapes_2020", max(length($"tsc_tree_cover_loss_drivers__type")).cast("boolean") as "tsc_tree_cover_loss_drivers__type", max($"is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", @@ -268,8 +279,6 @@ object AnnualUpdateMinimalDF { .cast("boolean") as "gfw_plantation__type", max($"is__gmw_mangroves_1996") as "is__gmw_mangroves_1996", max($"is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", - max(length($"ifl_intact_forest_landscape__year")) - .cast("boolean") as "ifl_intact_forest_landscape__year", max($"is__gfw_tiger_landscape") as "is__gfw_tiger_landscape", max($"is__landmark_land_right") as "is__landmark_land_right", max($"is__gfw_land_right") as "is__gfw_land_right", @@ -310,7 +319,6 @@ object AnnualUpdateMinimalDF { max($"gfw_planted_forests__type") as "gfw_planted_forests__type", max($"is__gmw_global_mangrove_extent_1996") as "is__gmw_global_mangrove_extent_1996", max($"is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", - max($"ifl_intact_forest_landscapes__year") as "ifl_intact_forest_landscapes__year", max($"is__wwf_tiger_conservation_landscapes") as "is__wwf_tiger_conservation_landscapes", max($"is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", max($"is__gfw_land_rights") as "is__gfw_land_rights", @@ -324,13 +332,17 @@ object AnnualUpdateMinimalDF { max($"is__gfw_managed_forests") as "is__gfw_managed_forests", max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain", max($"forest_age__category") as "forest_age__category", + max($"fao_ecozones__class") as "fao_ecozones__class", + max($"is__ifl_intact_forest_landscapes_2000") as "is__ifl_intact_forest_landscapes_2000", + max($"is__ifl_intact_forest_landscapes_2013") as "is__ifl_intact_forest_landscapes_2013", + max($"is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", + max($"is__ifl_intact_forest_landscapes_2020") as "is__ifl_intact_forest_landscapes_2020", max($"tsc_tree_cover_loss_drivers__type") as "tsc_tree_cover_loss_drivers__type", max($"is__birdlife_alliance_for_zero_extinction_site") as "is__birdlife_alliance_for_zero_extinction_site", max($"gfw_plantation__type") as "gfw_plantation__type", max($"is__gmw_mangroves_1996") as "is__gmw_mangroves_1996", max($"is__gmw_mangroves_2016") as "is__gmw_mangroves_2016", - max($"ifl_intact_forest_landscape__year") as "ifl_intact_forest_landscape__year", max($"is__gfw_tiger_landscape") as "is__gfw_tiger_landscape", max($"is__landmark_land_right") as "is__landmark_land_right", max($"is__gfw_land_right") as "is__gfw_land_right", diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala index cbd7d42a..2c819a05 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala @@ -10,7 +10,6 @@ case class AnnualUpdateMinimalDataGroup(lossYear: Integer, plantedForests: String, mangroves1996: Boolean, mangroves2016: Boolean, - intactForestLandscapes: String, tigerLandscapes: Boolean, landmark: Boolean, landRights: Boolean, @@ -23,4 +22,10 @@ case class AnnualUpdateMinimalDataGroup(lossYear: Integer, resourceRights: Boolean, logging: Boolean, isGain: Boolean, - forestAge: String) + forestAge: String, + faoEcozones: String, + intactForestLandscapes2000: Boolean, + intactForestLandscapes2013: Boolean, + intactForestLandscapes2016: Boolean, + intactForestLandscapes2020: Boolean, + ) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala index 396f6a14..387c3b03 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala @@ -24,7 +24,6 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val plantedForests: PlantedForests = PlantedForests(gridTile, kwargs) val mangroves1996: Mangroves1996 = Mangroves1996(gridTile, kwargs) val mangroves2016: Mangroves2016 = Mangroves2016(gridTile, kwargs) - val intactForestLandscapes: IntactForestLandscapes = IntactForestLandscapes(gridTile, kwargs) val tigerLandscapes: TigerLandscapes = TigerLandscapes(gridTile, kwargs) val landmark: Landmark = Landmark(gridTile, kwargs) val landRights: LandRights = LandRights(gridTile, kwargs) @@ -43,6 +42,11 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val grossEmissionsCo2eCo2Only: GrossEmissionsCo2OnlyCo2e = GrossEmissionsCo2OnlyCo2e(gridTile, kwargs = kwargs) val soilCarbon: SoilCarbon = SoilCarbon(gridTile, kwargs = kwargs) val forestAge: ForestAgeCategory = ForestAgeCategory(gridTile, kwargs = kwargs) + val faoEcozones: FaoEcozones = FaoEcozones(gridTile, kwargs = kwargs) + val intactForestLandscapes2000: IntactForestLandscapes2000 = IntactForestLandscapes2000(gridTile, kwargs) + val intactForestLandscapes2013: IntactForestLandscapes2013 = IntactForestLandscapes2013(gridTile, kwargs) + val intactForestLandscapes2016: IntactForestLandscapes2016 = IntactForestLandscapes2016(gridTile, kwargs) + val intactForestLandscapes2020: IntactForestLandscapes2020 = IntactForestLandscapes2020(gridTile, kwargs) def readWindow( windowKey: SpatialKey, windowLayout: LayoutDefinition @@ -71,7 +75,6 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val plantedForestsTile = plantedForests.fetchWindow(windowKey, windowLayout) val mangroves1996Tile = mangroves1996.fetchWindow(windowKey, windowLayout) val mangroves2016Tile = mangroves2016.fetchWindow(windowKey, windowLayout) - val intactForestLandscapesTile = intactForestLandscapes.fetchWindow(windowKey, windowLayout) val tigerLandscapesTile = tigerLandscapes.fetchWindow(windowKey, windowLayout) val landmarkTile = landmark.fetchWindow(windowKey, windowLayout) val landRightsTile = landRights.fetchWindow(windowKey, windowLayout) @@ -90,6 +93,11 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val grossEmissionsCo2eCo2OnlyTile = grossEmissionsCo2eCo2Only.fetchWindow(windowKey, windowLayout) val soilCarbonTile = soilCarbon.fetchWindow(windowKey, windowLayout) val forestAgeTile = forestAge.fetchWindow(windowKey, windowLayout) + val faoEcozonesTile = faoEcozones.fetchWindow(windowKey, windowLayout) + val intactForestLandscapes2000Tile = intactForestLandscapes2000.fetchWindow(windowKey, windowLayout) + val intactForestLandscapes2013Tile = intactForestLandscapes2013.fetchWindow(windowKey, windowLayout) + val intactForestLandscapes2016Tile = intactForestLandscapes2016.fetchWindow(windowKey, windowLayout) + val intactForestLandscapes2020Tile = intactForestLandscapes2020.fetchWindow(windowKey, windowLayout) val tile = AnnualUpdateMinimalTile( lossTile, @@ -105,7 +113,6 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String plantedForestsTile, mangroves1996Tile, mangroves2016Tile, - intactForestLandscapesTile, tigerLandscapesTile, landmarkTile, landRightsTile, @@ -124,6 +131,11 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String netFluxCo2Tile, soilCarbonTile, forestAgeTile, + faoEcozonesTile, + intactForestLandscapes2000Tile, + intactForestLandscapes2013Tile, + intactForestLandscapes2016Tile, + intactForestLandscapes2020Tile, ) Raster(tile, windowKey.extent(windowLayout)) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala index e6846ecd..edcf6b7f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala @@ -52,8 +52,6 @@ object AnnualUpdateMinimalSummary { val plantedForests: String = raster.tile.plantedForests.getData(col, row) val mangroves1996: Boolean = raster.tile.mangroves1996.getData(col, row) val mangroves2016: Boolean = raster.tile.mangroves2016.getData(col, row) - val intactForestLandscapes: String = - raster.tile.intactForestLandscapes.getData(col, row) val tigerLandscapes: Boolean = raster.tile.tigerLandscapes.getData(col, row) val landmark: Boolean = raster.tile.landmark.getData(col, row) @@ -70,6 +68,17 @@ object AnnualUpdateMinimalSummary { raster.tile.resourceRights.getData(col, row) val logging: Boolean = raster.tile.logging.getData(col, row) val forestAge: String = raster.tile.forestAge.getData(col, row) + val faoEcozones: String = raster.tile.faoEcozones.getData(col, row) + val intactForestLandscapes2000: Boolean = + raster.tile.intactForestLandscapes2000.getData(col, row) + val intactForestLandscapes2013: Boolean = + raster.tile.intactForestLandscapes2013.getData(col, row) + val intactForestLandscapes2016: Boolean = + raster.tile.intactForestLandscapes2016.getData(col, row) + val intactForestLandscapes2020: Boolean = + raster.tile.intactForestLandscapes2020.getData(col, row) + + val lat: Double = raster.rasterExtent.gridRowToMap(row) val area: Double = Geodesy.pixelArea(lat, raster.cellSize) // uses Pixel's center coordiate. +- raster.cellSize.height/2 doesn't make much of a difference @@ -120,7 +129,6 @@ object AnnualUpdateMinimalSummary { plantedForests, mangroves1996, mangroves2016, - intactForestLandscapes, tigerLandscapes, landmark, landRights, @@ -133,7 +141,12 @@ object AnnualUpdateMinimalSummary { resourceRights, logging, gain, - forestAge + forestAge, + faoEcozones, + intactForestLandscapes2000, + intactForestLandscapes2013, + intactForestLandscapes2016, + intactForestLandscapes2020, ) val summary: AnnualUpdateMinimalData = diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala index c1f1c08b..25553ac9 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala @@ -22,7 +22,6 @@ case class AnnualUpdateMinimalTile( plantedForests: PlantedForests#OptionalITile, mangroves1996: Mangroves1996#OptionalITile, mangroves2016: Mangroves2016#OptionalITile, - intactForestLandscapes: IntactForestLandscapes#OptionalITile, tigerLandscapes: TigerLandscapes#OptionalITile, landmark: Landmark#OptionalITile, landRights: LandRights#OptionalITile, @@ -41,6 +40,11 @@ case class AnnualUpdateMinimalTile( netFluxCo2: NetFluxCo2e#OptionalFTile, soilCarbon: SoilCarbon#OptionalFTile, forestAge: ForestAgeCategory#OptionalITile, + faoEcozones: FaoEcozones#OptionalITile, + intactForestLandscapes2000: IntactForestLandscapes2000#OptionalITile, + intactForestLandscapes2013: IntactForestLandscapes2013#OptionalITile, + intactForestLandscapes2016: IntactForestLandscapes2016#OptionalITile, + intactForestLandscapes2020: IntactForestLandscapes2020#OptionalITile, ) extends CellGrid[Int] { def cellType: CellType = loss.cellType def cols: Int = loss.cols From 67fd3e42a16dc5fe65f1970a2a60073f60c246ae Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Mon, 28 Mar 2022 09:07:28 -0700 Subject: [PATCH 10/21] Use gfw2-data emissions layer --- .../layers/GrossEmissionsCo2OnlyCo2e.scala | 2 +- .../annualupdate_minimal/AnnualUpdateMinimalDF.scala | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala index 667b93ea..14fdcf6c 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala @@ -10,5 +10,5 @@ case class GrossEmissionsCo2OnlyCo2e(gridTile: GridTile, val datasetName = "gfw_full_extent_co2_gross_emissions" val uri: String = - uriForGrid(gridTile) + s"s3://gfw2-data/climate/carbon_model/gross_emissions/all_drivers/CO2_only/biomass_soil/standard/20220316/${gridTile.tileId}_gross_emis_CO2_only_all_drivers_Mg_CO2e_ha_biomass_soil_2001_21.tif" } diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala index 094f3cd6..7efaf806 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala @@ -266,8 +266,10 @@ object AnnualUpdateMinimalDF { max($"is__gfw_resource_rights") as "is__gfw_resource_rights", max($"is__gfw_managed_forests") as "is__gfw_managed_forests", max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain", - max($"forest_age__category") as "forest_age__category", - max($"fao_ecozones__class") as "fao_ecozones__class", + max(length($"forest_age__category")) + .cast("boolean") as "forest_age__category", + max(length($"fao_ecozones__class")) + .cast("boolean") as "fao_ecozones__class", max($"is__ifl_intact_forest_landscapes_2000") as "is__ifl_intact_forest_landscapes_2000", max($"is__ifl_intact_forest_landscapes_2013") as "is__ifl_intact_forest_landscapes_2013", max($"is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", @@ -331,8 +333,10 @@ object AnnualUpdateMinimalDF { max($"is__gfw_resource_rights") as "is__gfw_resource_rights", max($"is__gfw_managed_forests") as "is__gfw_managed_forests", max($"is__umd_tree_cover_gain") as "is__umd_tree_cover_gain", - max($"forest_age__category") as "forest_age__category", - max($"fao_ecozones__class") as "fao_ecozones__class", + max(length($"forest_age__category")) + .cast("boolean") as "forest_age__category", + max(length($"fao_ecozones__class")) + .cast("boolean") as "fao_ecozones__class", max($"is__ifl_intact_forest_landscapes_2000") as "is__ifl_intact_forest_landscapes_2000", max($"is__ifl_intact_forest_landscapes_2013") as "is__ifl_intact_forest_landscapes_2013", max($"is__ifl_intact_forest_landscapes_2016") as "is__ifl_intact_forest_landscapes_2016", From 44e81fbcf1edf559bf11715886cd3fed708a1d66 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Mon, 28 Mar 2022 10:24:21 -0700 Subject: [PATCH 11/21] Update forest age --- src/main/resources/raster-catalog-default.json | 2 +- src/main/resources/raster-catalog-pro.json | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/resources/raster-catalog-default.json b/src/main/resources/raster-catalog-default.json index 4f55600e..a19c6619 100644 --- a/src/main/resources/raster-catalog-default.json +++ b/src/main/resources/raster-catalog-default.json @@ -306,7 +306,7 @@ }, { "name":"gfw_forest_age", - "source_uri": "s3://gfw-data-lake/gfw_forest_age/v20210621/raster/epsg-4326/{grid_size}/{row_count}/category/geotiff/{tile_id}.tif" + "source_uri": "s3://gfw-data-lake/gfw_forest_age/v20220309/raster/epsg-4326/{grid_size}/{row_count}/category/geotiff/{tile_id}.tif" }, { "name":"gfw_full_extent_net_flux", diff --git a/src/main/resources/raster-catalog-pro.json b/src/main/resources/raster-catalog-pro.json index 6e8d0f0f..b9b4b573 100644 --- a/src/main/resources/raster-catalog-pro.json +++ b/src/main/resources/raster-catalog-pro.json @@ -295,6 +295,10 @@ { "name":"gfwpro_forest_change_regions", "source_uri": "s3://gfw-data-lake/gfwpro_forest_change_regions/v20210129/raster/epsg-4326/{grid_size}/{row_count}/bit_encoding/gdal-geotiff/{tile_id}.tif" + }, + { + "name":"ifl_intact_forest_landscapes_2000", + "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2000/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" } ] } \ No newline at end of file From 7a02da63740eef7c1a2d306391dd440b45d0dd2f Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Mon, 28 Mar 2022 10:38:12 -0700 Subject: [PATCH 12/21] Fix David's comments --- .../annualupdate_minimal/AnnualUpdateMinimalDF.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala index 7efaf806..f4a579c2 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala @@ -103,7 +103,7 @@ object AnnualUpdateMinimalDF { $"data.totalGrossEmissionsCo2eCo2Only" as "gfw_full_extent_gross_emissions_CO2_only__Mg_CO2", $"data.totalGrossEmissionsCo2eNonCo2" as "gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e", $"data.totalGrossEmissionsCo2e" as "gfw_full_extent_gross_emissions__Mg_CO2e", - $"data.totalSoilCarbon" as "gfw_soil_carbon_stocks__Mg_CO2", + $"data.totalSoilCarbon" as "gfw_soil_carbon_stocks_2000__Mg_C", $"data.totalGrossCumulAboveBelowgroundRemovalsCo2" as "gfw_full_extent_gross_removals__Mg_CO2", $"data_group.threshold" as "umd_tree_cover_density__threshold", @@ -169,7 +169,7 @@ object AnnualUpdateMinimalDF { sum("gfw_full_extent_gross_emissions_CO2_only__Mg_CO2") as "gfw_full_extent_gross_emissions_CO2_only__Mg_CO2", sum("gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e") as "gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e", sum("gfw_full_extent_gross_emissions__Mg_CO2e") as "gfw_full_extent_gross_emissions__Mg_CO2e", - sum("gfw_soil_carbon_stocks__Mg_CO2") as "gfw_soil_carbon_stocks__Mg_CO2", + sum("gfw_soil_carbon_stocks_2000__Mg_C") as "gfw_soil_carbon_stocks_2000__Mg_C", sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", sum("gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", @@ -206,7 +206,7 @@ object AnnualUpdateMinimalDF { sum("gfw_full_extent_gross_emissions_CO2_only__Mg_CO2") as "gfw_full_extent_gross_emissions_CO2_only__Mg_CO2", sum("gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e") as "gfw_full_extent_gross_emissions_non_CO2__Mg_CO2e", sum("gfw_full_extent_gross_emissions__Mg_CO2e") as "gfw_full_extent_gross_emissions__Mg_CO2e", - sum("gfw_soil_carbon_stocks__Mg_CO2") as "gfw_soil_carbon_stocks__Mg_CO2", + sum("gfw_soil_carbon_stocks_2000__Mg_C") as "gfw_soil_carbon_stocks_2000__Mg_C", sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", sum("gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg") as "gfw_gross_cumulative_aboveground_belowground_co2_removals__Mg", From cc9922bd67b9f6ab29e257cd6769aaa02af24122 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Thu, 31 Mar 2022 10:50:49 -0700 Subject: [PATCH 13/21] Fix minor isssues --- .../layers/GrossEmissionsCo2OnlyCo2e.scala | 2 +- .../layers/TreeCoverGain.scala | 2 +- .../summarystats/SummaryRDD.scala | 20 +++++++++---------- .../AnnualUpdateMinimalGridSources.scala | 2 +- .../AnnualUpdateMinimalTile.scala | 2 +- .../CarbonSensitivityGridSources.scala | 4 +--- .../CarbonSensitivityTile.scala | 2 +- .../carbonflux/CarbonFluxGridSources.scala | 4 +--- .../carbonflux/CarbonFluxTile.scala | 2 +- .../treecoverloss/TreeLossGridSources.scala | 2 +- .../treecoverloss/TreeLossTile.scala | 2 +- 11 files changed, 19 insertions(+), 25 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala index 14fdcf6c..2ebde2ed 100644 --- a/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala +++ b/src/main/scala/org/globalforestwatch/layers/GrossEmissionsCo2OnlyCo2e.scala @@ -10,5 +10,5 @@ case class GrossEmissionsCo2OnlyCo2e(gridTile: GridTile, val datasetName = "gfw_full_extent_co2_gross_emissions" val uri: String = - s"s3://gfw2-data/climate/carbon_model/gross_emissions/all_drivers/CO2_only/biomass_soil/standard/20220316/${gridTile.tileId}_gross_emis_CO2_only_all_drivers_Mg_CO2e_ha_biomass_soil_2001_21.tif" + s"s3://gfw-files/flux_1_2_2/gross_emissions_co2_only_co2e/standard/${gridTile.tileId}.tif" } diff --git a/src/main/scala/org/globalforestwatch/layers/TreeCoverGain.scala b/src/main/scala/org/globalforestwatch/layers/TreeCoverGain.scala index 17a90888..a4cc6d73 100644 --- a/src/main/scala/org/globalforestwatch/layers/TreeCoverGain.scala +++ b/src/main/scala/org/globalforestwatch/layers/TreeCoverGain.scala @@ -4,7 +4,7 @@ import org.globalforestwatch.grids.GridTile case class TreeCoverGain(gridTile: GridTile, kwargs: Map[String, Any]) extends BooleanLayer - with RequiredILayer { + with OptionalILayer { val datasetName = "umd_tree_cover_gain" val uri: String = uriForGrid(gridTile) diff --git a/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala index 518694bc..d222b9c7 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala @@ -9,10 +9,11 @@ import geotrellis.raster.summary.polygonal.{NoIntersection, PolygonalSummaryResu import geotrellis.raster.summary.polygonal import geotrellis.store.index.zcurve.Z2 import geotrellis.vector._ -import org.apache.spark.RangePartitioner import org.apache.spark.rdd.RDD import org.globalforestwatch.features.FeatureId import org.globalforestwatch.grids.GridSources +import org.globalforestwatch.util.RepartitionSkewedRDD + import scala.reflect.ClassTag @@ -57,14 +58,13 @@ trait SummaryRDD extends LazyLogging with java.io.Serializable { */ val partitionedFeatureRDD = if (partition) { - val inputPartitionMultiplier = 64 - val rangePartitioner = - new RangePartitioner(featureRDD.getNumPartitions * inputPartitionMultiplier, keyedFeatureRDD) - keyedFeatureRDD.partitionBy(rangePartitioner) + RepartitionSkewedRDD.bySparseId(keyedFeatureRDD, 4096) } else { - keyedFeatureRDD + keyedFeatureRDD.values } + println(s"Number of partitions: ${partitionedFeatureRDD.getNumPartitions}") + /* * Here we're going to work with the features one partition at a time. * We're going to use the tile key from windowLayout to read pixels from appropriate raster. @@ -74,15 +74,13 @@ trait SummaryRDD extends LazyLogging with java.io.Serializable { */ val featuresWithSummaries: RDD[(FEATUREID, SUMMARY)] = partitionedFeatureRDD.mapPartitions { - featurePartition: Iterator[ - (Long, (SpatialKey, Feature[Geometry, FEATUREID])) - ] => - // Code inside .mapPartitions works in an Iterator of records + featurePartition: Iterator[(SpatialKey, Feature[Geometry, FEATUREID])] => + // Code inside .mapPartitions works in an Iterator of records // Doing things this way allows us to reuse resources and perform other optimizations // Grouping by spatial key allows us to minimize read thrashing from record to record val windowFeature = featurePartition.map { - case (z, (windowKey, feature)) => + case (windowKey, feature) => (windowKey, feature) } diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala index 387c3b03..29316686 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala @@ -56,7 +56,6 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String // Failure for any of these reads will result in function returning Left[Throwable] // These are effectively required fields without which we can't make sense of the analysis lossTile <- Either.catchNonFatal(treeCoverLoss.fetchWindow(windowKey, windowLayout)).right - gainTile <- Either.catchNonFatal(treeCoverGain.fetchWindow(windowKey, windowLayout)).right tcd2000Tile <- Either .catchNonFatal(treeCoverDensity2000.fetchWindow(windowKey, windowLayout)) .right @@ -66,6 +65,7 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String } yield { // Failure for these will be converted to optional result and propagated with TreeLossTile + val gainTile = treeCoverGain.fetchWindow(windowKey, windowLayout) val biomassTile = biomassPerHectar.fetchWindow(windowKey, windowLayout) val driversTile = treeCoverLossDrivers.fetchWindow(windowKey, windowLayout) val globalLandCoverTile = globalLandCover.fetchWindow(windowKey, windowLayout) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala index 25553ac9..0e367d0a 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala @@ -10,7 +10,7 @@ import org.globalforestwatch.layers._ */ case class AnnualUpdateMinimalTile( loss: TreeCoverLoss#ITile, - gain: TreeCoverGain#ITile, + gain: TreeCoverGain#OptionalITile, tcd2000: TreeCoverDensityThreshold#ITile, tcd2010: TreeCoverDensityThreshold#ITile, biomass: BiomassPerHectar#OptionalDTile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala index 11ab2dcd..189772cc 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityGridSources.scala @@ -69,9 +69,6 @@ case class CarbonSensitivityGridSources(gridTile: GridTile, lossTile <- Either .catchNonFatal(treeCoverLoss.fetchWindow(windowKey, windowLayout)) .right - gainTile <- Either - .catchNonFatal(treeCoverGain.fetchWindow(windowKey, windowLayout)) - .right tcd2000Tile <- Either .catchNonFatal( treeCoverDensity2000.fetchWindow(windowKey, windowLayout) @@ -80,6 +77,7 @@ case class CarbonSensitivityGridSources(gridTile: GridTile, } yield { // Failure for these will be converted to optional result and propagated with TreeLossTile + val gainTile = treeCoverGain.fetchWindow(windowKey, windowLayout) val biomassTile = biomassPerHectar.fetchWindow(windowKey, windowLayout) val grossCumulAbovegroundRemovalsCo2Tile = grossCumulAbovegroundRemovalsCo2.fetchWindow(windowKey, windowLayout) diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityTile.scala b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityTile.scala index c7a7a9c6..a223bed2 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbon_sensitivity/CarbonSensitivityTile.scala @@ -10,7 +10,7 @@ import org.globalforestwatch.layers._ */ case class CarbonSensitivityTile( loss: TreeCoverLoss#ITile, - gain: TreeCoverGain#ITile, + gain: TreeCoverGain#OptionalITile, tcd2000: TreeCoverDensityThreshold#ITile, biomass: BiomassPerHectar#OptionalDTile, grossCumulAbovegroundRemovalsCo2: GrossCumulAbovegroundRemovalsCo2#OptionalFTile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala index 1182d767..348d5852 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxGridSources.scala @@ -82,9 +82,6 @@ case class CarbonFluxGridSources(gridTile: GridTile, kwargs: Map[String, Any]) lossTile <- Either .catchNonFatal(treeCoverLoss.fetchWindow(windowKey, windowLayout)) .right - gainTile <- Either - .catchNonFatal(treeCoverGain.fetchWindow(windowKey, windowLayout)) - .right tcd2000Tile <- Either .catchNonFatal( treeCoverDensity2000.fetchWindow(windowKey, windowLayout) @@ -93,6 +90,7 @@ case class CarbonFluxGridSources(gridTile: GridTile, kwargs: Map[String, Any]) } yield { // Failure for these will be converted to optional result and propagated with TreeLossTile + val gainTile = treeCoverGain.fetchWindow(windowKey, windowLayout) val biomassTile = biomassPerHectar.fetchWindow(windowKey, windowLayout) val grossAnnualAbovegroundRemovalsCarbonTile = grossAnnualAbovegroundRemovalsCarbon.fetchWindow( diff --git a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxTile.scala b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxTile.scala index 36179663..d69b96ae 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/carbonflux/CarbonFluxTile.scala @@ -10,7 +10,7 @@ import org.globalforestwatch.layers._ */ case class CarbonFluxTile( loss: TreeCoverLoss#ITile, - gain: TreeCoverGain#ITile, + gain: TreeCoverGain#OptionalITile, tcd2000: TreeCoverDensityThreshold#ITile, biomass: BiomassPerHectar#OptionalDTile, grossAnnualAbovegroundRemovalsCarbon: GrossAnnualAbovegroundRemovalsCarbon#OptionalFTile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala index cd968ed3..39c0d6cf 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossGridSources.scala @@ -36,7 +36,6 @@ case class TreeLossGridSources(gridTile: GridTile, kwargs: Map[String, Any]) ext // Failure for any of these reads will result in function returning Left[Throwable] // These are effectively required fields without which we can't make sense of the analysis lossTile <- Either.catchNonFatal(treeCoverLoss.fetchWindow(windowKey, windowLayout)).right - gainTile <- Either.catchNonFatal(treeCoverGain.fetchWindow(windowKey, windowLayout)).right tcd2000Tile <- Either .catchNonFatal(treeCoverDensity2000.fetchWindow(windowKey, windowLayout)) .right @@ -46,6 +45,7 @@ case class TreeLossGridSources(gridTile: GridTile, kwargs: Map[String, Any]) ext } yield { // Failure for these will be converted to optional result and propagated with TreeLossTile + val gainTile = treeCoverGain.fetchWindow(windowKey, windowLayout) val primaryForestTile = primaryForest.fetchWindow(windowKey, windowLayout) val plantationsBoolTile = plantationsBool.fetchWindow(windowKey, windowLayout) diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossTile.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossTile.scala index 50063115..ff3c70e0 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossTile.scala @@ -9,7 +9,7 @@ import org.globalforestwatch.layers._ * We can not use GeoTrellis MultibandTile because it requires all bands share a CellType. */ case class TreeLossTile(loss: TreeCoverLoss#ITile, - gain: TreeCoverGain#ITile, + gain: TreeCoverGain#OptionalITile, tcd2000: TreeCoverDensityPercent2000#ITile, tcd2010: TreeCoverDensityPercent2010#ITile, biomass: BiomassPerHectar#OptionalDTile, From 6ed0dee9252bd0ff6c30aab6f35768a73bd08277 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Mon, 4 Apr 2022 11:44:57 -0700 Subject: [PATCH 14/21] Update IFL layers --- src/main/resources/raster-catalog-default.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/resources/raster-catalog-default.json b/src/main/resources/raster-catalog-default.json index a19c6619..db52e1c5 100644 --- a/src/main/resources/raster-catalog-default.json +++ b/src/main/resources/raster-catalog-default.json @@ -334,19 +334,19 @@ }, { "name":"ifl_intact_forest_landscapes_2000", - "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2000/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" + "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2000/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif" }, { "name":"ifl_intact_forest_landscapes_2013", - "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2013/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" + "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2013/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif" }, { "name":"ifl_intact_forest_landscapes_2016", - "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2016/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" + "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2016/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif" }, { "name":"ifl_intact_forest_landscapes_2020", - "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2020/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/geotiff/{tile_id}.tif" + "source_uri": "s3://gfw-data-lake/ifl_intact_forest_landscapes_2020/v2021/raster/epsg-4326/{grid_size}/{row_count}/is/gdal-geotiff/{tile_id}.tif" } ] } \ No newline at end of file From c9ea48d3a3ff0caf403ed2182c71cd055ff3a717 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Wed, 6 Apr 2022 13:44:58 -0700 Subject: [PATCH 15/21] Reduce number of output files --- .../org/globalforestwatch/summarystats/SummaryRDD.scala | 3 ++- .../annualupdate_minimal/AnnualUpdateMinimalExport.scala | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala index d222b9c7..5059fc0f 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala @@ -133,7 +133,8 @@ trait SummaryRDD extends LazyLogging with java.io.Serializable { println( s"There is an issue with geometry for ${feature.data}" ) - throw ise + // TODO some very invalid geoms are somehow getting here, skip for now + None } case te: org.locationtech.jts.geom.TopologyException => { println( diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala index f6fcbc47..35544e7c 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala @@ -100,7 +100,7 @@ object AnnualUpdateMinimalExport extends SummaryExport { .filter($"umd_tree_cover_loss__year".isNotNull && ($"umd_tree_cover_loss__ha" > 0 || $"gfw_full_extent_gross_emissions__Mg_CO2e" > 0)) .transform(AnnualUpdateMinimalDF.aggChange(List("iso", "adm1", "adm2"))) - .coalesce(133) // this should result in an avg file size of 100MB + .coalesce(65) // this should result in an avg file size of 100MB adm2ApiDF.write .options(csvOptions) @@ -108,7 +108,7 @@ object AnnualUpdateMinimalExport extends SummaryExport { val adm1ApiDF = adm2ApiDF .transform(AnnualUpdateMinimalDF.aggChange(List("iso", "adm1"))) - .coalesce(45) // this should result in an avg file size of 100MB + .coalesce(35) // this should result in an avg file size of 100MB adm1ApiDF.write .options(csvOptions) @@ -274,7 +274,7 @@ object AnnualUpdateMinimalExport extends SummaryExport { exportDF .transform(AnnualUpdateMinimalDF.aggSummary(idCols)) - .coalesce(4) // this should result in an avg file size of 100MB + .coalesce(10) // this should result in an avg file size of 100MB .write .options(csvOptions) .csv(path = outputUrl + "/geostore/summary") @@ -282,7 +282,7 @@ object AnnualUpdateMinimalExport extends SummaryExport { exportDF .filter($"umd_tree_cover_loss__year".isNotNull && $"umd_tree_cover_loss__ha" > 0) .transform(AnnualUpdateMinimalDF.aggChange(idCols)) - .coalesce(10) // this should result in an avg file size of 100MB + .coalesce(30) // this should result in an avg file size of 100MB .write .options(csvOptions) .csv(path = outputUrl + "/geostore/change") From c937db46019df105b11bc6b899160fb4e941fc07 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Tue, 12 Apr 2022 10:42:12 -0700 Subject: [PATCH 16/21] Remove Land Rights --- .../annualupdate_minimal/AnnualUpdateMinimalDF.scala | 3 +-- .../annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala | 1 - .../annualupdate_minimal/AnnualUpdateMinimalGridSources.scala | 3 --- .../annualupdate_minimal/AnnualUpdateMinimalSummary.scala | 2 -- .../annualupdate_minimal/AnnualUpdateMinimalTile.scala | 1 - 5 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala index f4a579c2..f371627a 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala @@ -71,7 +71,6 @@ object AnnualUpdateMinimalDF { $"data_group.mangroves2016" as "is__gmw_global_mangrove_extent_2016", $"data_group.tigerLandscapes" as "is__wwf_tiger_conservation_landscapes", $"data_group.landmark" as "is__landmark_indigenous_and_community_lands", - $"data_group.landRights" as "is__gfw_land_rights", $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_areas", $"data_group.mining" as "is__gfw_mining_concessions", $"data_group.peatlands" as "is__gfw_peatlands", @@ -114,7 +113,7 @@ object AnnualUpdateMinimalDF { $"data_group.mangroves2016" as "is__gmw_mangroves_2016", $"data_group.tigerLandscapes" as "is__gfw_tiger_landscape", $"data_group.landmark" as "is__landmark_land_right", - $"data_group.landRights" as "is__gfw_land_right", + $"data_group.landmark" as "is__gfw_land_right", $"data_group.keyBiodiversityAreas" as "is__birdlife_key_biodiversity_area", $"data_group.mining" as "is__gfw_mining", $"data_group.peatlands" as "is__peatland", diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala index 2c819a05..0aaa3734 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDataGroup.scala @@ -12,7 +12,6 @@ case class AnnualUpdateMinimalDataGroup(lossYear: Integer, mangroves2016: Boolean, tigerLandscapes: Boolean, landmark: Boolean, - landRights: Boolean, keyBiodiversityAreas: Boolean, mining: Boolean, peatlands: Boolean, diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala index 29316686..f2a7dba0 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalGridSources.scala @@ -26,7 +26,6 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val mangroves2016: Mangroves2016 = Mangroves2016(gridTile, kwargs) val tigerLandscapes: TigerLandscapes = TigerLandscapes(gridTile, kwargs) val landmark: Landmark = Landmark(gridTile, kwargs) - val landRights: LandRights = LandRights(gridTile, kwargs) val keyBiodiversityAreas: KeyBiodiversityAreas = KeyBiodiversityAreas(gridTile, kwargs) val mining: Mining = Mining(gridTile, kwargs) val peatlands: Peatlands = Peatlands(gridTile, kwargs) @@ -77,7 +76,6 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String val mangroves2016Tile = mangroves2016.fetchWindow(windowKey, windowLayout) val tigerLandscapesTile = tigerLandscapes.fetchWindow(windowKey, windowLayout) val landmarkTile = landmark.fetchWindow(windowKey, windowLayout) - val landRightsTile = landRights.fetchWindow(windowKey, windowLayout) val keyBiodiversityAreasTile = keyBiodiversityAreas.fetchWindow(windowKey, windowLayout) val miningTile = mining.fetchWindow(windowKey, windowLayout) val peatlandsTile = peatlands.fetchWindow(windowKey, windowLayout) @@ -115,7 +113,6 @@ case class AnnualUpdateMinimalGridSources(gridTile: GridTile, kwargs: Map[String mangroves2016Tile, tigerLandscapesTile, landmarkTile, - landRightsTile, keyBiodiversityAreasTile, miningTile, peatlandsTile, diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala index edcf6b7f..15ff94e8 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalSummary.scala @@ -55,7 +55,6 @@ object AnnualUpdateMinimalSummary { val tigerLandscapes: Boolean = raster.tile.tigerLandscapes.getData(col, row) val landmark: Boolean = raster.tile.landmark.getData(col, row) - val landRights: Boolean = raster.tile.landRights.getData(col, row) val keyBiodiversityAreas: Boolean = raster.tile.keyBiodiversityAreas.getData(col, row) val mining: Boolean = raster.tile.mining.getData(col, row) @@ -131,7 +130,6 @@ object AnnualUpdateMinimalSummary { mangroves2016, tigerLandscapes, landmark, - landRights, keyBiodiversityAreas, mining, peatlands, diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala index 0e367d0a..91c7b0a8 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalTile.scala @@ -24,7 +24,6 @@ case class AnnualUpdateMinimalTile( mangroves2016: Mangroves2016#OptionalITile, tigerLandscapes: TigerLandscapes#OptionalITile, landmark: Landmark#OptionalITile, - landRights: LandRights#OptionalITile, keyBiodiversityAreas: KeyBiodiversityAreas#OptionalITile, mining: Mining#OptionalITile, peatlands: Peatlands#OptionalITile, From f350b874f81529cc081931830ade9504fe938c47 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Tue, 12 Apr 2022 11:44:14 -0700 Subject: [PATCH 17/21] Update layers --- src/main/resources/raster-catalog-default.json | 2 +- .../annualupdate_minimal/AnnualUpdateMinimalDF.scala | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/resources/raster-catalog-default.json b/src/main/resources/raster-catalog-default.json index db52e1c5..9a06bc62 100644 --- a/src/main/resources/raster-catalog-default.json +++ b/src/main/resources/raster-catalog-default.json @@ -110,7 +110,7 @@ }, { "name":"tsc_tree_cover_loss_drivers", - "source_uri":"s3://gfw-data-lake/tsc_tree_cover_loss_drivers/v2021/raster/epsg-4326/{grid_size}/{row_count}/driver/geotiff/{tile_id}.tif" + "source_uri":"s3://gfw-data-lake/tsc_tree_cover_loss_drivers/v2021.1/raster/epsg-4326/{grid_size}/{row_count}/driver/geotiff/{tile_id}.tif" }, { "name":"umd_tree_cover_density_2010", diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala index f371627a..498a4be1 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalDF.scala @@ -17,7 +17,6 @@ object AnnualUpdateMinimalDF { "is__umd_regional_primary_forest_2001", "is__wwf_tiger_conservation_landscapes", "is__landmark_indigenous_and_community_lands", - "is__gfw_land_rights", "is__birdlife_key_biodiversity_areas", "is__gfw_mining_concessions", "is__gfw_peatlands", @@ -255,7 +254,6 @@ object AnnualUpdateMinimalDF { max($"is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", max($"is__wwf_tiger_conservation_landscapes") as "is__wwf_tiger_conservation_landscapes", max($"is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", - max($"is__gfw_land_rights") as "is__gfw_land_rights", max($"is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas", max($"is__gfw_mining_concessions") as "is__gfw_mining_concessions", max($"is__gfw_peatlands") as "is__gfw_peatlands", @@ -322,7 +320,6 @@ object AnnualUpdateMinimalDF { max($"is__gmw_global_mangrove_extent_2016") as "is__gmw_global_mangrove_extent_2016", max($"is__wwf_tiger_conservation_landscapes") as "is__wwf_tiger_conservation_landscapes", max($"is__landmark_indigenous_and_community_lands") as "is__landmark_indigenous_and_community_lands", - max($"is__gfw_land_rights") as "is__gfw_land_rights", max($"is__birdlife_key_biodiversity_areas") as "is__birdlife_key_biodiversity_areas", max($"is__gfw_mining_concessions") as "is__gfw_mining_concessions", max($"is__gfw_peatlands") as "is__gfw_peatlands", From 1715f130c59a2ac97b291fbdb0d901f038ab3bfd Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Wed, 13 Apr 2022 09:24:31 -0700 Subject: [PATCH 18/21] Update WDPA --- src/main/resources/raster-catalog-default.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/raster-catalog-default.json b/src/main/resources/raster-catalog-default.json index 9a06bc62..89a0a212 100644 --- a/src/main/resources/raster-catalog-default.json +++ b/src/main/resources/raster-catalog-default.json @@ -10,7 +10,7 @@ }, { "name":"wdpa_protected_areas", - "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202202/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/geotiff/{tile_id}.tif" + "source_uri":"s3://gfw-data-lake/wdpa_protected_areas/v202204/raster/epsg-4326/{grid_size}/{row_count}/iucn_cat/geotiff/{tile_id}.tif" }, { "name":"gfw_oil_gas", From bfdd9863ab1f80e47c08502be97e9db99625b541 Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Mon, 18 Apr 2022 08:17:26 -0700 Subject: [PATCH 19/21] Skip bad geometries --- .../summarystats/SummaryRDD.scala | 15 ++++++++++----- .../util/GeotrellisGeometryValidator.scala | 3 +++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala b/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala index 5059fc0f..d25e5225 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/SummaryRDD.scala @@ -138,17 +138,22 @@ trait SummaryRDD extends LazyLogging with java.io.Serializable { } case te: org.locationtech.jts.geom.TopologyException => { println( - s"There is an issue with geometry Topology for ${feature.data}" + s"There is an issue with geometry for ${feature.data}: ${feature.geom}" ) - throw te + None } case be: java.lang.ArrayIndexOutOfBoundsException => { println( - s"There is an issue with geometry ${feature.geom}" + s"There is an issue with geometry for ${feature.data}: ${feature.geom}" + ) + None + } + case ise: java.lang.IllegalArgumentException => { + println( + s"There is an issue with geometry for ${feature.data}: ${feature.geom}" ) - throw be + None } - case e: Throwable => throw e } diff --git a/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryValidator.scala b/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryValidator.scala index 8e686a40..f5c50940 100644 --- a/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryValidator.scala +++ b/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryValidator.scala @@ -24,6 +24,9 @@ object GeotrellisGeometryValidator extends java.io.Serializable { case _: java.lang.AssertionError => println("There was an empty geometry") None + case _: java.lang.IllegalArgumentException => + println("There was an empty geometry") + None } maybeGeom match { From 1fa4c01b05a150691416b4dae92790af4fea18ed Mon Sep 17 00:00:00 2001 From: Justin Terry Date: Fri, 22 Apr 2022 11:24:49 -0700 Subject: [PATCH 20/21] Only fix Pro geoms --- .../AnnualUpdateMinimalExport.scala | 20 ++++++++-------- .../util/GeotrellisGeometryValidator.scala | 23 +++++++++---------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala index 35544e7c..e6b3a0a7 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/annualupdate_minimal/AnnualUpdateMinimalExport.scala @@ -194,11 +194,11 @@ object AnnualUpdateMinimalExport extends SummaryExport { import spark.implicits._ val idCols: List[String] = List( - "wdpa_protected_areas__id", - "wdpa_protected_areas__name", - "wdpa_protected_areas__iucn_cat", - "wdpa_protected_areas__iso", - "wdpa_protected_areas__status" + "wdpa_protected_area__id", + "wdpa_protected_area__name", + "wdpa_protected_area__iucn_cat", + "wdpa_protected_area__iso", + "wdpa_protected_area__status" ) val changeOnly: Boolean = @@ -208,11 +208,11 @@ object AnnualUpdateMinimalExport extends SummaryExport { .transform( AnnualUpdateMinimalDF.unpackValues( List( - $"id.wdpaId" as "wdpa_protected_areas__id", - $"id.name" as "wdpa_protected_areas__name", - $"id.iucnCat" as "wdpa_protected_areas__iucn_cat", - $"id.iso" as "wdpa_protected_areas__iso", - $"id.status" as "wdpa_protected_areas__status" + $"id.wdpaId" as "wdpa_protected_area__id", + $"id.name" as "wdpa_protected_area__name", + $"id.iucnCat" as "wdpa_protected_area__iucn_cat", + $"id.iso" as "wdpa_protected_area__iso", + $"id.status" as "wdpa_protected_area__status" ), wdpa = true ) diff --git a/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryValidator.scala b/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryValidator.scala index f5c50940..d19c54e3 100644 --- a/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryValidator.scala +++ b/src/main/scala/org/globalforestwatch/util/GeotrellisGeometryValidator.scala @@ -1,17 +1,10 @@ package org.globalforestwatch.util import org.apache.log4j.Logger -import geotrellis.vector.{ - Geometry, - GeomFactory, - LineString, - MultiPoint, - Point, - Polygon, - MultiLineString, - MultiPolygon -} +import geotrellis.vector.{GeomFactory, Geometry, LineString, MultiLineString, MultiPoint, MultiPolygon, Point, Polygon} import geotrellis.vector.io.wkb.WKB +import org.globalforestwatch.config.GfwConfig import org.globalforestwatch.util.GeotrellisGeometryReducer.{gpr, reduce} + import scala.util.Try object GeotrellisGeometryValidator extends java.io.Serializable { @@ -58,8 +51,14 @@ object GeotrellisGeometryValidator extends java.io.Serializable { } def makeValidGeom(wkb: String): Geometry = { - val geom: Option[Geometry] = Try(WKB.read(wkb)).toOption - geom.map(makeValidGeom(_)).getOrElse(GeomFactory.factory.createPoint()) + if (GfwConfig.isGfwPro) { + // Pro is getting direct geometries from users, geometries may be invalid + val geom: Option[Geometry] = Try(WKB.read(wkb)).toOption + geom.map(makeValidGeom(_)).getOrElse(GeomFactory.factory.createPoint()) + } else { + // Flagship has already preprocessed geometries to make them valid + WKB.read(wkb) + } } def makeMultiGeom(geom: Geometry): Geometry = { From b533300ccdf1fc334f421ac10d1e5cdf7743d25f Mon Sep 17 00:00:00 2001 From: "David.Gibbs" Date: Wed, 27 Apr 2022 15:50:29 -0400 Subject: [PATCH 21/21] Updated treecoverloss package that is used in ArcPy client. Ran locally for GMB in sbt shell and results matched the annualupdate_minimal and carbonflux results. Created jar to use in ArcPy client repo. --- .../treecoverloss/TreeLossDF.scala | 59 ++++++++++--------- .../treecoverloss/TreeLossYearDataMap.scala | 3 +- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossDF.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossDF.scala index 72f9b7d9..c1730df4 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossDF.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossDF.scala @@ -8,7 +8,7 @@ import org.apache.spark.sql.{DataFrame, SparkSession} object TreeLossDF { val treecoverLossMinYear = 2001 - val treecoverLossMaxYear = 2020 + val treecoverLossMaxYear = 2021 def unpackValues(carbonPools: Boolean)(df: DataFrame): DataFrame = { val spark: SparkSession = df.sparkSession @@ -46,24 +46,24 @@ object TreeLossDF { $"data.treecoverExtent2000" as "umd_tree_cover_extent_2000__ha", $"data.treecoverExtent2010" as "umd_tree_cover_extent_2010__ha", $"data.totalArea" as "area__ha", - $"data.totalGainArea" as "umd_tree_cover_gain_2000-2012__ha", + $"data.totalGainArea" as "umd_tree_cover_gain__ha", $"data.totalBiomass" as "whrc_aboveground_biomass_stock_2000__Mg", $"data.avgBiomass" as "avg_whrc_aboveground_biomass_stock_2000__Mg_ha-1", - $"data.totalGrossCumulAbovegroundRemovalsCo2" as "gfw_gross_cumulative_aboveground_co2_removals_2001-2020__Mg", - $"data.totalGrossCumulBelowgroundRemovalsCo2" as "gfw_gross_cumulative_belowground_co2_removals_2001-2020__Mg", - $"data.totalGrossCumulAboveBelowgroundRemovalsCo2" as "gfw_gross_cumulative_aboveground_belowground_co2_removals_2001-2020__Mg", - $"data.totalGrossEmissionsCo2eCo2Only" as "gfw_gross_emissions_co2e_co2_only_2001-2020__Mg", - $"data.totalGrossEmissionsCo2eNonCo2" as "gfw_gross_emissions_co2e_non_co2_2001-2020__Mg", - $"data.totalGrossEmissionsCo2eAllGases" as "gfw_gross_emissions_co2e_all_gases_2001-2020__Mg", - $"data.totalNetFluxCo2" as "gfw_net_flux_co2e_2001-2020__Mg", + $"data.totalGrossCumulAbovegroundRemovalsCo2" as s"gfw_forest_carbon_gross_removals_aboveground_2001_${treecoverLossMaxYear}__Mg_CO2", + $"data.totalGrossCumulBelowgroundRemovalsCo2" as s"gfw_forest_carbon_gross_removals_belowground_2001_${treecoverLossMaxYear}__Mg_CO2", + $"data.totalGrossCumulAboveBelowgroundRemovalsCo2" as s"gfw_forest_carbon_gross_removals_2001_${treecoverLossMaxYear}__Mg_CO2", + $"data.totalGrossEmissionsCo2eCo2Only" as s"gfw_forest_carbon_gross_emissions_co2_only_2001_${treecoverLossMaxYear}__Mg_CO2e", + $"data.totalGrossEmissionsCo2eNonCo2" as s"gfw_forest_carbon_gross_emissions_non_co2_2001_${treecoverLossMaxYear}__Mg_CO2e", + $"data.totalGrossEmissionsCo2eAllGases" as s"gfw_forest_carbon_gross_emissions_2001_${treecoverLossMaxYear}__Mg_CO2e", + $"data.totalNetFluxCo2" as s"gfw_forest_carbon_net_flux_2001_${treecoverLossMaxYear}__Mg_CO2e", $"data.totalFluxModelExtentArea" as "gfw_flux_model_extent__ha" ) val carbonPoolCols = if (carbonPools) { List( - $"data.totalAgc2000" as "gfw_aboveground_carbon_stock_2000__Mg", - $"data.totalBgc2000" as "gfw_belowground_carbon_stock_2000__Mg", - $"data.totalSoilCarbon2000" as "gfw_soil_carbon_stock_2000__Mg" + $"data.totalAgc2000" as "gfw_aboveground_carbon_stock_2000__Mg_C", + $"data.totalBgc2000" as "gfw_belowground_carbon_stock_2000__Mg_C", + $"data.totalSoilCarbon2000" as "gfw_soil_carbon_stock_2000__Mg_C" ) } else { List() @@ -103,32 +103,33 @@ object TreeLossDF { sum("area__ha") as "area__ha", sum("umd_tree_cover_extent_2000__ha") as "umd_tree_cover_extent_2000__ha", sum("umd_tree_cover_extent_2010__ha") as "umd_tree_cover_extent_2010__ha", - sum("umd_tree_cover_gain_2000-2012__ha") as "umd_tree_cover_gain_2000-2012__ha", + sum("umd_tree_cover_gain__ha") as "umd_tree_cover_gain__ha", sum("whrc_aboveground_biomass_stock_2000__Mg") as "whrc_aboveground_biomass_stock_2000__Mg", sum($"avg_whrc_aboveground_biomass_stock_2000__Mg_ha-1" * $"umd_tree_cover_extent_2000__ha") / sum($"umd_tree_cover_extent_2000__ha") as "avg_whrc_aboveground_biomass_density_2000__Mg_ha-1", - sum("gfw_gross_cumulative_aboveground_co2_removals_2001-2020__Mg") - as "gfw_gross_cumulative_aboveground_co2_removals_2001-2020__Mg", - sum("gfw_gross_cumulative_belowground_co2_removals_2001-2020__Mg") - as "gfw_gross_cumulative_belowground_co2_removals_2001-2020__Mg", - sum("gfw_gross_cumulative_aboveground_belowground_co2_removals_2001-2020__Mg") - as "gfw_gross_cumulative_aboveground_belowground_co2_removals_2001-2020__Mg", - sum("gfw_gross_emissions_co2e_co2_only_2001-2020__Mg") - as "gfw_gross_emissions_co2e_co2_only_2001-2020__Mg", - sum("gfw_gross_emissions_co2e_non_co2_2001-2020__Mg") - as "gfw_gross_emissions_co2e_non_co2_2001-2020__Mg", - sum("gfw_gross_emissions_co2e_all_gases_2001-2020__Mg") - as "gfw_gross_emissions_co2e_all_gases_2001-2020__Mg", - sum("gfw_net_flux_co2e_2001-2020__Mg") as "gfw_net_flux_co2e_2001-2020__Mg", + sum(s"gfw_forest_carbon_gross_removals_aboveground_2001_${treecoverLossMaxYear}__Mg_CO2") + as s"gfw_forest_carbon_gross_removals_aboveground_2001_${treecoverLossMaxYear}__Mg_CO2", + sum(s"gfw_forest_carbon_gross_removals_belowground_2001_${treecoverLossMaxYear}__Mg_CO2") + as s"gfw_forest_carbon_gross_removals_belowground_2001_${treecoverLossMaxYear}__Mg_CO2", + sum(s"gfw_forest_carbon_gross_removals_2001_${treecoverLossMaxYear}__Mg_CO2") + as s"gfw_forest_carbon_gross_removals_2001_${treecoverLossMaxYear}__Mg_CO2", + sum(s"gfw_forest_carbon_gross_emissions_co2_only_2001_${treecoverLossMaxYear}__Mg_CO2e") + as s"gfw_forest_carbon_gross_emissions_co2_only_2001_${treecoverLossMaxYear}__Mg_CO2e", + sum(s"gfw_forest_carbon_gross_emissions_non_co2_2001_${treecoverLossMaxYear}__Mg_CO2e") + as s"gfw_forest_carbon_gross_emissions_non_co2_2001_${treecoverLossMaxYear}__Mg_CO2e", + sum(s"gfw_forest_carbon_gross_emissions_2001_${treecoverLossMaxYear}__Mg_CO2e") + as s"gfw_forest_carbon_gross_emissions_2001_${treecoverLossMaxYear}__Mg_CO2e", + sum(s"gfw_forest_carbon_net_flux_2001_${treecoverLossMaxYear}__Mg_CO2e") + as s"gfw_forest_carbon_net_flux_2001_${treecoverLossMaxYear}__Mg_CO2e", sum("gfw_flux_model_extent__ha") as "gfw_flux_model_extent__ha" ) val carbonPoolCols = if (carbonPools) { List( - sum("gfw_aboveground_carbon_stock_2000__Mg") as "gfw_aboveground_carbon_stock_2000__Mg", - sum("gfw_belowground_carbon_stock_2000__Mg") as "gfw_belowground_carbon_stock_2000__Mg", - sum("gfw_soil_carbon_stock_2000__Mg") as "gfw_soil_carbon_stock_2000__Mg" + sum("gfw_aboveground_carbon_stock_2000__Mg_C") as "gfw_aboveground_carbon_stock_2000__Mg_C", + sum("gfw_belowground_carbon_stock_2000__Mg_C") as "gfw_belowground_carbon_stock_2000__Mg_C", + sum("gfw_soil_carbon_stock_2000__Mg_C") as "gfw_soil_carbon_stock_2000__Mg_C" ) } else { List() diff --git a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossYearDataMap.scala b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossYearDataMap.scala index 99e33732..fe4f459a 100644 --- a/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossYearDataMap.scala +++ b/src/main/scala/org/globalforestwatch/summarystats/treecoverloss/TreeLossYearDataMap.scala @@ -24,7 +24,8 @@ object TreeLossYearDataMap { 2017 -> TreeLossYearData(2017, 0, 0, 0, 0, 0), 2018 -> TreeLossYearData(2018, 0, 0, 0, 0, 0), 2019 -> TreeLossYearData(2019, 0, 0, 0, 0, 0), - 2020 -> TreeLossYearData(2020, 0, 0, 0, 0, 0) + 2020 -> TreeLossYearData(2020, 0, 0, 0, 0, 0), + 2021 -> TreeLossYearData(2021, 0, 0, 0, 0, 0) ) //def mapValuesToList(map: Map[Int, LossYearData]): List[LossYearData] = ???