notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example notebook to use the GloUrbEE extraction tool\n",
    "\n",
    "First, you need to create a Google Cloud Project with the EarthEngine API enabled. \n",
    "Then create a service account with the EarthEngine Viewer role, and download a login json key for this service account. \n",
    "\n",
    "## Login to Google Cloud\n",
    "\n",
    "The next cell load Earth Engine package and login to the API using the service account."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import ee\n",
    "\n",
    "# Set your Google Cloud EarthEngine-enabled project name\n",
    "ee_project_name='ee-glourb'\n",
    "\n",
    "# Set your credentials\n",
    "credentials = ee.ServiceAccountCredentials('samuel@ee-glourb.iam.gserviceaccount.com', './earthengine-key.json')\n",
    "ee.Initialize(credentials, project=ee_project_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import the required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from glourbee import (\n",
    "    assets_management,\n",
    "    workflow,\n",
    "    collection\n",
    ")\n",
    "\n",
    "# The next lines are useful if you make modifications in the module files and want to reload it without restarting the notebook kernel\n",
    "import importlib\n",
    "importlib.reload(assets_management)\n",
    "importlib.reload(workflow)\n",
    "importlib.reload(collection)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Extraction zones management"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use your own extraction zones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define your extraction zones\n",
    "extraction_zones = assets_management.ExtractionZones(local_file='./data/dgo_2km_pilcomayo18_06_2024_rives_V2.gpkg', # Local file containing extraction features\n",
    "                                                     fid_field='id', # Unique identifier field name (must be integer)\n",
    "                                                     zone_type='DGOs', # Literal description of the zones types (eg. DGOs, Cites, Gravel quarries, ...)\n",
    "                                                     description='2km DGOs for Pilcomayo, separated by right and left banks', # Literal description of the zones. Give a maximum details so other GloUrb researcher can understand chat are these zones.\n",
    "                                                     author='Louis Rey' # Identify the producer of those extraction zones\n",
    "                                                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check your assigned UUID, you can use it to retrieve your zones the next time you will start GloUrbEE\n",
    "zones_uuid = extraction_zones.asset_uuid\n",
    "print(zones_uuid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace this uuid with the output of the previous cell to keep trace of your current work\n",
    "zones_uuid = 'b744d80a92a344a78ba8748507372516'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check if your zones are already uploaded to GEE\n",
    "extraction_zones.gee_state # 'none', 'partial' or 'complete'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Upload the zones to GEE\n",
    "extraction_zones.upload_to_gee()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check again if your zones are already uploaded to GEE\n",
    "extraction_zones.update_gee_state()\n",
    "extraction_zones.gee_state # 'none', 'partial' or 'complete'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Once the upload process is started, you can retrieve your asset with it's UUID\n",
    "extraction_zones = assets_management.ExtractionZones(asset_uuid=zones_uuid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# More details on upload tasks\n",
    "extraction_zones.update_gee_state()\n",
    "extraction_zones.linked_tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can use this to wait for upload to complete\n",
    "extraction_zones.wait_for_tasks()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can cancel upload tasks if needed\n",
    "extraction_zones.cancel_linked_tasks()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use other extraction zones already uploaded to the GEE project\n",
    "\n",
    "You can explore other dataset created by GloUrb researchers and uploaded to the GEE project with the two following methods."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get all the extraction zones available on the GEE project\n",
    "glourbee_zones = collection.getGlourbeeExtractionZones()\n",
    "glourbee_zones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use one of the extraction zones for your current work\n",
    "extraction_zones = assets_management.ExtractionZones(asset_uuid='4923d2bac3f444219c4997822ecc7c7c')\n",
    "\n",
    "# Don't forget to check the state of the asset upload before doing anything\n",
    "extraction_zones.gee_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get all the metrics datasets already calculated for an extraction zone dataset\n",
    "glourbee_metrics = collection.getGlourbeeMetrics(zones_uuid='4923d2bac3f444219c4997822ecc7c7c')\n",
    "glourbee_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download one of the metrics datasets\n",
    "metrics = assets_management.MetricsDataset(asset_uuid='79d5116ef1bb47988eb2db32246268a6', parent_zones='4923d2bac3f444219c4997822ecc7c7c')\n",
    "metrics.download(output_file='./data/lhasa_2km_metrics.csv',\n",
    "                 overwrite=False # By setting this to False, download will resume at his previous step if it was interrupted\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = assets_management.MetricsDataset(asset_uuid='79d5116ef1bb47988eb2db32246268a6', parent_zones='4923d2bac3f444219c4997822ecc7c7c')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Delete extraction zones and associated metrics from the GEE project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# WARNING: unreversible operation\n",
    "extraction_zones.delete()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 2. Indicators workflow\n",
    "\n",
    "It is possible to extract indicators based on the JRC Global Surface Water Mapping Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "workflow.indicatorsWorkflow(extraction_zones=extraction_zones, \n",
    "                            output_csv='./data/villes_fanny_jrc_indicators.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Metrics workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start a new matrics calculation\n",
    "metrics = workflow.startWorkflow(zones_dataset=extraction_zones, # Your extraction zones dataset\n",
    "                                 satellite_type='Landsat', # Landsat | Sentinel-2\n",
    "                                 start='1980-01-01',\n",
    "                                 end='2025-01-31',\n",
    "                                 cloud_filter=80, # Maximum cloud coverage for an image\n",
    "                                 cloud_masking=True, # Masking the clouds on the metrics results (recommended)\n",
    "                                 mosaic_same_day=False, # Merge all images of same day into one\n",
    "\n",
    "                                 # With those three last parameters, you can customize the way to define each mask created to calculate metrics. Check README for more details\n",
    "                                 watermask_expression=None,\n",
    "                                 activechannel_expression=None,\n",
    "                                 vegetation_expression=None\n",
    "                                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check your assigned UUID, you can use it to retrieve your metrics the next time you will start GloUrbEE\n",
    "metrics_uuid = metrics.asset_uuid\n",
    "print(metrics_uuid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace this uuid with the output of the previous cell to keep trace of your current work\n",
    "metrics_uuid = 'b2ed1f33cec1493099d557aeff34812c'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Retrieve metrics dataset with UUID\n",
    "metrics = assets_management.MetricsDataset(parent_zones=extraction_zones,\n",
    "                                           asset_uuid=metrics_uuid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check your metrics calculation state\n",
    "metrics.update_gee_state()\n",
    "metrics.gee_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ee.data.lismetrics.gee_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# More details on calculation tasks\n",
    "metrics.update_gee_state()\n",
    "metrics.linked_tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len([t for t in metrics.linked_tasks if t['metadata']['state'] == 'SUCCEEDED'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cancel calculation tasks if needed\n",
    "metrics.cancel_linked_tasks()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Delete the metrics dataset from GEE\n",
    "# WARNING: non reversible operation\n",
    "metrics.delete()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export results\n",
    "When all the computation tasks are complete, use the follwing to merge the result into one file and download it locally."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics.download(output_file='./data/lhasa_2km_metrics.csv',\n",
    "                 overwrite=False # By setting this to False, download will resume at his previous step if it was interrupted\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glourbee import zones_indicators\n",
    "from urllib.request import urlretrieve, urlopen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "    indics = []\n",
    "\n",
    "    for asset in [ee.FeatureCollection(asset['id']) for asset in extraction_zones.gee_assets]:\n",
    "        unit_indics = zones_indicators.calculateGSWindicators(asset)\n",
    "        unit_df = pd.read_csv(StringIO(urlopen(unit_indics.getDownloadUrl()).read().decode('utf-8')))\n",
    "        indics.append(unit_df)\n",
    "        \n",
    "    df = pd.concat(indics)\n",
    "    df.drop(['system:index', '.geo'], axis=1, inplace=True)\n",
    "    df.to_csv(output_csv)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from io import StringIO\n",
    "pd.read_csv(StringIO(indics[1].decode('utf-8')))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "8aeb1f878ba8158bc058a932ecec9da3751cdfd192578fad3204ee3a44e5792c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}