add docs

bugfixes
update notebooks
add paper

aces/model_trainer.py

@@ -33,6 +33,8 @@ def __init__(self, config: Config, use_seed: bool = True, seed: int = 42):
 
         Args:
             config: An object containing the configuration settings for model training.
+            use_seed: A flag indicating whether to use a seed for reproducibility (needs fixing).
+            seed: The seed value to use for reproducibility.
 
         Attributes:
             config: The configuration settings for model training.
@@ -148,7 +150,8 @@ def prepare_output_dir(self) -> None:
         if not self.config.AUTO_MODEL_DIR_NAME:
             self.config.MODEL_SAVE_DIR = self.config.OUTPUT_DIR / self.config.MODEL_DIR_NAME
             print(f"> Saving models and results at {self.config.MODEL_SAVE_DIR}...")
-            os.mkdir(self.config.MODEL_SAVE_DIR)
+            if not os.path.exists(self.config.MODEL_SAVE_DIR):
+                os.mkdir(self.config.MODEL_SAVE_DIR)
         else:
             today = datetime.date.today().strftime("%Y_%m_%d")
             iterator = 1

paper/paper.bib

@@ -0,0 +1,80 @@
+@article{Gorelick2017,
+  title     = "{Google Earth Engine: Planetary-scale geospatial analysis for
+               everyone}",
+  author    = "Gorelick, Noel and Hancher, Matt and Dixon, Mike and
+               Ilyushchenko, Simon and Thau, David and Moore, Rebecca",
+  journal   = "Remote Sensing of Environment",
+  publisher = "Elsevier",
+  volume    =  202,
+  pages     = "18--27",
+  month     =  dec,
+  year      =  2017,
+  url       = "http://www.sciencedirect.com/science/article/pii/S0034425717302900",
+  keywords  = "Cloud computing; Big data; Analysis; Platform; Data
+               democratization; Earth Engine",
+  issn      = "0034-4257",
+  doi       = "10.1016/j.rse.2017.06.031"
+}
+
+@article{Mayer2021,
+  title={Deep learning approach for Sentinel-1 surface water mapping leveraging Google Earth Engine},
+  author={Mayer, Timothy and Poortinga, Ate and Bhandari, Biplov and Nicolau, Andrea P and Markert, Kel and Thwal, Nyein Soe and Markert, Amanda and Haag, Arjen and Kilbride, John and Chishtie, Farrukh and others},
+  journal={ISPRS Open Journal of Photogrammetry and Remote Sensing},
+  volume={2},
+  pages={100005},
+  year={2021},
+  publisher={Elsevier}
+}
+
+@article{Mayer2023,
+  title={Employing the agricultural classification and estimation service (ACES) for mapping smallholder rice farms in Bhutan},
+  author={Mayer, Timothy and Bhandari, Biplov and Mart{\'\i}nez, Filoteo G{\'o}mez and Walker, Kaitlin and Jim{\'e}nez, Stephanie A and Kruskopf, Meryl and Maganini, Micky and Phalke, Aparna and Wangchen, Tshering and Phuntsho, Loday and others},
+  journal={ISPRS Open Journal of Photogrammetry and Remote Sensing},
+  volume={11},
+  pages={1137835},
+  year={2023},
+  publisher={Frontiers}
+}
+
+@article{Bhandari2024,
+  title={Comparing Deep Learning Models for Rice Mapping in Bhutan using High Resolution Satellite Imagery},
+  author={Bhandari, Biplov and Mayer, Timothy},
+  journal={Frontiers in Environmental Science},
+  volume={to appear},
+  pages={to appear},
+  year={2024},
+  publisher={Elsevier},
+  note={under review}
+}
+
+@article{Bakkestuen2023,
+  title={Delineation of Wetland Areas in South Norway from Sentinel-2 Imagery and LiDAR Using TensorFlow, U-Net, and Google Earth Engine},
+  author={Bakkestuen, Vegar and Venter, Zander and Ganer{\o}d, Alexandra Jarna and Framstad, Erik},
+  journal={Remote Sensing},
+  volume={15},
+  number={5},
+  pages={1203},
+  year={2023},
+  publisher={MDPI}
+}
+
+@article{Poortinga2021,
+  title={Mapping sugarcane in Thailand using transfer learning, a lightweight convolutional neural network, NICFI high resolution satellite imagery and Google Earth Engine},
+  author={Poortinga, Ate and Thwal, Nyein Soe and Khanal, Nishanta and Mayer, Timothy and Bhandari, Biplov and Markert, Kel and Nicolau, Andrea P and Dilger, John and Tenneson, Karis and Clinton, Nicholas and others},
+  journal={ISPRS Open Journal of Photogrammetry and Remote Sensing},
+  volume={1},
+  pages={100003},
+  year={2021},
+  publisher={Elsevier}
+}
+
+@article{Parekh2021,
+  title={Automatic detection of impervious surfaces from remotely sensed data using deep learning},
+  author={Parekh, Jash R and Poortinga, Ate and Bhandari, Biplov and Mayer, Timothy and Saah, David and Chishtie, Farrukh},
+  journal={Remote Sensing},
+  volume={13},
+  number={16},
+  pages={3166},
+  year={2021},
+  publisher={MDPI}
+}

paper/paper.md

@@ -0,0 +1,63 @@
+---
+title: "servir-aces: A Python Package for Training Machine Learning Models for Remote Sensing Applications"
+tags:
+    - Python
+    - Remote Sensing
+    - Crop Mapping
+    - Machine Learning
+    - Tensorflow
+    - Google Earth Engine
+    - mapping
+    - Jupyter notebook
+authors:
+    - name: Biplov Bhandari
+      orcid: 0000-0001-6169-8236
+      affiliation: "1, 2"
+    - name: Timothy Mayer
+      orcid: 0000-0001-9489-9392
+      affiliation: "1, 2"
+affiliations:
+    - name: Earth System Science Center, The University of Alabama in Huntsville, 320 Sparkman Drive, Huntsville, AL 35805, USA
+      index: 1
+    - name: SERVIR Science Coordination Office, NASA Marshall Space Flight Center, 320 Sparkman Drive, Huntsville, AL 35805, USA
+      index: 2
+date: 16 April 2024
+bibliography: paper.bib
+---
+
+# Summary
+
+**`servir-aces`** (ACES for Agricultural Classification and Estimation Service) is a Python package for generating training data using highly parallelized [apache-beam](https://beam.apache.org/) and [Google Earth Engine](https://earthengine.google.com/) [@Gorelick2017] workflows as well as for training various Machine Learning (ML) and Deep Learning (DL) Models for Remote Sensing Applications [@Mayer2023], [@Bhandari2024]. With various type of data available via GEE, and the smooth integration between GEE and TensorFlow (TF); models trained in TF can be easily loaded into GEE. This package provides functionalities for data processing, data loading from Earth Engine, feature extraction, and model training. The use of combining TF and GEE for has enabled several large scale ML and DL Remote Sensing Applications. Some of them include Wetland Areas Mapping [@Bakkestuen2023], Crop Type Mapping [@Poortinga2021], Surface Water Mapping [@Mayer2021], and Urban Mapping [@Parekh2021].
+
+Despite prevalent research, specialized technical knowledge is required to setup and run various ML/DL models. Many of the practitioner/ scientist find it difficult to use DL framework. The **`servir-aces`** Python package is created to fill this grap. It makes it easier to export training data and train a DL models using cloud-based technology using GEE. Several examples are provided to make it easier for scientists to use it.
+
+# **`servir-aces`** Audience
+
+**`servir-aces`** is intended for development practitioner, researchers, and student who would like to utilize various freely available Earth Observation (EO) data using cloud-based GEE and TensorFlow ecosystem to perform large scale ML/DL related Remote Sensing Applications.
+
+We also provide several notebook examples to showcase the usage of the **`servir-aces`**. Here we show how **`servir-aces`** can be used for crop-mapping related application. Ideally, the same process can be repeated for any kind of the image segmentation task.
+
+# **`servir-aces`** Functionality
+
+The major high-level functionality of the **servir-aces** packages are:
+- Data loading and processing from GEE.
+- Generation of training data for various Machine Learning (ML) and Deep Learning (DL) Models.
+- Training and evaluation of ML/DL Models.
+- Support for remote sensing feature extraction.
+- Integration with Apache Beam for data processing.
+
+The key functionality of **`servir-aces`** is organized into several modules:
+
+-   [data_processor](https://servir.github.io/servir-aces/data_processor/): this module provides functions for data input/output and preprocessing for the image segmentation project.
+
+-   [model_builder](https://servir.github.io/servir-aces/model_builder/): this module provides functionality for creating and compiling various Neural Network Models, including DNN, CNN, U-Net.
+
+-   [model_trainer](https://servir.github.io/servir-aces/model_trainer/): this module provides functionality for training, buidling, compiling, and running specified deep learning models.
+
+-   [metrics](https://servir.github.io/servir-aces/metrics/): this module provides custom metrics for evaluating model performance and provide utility functions for plotting and visualizing model metrics during training.
+
+-   [ee_utils](https://servir.github.io/servir-aces/ee_utils/): a module for providing utility functions to handle Earth Engine API information and make authenticated requests.
+
+-   [remote_sensing](https://servir.github.io/servir-aces/remote_sensing/): a module that provides various static methods to compute various remote sensing indices and analyse them.
+
+# References