Merge pull request #18 from nasa-nccs-hpda/develop

Develop

README.rst

@@ -0,0 +1,29 @@
+=============
+vhr-cloudmask
+=============
+
+.. image:: https://github.com/nasa-nccs-hpda/vhr-cloudmask/actions/workflows/lint.yml/badge.svg
+        :target: https://github.com/nasa-nccs-hpda/vhr-cloudmask/actions/workflows/lint.yml
+
+**Very High-Resolution Cloud Masking Framework**
+
+* GitHub repo: https://github.com/nasa-nccs-hpda/vhr-cloudmask
+* Documentation: https://nasa-nccs-hpda.github.io/vhr-cloudmask
+
+**Contents**
+
+- `Objectives`_
+- `Installation`_
+- `Infrastructure`_
+- `Examples`_
+- `Best Practices`_
+- `Contributing`_
+- `Contributors`_
+- `References`_
+
+Objectives
+------------
+
+- Library to process remote sensing imagery using GPU and CPU parallelization
+- Machine learning and deep learning cloud segmentation
+- Large-scale image inference

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,51 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('..'))
+
+import vhr_cloudmask
+
+project = 'vhr-cloudmask'
+copyright = '2022, Jordan A. Caraballo-Vega'
+author = 'Jordan A. Caraballo-Vega'
+
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx_autodoc_typehints',
+    'jupyter_sphinx.execute',
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.napoleon",
+    "sphinx_click.ext",
+    "sphinx.ext.githubpages",
+    "nbsphinx",
+    "autodocsumm"
+]
+
+autodoc_default_options = {"autosummary": True}
+
+intersphinx_mapping = {
+    "pyproj": ("https://pyproj4.github.io/pyproj/stable/", None),
+    "rasterio": ("https://rasterio.readthedocs.io/en/stable/", None),
+    "xarray": ("http://xarray.pydata.org/en/stable/", None),
+}
+
+templates_path = ['_templates']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+master_doc = "index"
+version = release = vhr_cloudmask.__version__
+pygments_style = "sphinx"
+todo_include_todos = False
+html_theme = 'sphinx_rtd_theme'
+
+# html_static_path = ['_static/']
+
+myst_enable_extensions = [
+    "amsmath",
+    "colon_fence",
+    "deflist",
+    "dollarmath",
+    "html_image",
+]
+myst_url_schemes = ("http", "https", "mailto")

docs/examples.rst

@@ -0,0 +1,3 @@
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:

docs/index.rst

@@ -0,0 +1,22 @@
+.. vhr-cloudmask documentation master file, created by
+   sphinx-quickstart on Fri Jun 23 11:32:18 2023.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to vhr-cloudmask's documentation!
+=========================================
+
+.. toctree::
+   :maxdepth: 4
+   :caption: Contents:
+
+   readme
+   examples
+   modules
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/readme.rst

@@ -0,0 +1 @@
+.. include:: ../README.rst

docs/requirements.txt

@@ -0,0 +1,4 @@
+sphinx
+sphinx-autodoc-typehints
+jupyter-sphinx
+sphinx-rtd-theme

projects/cloud_cnn/configs/experiments/ethiopia/vietnam_senegal_ethiopia_clouds.yaml

@@ -61,10 +61,15 @@ max_epochs: 6000
 #inference_regex: '/adapt/nobackup/projects/ilab/projects/Senegal/Tapan_data/Tappan12_WV02_20140925_M1BS_103001003775F400_data.tif'
 
 #model_filename: /adapt/nobackup/projects/ilab/projects/VHRCloudMask/Vietnam/cnn/models/23-0.05-vietnam-only-10000norm-binarycross.hdf5
-model_filename: /adapt/nobackup/projects/ilab/projects/VHRCloudMask/Senegal/cnn/models/46-0.04-vietnam-senegal-10000norm-binarycross.hdf5
+#model_filename: /adapt/nobackup/projects/ilab/projects/VHRCloudMask/Senegal/cnn/models/46-0.04-vietnam-senegal-10000norm-binarycross.hdf5
+model_filename: /explore/nobackup/projects/ilab/projects/CloudMask/VHRCloudMask-Paper/Vietnam/cnn/models/22-0.06-vietnam-senegal-128ts-10000norm-binarycross.hdf5
 #model_filename: /lscratch/jacaraba/vietnam-senegal-lcluc/clouds-iuoloss/model/42-0.04.hdf5
-inference_regex: '/adapt/nobackup/projects/ilab/projects/VHRCloudMask/Ethiopia/data_fixed/*.tif'
-inference_save_dir: /adapt/nobackup/projects/ilab/projects/VHRCloudMask/Ethiopia/cnn/predictions/46-0.04-vietnam-senegal-10000norm-binarycross-0.40thresh
+inference_regex_list:
+  #- '/adapt/nobackup/projects/ilab/projects/VHRCloudMask/Ethiopia/data_fixed/*.tif'
+  - '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV02_20210601_M1BS_10300100BF816100-toa.tif'
+  - '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV02_20100331_M1BS_103001000470DA00-toa.tif'
+  - '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV03_20220204_M1BS_1040010073369100-toa.tif'
+inference_save_dir: /explore/nobackup/projects/ilab/projects/VHRCloudMask/Ethiopia/cnn/predictions/46-0.04-vietnam-senegal-10000norm-binarycross-0.40thresh
 #inference_save_dir: /adapt/nobackup/projects/ilab/projects/VHRCloudMask/Ethiopia/cnn/predictions/42-0.04-vietnam-senegal-128ts-10000norm-tversky
 
 window_size: 8192

projects/cloud_cnn/configs/production/cloud_mask_alaska_laselva.csv

@@ -1,10 +1,10 @@
 data,label,ntiles
-/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20100310_M1BS_1050410004191E00-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20100310_M1BS_1050410004191E00-toa.cloudmask.v1.2.tif,1000
-/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20150621_M1BS_10504100130C2700-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20150621_M1BS_10504100130C2700-toa.cloudmask.v1.2.tif,1000
-/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160531_M1BS_10500100049D6600-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160531_M1BS_10500100049D6600-toa.cloudmask.v1.2.tif,1000
-/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160531_M1BS_10500100049D6800-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160531_M1BS_10500100049D6800-toa.cloudmask.v1.2.tif,1000
-/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160608_M1BS_1050010004BA6D00-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160608_M1BS_1050010004BA6D00-toa.cloudmask.v1.2.tif,1000
-/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20210214_M1BS_10500100227B0500-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20210214_M1BS_10500100227B0500-toa.cloudmask.v1.2.tif,1000
-/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20020606_M1BS_1010010000918F00-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20020606_M1BS_1010010000918F00-toa.cloudmask.v1.2.tif,1000
-/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20070419_M1BS_1010010005952200-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20070419_M1BS_1010010005952200-toa.cloudmask.v1.2.tif,1000
-/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20080408_M1BS_1010010007E4C300-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20080408_M1BS_1010010007E4C300-toa.cloudmask.v1.2.tif,1000
+/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20100310_M1BS_1050410004191E00-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20100310_M1BS_1050410004191E00-toa.cloudmask.v1.2.tif,1000
+/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20150621_M1BS_10504100130C2700-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20150621_M1BS_10504100130C2700-toa.cloudmask.v1.2.tif,1000
+/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160531_M1BS_10500100049D6600-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160531_M1BS_10500100049D6600-toa.cloudmask.v1.2.tif,1000
+/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160531_M1BS_10500100049D6800-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160531_M1BS_10500100049D6800-toa.cloudmask.v1.2.tif,1000
+/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160608_M1BS_1050010004BA6D00-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160608_M1BS_1050010004BA6D00-toa.cloudmask.v1.2.tif,1000
+/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20210214_M1BS_10500100227B0500-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20210214_M1BS_10500100227B0500-toa.cloudmask.v1.2.tif,1000
+/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20020606_M1BS_1010010000918F00-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20020606_M1BS_1010010000918F00-toa.cloudmask.v1.2.tif,1000
+/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20070419_M1BS_1010010005952200-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20070419_M1BS_1010010005952200-toa.cloudmask.v1.2.tif,1000
+/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20080408_M1BS_1010010007E4C300-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20080408_M1BS_1010010007E4C300-toa.cloudmask.v1.2.tif,1000

projects/cloud_cnn/configs/production/cloud_mask_alaska_senegal_3sl_cas.yaml

@@ -2,7 +2,7 @@ experiment_name: cloudmask-alaska
 experiment_type: cloudmask
 
 data_dir: /lscratch/jacaraba/srlite/clouds-binary-alaska
-model_dir: /adapt/nobackup/projects/ilab/projects/CloudMask/development/alaska-norm
+model_dir: /explore/nobackup/projects/ilab/projects/CloudMask/development/alaska-norm
 
 seed: 42
 gpu_devices: 0,1,2,3
@@ -81,15 +81,18 @@ callbacks:
   - "tf.keras.callbacks.TerminateOnNaN()"
 
 # window_size: 8192
-window_size: 5000
-pred_batch_size: 64
-inference_overlap: 2
+window_size: 10000
+pred_batch_size: 32
+inference_overlap: 0.50
 inference_treshold: 0.50
+probability_map: False
 
 # Prediction location
 #model_filename: /adapt/nobackup/projects/ilab/projects/CloudMask/development/alaska/cloudmask-alaska36-0.02.hdf5
 #model_filename: /adapt/nobackup/projects/ilab/projects/CloudMask/development/alaska-std/cloudmask-alaska89-0.01.hdf5
-model_filename: /adapt/nobackup/projects/ilab/projects/CloudMask/development/alaska-norm/cloudmask-alaska25-0.02.hdf5
+#model_filename: /explore/nobackup/projects/ilab/projects/CloudMask/development/alaska-norm/cloudmask-alaska25-0.02.hdf5
+#model_filename: /explore/nobackup/projects/ilab/projects/CloudMask/VHRCloudMask-Paper/Vietnam/cnn/models/23-0.05-vietnam-only-10000norm-binarycross.hdf5
+model_filename: /explore/nobackup/projects/ilab/projects/CloudMask/VHRCloudMask-Paper/Senegal/cnn/models/46-0.04-vietnam-senegal-10000norm-binarycross.hdf5
 #inference_regex: '/adapt/nobackup/people/mwooten3/Senegal_LCLUC/testForMark/5-toas/WV02_20101020_M1BS_1030010007BBFA00-toa.tif'
 #inference_regex: '/adapt/nobackup/projects/ilab/projects/srlite/input/TOA_v4/ahri_plus_p1_UTM3N/5-toas/*.tif'
 #inference_save_dir: /adapt/nobackup/projects/ilab/projects/Vietnam/Jordan/VIETNAM_PRIORITY/clouds-binary-tensorflow-2022-05-02-senegal
@@ -99,5 +102,14 @@ model_filename: /adapt/nobackup/projects/ilab/projects/CloudMask/development/ala
 #inference_regex: '/adapt/nobackup/projects/ilab/projects/srlite/input/TOA_v2/Siberia/5-toas/WV02_20130218_M1BS_103001001F5AEF00-toa.tif'
 #inference_save_dir: /adapt/nobackup/projects/ilab/projects/Vietnam/Jordan/VIETNAM_PRIORITY/clouds-binary-tensorflow-2022-05-02-siberia
 #inference_regex: '/adapt/nobackup/projects/ilab/projects/srlite/input/YukonDelta/*-toa.tif'
-inference_regex: '/adapt/nobackup/projects/3sl/data/VHR/CAS/M1BS/*M1BS*-toa.tif'
-inference_save_dir: '/adapt/nobackup/projects/ilab/projects/CloudMask/products/3sl/v1/CAS'
+inference_regex_list:
+  #- '/adapt/nobackup/projects/3sl/data/VHR/CAS/M1BS/*M1BS*-toa.tif'
+  #- '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV02_20210601_M1BS_10300100BF816100-toa.tif'
+  #- '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV02_20100331_M1BS_103001000470DA00-toa.tif'
+  #- '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV03_20220204_M1BS_1040010073369100-toa.tif'
+  #- '/explore/nobackup/projects/hls/EVHR/Amhara-MS/*-toa.tif'
+  #- '/explore/nobackup/people/ameddens/1_SmallSat/2_Neigh_API/cloud_mask_img/*-toa.tif'
+  - '/explore/nobackup/projects/3sl/data/VHR/CAS/M1BS/*-toa.tif'
+  - '/explore/nobackup/projects/3sl/data/VHR/ETZ/M1BS/*-toa.tif'
+  - '/explore/nobackup/projects/3sl/data/VHR/SRV/M1BS/*-toa.tif'
+inference_save_dir: '/explore/nobackup/projects/ilab/projects/CloudMask/products/3sl/v2'

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vhr-cloudmask
-version = attr: vhr-cloudmask.__version__
+version = attr: vhr_cloudmask.__version__
 author = Jordan A. Caraballo-Vega
 author_email = jordan.a.caraballo-vega@nasa.gov
 url = https://github.com/nasa-nccs-hpda/vhr-cloudmask

tools/fix_labels.py

@@ -28,7 +28,7 @@ def arr_to_tif(raster_f, segments, out_tif='s.tif', ndval=-10001):
         nodatavals = src.read_masks(1).astype('int16')
 
     # load numpy array if file is given
-    if type(segments) == str:
+    if type(segments) == 'str':
         segments = np.load(segments)
     segments = segments.astype('int16')
 

tools/validation.py

@@ -38,7 +38,7 @@ def arr_to_tif(raster_f, segments, out_tif='s.tif', ndval=-10001):
         nodatavals = src.read_masks(1).astype('int16')
 
     # load numpy array if file is given
-    if type(segments) == str:
+    if type(segments) == 'str':
         segments = np.load(segments)
     segments = segments.astype('int16')