From 5f0d4cc223d67e383ba7528a517a9510456db7e4 Mon Sep 17 00:00:00 2001 From: jordancaraballo Date: Mon, 18 Dec 2023 23:55:04 -0500 Subject: [PATCH 1/2] Adding changes to support READTHEDOCS --- README.rst | 29 ++++++++++++++++++++++++ docs/Makefile | 20 +++++++++++++++++ docs/conf.py | 51 +++++++++++++++++++++++++++++++++++++++++++ docs/examples.rst | 3 +++ docs/index.rst | 22 +++++++++++++++++++ docs/make.bat | 35 +++++++++++++++++++++++++++++ docs/readme.rst | 1 + docs/requirements.txt | 4 ++++ setup.cfg | 2 +- tools/fix_labels.py | 2 +- tools/validation.py | 2 +- 11 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 README.rst create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/examples.rst create mode 100644 docs/index.rst create mode 100644 docs/make.bat create mode 100644 docs/readme.rst create mode 100644 docs/requirements.txt diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..abc5b9b --- /dev/null +++ b/README.rst @@ -0,0 +1,29 @@ +============= +vhr-cloudmask +============= + +.. image:: https://github.com/nasa-nccs-hpda/vhr-cloudmask/actions/workflows/lint.yml/badge.svg + :target: https://github.com/nasa-nccs-hpda/vhr-cloudmask/actions/workflows/lint.yml + +**Very High-Resolution Cloud Masking Framework** + +* GitHub repo: https://github.com/nasa-nccs-hpda/vhr-cloudmask +* Documentation: https://nasa-nccs-hpda.github.io/vhr-cloudmask + +**Contents** + +- `Objectives`_ +- `Installation`_ +- `Infrastructure`_ +- `Examples`_ +- `Best Practices`_ +- `Contributing`_ +- `Contributors`_ +- `References`_ + +Objectives +------------ + +- Library to process remote sensing imagery using GPU and CPU parallelization +- Machine learning and deep learning cloud segmentation +- Large-scale image inference \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..41c270b --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..a222a88 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,51 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath('..')) + +import vhr_cloudmask + +project = 'vhr-cloudmask' +copyright = '2022, Jordan A. Caraballo-Vega' +author = 'Jordan A. Caraballo-Vega' + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx_autodoc_typehints', + 'jupyter_sphinx.execute', + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", + "sphinx_click.ext", + "sphinx.ext.githubpages", + "nbsphinx", + "autodocsumm" +] + +autodoc_default_options = {"autosummary": True} + +intersphinx_mapping = { + "pyproj": ("https://pyproj4.github.io/pyproj/stable/", None), + "rasterio": ("https://rasterio.readthedocs.io/en/stable/", None), + "xarray": ("http://xarray.pydata.org/en/stable/", None), +} + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +master_doc = "index" +version = release = vhr_cloudmask.__version__ +pygments_style = "sphinx" +todo_include_todos = False +html_theme = 'sphinx_rtd_theme' + +# html_static_path = ['_static/'] + +myst_enable_extensions = [ + "amsmath", + "colon_fence", + "deflist", + "dollarmath", + "html_image", +] +myst_url_schemes = ("http", "https", "mailto") diff --git a/docs/examples.rst b/docs/examples.rst new file mode 100644 index 0000000..6c2cab1 --- /dev/null +++ b/docs/examples.rst @@ -0,0 +1,3 @@ +.. toctree:: + :maxdepth: 2 + :caption: Contents: \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..ae47677 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,22 @@ +.. vhr-cloudmask documentation master file, created by + sphinx-quickstart on Fri Jun 23 11:32:18 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to vhr-cloudmask's documentation! +========================================= + +.. toctree:: + :maxdepth: 4 + :caption: Contents: + + readme + examples + modules + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..b4f380c --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd \ No newline at end of file diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 0000000..6b2b3ec --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1 @@ +.. include:: ../README.rst \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..0134670 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +sphinx +sphinx-autodoc-typehints +jupyter-sphinx +sphinx-rtd-theme \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index dcbeab4..ab79ddf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = vhr-cloudmask -version = attr: vhr-cloudmask.__version__ +version = attr: vhr_cloudmask.__version__ author = Jordan A. Caraballo-Vega author_email = jordan.a.caraballo-vega@nasa.gov url = https://github.com/nasa-nccs-hpda/vhr-cloudmask diff --git a/tools/fix_labels.py b/tools/fix_labels.py index 2a8ffd7..b7161b2 100644 --- a/tools/fix_labels.py +++ b/tools/fix_labels.py @@ -28,7 +28,7 @@ def arr_to_tif(raster_f, segments, out_tif='s.tif', ndval=-10001): nodatavals = src.read_masks(1).astype('int16') # load numpy array if file is given - if type(segments) == str: + if type(segments) == 'str': segments = np.load(segments) segments = segments.astype('int16') diff --git a/tools/validation.py b/tools/validation.py index 9bc0861..555ac65 100644 --- a/tools/validation.py +++ b/tools/validation.py @@ -38,7 +38,7 @@ def arr_to_tif(raster_f, segments, out_tif='s.tif', ndval=-10001): nodatavals = src.read_masks(1).astype('int16') # load numpy array if file is given - if type(segments) == str: + if type(segments) == 'str': segments = np.load(segments) segments = segments.astype('int16') From 277d49a5f996bef6f73ee6514f5b51d190b60910 Mon Sep 17 00:00:00 2001 From: jordancaraballo Date: Mon, 18 Dec 2023 23:56:26 -0500 Subject: [PATCH 2/2] Adding configuration files --- .../vietnam_senegal_ethiopia_clouds.yaml | 11 +++++--- .../production/cloud_mask_alaska_laselva.csv | 18 ++++++------- .../cloud_mask_alaska_senegal_3sl_cas.yaml | 26 ++++++++++++++----- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/projects/cloud_cnn/configs/experiments/ethiopia/vietnam_senegal_ethiopia_clouds.yaml b/projects/cloud_cnn/configs/experiments/ethiopia/vietnam_senegal_ethiopia_clouds.yaml index 937df0c..7f6cbeb 100755 --- a/projects/cloud_cnn/configs/experiments/ethiopia/vietnam_senegal_ethiopia_clouds.yaml +++ b/projects/cloud_cnn/configs/experiments/ethiopia/vietnam_senegal_ethiopia_clouds.yaml @@ -61,10 +61,15 @@ max_epochs: 6000 #inference_regex: '/adapt/nobackup/projects/ilab/projects/Senegal/Tapan_data/Tappan12_WV02_20140925_M1BS_103001003775F400_data.tif' #model_filename: /adapt/nobackup/projects/ilab/projects/VHRCloudMask/Vietnam/cnn/models/23-0.05-vietnam-only-10000norm-binarycross.hdf5 -model_filename: /adapt/nobackup/projects/ilab/projects/VHRCloudMask/Senegal/cnn/models/46-0.04-vietnam-senegal-10000norm-binarycross.hdf5 +#model_filename: /adapt/nobackup/projects/ilab/projects/VHRCloudMask/Senegal/cnn/models/46-0.04-vietnam-senegal-10000norm-binarycross.hdf5 +model_filename: /explore/nobackup/projects/ilab/projects/CloudMask/VHRCloudMask-Paper/Vietnam/cnn/models/22-0.06-vietnam-senegal-128ts-10000norm-binarycross.hdf5 #model_filename: /lscratch/jacaraba/vietnam-senegal-lcluc/clouds-iuoloss/model/42-0.04.hdf5 -inference_regex: '/adapt/nobackup/projects/ilab/projects/VHRCloudMask/Ethiopia/data_fixed/*.tif' -inference_save_dir: /adapt/nobackup/projects/ilab/projects/VHRCloudMask/Ethiopia/cnn/predictions/46-0.04-vietnam-senegal-10000norm-binarycross-0.40thresh +inference_regex_list: + #- '/adapt/nobackup/projects/ilab/projects/VHRCloudMask/Ethiopia/data_fixed/*.tif' + - '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV02_20210601_M1BS_10300100BF816100-toa.tif' + - '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV02_20100331_M1BS_103001000470DA00-toa.tif' + - '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV03_20220204_M1BS_1040010073369100-toa.tif' +inference_save_dir: /explore/nobackup/projects/ilab/projects/VHRCloudMask/Ethiopia/cnn/predictions/46-0.04-vietnam-senegal-10000norm-binarycross-0.40thresh #inference_save_dir: /adapt/nobackup/projects/ilab/projects/VHRCloudMask/Ethiopia/cnn/predictions/42-0.04-vietnam-senegal-128ts-10000norm-tversky window_size: 8192 diff --git a/projects/cloud_cnn/configs/production/cloud_mask_alaska_laselva.csv b/projects/cloud_cnn/configs/production/cloud_mask_alaska_laselva.csv index 07cb428..1348619 100644 --- a/projects/cloud_cnn/configs/production/cloud_mask_alaska_laselva.csv +++ b/projects/cloud_cnn/configs/production/cloud_mask_alaska_laselva.csv @@ -1,10 +1,10 @@ data,label,ntiles -/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20100310_M1BS_1050410004191E00-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20100310_M1BS_1050410004191E00-toa.cloudmask.v1.2.tif,1000 -/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20150621_M1BS_10504100130C2700-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20150621_M1BS_10504100130C2700-toa.cloudmask.v1.2.tif,1000 -/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160531_M1BS_10500100049D6600-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160531_M1BS_10500100049D6600-toa.cloudmask.v1.2.tif,1000 -/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160531_M1BS_10500100049D6800-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160531_M1BS_10500100049D6800-toa.cloudmask.v1.2.tif,1000 -/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160608_M1BS_1050010004BA6D00-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160608_M1BS_1050010004BA6D00-toa.cloudmask.v1.2.tif,1000 -/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20210214_M1BS_10500100227B0500-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20210214_M1BS_10500100227B0500-toa.cloudmask.v1.2.tif,1000 -/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20020606_M1BS_1010010000918F00-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20020606_M1BS_1010010000918F00-toa.cloudmask.v1.2.tif,1000 -/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20070419_M1BS_1010010005952200-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20070419_M1BS_1010010005952200-toa.cloudmask.v1.2.tif,1000 -/adapt/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20080408_M1BS_1010010007E4C300-toa.tif,/adapt/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20080408_M1BS_1010010007E4C300-toa.cloudmask.v1.2.tif,1000 \ No newline at end of file +/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20100310_M1BS_1050410004191E00-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20100310_M1BS_1050410004191E00-toa.cloudmask.v1.2.tif,1000 +/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20150621_M1BS_10504100130C2700-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20150621_M1BS_10504100130C2700-toa.cloudmask.v1.2.tif,1000 +/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160531_M1BS_10500100049D6600-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160531_M1BS_10500100049D6600-toa.cloudmask.v1.2.tif,1000 +/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160531_M1BS_10500100049D6800-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160531_M1BS_10500100049D6800-toa.cloudmask.v1.2.tif,1000 +/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20160608_M1BS_1050010004BA6D00-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20160608_M1BS_1050010004BA6D00-toa.cloudmask.v1.2.tif,1000 +/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/GE01_20210214_M1BS_10500100227B0500-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/GE01_20210214_M1BS_10500100227B0500-toa.cloudmask.v1.2.tif,1000 +/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20020606_M1BS_1010010000918F00-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20020606_M1BS_1010010000918F00-toa.cloudmask.v1.2.tif,1000 +/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20070419_M1BS_1010010005952200-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20070419_M1BS_1010010005952200-toa.cloudmask.v1.2.tif,1000 +/explore/nobackup/projects/ilab/projects/srlite/input/Laselva/QB02_20080408_M1BS_1010010007E4C300-toa.tif,/explore/nobackup/projects/ilab/projects/CloudMask/products/srlite/v1.2/Laselva/QB02_20080408_M1BS_1010010007E4C300-toa.cloudmask.v1.2.tif,1000 \ No newline at end of file diff --git a/projects/cloud_cnn/configs/production/cloud_mask_alaska_senegal_3sl_cas.yaml b/projects/cloud_cnn/configs/production/cloud_mask_alaska_senegal_3sl_cas.yaml index 2987e62..d26a9e8 100644 --- a/projects/cloud_cnn/configs/production/cloud_mask_alaska_senegal_3sl_cas.yaml +++ b/projects/cloud_cnn/configs/production/cloud_mask_alaska_senegal_3sl_cas.yaml @@ -2,7 +2,7 @@ experiment_name: cloudmask-alaska experiment_type: cloudmask data_dir: /lscratch/jacaraba/srlite/clouds-binary-alaska -model_dir: /adapt/nobackup/projects/ilab/projects/CloudMask/development/alaska-norm +model_dir: /explore/nobackup/projects/ilab/projects/CloudMask/development/alaska-norm seed: 42 gpu_devices: 0,1,2,3 @@ -81,15 +81,18 @@ callbacks: - "tf.keras.callbacks.TerminateOnNaN()" # window_size: 8192 -window_size: 5000 -pred_batch_size: 64 -inference_overlap: 2 +window_size: 10000 +pred_batch_size: 32 +inference_overlap: 0.50 inference_treshold: 0.50 +probability_map: False # Prediction location #model_filename: /adapt/nobackup/projects/ilab/projects/CloudMask/development/alaska/cloudmask-alaska36-0.02.hdf5 #model_filename: /adapt/nobackup/projects/ilab/projects/CloudMask/development/alaska-std/cloudmask-alaska89-0.01.hdf5 -model_filename: /adapt/nobackup/projects/ilab/projects/CloudMask/development/alaska-norm/cloudmask-alaska25-0.02.hdf5 +#model_filename: /explore/nobackup/projects/ilab/projects/CloudMask/development/alaska-norm/cloudmask-alaska25-0.02.hdf5 +#model_filename: /explore/nobackup/projects/ilab/projects/CloudMask/VHRCloudMask-Paper/Vietnam/cnn/models/23-0.05-vietnam-only-10000norm-binarycross.hdf5 +model_filename: /explore/nobackup/projects/ilab/projects/CloudMask/VHRCloudMask-Paper/Senegal/cnn/models/46-0.04-vietnam-senegal-10000norm-binarycross.hdf5 #inference_regex: '/adapt/nobackup/people/mwooten3/Senegal_LCLUC/testForMark/5-toas/WV02_20101020_M1BS_1030010007BBFA00-toa.tif' #inference_regex: '/adapt/nobackup/projects/ilab/projects/srlite/input/TOA_v4/ahri_plus_p1_UTM3N/5-toas/*.tif' #inference_save_dir: /adapt/nobackup/projects/ilab/projects/Vietnam/Jordan/VIETNAM_PRIORITY/clouds-binary-tensorflow-2022-05-02-senegal @@ -99,5 +102,14 @@ model_filename: /adapt/nobackup/projects/ilab/projects/CloudMask/development/ala #inference_regex: '/adapt/nobackup/projects/ilab/projects/srlite/input/TOA_v2/Siberia/5-toas/WV02_20130218_M1BS_103001001F5AEF00-toa.tif' #inference_save_dir: /adapt/nobackup/projects/ilab/projects/Vietnam/Jordan/VIETNAM_PRIORITY/clouds-binary-tensorflow-2022-05-02-siberia #inference_regex: '/adapt/nobackup/projects/ilab/projects/srlite/input/YukonDelta/*-toa.tif' -inference_regex: '/adapt/nobackup/projects/3sl/data/VHR/CAS/M1BS/*M1BS*-toa.tif' -inference_save_dir: '/adapt/nobackup/projects/ilab/projects/CloudMask/products/3sl/v1/CAS' +inference_regex_list: + #- '/adapt/nobackup/projects/3sl/data/VHR/CAS/M1BS/*M1BS*-toa.tif' + #- '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV02_20210601_M1BS_10300100BF816100-toa.tif' + #- '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV02_20100331_M1BS_103001000470DA00-toa.tif' + #- '/explore/nobackup/projects/hls/EVHR/Amhara-MS/WV03_20220204_M1BS_1040010073369100-toa.tif' + #- '/explore/nobackup/projects/hls/EVHR/Amhara-MS/*-toa.tif' + #- '/explore/nobackup/people/ameddens/1_SmallSat/2_Neigh_API/cloud_mask_img/*-toa.tif' + - '/explore/nobackup/projects/3sl/data/VHR/CAS/M1BS/*-toa.tif' + - '/explore/nobackup/projects/3sl/data/VHR/ETZ/M1BS/*-toa.tif' + - '/explore/nobackup/projects/3sl/data/VHR/SRV/M1BS/*-toa.tif' +inference_save_dir: '/explore/nobackup/projects/ilab/projects/CloudMask/products/3sl/v2'