diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..84a21fb --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,283 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "terminal.integrated.env.windows": { + "PYTHONPATH": "/explore/nobackup/people/gtamkin/dev/vhr-toolkit/srlite:${workspaceFolder}" + }, + "configurations": [ + { + "name": "srlite-2.0-rma-baseline", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "-toa_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-target_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-cloudmask_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-bandpairs", + "[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G'], ['red_ccdc', 'BAND-R'], ['nir_ccdc', 'BAND-N'],['blue_ccdc', 'BAND-C'], ['green_ccdc', 'BAND-Y'], ['red_ccdc', 'BAND-RE'], ['nir_ccdc', 'BAND-N2']]", + "-output_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240321_2.0-rma-baseline_VHR", + "--regressor", + "rma", + "--debug", + "1", + "--pmask", + "--cloudmask", + "--csv", + "--band8" + ] + }, + { + "name": "srlite-2.0-rma-baseline-BAND-B", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "-toa_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-target_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-cloudmask_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-bandpairs", + "[['blue_ccdc', 'BAND-B']]", + "-output_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240314-BAND-B", + "--regressor", + "rma", + "--debug", + "1", + "--pmask", + "--cloudmask", + ] + }, + { + "name": "srlite-2.0-rma-baseline-err", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "-toa_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-target_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-cloudmask_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-bandpairs", + "[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G'], ['red_ccdc', 'BAND-R'], ['nir_ccdc', 'BAND-N'],['blue_ccdc', 'BAND-C'], ['green_ccdc', 'BAND-Y'], ['red_ccdc', 'BAND-RE'], ['nir_ccdc', 'BAND-N2']]", + "-output_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240314-err", + "--regressor", + "rma", + "--debug", + "1", + "--pmask", + "--cloudmask", + "--csv", + "--band8", + "--err" + ] + }, + { + "name": "srlite-2.0-rma-baseline-no-bandpairs", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "-toa_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-target_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-cloudmask_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-output_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240314-no-bandpairs", + "--regressor", + "rma", + "--debug", + "1", + "--pmask", + "--cloudmask", + "--csv", + "--band8" + ] + }, + { + "name": "srlite-2.0-rma-baseline-BAND-bad", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "-toa_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-target_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-cloudmask_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-bandpairs", + "[['blue_ccdc', 'BAND-bad']]", + "-output_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240313-BAND-bad-err", + "--regressor", + "rma", + "--debug", + "1", + "--pmask", + "--cloudmask", + "--csv", + "--band8" + ] + }, + { + "name": "srlite-2.0-rma-baseline-BAND-B-clean", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "-toa_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-target_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-cloudmask_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-bandpairs", + "[['blue_ccdc', 'BAND-B']]", + "-output_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240314-BAND-B-clean", + "--regressor", + "rma", + "--debug", + "1", + "--pmask", + "--cloudmask", + "--clean", + ] + }, + { + "name": "srlite-GI#2_SRLite_running_out_of_memory_err", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "-toa_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#2_SRLite_running_out_of_memory", + "-target_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#2_SRLite_running_out_of_memory", + "-cloudmask_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#2_SRLite_running_out_of_memory", + "-bandpairs", + "[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G'], ['red_ccdc', 'BAND-R'], ['nir_ccdc', 'BAND-N'],['blue_ccdc', 'BAND-C'], ['green_ccdc', 'BAND-Y'], ['red_ccdc', 'BAND-RE'], ['nir_ccdc', 'BAND-N2']]", + "-output_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#2_SRLite_running_out_of_memory/20240319", + "--regressor", + "rma", + "--debug", + "1", + "--pmask", + "--cloudmask", + "--csv", + "--clean", + "--band8", + "--cloudmask_suffix", + "toa.cloudmask.tif", + "--target_suffix", + "ccdc.tif" + ] + }, + { + "name": "srlite-GI#15_Streamline_to_remove_duplicate_warp()_calls", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "-toa_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-target_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-cloudmask_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-bandpairs", + "[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G'], ['red_ccdc', 'BAND-R'], ['nir_ccdc', 'BAND-N'],['blue_ccdc', 'BAND-C'], ['green_ccdc', 'BAND-Y'], ['red_ccdc', 'BAND-RE'], ['nir_ccdc', 'BAND-N2']]", + "-output_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#15_Streamline_to_remove_duplicate_warp_calls/20240321", + "--regressor", + "rma", + "--debug", + "1", + "--pmask", + "--cloudmask", + "--csv", + "--band8" + ] + }, + { + "name": "srlite-GI#16_Parameterize_COG_flag_appropriately", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "-toa_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-target_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-cloudmask_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-bandpairs", + "[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G']]", + "-output_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#16_Parameterize_COG_flag_appropriately/20240319", + "--regressor", + "rma", + "--debug", + "1", + "--pmask", + "--cloudmask", + "--band8", + "--clean", + // "--noncog" + ] + }, + { + "name": "srlite-GI#23_Generalize_CSV_logic", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "args": [ + "-toa_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-target_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-cloudmask_dir", + "/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline", + "-bandpairs", + "[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G']]", + "-output_dir", + "/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#23_Generalize_CSV_logic/20240319", + "--regressor", + "rma", + "--debug", + "1", + "--pmask", + "--cloudmask", + "--csv", + // "--band8" + ] + } + ] +} \ No newline at end of file diff --git a/srlite/model/RasterLib.py b/srlite/model/RasterLib.py index 4b33fd9..66ad613 100644 --- a/srlite/model/RasterLib.py +++ b/srlite/model/RasterLib.py @@ -336,10 +336,10 @@ def getReprojection(self, context): self._validateParms(context, [Context.FN_LIST, Context.FN_REPROJECTION_LIST, Context.TARGET_FN, Context.TARGET_SAMPLING_METHOD]) - # Reproject inputs to TOA attributes (res, extent, srs, nodata) + # Ensure equivalent NoDataValues dst_ndv = self.alignNoDataValues(context) - + # Reproject inputs to TOA attributes (res, extent, srs, nodata) src_fn = context[Context.FN_REPROJECTION_LIST] warp_ds_list = [] for fn in src_fn: @@ -570,17 +570,21 @@ def calculateStatistics(self, context): NIR1Slope = sr_metrics_list['slope'][3] NIR1Intercept = sr_metrics_list['intercept'][3] - # Read CCDC, SRLite, and Cloud images - ccdcImage = os.path.join(context[Context.FN_TARGET]) + # Reproject newly minted sr-lite output for status calculations evhrSrliteImage = os.path.join(context[Context.FN_COG]) - cloudImage = os.path.join(context[Context.FN_CLOUDMASK]) - - # _ndv=-9999 - _ndv= context[Context.TARGET_NODATA_VALUE] - fn_list = [ccdcImage, evhrSrliteImage, cloudImage] - warp_ds_list = warplib.memwarp_multi_fn(fn_list, res=30, extent=evhrSrliteImage, - t_srs=evhrSrliteImage, r='average', dst_ndv=_ndv) - + fn_list = [evhrSrliteImage] + warp_sr_ds_list = warplib.memwarp_multi_fn(fn_list, + res=30, + extent=evhrSrliteImage, + t_srs=evhrSrliteImage, + r='average', + dst_ndv=context[Context.TARGET_NODATA_VALUE]) + + # Aggregate warped CCDC and Cloudmask from initialization with reprojected SR sandwiched between + warp_ds_list = [context[Context.DS_WARP_LIST][0], warp_sr_ds_list[0], + context[Context.DS_WARP_CLOUD_LIST][0]] + + # Generate masked arrays for calculations warp_ds_list_multiband = warp_ds_list[0:2] warp_ma_list_blu = [self.ds_getma(ds, 1) for ds in warp_ds_list_multiband] warp_ma_list_grn = [self.ds_getma(ds, 2) for ds in warp_ds_list_multiband] @@ -604,17 +608,6 @@ def calculateStatistics(self, context): common_mask = malib.common_mask(warp_ma_list_cloudfree) warp_ma_masked_list = [np.ma.array(ma, mask=common_mask) for ma in warp_ma_list_cloudfree] - # Pull out flat masked arrays - ccdc_blu = self.ma2_1d(warp_ma_masked_list[0]) - ccdc_grn = self.ma2_1d(warp_ma_masked_list[1]) - ccdc_red = self.ma2_1d(warp_ma_masked_list[2]) - ccdc_nir = self.ma2_1d(warp_ma_masked_list[3]) - - evhr_srlite_blu = self.ma2_1d(warp_ma_masked_list[4]) - evhr_srlite_grn = self.ma2_1d(warp_ma_masked_list[5]) - evhr_srlite_red = self.ma2_1d(warp_ma_masked_list[6]) - evhr_srlite_nir = self.ma2_1d(warp_ma_masked_list[7]) - # Create a dataframe with the arrays reflect_df = pd.concat([ self.ma2df(warp_ma_masked_list[0], 'CCDC_SR', 'Blue'), diff --git a/srlite/view/SrliteWorkflowCommandLineView.py b/srlite/view/SrliteWorkflowCommandLineView.py index cd1e252..fe3d218 100644 --- a/srlite/view/SrliteWorkflowCommandLineView.py +++ b/srlite/view/SrliteWorkflowCommandLineView.py @@ -75,7 +75,7 @@ def main(): context[Context.LIST_BAND_PAIR_INDICES] = rasterLib.getBandIndices(context) # Reproject (downscale) TOA to CCDC resolution (30m) - use 'average' for resampling method - # Reproject TARGET (CCDC) to remaining attributes of EVHR TOA Downscale (extent, srs, etc.) - use 'average' for resampling method + # Reproject TARGET (CCDC) to remaining attributes of EVHR TOA Downscale (extent, srs, etc.) context[Context.FN_REPROJECTION_LIST] = [str(context[Context.FN_TARGET]), str(context[Context.FN_TOA])] context[Context.TARGET_FN] = str(context[Context.FN_TOA]) context[Context.TARGET_SAMPLING_METHOD] = 'average' @@ -86,6 +86,8 @@ def main(): context[Context.FN_LIST].append(str(context[Context.FN_CLOUDMASK])) context[Context.FN_REPROJECTION_LIST] = [str(context[Context.FN_CLOUDMASK])] context[Context.TARGET_FN] = str(context[Context.FN_TOA]) + + # Reproject to 'mode' sampling for regression context[Context.TARGET_SAMPLING_METHOD] = 'mode' context[Context.DS_WARP_CLOUD_LIST], context[ Context.MA_WARP_CLOUD_LIST] = rasterLib.getReprojection(context)