Skip to content

Commit

Permalink
srlite-GI#15_Streamline_to_remove_duplicate_warp()_calls
Browse files Browse the repository at this point in the history
  • Loading branch information
gtamkin authored and gtamkin committed Mar 21, 2024
1 parent 76e27d5 commit 23a965e
Show file tree
Hide file tree
Showing 3 changed files with 302 additions and 24 deletions.
283 changes: 283 additions & 0 deletions .vscode/launch.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,283 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"terminal.integrated.env.windows": {
"PYTHONPATH": "/explore/nobackup/people/gtamkin/dev/vhr-toolkit/srlite:${workspaceFolder}"
},
"configurations": [
{
"name": "srlite-2.0-rma-baseline",
"type": "debugpy",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"-toa_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-target_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-cloudmask_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-bandpairs",
"[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G'], ['red_ccdc', 'BAND-R'], ['nir_ccdc', 'BAND-N'],['blue_ccdc', 'BAND-C'], ['green_ccdc', 'BAND-Y'], ['red_ccdc', 'BAND-RE'], ['nir_ccdc', 'BAND-N2']]",
"-output_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240321_2.0-rma-baseline_VHR",
"--regressor",
"rma",
"--debug",
"1",
"--pmask",
"--cloudmask",
"--csv",
"--band8"
]
},
{
"name": "srlite-2.0-rma-baseline-BAND-B",
"type": "debugpy",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"-toa_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-target_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-cloudmask_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-bandpairs",
"[['blue_ccdc', 'BAND-B']]",
"-output_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240314-BAND-B",
"--regressor",
"rma",
"--debug",
"1",
"--pmask",
"--cloudmask",
]
},
{
"name": "srlite-2.0-rma-baseline-err",
"type": "debugpy",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"-toa_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-target_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-cloudmask_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-bandpairs",
"[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G'], ['red_ccdc', 'BAND-R'], ['nir_ccdc', 'BAND-N'],['blue_ccdc', 'BAND-C'], ['green_ccdc', 'BAND-Y'], ['red_ccdc', 'BAND-RE'], ['nir_ccdc', 'BAND-N2']]",
"-output_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240314-err",
"--regressor",
"rma",
"--debug",
"1",
"--pmask",
"--cloudmask",
"--csv",
"--band8",
"--err"
]
},
{
"name": "srlite-2.0-rma-baseline-no-bandpairs",
"type": "debugpy",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"-toa_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-target_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-cloudmask_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-output_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240314-no-bandpairs",
"--regressor",
"rma",
"--debug",
"1",
"--pmask",
"--cloudmask",
"--csv",
"--band8"
]
},
{
"name": "srlite-2.0-rma-baseline-BAND-bad",
"type": "debugpy",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"-toa_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-target_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-cloudmask_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-bandpairs",
"[['blue_ccdc', 'BAND-bad']]",
"-output_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240313-BAND-bad-err",
"--regressor",
"rma",
"--debug",
"1",
"--pmask",
"--cloudmask",
"--csv",
"--band8"
]
},
{
"name": "srlite-2.0-rma-baseline-BAND-B-clean",
"type": "debugpy",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"-toa_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-target_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-cloudmask_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-bandpairs",
"[['blue_ccdc', 'BAND-B']]",
"-output_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-2.0-rma-baseline/20240314-BAND-B-clean",
"--regressor",
"rma",
"--debug",
"1",
"--pmask",
"--cloudmask",
"--clean",
]
},
{
"name": "srlite-GI#2_SRLite_running_out_of_memory_err",
"type": "debugpy",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"-toa_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#2_SRLite_running_out_of_memory",
"-target_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#2_SRLite_running_out_of_memory",
"-cloudmask_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#2_SRLite_running_out_of_memory",
"-bandpairs",
"[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G'], ['red_ccdc', 'BAND-R'], ['nir_ccdc', 'BAND-N'],['blue_ccdc', 'BAND-C'], ['green_ccdc', 'BAND-Y'], ['red_ccdc', 'BAND-RE'], ['nir_ccdc', 'BAND-N2']]",
"-output_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#2_SRLite_running_out_of_memory/20240319",
"--regressor",
"rma",
"--debug",
"1",
"--pmask",
"--cloudmask",
"--csv",
"--clean",
"--band8",
"--cloudmask_suffix",
"toa.cloudmask.tif",
"--target_suffix",
"ccdc.tif"
]
},
{
"name": "srlite-GI#15_Streamline_to_remove_duplicate_warp()_calls",
"type": "debugpy",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"-toa_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-target_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-cloudmask_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-bandpairs",
"[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G'], ['red_ccdc', 'BAND-R'], ['nir_ccdc', 'BAND-N'],['blue_ccdc', 'BAND-C'], ['green_ccdc', 'BAND-Y'], ['red_ccdc', 'BAND-RE'], ['nir_ccdc', 'BAND-N2']]",
"-output_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#15_Streamline_to_remove_duplicate_warp_calls/20240321",
"--regressor",
"rma",
"--debug",
"1",
"--pmask",
"--cloudmask",
"--csv",
"--band8"
]
},
{
"name": "srlite-GI#16_Parameterize_COG_flag_appropriately",
"type": "debugpy",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"-toa_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-target_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-cloudmask_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-bandpairs",
"[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G']]",
"-output_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#16_Parameterize_COG_flag_appropriately/20240319",
"--regressor",
"rma",
"--debug",
"1",
"--pmask",
"--cloudmask",
"--band8",
"--clean",
// "--noncog"
]
},
{
"name": "srlite-GI#23_Generalize_CSV_logic",
"type": "debugpy",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"args": [
"-toa_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-target_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-cloudmask_dir",
"/panfs/ccds02/nobackup/people/iluser/projects/srlite/test/input/baseline",
"-bandpairs",
"[['blue_ccdc', 'BAND-B'], ['green_ccdc', 'BAND-G']]",
"-output_dir",
"/explore/nobackup/people/gtamkin/dev/srlite/test/srlite-GI#23_Generalize_CSV_logic/20240319",
"--regressor",
"rma",
"--debug",
"1",
"--pmask",
"--cloudmask",
"--csv",
// "--band8"
]
}
]
}
39 changes: 16 additions & 23 deletions srlite/model/RasterLib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,10 +336,10 @@ def getReprojection(self, context):
self._validateParms(context, [Context.FN_LIST, Context.FN_REPROJECTION_LIST, Context.TARGET_FN,
Context.TARGET_SAMPLING_METHOD])

# Reproject inputs to TOA attributes (res, extent, srs, nodata)
# Ensure equivalent NoDataValues
dst_ndv = self.alignNoDataValues(context)


# Reproject inputs to TOA attributes (res, extent, srs, nodata)
src_fn = context[Context.FN_REPROJECTION_LIST]
warp_ds_list = []
for fn in src_fn:
Expand Down Expand Up @@ -570,17 +570,21 @@ def calculateStatistics(self, context):
NIR1Slope = sr_metrics_list['slope'][3]
NIR1Intercept = sr_metrics_list['intercept'][3]

# Read CCDC, SRLite, and Cloud images
ccdcImage = os.path.join(context[Context.FN_TARGET])
# Reproject newly minted sr-lite output for status calculations
evhrSrliteImage = os.path.join(context[Context.FN_COG])
cloudImage = os.path.join(context[Context.FN_CLOUDMASK])

# _ndv=-9999
_ndv= context[Context.TARGET_NODATA_VALUE]
fn_list = [ccdcImage, evhrSrliteImage, cloudImage]
warp_ds_list = warplib.memwarp_multi_fn(fn_list, res=30, extent=evhrSrliteImage,
t_srs=evhrSrliteImage, r='average', dst_ndv=_ndv)

fn_list = [evhrSrliteImage]
warp_sr_ds_list = warplib.memwarp_multi_fn(fn_list,
res=30,
extent=evhrSrliteImage,
t_srs=evhrSrliteImage,
r='average',
dst_ndv=context[Context.TARGET_NODATA_VALUE])

# Aggregate warped CCDC and Cloudmask from initialization with reprojected SR sandwiched between
warp_ds_list = [context[Context.DS_WARP_LIST][0], warp_sr_ds_list[0],
context[Context.DS_WARP_CLOUD_LIST][0]]

# Generate masked arrays for calculations
warp_ds_list_multiband = warp_ds_list[0:2]
warp_ma_list_blu = [self.ds_getma(ds, 1) for ds in warp_ds_list_multiband]
warp_ma_list_grn = [self.ds_getma(ds, 2) for ds in warp_ds_list_multiband]
Expand All @@ -604,17 +608,6 @@ def calculateStatistics(self, context):
common_mask = malib.common_mask(warp_ma_list_cloudfree)
warp_ma_masked_list = [np.ma.array(ma, mask=common_mask) for ma in warp_ma_list_cloudfree]

# Pull out flat masked arrays
ccdc_blu = self.ma2_1d(warp_ma_masked_list[0])
ccdc_grn = self.ma2_1d(warp_ma_masked_list[1])
ccdc_red = self.ma2_1d(warp_ma_masked_list[2])
ccdc_nir = self.ma2_1d(warp_ma_masked_list[3])

evhr_srlite_blu = self.ma2_1d(warp_ma_masked_list[4])
evhr_srlite_grn = self.ma2_1d(warp_ma_masked_list[5])
evhr_srlite_red = self.ma2_1d(warp_ma_masked_list[6])
evhr_srlite_nir = self.ma2_1d(warp_ma_masked_list[7])

# Create a dataframe with the arrays
reflect_df = pd.concat([
self.ma2df(warp_ma_masked_list[0], 'CCDC_SR', 'Blue'),
Expand Down
4 changes: 3 additions & 1 deletion srlite/view/SrliteWorkflowCommandLineView.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def main():
context[Context.LIST_BAND_PAIR_INDICES] = rasterLib.getBandIndices(context)

# Reproject (downscale) TOA to CCDC resolution (30m) - use 'average' for resampling method
# Reproject TARGET (CCDC) to remaining attributes of EVHR TOA Downscale (extent, srs, etc.) - use 'average' for resampling method
# Reproject TARGET (CCDC) to remaining attributes of EVHR TOA Downscale (extent, srs, etc.)
context[Context.FN_REPROJECTION_LIST] = [str(context[Context.FN_TARGET]), str(context[Context.FN_TOA])]
context[Context.TARGET_FN] = str(context[Context.FN_TOA])
context[Context.TARGET_SAMPLING_METHOD] = 'average'
Expand All @@ -86,6 +86,8 @@ def main():
context[Context.FN_LIST].append(str(context[Context.FN_CLOUDMASK]))
context[Context.FN_REPROJECTION_LIST] = [str(context[Context.FN_CLOUDMASK])]
context[Context.TARGET_FN] = str(context[Context.FN_TOA])

# Reproject to 'mode' sampling for regression
context[Context.TARGET_SAMPLING_METHOD] = 'mode'
context[Context.DS_WARP_CLOUD_LIST], context[
Context.MA_WARP_CLOUD_LIST] = rasterLib.getReprojection(context)
Expand Down

0 comments on commit 23a965e

Please sign in to comment.