Skip to content

Commit

Permalink
Merge branch 'release' into 'master'
Browse files Browse the repository at this point in the history
Release

See merge request 3d/demcompare!195
  • Loading branch information
adebardo committed May 29, 2024
2 parents 2e031e2 + 565098c commit 1c96681
Show file tree
Hide file tree
Showing 41 changed files with 451 additions and 468 deletions.
23 changes: 7 additions & 16 deletions demcompare/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def run(
Demcompare RUN execution.
:param json_file_path: Input Json configuration file
:type json_file: str
:type json_file_path: str
:param loglevel: Choose Loglevel (default: WARNING)
:type loglevel: int
"""
Expand Down Expand Up @@ -311,7 +311,7 @@ def load_input_dems(
:param cfg: input configuration
:type cfg: ConfigType
:return: input_ref and input_dem datasets or None
:rtype: Tuple(xr.Dataset, xr.dataset)
:rtype: Tuple(xr.Dataset, xr.Dataset)
The xr.Datasets containing :
- im : 2D (row, col) xarray.DataArray float32
Expand Down Expand Up @@ -403,7 +403,7 @@ def run_coregistration(
- im : 2D (row, col) xarray.DataArray float32
- trans: 1D (trans_len) xarray.DataArray
:return: reproj_coreg_sec, reproj_coreg_ref
:rtype: Tuple(xr.Dataset, xr.dataset)
:rtype: Tuple(xr.Dataset, xr.Dataset)
The xr.Datasets containing :
- im : 2D (row, col) xarray.DataArray float32
Expand All @@ -412,22 +412,13 @@ def run_coregistration(
logging.info("[Coregistration]")
# Create coregistration object
coregistration_ = Coregistration(cfg)
# Compute coregistration to get applicable transformation object
transformation = coregistration_.compute_coregistration(
input_sec, input_ref
)
# Apply coregistration offsets to the original DEM and store it
# reprojection is also done.
coreg_sec = transformation.apply_transform(input_sec)

# Compute coregistration
_ = coregistration_.compute_coregistration(input_sec, input_ref)

# Get coregistration_results dict
coregistration_results = coregistration_.coregistration_results

# Save the coregistered DEM (even without save_optional_outputs option)
# - coreg_SEC.tif -> coregistered sec
save_dem(
coreg_sec,
os.path.join(cfg["output_dir"], "./coreg_SEC.tif"),
)
# Save coregistration_results
save_config_file(
os.path.join(
Expand Down
3 changes: 3 additions & 0 deletions demcompare/classification_layer/slope_classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ def __init__(

# Ranges
self.ranges: List = self.cfg["ranges"]
if len(self.ranges) == 1 and self.ranges[0] != 0:
self.ranges.insert(0, 0)

# Checking configuration during initialisation step
# doesn't require classification layers
if dem is not None:
Expand Down
11 changes: 11 additions & 0 deletions demcompare/coregistration/coregistration_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,11 +311,16 @@ def compute_coregistration(
self.reproj_coreg_ref,
) = self._coregister_dems_algorithm(self.reproj_sec, self.reproj_ref)

# Apply coregistration offsets to the original DEM and store it
# reprojection is also done.
self.coreg_sec = self.transform.apply_transform(sec)

# Compute and store the coregistration_results dict
self.save_results_dict()
# Save internal_dems if the option was chosen
if self.save_optional_outputs:
self.save_internal_outputs()

# Return the transform
return self.transform

Expand Down Expand Up @@ -365,6 +370,7 @@ def save_internal_outputs(self):
coregistered sec
- ./coregistration/reproj_coreg_REF.tif -> reprojected
coregistered ref
- ./coregistration/coreg_sec.tif -> coregistered ref
:return: None
"""
Expand All @@ -388,6 +394,11 @@ def save_internal_outputs(self):
self.reproj_coreg_ref,
os.path.join(self.output_dir, "reproj_coreg_REF.tif"),
)
# Save the coregistered DEM
self.coreg_sec = save_dem(
self.coreg_sec,
os.path.join(self.output_dir, "coreg_SEC.tif"),
)
# Update path on coregistration_results file
if self.coregistration_results:
self.coregistration_results["coregistration_results"][
Expand Down
4 changes: 0 additions & 4 deletions demcompare/demcompare_tiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -258,10 +258,6 @@ def run_tiles(tiles_config, loglevel): # pylint:disable=too-many-locals

os.remove(config)

x_2d = np.flip(x_2d, axis=0)
y_2d = np.flip(y_2d, axis=0)
z_2d = np.flip(z_2d, axis=0)

np.save(output_dir + "/coreg_results_x2D.npy", x_2d)
np.save(output_dir + "/coreg_results_y2D.npy", y_2d)
np.save(output_dir + "/coreg_results_z2D.npy", z_2d)
Expand Down
18 changes: 16 additions & 2 deletions docs/source/faq.rst
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
Frequently Asked Questions
===========================

Installation troubleshootings
*****************************
Installation troubleshooting
****************************

Depending on pip version, installation problems can happen with packages dependencies installation order. Install and upgrade pip and numpy if demcompare installation crashes:

Expand All @@ -23,3 +23,17 @@ Depending on pip version, installation problems can happen with packages depende
python3 -m pip install --no-binary rasterio rasterio
Data Dimension Management
*************************

There are no constraints on the dimensions of input data, except that they must share a geographical footprint (cf: Intersection DEM schema).
However, classification masks must have the same dimensions as the associated DEM.
The constraints and returned error we impose are directly derived from the rasterio mask class . `Documentation rasterio mask`_

.. figure:: /images/dem_intersection.png
:width: 1000px
:align: center

Intersection DEM schema

.. _`Documentation rasterio mask`: https://rasterio.readthedocs.io/en/stable/api/rasterio.mask.html
Binary file added docs/source/images/dem_intersection.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
4 changes: 2 additions & 2 deletions docs/source/userguide/coregistration.rst
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ iii) Shift of the ``input_sec`` DEM from ``x`` and ``y`` planimetric offsets

The last step consists in changing the origin of the ``input_sec`` DEM so that it now matches the ``input_ref`` ones.

The output of this step is then a ``coreg_SEC.tif`` DEM. One can now open the ``input_ref`` and the ``coreg_SEC.tif`` in a GIS viewer and, hopefully, observe no residual shift.
The output of this step is then a ``coreg_SEC.tif`` DEM. One can now open the ``input_ref`` and the ``coreg_SEC.tif`` in a GIS viewer and, hopefully, observe no residual shift.


Schematic overview
Expand Down Expand Up @@ -228,7 +228,6 @@ The coregistration images and files saved to disk :
:widths: auto
:align: left

``coreg_SEC.tif``,Coregistered secondary DEM
``coregistration_results.json``,Output json file containing coregistration offsets
``logs.log``,Logging file

Expand Down Expand Up @@ -262,6 +261,7 @@ The coregistration images saved with the ``coregistration`` ``save_optional_outp
*reproj_coreg_REF.tif*,Intermediate coregistered reference DEM.
*reproj_SEC.tif*,Intermediate reprojected secondary DEM.
*reproj_REF.tif*, Intermediate reprojected reference DEM.
*coreg_SEC.tif*, Coregistered secondary DEM
*nuth_kaab_iter#.png*,Iteration fit plot
*ElevationDiff_AfterCoreg.png*,Elevation difference plot after coregistration
*ElevationDiff_BeforeCoreg.png*,Elevation difference plot before coregistration
Expand Down
3 changes: 3 additions & 0 deletions docs/source/userguide/statistics/classification_layers.rst
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,10 @@ Four types of classification layers exist:
}
}
.. note::

- If only one value [X] is entered in the ranges, then the calculated intervals are [0%, X[; [X, inf[
- If range is [0], then the calculated interval is [0%, inf[

.. tab:: Fusion

Expand Down
2 changes: 1 addition & 1 deletion docs/source/userguide/statistics/dem_processing.rst
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ The DEM processing methods can be split in two categories: DEM processing method

.. note::

More informations about the curvature, the difference in altitude between the two input DEMs normalized by the slope and the angular difference can be found in :ref:`curvature`, :ref:`slope_normalized_elevation_difference` and :ref:`angular_difference` respectively.
More information about the curvature, the difference in altitude between the two input DEMs normalized by the slope and the angular difference can be found in :ref:`curvature`, :ref:`slope_normalized_elevation_difference` and :ref:`angular_difference` respectively.

After the DEM processing methods, statistics can be computed on the resulting DEM.

Expand Down
18 changes: 6 additions & 12 deletions tests/classification_layer/test_classification_layer.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,7 @@ def test_get_outliers_free_mask(get_default_metrics):
gt_filtered_mask[np.where(data > upper_threshold)] = False
gt_filtered_mask[np.where(data < lower_threshold)] = False

np.testing.assert_allclose(
gt_filtered_mask, output_filtered_mask, rtol=1e-02
)
np.testing.assert_equal(gt_filtered_mask, output_filtered_mask)


@pytest.mark.unit_tests
Expand Down Expand Up @@ -168,7 +166,7 @@ def test_get_nonan_mask_custom_nodata(initialize_slope_layer):
dtype=np.float32,
)
# Test that the computed no nan mask is equal to ground truth
np.testing.assert_allclose(gt_nonan_mask, output_nonan_mask, rtol=1e-02)
np.testing.assert_equal(gt_nonan_mask, output_nonan_mask)


@pytest.mark.unit_tests
Expand Down Expand Up @@ -210,7 +208,7 @@ def test_get_nonan_mask_defaut_nodata(initialize_slope_layer):
dtype=np.float32,
)
# Test that the computed no nan mask is equal to ground truth
np.testing.assert_allclose(gt_nonan_mask, output_nonan_mask, rtol=1e-02)
np.testing.assert_equal(gt_nonan_mask, output_nonan_mask)


@pytest.mark.unit_tests
Expand Down Expand Up @@ -295,13 +293,9 @@ def test_create_mode_masks(get_default_metrics):
)

# Test that the computed masks_modes are the same as ground truth
np.testing.assert_allclose(gt_standard_mode_mask, mode_masks[0], rtol=1e-02)
np.testing.assert_allclose(
gt_intersection_mode_mask, mode_masks[1], rtol=1e-02
)
np.testing.assert_allclose(
gt_exclusion_mode_mask, mode_masks[2], rtol=1e-02
)
np.testing.assert_equal(gt_standard_mode_mask, mode_masks[0])
np.testing.assert_equal(gt_intersection_mode_mask, mode_masks[1])
np.testing.assert_equal(gt_exclusion_mode_mask, mode_masks[2])


@pytest.mark.unit_tests
Expand Down
10 changes: 3 additions & 7 deletions tests/classification_layer/test_fusion_classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,8 @@ def test_merge_classes_and_create_sets_masks(initialize_fusion_layer):
"fusion_layer": gt_sets_masks,
}
# Test that the computed sets_masks_dict is the same as gt
np.testing.assert_allclose(
gt_classes_masks["fusion_layer"],
fusion_layer_.classes_masks["sec"],
rtol=1e-02,
np.testing.assert_equal(
gt_classes_masks["fusion_layer"], fusion_layer_.classes_masks["sec"]
)


Expand Down Expand Up @@ -188,6 +186,4 @@ def test_create_labelled_map(initialize_fusion_layer):
[[1, 4, 5], [5, 6, 3], [-9999, -9999, -9999], [-9999, 6, 2]]
)

np.testing.assert_allclose(
gt_map_image, fusion_layer_.map_image["sec"], rtol=1e-02
)
np.testing.assert_equal(gt_map_image, fusion_layer_.map_image["sec"])
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,7 @@ def test_create_labelled_map(initialize_segmentation_classification):
# Test that the test_first_classif's map image has been correctly loaded
# on the dataset
gt_map_image = classif_data[:, :, 0]
np.testing.assert_allclose(
gt_map_image, classif_layer_.map_image["ref"], rtol=1e-02
)
np.testing.assert_equal(gt_map_image, classif_layer_.map_image["ref"])


@pytest.mark.unit_tests
Expand Down Expand Up @@ -109,8 +107,7 @@ def _test_create_class_masks(initialize_segmentation_classification):
}

# Test that the computed classes_masks are the same as gt
np.testing.assert_allclose(
np.testing.assert_equal(
gt_classes_masks["test_first_classif"],
classif_layer_.classes_masks["ref"],
rtol=1e-02,
)
10 changes: 3 additions & 7 deletions tests/classification_layer/test_slope_classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,7 @@ def test_classify_slope_by_ranges(initialize_slope_layer):
)

# Test that the output classified slope is the same as gt
np.testing.assert_allclose(
gt_classified_slope, output_classified_slope, rtol=1e-02
)
np.testing.assert_equal(gt_classified_slope, output_classified_slope)


@pytest.mark.unit_tests
Expand Down Expand Up @@ -135,8 +133,6 @@ def test_create_class_masks(initialize_slope_layer):
]

# Test that the computed sets_masks_dict is the same as ground truth
np.testing.assert_allclose(
gt_classes_masks_masks,
slope_classif_layer_.classes_masks["ref"],
rtol=1e-02,
np.testing.assert_equal(
gt_classes_masks_masks, slope_classif_layer_.classes_masks["ref"]
)
18 changes: 9 additions & 9 deletions tests/coregistration/test_coregister_dems.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
from demcompare.helpers_init import read_config_file

# Tests helpers
from tests.helpers import demcompare_test_data_path
from tests.helpers import BOUNDS_TOL, TRANSFORM_TOL, demcompare_test_data_path


@pytest.mark.unit_tests
Expand Down Expand Up @@ -104,9 +104,9 @@ def test_coregister_dems_algorithm_gironde_sampling_sec():

# Test that the outputs match the ground truth
assert rotation == transform.rotation
np.testing.assert_allclose(x_offset, transform.x_offset, rtol=1e-02)
np.testing.assert_allclose(y_offset, transform.y_offset, rtol=1e-02)
np.testing.assert_allclose(z_offset, transform.z_offset, rtol=1e-02)
np.testing.assert_allclose(x_offset, transform.x_offset, rtol=TRANSFORM_TOL)
np.testing.assert_allclose(y_offset, transform.y_offset, rtol=TRANSFORM_TOL)
np.testing.assert_allclose(z_offset, transform.z_offset, rtol=TRANSFORM_TOL)


@pytest.mark.unit_tests
Expand Down Expand Up @@ -176,10 +176,10 @@ def test_bounds_in_coregister_dems_algorithm_gironde_sampling_sec():

# Test that the outputs match the ground truth
np.testing.assert_allclose(
coreg_sec_dataset.bounds, (ulx, uly, lrx, lry), rtol=1e-02
coreg_sec_dataset.bounds, (ulx, uly, lrx, lry), rtol=BOUNDS_TOL
)
np.testing.assert_allclose(
coreg_ref_dataset.bounds, (ulx, uly, lrx, lry), rtol=1e-02
coreg_ref_dataset.bounds, (ulx, uly, lrx, lry), rtol=BOUNDS_TOL
)


Expand Down Expand Up @@ -251,6 +251,6 @@ def test_coregister_dems_algorithm_gironde_sampling_ref():

# Test that the outputs match the ground truth
assert rotation == transform.rotation
np.testing.assert_allclose(x_offset, transform.x_offset, rtol=1e-02)
np.testing.assert_allclose(y_offset, transform.y_offset, rtol=1e-02)
np.testing.assert_allclose(z_offset, transform.z_offset, rtol=1e-02)
np.testing.assert_allclose(x_offset, transform.x_offset, rtol=TRANSFORM_TOL)
np.testing.assert_allclose(y_offset, transform.y_offset, rtol=TRANSFORM_TOL)
np.testing.assert_allclose(z_offset, transform.z_offset, rtol=TRANSFORM_TOL)
8 changes: 4 additions & 4 deletions tests/coregistration/test_coregistration_cropped_dem.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def test_crop_dem_with_offset_pos_x_pos_y(initialize_dem_and_coreg):
int(np.floor(y_offset)) : input_dem.shape[0],
0 : input_dem.shape[1] - int(np.ceil(x_offset)),
]
np.testing.assert_allclose(gt_cropped_dem, output_cropped_dem, rtol=1e-02)
np.testing.assert_equal(gt_cropped_dem, output_cropped_dem)


@pytest.mark.unit_tests
Expand Down Expand Up @@ -89,7 +89,7 @@ def test_crop_dem_with_offset_pos_x_neg_y(initialize_dem_and_coreg):
0 : input_dem.shape[0] - int(np.ceil(-y_offset)),
0 : input_dem.shape[1] - int(np.ceil(x_offset)),
]
np.testing.assert_allclose(gt_cropped_dem, output_cropped_dem, rtol=1e-02)
np.testing.assert_equal(gt_cropped_dem, output_cropped_dem)


@pytest.mark.unit_tests
Expand Down Expand Up @@ -119,7 +119,7 @@ def test_crop_dem_with_offset_neg_x_pos_y(initialize_dem_and_coreg):
int(np.floor(y_offset)) : input_dem.shape[0],
int(np.floor(-x_offset)) : input_dem.shape[1],
]
np.testing.assert_allclose(gt_cropped_dem, output_cropped_dem, rtol=1e-02)
np.testing.assert_equal(gt_cropped_dem, output_cropped_dem)


@pytest.mark.unit_tests
Expand Down Expand Up @@ -149,4 +149,4 @@ def test_crop_dem_with_offset_neg_x_neg_y(initialize_dem_and_coreg):
0 : input_dem.shape[0] - int(np.ceil(-y_offset)),
int(np.floor(-x_offset)) : input_dem.shape[1],
]
np.testing.assert_allclose(gt_cropped_dem, output_cropped_dem, rtol=1e-02)
np.testing.assert_equal(gt_cropped_dem, output_cropped_dem)
4 changes: 1 addition & 3 deletions tests/coregistration/test_coregistration_errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def test_coregistration_with_output_dir():
- Create temporary_dir named tmp_dir
- Loads the data present in the test root data directory
- Creates a coregistration object and does compute_coregistration
- Verify that coreg_sec.tif and coregistration_results.json are saved
- Verify that coregistration_results.json are saved
- parameter output_dir not being specified and save_optional_outputs
set to True
- Creates a new coregistration object and does compute_coregistration
Expand Down Expand Up @@ -140,8 +140,6 @@ def test_coregistration_with_output_dir():
# compute coregistration
_ = coregistration_.compute_coregistration(sec, ref)

# test output_dir/coregistration/coreg_SEC.tif creation
assert os.path.isfile(tmp_dir + "/coregistration/coreg_SEC.tif") is True
# test output_dir/coregistration/coreg_SEC.tif creation
assert (
os.path.isfile(
Expand Down
Loading

0 comments on commit 1c96681

Please sign in to comment.