diff --git a/cars/applications/__init__.py b/cars/applications/__init__.py index c0a3f9d3..16b82a88 100644 --- a/cars/applications/__init__.py +++ b/cars/applications/__init__.py @@ -31,7 +31,7 @@ from . import holes_detection # noqa: F401 from . import point_cloud_denoising # noqa: F401 from . import point_cloud_fusion # noqa: F401 -from . import point_cloud_outliers_removing # noqa: F401 +from . import point_cloud_outlier_removal # noqa: F401 from . import rasterization # noqa: F401 from . import resampling # noqa: F401 from . import sparse_matching # noqa: F401 diff --git a/cars/applications/point_cloud_outliers_removing/__init__.py b/cars/applications/point_cloud_outlier_removal/__init__.py similarity index 88% rename from cars/applications/point_cloud_outliers_removing/__init__.py rename to cars/applications/point_cloud_outlier_removal/__init__.py index 61761344..89e56e2b 100644 --- a/cars/applications/point_cloud_outliers_removing/__init__.py +++ b/cars/applications/point_cloud_outlier_removal/__init__.py @@ -23,8 +23,8 @@ """ # flake8: noqa: F401 -from cars.applications.point_cloud_outliers_removing.pc_out_removing import ( - PointCloudOutliersRemoving, +from cars.applications.point_cloud_outlier_removal.pc_out_removal import ( + PointCloudOutlierRemoval, ) from . import small_components, statistical diff --git a/cars/applications/point_cloud_outliers_removing/outlier_removing_tools.py b/cars/applications/point_cloud_outlier_removal/outlier_removal_tools.py similarity index 61% rename from cars/applications/point_cloud_outliers_removing/outlier_removing_tools.py rename to cars/applications/point_cloud_outlier_removal/outlier_removal_tools.py index 4e2c019e..d08d62ff 100644 --- a/cars/applications/point_cloud_outliers_removing/outlier_removing_tools.py +++ b/cars/applications/point_cloud_outlier_removal/outlier_removal_tools.py @@ -19,26 +19,28 @@ # limitations under the License. # """ -This module contains functions used in outlier removing +This module contains functions used in outlier removal """ # Standard imports -import logging from typing import List, Tuple, Union # Third party imports import numpy as np +import outlier_filter # pylint:disable=E0401 import pandas -import xarray as xr from scipy.spatial import cKDTree # pylint: disable=no-name-in-module +from cars.applications.point_cloud_fusion.point_cloud_tools import filter_cloud + # CARS imports from cars.core import constants as cst +from cars.core import projection -# ##### Small components filtering ###### +# ##### Small component filtering ###### -def small_components_filtering( +def small_component_filtering( cloud: pandas.DataFrame, connection_val: float, nb_pts_threshold: int, @@ -64,9 +66,20 @@ def small_components_filtering( :return: Tuple made of the filtered cloud and the removed elements positions in their epipolar images """ - cloud_xyz = cloud.loc[:, [cst.X, cst.Y, cst.Z]].values - index_elt_to_remove = detect_small_components( - cloud_xyz, connection_val, nb_pts_threshold, clusters_distance_threshold + + clusters_distance_threshold_float = ( + np.nan + if clusters_distance_threshold is None + else clusters_distance_threshold + ) + + index_elt_to_remove = outlier_filter.pc_small_component_outlier_filtering( + cloud.loc[:, cst.X].values, + cloud.loc[:, cst.Y].values, + cloud.loc[:, cst.Z].values, + radius=connection_val, + min_cluster_size=nb_pts_threshold, + clusters_distance_threshold=clusters_distance_threshold_float, ) return filter_cloud(cloud, index_elt_to_remove, filtered_elt_pos) @@ -175,7 +188,7 @@ def detect_small_components( # ##### statistical filtering ###### -def statistical_outliers_filtering( +def statistical_outlier_filtering( cloud: pandas.DataFrame, k: int, dev_factor: float, @@ -198,9 +211,14 @@ def statistical_outliers_filtering( :return: Tuple made of the filtered cloud and the removed elements positions in their epipolar images """ - cloud_xyz = cloud.loc[:, [cst.X, cst.Y, cst.Z]].values - index_elt_to_remove = detect_statistical_outliers( - cloud_xyz, k, dev_factor, use_median + + index_elt_to_remove = outlier_filter.pc_statistical_outlier_filtering( + cloud.loc[:, cst.X].values, + cloud.loc[:, cst.Y].values, + cloud.loc[:, cst.Z].values, + dev_factor=dev_factor, + k=k, + use_median=use_median, ) return filter_cloud(cloud, index_elt_to_remove, filtered_elt_pos) @@ -247,6 +265,7 @@ def detect_statistical_outliers( iqr_distances = np.percentile( mean_neighbors_distances, 75 ) - np.percentile(mean_neighbors_distances, 25) + # compute distance threshold and # apply it to determine which points will be removed dist_thresh = median_distances + dev_factor * iqr_distances @@ -269,151 +288,92 @@ def detect_statistical_outliers( return detected_points -# ##### common filtering tools ###### - +def epipolar_small_components( + cloud, + epsg, + min_cluster_size=15, + radius=1.0, + half_window_size=5, + clusters_distance_threshold=np.nan, +): + """ + Filter outliers using the small components method in epipolar geometry + + :param epipolar_ds: epipolar dataset to filter + :type epipolar_ds: xr.Dataset + :param epsg: epsg code of the CRS used to compute distances + :type epsg: int + :param statistical_k: k + :type statistical_k: int + :param std_dev_factor: std factor + :type std_dev_factor: float + :param half_window_size: use median and quartile instead of mean and std + :type half_window_size: int + :param use_median: use median and quartile instead of mean and std + :type use_median: bool + + :return: filtered dataset + :rtype: xr.Dataset -def filter_cloud( - cloud: pandas.DataFrame, - index_elt_to_remove: List[int], - filtered_elt_pos: bool = False, -) -> Tuple[pandas.DataFrame, Union[None, pandas.DataFrame]]: """ - Filter all points of the cloud DataFrame - which index is in the index_elt_to_remove list. - If filtered_elt_pos is set to True, the information of the removed elements - positions in their original epipolar images are returned. + projection.points_cloud_conversion_dataset(cloud, epsg) - To do so the cloud DataFrame has to be build - with the 'with_coords' option activated. + if clusters_distance_threshold is None: + clusters_distance_threshold = np.nan - :param cloud: combined cloud - as returned by the create_combined_cloud function - :param index_elt_to_remove: indexes of lines - to filter in the cloud DataFrame - :param filtered_elt_pos: if filtered_elt_pos is set to True, - the removed points positions in their original epipolar images are - returned, otherwise it is set to None - :return: Tuple composed of the filtered cloud DataFrame and - the filtered elements epipolar position information - (or None for the latter if filtered_elt_pos is set to False - or if the cloud Dataframe has not been build with with_coords option) - """ - if filtered_elt_pos and not ( - cst.POINTS_CLOUD_COORD_EPI_GEOM_I in cloud.columns - and cst.POINTS_CLOUD_COORD_EPI_GEOM_J in cloud.columns - and cst.POINTS_CLOUD_ID_IM_EPI in cloud.columns - ): - logging.warning( - "In filter_cloud: the filtered_elt_pos has been activated but " - "the cloud Datafram has not been build with option with_coords. " - "The positions cannot be retrieved." - ) - filtered_elt_pos = False - - # retrieve removed points position in their original epipolar images - if filtered_elt_pos: - labels = [ - cst.POINTS_CLOUD_COORD_EPI_GEOM_I, - cst.POINTS_CLOUD_COORD_EPI_GEOM_J, - cst.POINTS_CLOUD_ID_IM_EPI, - ] - - removed_elt_pos_infos = cloud.loc[ - cloud.index.values[index_elt_to_remove], labels - ].values - - removed_elt_pos_infos = pandas.DataFrame( - removed_elt_pos_infos, columns=labels - ) - else: - removed_elt_pos_infos = None - - # remove points from the cloud - cloud = cloud.drop(index=cloud.index.values[index_elt_to_remove]) + outlier_filter.epipolar_small_component_outlier_filtering( + cloud[cst.X], + cloud[cst.Y], + cloud[cst.Z], + min_cluster_size, + radius, + half_window_size, + clusters_distance_threshold, + ) - return cloud, removed_elt_pos_infos + return cloud -def add_cloud_filtering_msk( - clouds_list: List[xr.Dataset], - elt_pos_infos: pandas.DataFrame, - mask_label: str, - mask_value: int = 255, +def epipolar_statistical_filtering( + epipolar_ds, + epsg, + k=15, + dev_factor=1.0, + half_window_size=5, + use_median=False, ): """ - Add a uint16 mask labeled 'mask_label' to the clouds in clouds_list. - (in-line function) - - :param clouds_list: Input list of clouds - :param elt_pos_infos: pandas dataframe - composed of cst.POINTS_CLOUD_COORD_EPI_GEOM_I, - cst.POINTS_CLOUD_COORD_EPI_GEOM_J, cst.POINTS_CLOUD_ID_IM_EPI columns - as computed in the create_combined_cloud function. - Those information are used to retrieve the point position - in its original epipolar image. - :param mask_label: label to give to the mask in the datasets - :param mask_value: filtered elements value in the mask - """ - - # Verify that the elt_pos_infos is consistent - if ( - elt_pos_infos is None - or cst.POINTS_CLOUD_COORD_EPI_GEOM_I not in elt_pos_infos.columns - or cst.POINTS_CLOUD_COORD_EPI_GEOM_J not in elt_pos_infos.columns - or cst.POINTS_CLOUD_ID_IM_EPI not in elt_pos_infos.columns - ): - logging.warning( - "Cannot generate filtered elements mask, " - "no information about the point's" - " original position in the epipolar image is given" - ) - - else: - elt_index = elt_pos_infos.loc[:, cst.POINTS_CLOUD_ID_IM_EPI].to_numpy() - - min_elt_index = np.min(elt_index) - max_elt_index = np.max(elt_index) + Filter outliers using the statistical method in epipolar geometry + + :param epipolar_ds: epipolar dataset to filter + :type epipolar_ds: xr.Dataset + :param epsg: epsg code of the CRS used to compute distances + :type epsg: int + :param statistical_k: k + :type statistical_k: int + :param std_dev_factor: std factor + :type std_dev_factor: float + :param half_window_size: use median and quartile instead of mean and std + :type half_window_size: int + :param use_median: use median and quartile instead of mean and std + :type use_median: bool + + :return: filtered dataset + :rtype: xr.Dataset - if min_elt_index < 0 or max_elt_index > len(clouds_list) - 1: - raise RuntimeError( - "Index indicated in the elt_pos_infos pandas. " - "DataFrame is not coherent with the clouds list given in input" - ) - - # create and add mask to each element of clouds_list - for cloud_idx, cloud_item in enumerate(clouds_list): - if mask_label not in cloud_item: - nb_row = cloud_item.coords[cst.ROW].data.shape[0] - nb_col = cloud_item.coords[cst.COL].data.shape[0] - msk = np.zeros((nb_row, nb_col), dtype=np.uint16) - else: - msk = cloud_item[mask_label].values + """ - cur_elt_index = np.argwhere(elt_index == cloud_idx) + projection.points_cloud_conversion_dataset(epipolar_ds, epsg) - for elt_pos in range(cur_elt_index.shape[0]): - i = int( - elt_pos_infos.loc[ - cur_elt_index[elt_pos], - cst.POINTS_CLOUD_COORD_EPI_GEOM_I, - ].iat[0] - ) - j = int( - elt_pos_infos.loc[ - cur_elt_index[elt_pos], - cst.POINTS_CLOUD_COORD_EPI_GEOM_J, - ].iat[0] - ) + outlier_filter.epipolar_statistical_outlier_filtering( + epipolar_ds[cst.X], + epipolar_ds[cst.Y], + epipolar_ds[cst.Z], + k, + half_window_size, + dev_factor, + use_median, + ) - try: - msk[i, j] = mask_value - except Exception as index_error: - raise RuntimeError( - "Point at location ({},{}) is not accessible " - "in an image of size ({},{})".format( - i, j, msk.shape[0], msk.shape[1] - ) - ) from index_error - - cloud_item[mask_label] = ([cst.ROW, cst.COL], msk) + return epipolar_ds diff --git a/cars/applications/point_cloud_outliers_removing/pc_out_removing.py b/cars/applications/point_cloud_outlier_removal/pc_out_removal.py similarity index 50% rename from cars/applications/point_cloud_outliers_removing/pc_out_removing.py rename to cars/applications/point_cloud_outlier_removal/pc_out_removal.py index 8e3aea2b..eb6a7e09 100644 --- a/cars/applications/point_cloud_outliers_removing/pc_out_removing.py +++ b/cars/applications/point_cloud_outlier_removal/pc_out_removal.py @@ -19,7 +19,7 @@ # limitations under the License. # """ -this module contains the abstract PointsCloudOutlierRemoving application class. +this module contains the abstract PointsCloudOutlierRemoval application class. """ import logging @@ -27,17 +27,23 @@ from abc import ABCMeta, abstractmethod from typing import Dict +import numpy as np + from cars.applications import application_constants from cars.applications.application import Application from cars.applications.application_template import ApplicationTemplate +from cars.applications.point_cloud_outlier_removal import ( + point_removal_constants as pr_cst, +) +from cars.core import constants as cst from cars.core.utils import safe_makedirs from cars.data_structures import cars_dataset -@Application.register("point_cloud_outliers_removing") -class PointCloudOutliersRemoving(ApplicationTemplate, metaclass=ABCMeta): +@Application.register("point_cloud_outlier_removal") +class PointCloudOutlierRemoval(ApplicationTemplate, metaclass=ABCMeta): """ - PointCloudOutliersRemoving + PointCloudOutlierRemoval """ available_applications: Dict = {} @@ -49,38 +55,38 @@ def __new__(cls, conf=None): # pylint: disable=W0613 :raises: - KeyError when the required application is not registered - :param conf: configuration for points removing + :param conf: configuration for points removal :return: a application_to_use object """ - points_removing_method = cls.default_application + points_removal_method = cls.default_application if bool(conf) is False or "method" not in conf: logging.info( - "Points removing method not specified, " - "default {} is used".format(points_removing_method) + "Points removal method not specified, " + "default {} is used".format(points_removal_method) ) else: - points_removing_method = conf.get("method", cls.default_application) + points_removal_method = conf.get("method", cls.default_application) - if points_removing_method not in cls.available_applications: + if points_removal_method not in cls.available_applications: logging.error( - "No Points removing application named {} registered".format( - points_removing_method + "No Points removal application named {} registered".format( + points_removal_method ) ) raise KeyError( - "No Points removing application named {} registered".format( - points_removing_method + "No Points removal application named {} registered".format( + points_removal_method ) ) logging.info( - "The PointCloudOutliersRemoving({}) application" - " will be used".format(points_removing_method) + "The PointCloudOutlierRemoval({}) application" + " will be used".format(points_removal_method) ) - return super(PointCloudOutliersRemoving, cls).__new__( - cls.available_applications[points_removing_method] + return super(PointCloudOutlierRemoval, cls).__new__( + cls.available_applications[points_removal_method] ) def __init_subclass__(cls, short_name, **kwargs): # pylint: disable=E0302 @@ -91,7 +97,7 @@ def __init_subclass__(cls, short_name, **kwargs): # pylint: disable=E0302 def __init__(self, conf=None): """ - Init function of PointCloudOutliersRemoving + Init function of PointCloudOutlierRemoval :param conf: configuration :return: an application_to_use object @@ -141,10 +147,90 @@ def get_optimal_tile_size( """ - def __register_dataset__( + def __register_epipolar_dataset__( + self, merged_points_cloud, output_dir=None, dump_dir=None, app_name="" + ): + """ + Create dataset and registered the output in the orchestrator. the output + X, Y and Z ground coordinates will be saved in output_dir if the + parameter is no None. Alternatively it will be saved to dump_dir if + save_intermediate_data is set and output_dir is None. + + :param merged_points_cloud: Merged point cloud + :type merged_points_cloud: CarsDataset + :param output_dir: output depth map directory. If None output will be + written in dump_dir if intermediate data is requested + :type output_dir: str + :param dump_dir: dump dir for output (except depth map) if intermediate + data is requested + :type dump_dir: str + :param app_name: application name for file names + :type app_name: str + + :return: Filtered point cloud + :rtype: CarsDataset + + """ + + # Create epipolar point cloud CarsDataset + filtered_point_cloud = cars_dataset.CarsDataset( + merged_points_cloud.dataset_type, name=app_name + ) + + filtered_point_cloud.create_empty_copy(merged_points_cloud) + filtered_point_cloud.overlaps *= 0 # Margins removed (for now) + + # Update attributes to get epipolar info + filtered_point_cloud.attributes.update(merged_points_cloud.attributes) + + if output_dir or self.used_config.get( + application_constants.SAVE_INTERMEDIATE_DATA + ): + filtered_dir = output_dir if output_dir is not None else dump_dir + safe_makedirs(filtered_dir) + self.orchestrator.add_to_save_lists( + os.path.join(filtered_dir, "X.tif"), + cst.X, + filtered_point_cloud, + cars_ds_name="depth_map_x_filtered_" + app_name, + dtype=np.float64, + ) + + self.orchestrator.add_to_save_lists( + os.path.join(filtered_dir, "Y.tif"), + cst.Y, + filtered_point_cloud, + cars_ds_name="depth_map_y_filtered_" + app_name, + dtype=np.float64, + ) + self.orchestrator.add_to_save_lists( + os.path.join(filtered_dir, "Z.tif"), + cst.Z, + filtered_point_cloud, + cars_ds_name="depth_map_z_filtered_" + app_name, + dtype=np.float64, + ) + + # Get saving infos in order to save tiles when they are computed + [saving_info] = self.orchestrator.get_saving_infos( + [filtered_point_cloud] + ) + + # Add infos to orchestrator.out_json + updating_dict = { + application_constants.APPLICATION_TAG: { + pr_cst.CLOUD_OUTLIER_REMOVAL_RUN_TAG: {}, + } + } + self.orchestrator.update_out_info(updating_dict) + + return filtered_point_cloud, saving_info + + def __register_pc_dataset__( self, merged_points_cloud, - save_laz_output=False, + output_dir=None, + dump_dir=None, app_name=None, ): """ @@ -156,8 +242,12 @@ def __register_dataset__( :param merged_points_cloud: Merged point cloud :type merged_points_cloud: CarsDataset - :param save_laz_output: true if save to laz as official product - :type save_laz_output: bool + :param output_dir: output depth map directory. If None output will be + written in dump_dir if intermediate data is requested + :type output_dir: str + :param dump_dir: dump dir for output (except depth map) if intermediate + data is requested + :type dump_dir: str :param app_name: application name for file names :type app_name: str @@ -172,14 +262,17 @@ def __register_dataset__( application_constants.SAVE_INTERMEDIATE_DATA, False ) # Save laz point cloud if save_intermediate_date is activated (dump_dir) - # or if save_laz_output is activated (official product) - save_point_cloud_as_laz = save_laz_output or self.used_config.get( - application_constants.SAVE_INTERMEDIATE_DATA, False + # or if output_dir is provided is activated (save as official product) + save_point_cloud_as_laz = ( + output_dir is not None + or self.used_config.get( + application_constants.SAVE_INTERMEDIATE_DATA, False + ) ) # Create CarsDataset filtered_point_cloud = cars_dataset.CarsDataset( - "points", name="point_cloud_removing_" + app_name + "points", name="point_cloud_removal_" + app_name ) # Get tiling grid @@ -191,14 +284,10 @@ def __register_dataset__( pc_laz_file_name = None if save_point_cloud_as_laz: # Points cloud file name - if save_laz_output: - pc_laz_file_name = os.path.join( - self.orchestrator.out_dir, "point_cloud" - ) + if output_dir is not None: + pc_laz_file_name = output_dir else: - pc_laz_file_name = os.path.join( - self.orchestrator.out_dir, "dump_dir", app_name, "laz" - ) + pc_laz_file_name = os.path.join(dump_dir, "laz") safe_makedirs(pc_laz_file_name) pc_laz_file_name = os.path.join(pc_laz_file_name, "pc") self.orchestrator.add_to_compute_lists( @@ -209,9 +298,7 @@ def __register_dataset__( pc_csv_file_name = None if save_point_cloud_as_csv: # Points cloud file name - pc_csv_file_name = os.path.join( - self.orchestrator.out_dir, "dump_dir", app_name, "csv" - ) + pc_csv_file_name = os.path.join(dump_dir, "csv") safe_makedirs(pc_csv_file_name) pc_csv_file_name = os.path.join(pc_csv_file_name, "pc") self.orchestrator.add_to_compute_lists( @@ -223,10 +310,16 @@ def __register_dataset__( @abstractmethod def run( - self, merged_points_cloud, orchestrator=None, save_laz_output=False + self, + merged_points_cloud, + orchestrator=None, + save_laz_output=False, + output_dir=None, + dump_dir=None, + epsg=None, ): """ - Run PointCloudOutliersRemoving application. + Run PointCloudOutlierRemoval application. Creates a CarsDataset filled with new point cloud tiles. @@ -235,6 +328,14 @@ def run( :param orchestrator: orchestrator used :param save_laz_output: save output point cloud as laz :type save_laz_output: bool + :param output_dir: output depth map directory. If None output will be + written in dump_dir if intermediate data is requested + :type output_dir: str + :param dump_dir: dump dir for output (except depth map) if intermediate + data is requested + :type dump_dir: str + :param epsg: cartographic reference for the point cloud (array input) + :type epsg: int :return: filtered merged points cloud :rtype: CarsDataset filled with xr.Dataset diff --git a/cars/applications/point_cloud_outliers_removing/points_removing_constants.py b/cars/applications/point_cloud_outlier_removal/point_removal_constants.py similarity index 90% rename from cars/applications/point_cloud_outliers_removing/points_removing_constants.py rename to cars/applications/point_cloud_outlier_removal/point_removal_constants.py index 75c362f9..11159fc1 100644 --- a/cars/applications/point_cloud_outliers_removing/points_removing_constants.py +++ b/cars/applications/point_cloud_outlier_removal/point_removal_constants.py @@ -23,14 +23,14 @@ """ -CLOUD_OUTLIER_REMOVING_RUN_TAG = "points_removing_constants_run" +CLOUD_OUTLIER_REMOVAL_RUN_TAG = "point_removal_constants_run" # Params METHOD = "method" # small components -SMALL_COMPONENTS_FILTER = "small_components_filter_activated" +SMALL_COMPONENT_FILTER = "small_components_filter_activated" SC_ON_GROUND_MARGIN = "on_ground_margin" SC_CONNECTION_DISTANCE = "connection_distance" SC_NB_POINTS_THRESHOLD = "nb_points_threshold" diff --git a/cars/applications/point_cloud_outliers_removing/small_components.py b/cars/applications/point_cloud_outlier_removal/small_components.py similarity index 70% rename from cars/applications/point_cloud_outliers_removing/small_components.py rename to cars/applications/point_cloud_outlier_removal/small_components.py index d451cc31..1ab4c765 100644 --- a/cars/applications/point_cloud_outliers_removing/small_components.py +++ b/cars/applications/point_cloud_outlier_removal/small_components.py @@ -19,7 +19,7 @@ # limitations under the License. # """ -this module contains the statistical points removing application class. +this module contains the small_components point removal application class. """ @@ -38,33 +38,31 @@ # CARS imports import cars.orchestrator.orchestrator as ocht from cars.applications import application_constants -from cars.applications.point_cloud_outliers_removing import ( - outlier_removing_tools, +from cars.applications.point_cloud_outlier_removal import outlier_removal_tools +from cars.applications.point_cloud_outlier_removal import ( + pc_out_removal as pc_removal, ) -from cars.applications.point_cloud_outliers_removing import ( - pc_out_removing as pc_removing, -) -from cars.applications.point_cloud_outliers_removing import ( - points_removing_constants as pr_cst, +from cars.applications.point_cloud_outlier_removal import ( + point_removal_constants as pr_cst, ) from cars.core import projection from cars.data_structures import cars_dataset class SmallComponents( - pc_removing.PointCloudOutliersRemoving, short_name="small_components" + pc_removal.PointCloudOutlierRemoval, short_name="small_components" ): # pylint: disable=R0903 """ - PointCloudOutliersRemoving + SmallComponents """ # pylint: disable=too-many-instance-attributes def __init__(self, conf=None): """ - Init function of PointCloudOutliersRemoving + Init function of SmallComponents - :param conf: configuration for points outliers removing + :param conf: configuration for points outlier removal :return: an application_to_use object """ @@ -80,6 +78,7 @@ def __init__(self, conf=None): self.clusters_distance_threshold = self.used_config[ "clusters_distance_threshold" ] + self.half_epipolar_size = self.used_config["half_epipolar_size"] # Saving files self.save_by_pair = self.used_config.get("save_by_pair", False) @@ -144,6 +143,13 @@ def check_conf(self, conf): "clusters_distance_threshold", None ) + # half_epipolar_size: + # Half size of the epipolar window used for neighobr search (depth map + # input only) + overloaded_conf["half_epipolar_size"] = conf.get( + "half_epipolar_size", 5 + ) + points_cloud_fusion_schema = { "method": str, "save_by_pair": bool, @@ -152,6 +158,7 @@ def check_conf(self, conf): "connection_distance": And(float, lambda x: x > 0), "nb_points_threshold": And(int, lambda x: x > 0), "clusters_distance_threshold": Or(None, float), + "half_epipolar_size": int, application_constants.SAVE_INTERMEDIATE_DATA: bool, } @@ -196,7 +203,7 @@ def get_optimal_tile_size( logging.info( "Estimated optimal tile size for small" - "components removing: {} meters".format(tile_size) + "component removal: {} meters".format(tile_size) ) return tile_size @@ -229,10 +236,15 @@ def get_on_ground_margin(self, resolution=0.5): return on_ground_margin def run( - self, merged_points_cloud, orchestrator=None, save_laz_output=False + self, + merged_points_cloud, + orchestrator=None, + output_dir=None, + dump_dir=None, + epsg=None, ): """ - Run PointCloudOutliersRemoving application. + Run PointCloudOutlierRemoval application. Creates a CarsDataset filled with new point cloud tiles. @@ -249,8 +261,14 @@ def run( :type merged_points_cloud: CarsDataset filled with pandas.DataFrame :param orchestrator: orchestrator used - :param save_laz_output: save output point cloud as laz - :type save_laz_output: bool + :param output_dir: output depth map directory. If None output will be + written in dump_dir if intermediate data is requested + :type output_dir: str + :param dump_dir: dump dir for output (except depth map) if intermediate + data is requested + :type dump_dir: str + :param epsg: cartographic reference for the point cloud (array input) + :type epsg: int :return: filtered merged points cloud. CarsDataset contains: @@ -285,9 +303,10 @@ def run( filtered_point_cloud, point_cloud_laz_file_name, point_cloud_csv_file_name, - ) = self.__register_dataset__( + ) = self.__register_pc_dataset__( merged_points_cloud, - save_laz_output, + output_dir, + dump_dir, app_name="small_components", ) @@ -299,7 +318,7 @@ def run( # Add infos to orchestrator.out_json updating_dict = { application_constants.APPLICATION_TAG: { - pr_cst.CLOUD_OUTLIER_REMOVING_RUN_TAG: {}, + pr_cst.CLOUD_OUTLIER_REMOVAL_RUN_TAG: {}, } } orchestrator.update_out_info(updating_dict) @@ -316,7 +335,7 @@ def run( filtered_point_cloud[ row, col ] = self.orchestrator.cluster.create_task( - small_components_removing_wrapper + small_component_removal_wrapper )( merged_points_cloud[row, col], self.connection_distance, @@ -327,10 +346,48 @@ def run( point_cloud_laz_file_name=point_cloud_laz_file_name, saving_info=full_saving_info, ) + elif merged_points_cloud.dataset_type == "arrays": + filtered_point_cloud, saving_info = ( + self.__register_epipolar_dataset__( + merged_points_cloud, + output_dir, + dump_dir, + app_name="small_components", + ) + ) + + # Generate rasters + for col in range(filtered_point_cloud.shape[1]): + for row in range(filtered_point_cloud.shape[0]): + + # update saving infos for potential replacement + full_saving_info = ocht.update_saving_infos( + saving_info, row=row, col=col + ) + if merged_points_cloud[row][col] is not None: + + window = merged_points_cloud.tiling_grid[row, col] + overlap = merged_points_cloud.overlaps[row, col] + # Delayed call to cloud filtering + filtered_point_cloud[ + row, col + ] = self.orchestrator.cluster.create_task( + epipolar_small_component_removal_wrapper + )( + merged_points_cloud[row, col], + self.connection_distance, + self.nb_points_threshold, + self.clusters_distance_threshold, + self.half_epipolar_size, + window, + overlap, + epsg=epsg, + saving_info=full_saving_info, + ) else: logging.error( - "PointCloudOutliersRemoving application doesn't support" + "PointCloudOutlierRemoval application doesn't support " "this input data " "format" ) @@ -338,7 +395,7 @@ def run( return filtered_point_cloud -def small_components_removing_wrapper( +def small_component_removal_wrapper( cloud, connection_distance, nb_points_threshold, @@ -349,7 +406,7 @@ def small_components_removing_wrapper( saving_info=None, ): """ - Statistical outlier removing + Statistical outlier removal :param cloud: cloud to filter :type cloud: pandas DataFrame @@ -406,7 +463,7 @@ def small_components_removing_wrapper( ( new_cloud, _, - ) = outlier_removing_tools.small_components_filtering( + ) = outlier_removal_tools.small_component_filtering( new_cloud, connection_distance, nb_points_threshold, @@ -414,7 +471,7 @@ def small_components_removing_wrapper( ) toc = time.process_time() logging.debug( - "Small components cloud filtering done in {} seconds".format(toc - tic) + "Small component cloud filtering done in {} seconds".format(toc - tic) ) # Conversion to UTM @@ -446,3 +503,65 @@ def small_components_removing_wrapper( ) return new_cloud + + +def epipolar_small_component_removal_wrapper( + cloud, + connection_distance, + nb_points_threshold, + clusters_distance_threshold, + half_epipolar_size, + window, + overlap, + epsg, + saving_info=None, +): + """ + Small component outlier removal in epipolar geometry + + :param epipolar_ds: epipolar dataset to filter + :type epipolar_ds: xr.Dataset + :param connection_distance: minimum distance of two connected points + :type connection_distance: float + :param nb_points_threshold: minimum valid cluster size + :type nb_points_threshold: int + :param clusters_distance_threshold: max distance between an outlier cluster + and other points + :type clusters_distance_threshold: float + :param half_epipolar_size: half size of the window used to search neighbors + :type half_epipolar_size: int + :param window: window of base tile [row min, row max, col min col max] + :type window: list + :param overlap: overlap [row min, row max, col min col max] + :type overlap: list + :param epsg: epsg code of the CRS used to compute distances + :type epsg: int + + :return: filtered dataset + :rtype: xr.Dataset + + """ + + # Copy input cloud + filtered_cloud = copy.copy(cloud) + + outlier_removal_tools.epipolar_small_components( + filtered_cloud, + epsg, + min_cluster_size=nb_points_threshold, + radius=connection_distance, + half_window_size=half_epipolar_size, + clusters_distance_threshold=clusters_distance_threshold, + ) + + # Fill with attributes + cars_dataset.fill_dataset( + filtered_cloud, + saving_info=saving_info, + window=cars_dataset.window_array_to_dict(window), + profile=None, + attributes=None, + overlaps=cars_dataset.overlap_array_to_dict(overlap), + ) + + return filtered_cloud diff --git a/cars/applications/point_cloud_outliers_removing/statistical.py b/cars/applications/point_cloud_outlier_removal/statistical.py similarity index 68% rename from cars/applications/point_cloud_outliers_removing/statistical.py rename to cars/applications/point_cloud_outlier_removal/statistical.py index cf9c1815..f6625720 100644 --- a/cars/applications/point_cloud_outliers_removing/statistical.py +++ b/cars/applications/point_cloud_outlier_removal/statistical.py @@ -19,7 +19,7 @@ # limitations under the License. # """ -this module contains the statistical points removing application class. +this module contains the statistical point removal application class. """ @@ -39,14 +39,12 @@ # CARS imports import cars.orchestrator.orchestrator as ocht from cars.applications import application_constants -from cars.applications.point_cloud_outliers_removing import ( - outlier_removing_tools, +from cars.applications.point_cloud_outlier_removal import outlier_removal_tools +from cars.applications.point_cloud_outlier_removal import ( + pc_out_removal as pc_removal, ) -from cars.applications.point_cloud_outliers_removing import ( - pc_out_removing as pc_removing, -) -from cars.applications.point_cloud_outliers_removing import ( - points_removing_constants as pr_cst, +from cars.applications.point_cloud_outlier_removal import ( + point_removal_constants as pr_cst, ) from cars.core import projection from cars.data_structures import cars_dataset @@ -56,19 +54,19 @@ class Statistical( - pc_removing.PointCloudOutliersRemoving, short_name="statistical" + pc_removal.PointCloudOutlierRemoval, short_name="statistical" ): # pylint: disable=R0903 """ - PointCloudOutliersRemoving + Statistical """ # pylint: disable=too-many-instance-attributes def __init__(self, conf=None): """ - Init function of PointCloudOutliersRemoving + Init function of Statistical - :param conf: configuration for points outliers removing + :param conf: configuration for points outlier removal :return: a application_to_use object """ @@ -80,6 +78,8 @@ def __init__(self, conf=None): self.activated = self.used_config["activated"] self.k = self.used_config["k"] self.std_dev_factor = self.used_config["std_dev_factor"] + self.use_median = self.used_config["use_median"] + self.half_epipolar_size = self.used_config["half_epipolar_size"] # Saving files self.save_by_pair = self.used_config.get("save_by_pair", False) @@ -112,6 +112,7 @@ def check_conf(self, conf): conf.get(application_constants.SAVE_INTERMEDIATE_DATA, False) ) overloaded_conf["save_by_pair"] = conf.get("save_by_pair", False) + overloaded_conf["use_median"] = conf.get("use_median", True) # statistical outlier filtering overloaded_conf["activated"] = conf.get( @@ -123,12 +124,21 @@ def check_conf(self, conf): # stdev_factor: factor to apply in the distance threshold computation overloaded_conf["std_dev_factor"] = conf.get("std_dev_factor", 5.0) + # half_epipolar_size: + # Half size of the epipolar window used for neighobr search (depth map + # input only) + overloaded_conf["half_epipolar_size"] = conf.get( + "half_epipolar_size", 5 + ) + points_cloud_fusion_schema = { "method": str, "save_by_pair": bool, "activated": bool, "k": And(int, lambda x: x > 0), "std_dev_factor": And(float, lambda x: x > 0), + "use_median": bool, + "half_epipolar_size": int, application_constants.SAVE_INTERMEDIATE_DATA: bool, } @@ -173,7 +183,7 @@ def get_optimal_tile_size( logging.info( "Estimated optimal tile size for statistical " - "removing: {} meters".format(tile_size) + "removal: {} meters".format(tile_size) ) return tile_size @@ -204,10 +214,12 @@ def run( self, merged_points_cloud, orchestrator=None, - save_laz_output=False, + output_dir=None, + dump_dir=None, + epsg=None, ): """ - Run PointCloudOutliersRemoving application. + Run PointCloudOutlierRemoval application. Creates a CarsDataset filled with new point cloud tiles. @@ -224,8 +236,14 @@ def run( :type merged_points_cloud: CarsDataset filled with pandas.DataFrame :param orchestrator: orchestrator used - :param save_laz_output: save output point cloud as laz - :type save_laz_output: bool + :param output_dir: output depth map directory. If None output will be + written in dump_dir if intermediate data is requested + :type output_dir: str + :param dump_dir: dump dir for output (except depth map) if intermediate + data is requested + :type dump_dir: str + :param epsg: cartographic reference for the point cloud (array input) + :type epsg: int :return: filtered merged points cloud. CarsDataset contains: @@ -260,9 +278,10 @@ def run( filtered_point_cloud, point_cloud_laz_file_name, point_cloud_csv_file_name, - ) = self.__register_dataset__( + ) = self.__register_pc_dataset__( merged_points_cloud, - save_laz_output, + output_dir, + dump_dir, app_name="statistical", ) @@ -274,7 +293,7 @@ def run( # Add infos to orchestrator.out_json updating_dict = { application_constants.APPLICATION_TAG: { - pr_cst.CLOUD_OUTLIER_REMOVING_RUN_TAG: {}, + pr_cst.CLOUD_OUTLIER_REMOVAL_RUN_TAG: {}, } } orchestrator.update_out_info(updating_dict) @@ -297,20 +316,59 @@ def run( filtered_point_cloud[ row, col ] = self.orchestrator.cluster.create_task( - statistical_removing_wrapper + statistical_removal_wrapper )( merged_points_cloud[row, col], self.k, self.std_dev_factor, + self.use_median, save_by_pair=(self.save_by_pair), point_cloud_csv_file_name=point_cloud_csv_file_name, point_cloud_laz_file_name=point_cloud_laz_file_name, saving_info=full_saving_info, ) + elif merged_points_cloud.dataset_type == "arrays": + filtered_point_cloud, saving_info = ( + self.__register_epipolar_dataset__( + merged_points_cloud, + output_dir, + dump_dir, + app_name="statistical", + ) + ) + + # Generate rasters + for col in range(filtered_point_cloud.shape[1]): + for row in range(filtered_point_cloud.shape[0]): + + # update saving infos for potential replacement + full_saving_info = ocht.update_saving_infos( + saving_info, row=row, col=col + ) + if merged_points_cloud[row][col] is not None: + + window = merged_points_cloud.tiling_grid[row, col] + overlap = merged_points_cloud.overlaps[row, col] + # Delayed call to cloud filtering + filtered_point_cloud[ + row, col + ] = self.orchestrator.cluster.create_task( + epipolar_statistical_removal_wrapper + )( + merged_points_cloud[row, col], + self.k, + self.std_dev_factor, + self.use_median, + self.half_epipolar_size, + window, + overlap, + epsg=epsg, + saving_info=full_saving_info, + ) else: logging.error( - "PointCloudOutliersRemoving application doesn't support" + "PointCloudOutlierRemoval application doesn't support" "this input data " "format" ) @@ -318,24 +376,27 @@ def run( return filtered_point_cloud -def statistical_removing_wrapper( +def statistical_removal_wrapper( cloud, statistical_k, std_dev_factor, + use_median, save_by_pair: bool = False, point_cloud_csv_file_name=None, point_cloud_laz_file_name=None, saving_info=None, ): """ - Statistical outlier removing + Statistical outlier removal :param cloud: cloud to filter :type cloud: pandas DataFrame :param statistical_k: k - :type statistical_k: float + :type statistical_k: int :param std_dev_factor: std factor :type std_dev_factor: float + :param use_median: use median and quartile instead of mean and std + :type use median: bool :param save_by_pair: save point cloud as pair :type save_by_pair: bool :param point_cloud_csv_file_name: write point cloud as CSV in filename @@ -379,8 +440,8 @@ def statistical_removing_wrapper( # Filter point cloud tic = time.process_time() - (new_cloud, _) = outlier_removing_tools.statistical_outliers_filtering( - new_cloud, statistical_k, std_dev_factor + (new_cloud, _) = outlier_removal_tools.statistical_outlier_filtering( + new_cloud, statistical_k, std_dev_factor, use_median ) toc = time.process_time() logging.debug( @@ -417,3 +478,64 @@ def statistical_removing_wrapper( ) return new_cloud + + +def epipolar_statistical_removal_wrapper( + epipolar_ds, + statistical_k, + std_dev_factor, + use_median, + half_epipolar_size, + window, + overlap, + epsg, + saving_info=None, +): + """ + Statistical outlier removal in epipolar geometry + + :param epipolar_ds: epipolar dataset to filter + :type epipolar_ds: xr.Dataset + :param statistical_k: k + :type statistical_k: int + :param std_dev_factor: std factor + :type std_dev_factor: float + :param use_median: use median and quartile instead of mean and std + :type use median: bool + :param half_epipolar_size: half size of the window used to search neighbors + :type half_epipolar_size: int + :param window: window of base tile [row min, row max, col min col max] + :type window: list + :param overlap: overlap [row min, row max, col min col max] + :type overlap: list + :param epsg: epsg code of the CRS used to compute distances + :type epsg: int + + :return: filtered dataset + :rtype: xr.Dataset + + """ + + # Copy input cloud + filtered_cloud = copy.copy(epipolar_ds) + + outlier_removal_tools.epipolar_statistical_filtering( + filtered_cloud, + epsg, + k=statistical_k, + dev_factor=std_dev_factor, + use_median=use_median, + half_window_size=half_epipolar_size, + ) + + # Fill with attributes + cars_dataset.fill_dataset( + filtered_cloud, + saving_info=saving_info, + window=cars_dataset.window_array_to_dict(window), + profile=None, + attributes=None, + overlaps=cars_dataset.overlap_array_to_dict(overlap), + ) + + return filtered_cloud diff --git a/cars/applications/sparse_matching/sparse_matching_tools.py b/cars/applications/sparse_matching/sparse_matching_tools.py index 2bbaff2a..9fe2c80c 100644 --- a/cars/applications/sparse_matching/sparse_matching_tools.py +++ b/cars/applications/sparse_matching/sparse_matching_tools.py @@ -35,9 +35,7 @@ # CARS imports import cars.applications.sparse_matching.sparse_matching_constants as sm_cst from cars.applications import application_constants -from cars.applications.point_cloud_outliers_removing import ( - outlier_removing_tools, -) +from cars.applications.point_cloud_outlier_removal import outlier_removal_tools def euclidean_matrix_distance(descr1: np.array, descr2: np.array): @@ -403,7 +401,7 @@ def filter_point_cloud_matches( """ # Statistical filtering - filter_cloud, _ = outlier_removing_tools.statistical_outliers_filtering( + filter_cloud, _ = outlier_removal_tools.statistical_outlier_filtering( pd_cloud, k=matches_filter_knn, dev_factor=matches_filter_dev_factor, diff --git a/cars/applications/triangulation/line_of_sight_intersection.py b/cars/applications/triangulation/line_of_sight_intersection.py index 7c4779c8..f271d4f2 100644 --- a/cars/applications/triangulation/line_of_sight_intersection.py +++ b/cars/applications/triangulation/line_of_sight_intersection.py @@ -130,6 +130,7 @@ def save_triangulation_output( output_dir, dump_dir=None, intervals=None, + save_output_coordinates=True, save_output_color=True, save_output_classification=False, save_output_mask=False, @@ -158,6 +159,8 @@ def save_triangulation_output( :type dump_dir: str :param intervals: Either None or a List of 2 intervals indicators :type intervals: None or [str, str] + :param save_output_coordinates: Save X, Y and Z coords in output_dir + :type save_output_coordinates: bool :param save_output_color: Save color depth map in output_dir :type save_output_color: bool :param save_output_classification: Save classification depth map in @@ -187,9 +190,13 @@ def save_triangulation_output( if output_dir is None: output_dir = dump_dir - if output_dir: + + if save_output_coordinates or dump_dir: + coords_output_dir = ( + output_dir if save_output_coordinates else dump_dir + ) self.orchestrator.add_to_save_lists( - os.path.join(output_dir, "X.tif"), + os.path.join(coords_output_dir, "X.tif"), cst.X, epipolar_points_cloud, cars_ds_name="depth_map_x", @@ -197,7 +204,7 @@ def save_triangulation_output( ) self.orchestrator.add_to_save_lists( - os.path.join(output_dir, "Y.tif"), + os.path.join(coords_output_dir, "Y.tif"), cst.Y, epipolar_points_cloud, cars_ds_name="depth_map_y", @@ -205,7 +212,7 @@ def save_triangulation_output( ) self.orchestrator.add_to_save_lists( - os.path.join(output_dir, "Z.tif"), + os.path.join(coords_output_dir, "Z.tif"), cst.Z, epipolar_points_cloud, cars_ds_name="depth_map_z", @@ -315,6 +322,7 @@ def run( # noqa: C901 cloud_id=None, intervals=None, pair_output_dir=None, + save_output_coordinates=False, save_output_color=False, save_output_classification=False, save_output_mask=False, @@ -397,6 +405,8 @@ def run( # noqa: C901 :param pair_output_dir: directory to write triangulation output depth map. :type pair_output_dir: None or str + :param save_output_coordinates: Save X, Y, Z coords in pair_output_dir + :type save_output_coordinates: bool :param save_output_color: Save color depth map in pair_output_dir :type save_output_color: bool :param save_output_classification: Save classification depth map in @@ -563,6 +573,7 @@ def run( # noqa: C901 pair_output_dir, pair_dump_dir if self.save_intermediate_data else None, intervals, + save_output_coordinates, save_output_color, save_output_classification, save_output_mask, diff --git a/cars/applications/triangulation/triangulation.py b/cars/applications/triangulation/triangulation.py index c6c52a0b..3cbc6a64 100644 --- a/cars/applications/triangulation/triangulation.py +++ b/cars/applications/triangulation/triangulation.py @@ -114,6 +114,7 @@ def run( # noqa: C901 cloud_id=None, intervals=None, pair_output_dir=None, + save_output_coordinates=False, save_output_color=False, save_output_classification=False, save_output_mask=False, @@ -196,6 +197,8 @@ def run( # noqa: C901 :param pair_output_dir: directory to write triangulation output depth map. :type pair_output_dir: None or str + :param save_output_coordinates: Save X, Y, Z coords in pair_output_dir + :type save_output_coordinates: bool :param save_output_color: Save color depth map in pair_output_dir :type save_output_color: bool :param save_output_classification: Save classification depth map in diff --git a/cars/pipelines/default/default_pipeline.py b/cars/pipelines/default/default_pipeline.py index 6d6ee75d..533b0ab4 100644 --- a/cars/pipelines/default/default_pipeline.py +++ b/cars/pipelines/default/default_pipeline.py @@ -259,8 +259,8 @@ def infer_conditions_from_applications(self, conf): depth_merge_apps = { "point_cloud_fusion": 12, - "point_cloud_outliers_removing.1": 13, - "point_cloud_outliers_removing.2": 14, + "point_cloud_outlier_removal.1": 13, + "point_cloud_outlier_removal.2": 14, } depth_to_dsm_apps = { @@ -441,6 +441,14 @@ def check_applications( "dem_generation", ] + # TODO: point_cloud_outlier_removal is always needed for now, but + # after merging mode removal it would only be required if + # self.sensors_in_inputs. + needed_applications += [ + "point_cloud_outlier_removal.1", + "point_cloud_outlier_removal.2", + ] + if self.save_output_dsm or self.save_output_point_cloud: needed_applications += ["pc_denoising"] @@ -452,11 +460,7 @@ def check_applications( ] if self.merging: # we have to merge point clouds, add merging apps - needed_applications += [ - "point_cloud_fusion", - "point_cloud_outliers_removing.1", - "point_cloud_outliers_removing.2", - ] + needed_applications += ["point_cloud_fusion"] for app_key in conf.keys(): if app_key not in needed_applications: @@ -478,8 +482,8 @@ def check_applications( for app_key in [ "point_cloud_fusion", - "point_cloud_outliers_removing.1", - "point_cloud_outliers_removing.2", + "point_cloud_outlier_removal.1", + "point_cloud_outlier_removal.2", "pc_denoising", ]: if app_key in needed_applications: @@ -497,8 +501,8 @@ def check_applications( self.triangulation_application = None self.dem_generation_application = None self.pc_denoising_application = None - self.pc_outliers_removing_1_app = None - self.pc_outliers_removing_2_app = None + self.pc_outlier_removal_1_app = None + self.pc_outlier_removal_2_app = None self.rasterization_application = None self.pc_fusion_application = None self.dsm_filling_application = None @@ -584,6 +588,30 @@ def check_applications( self.dem_generation_application.get_conf() ) + # Points cloud small component outlier removal + self.pc_outlier_removal_1_app = Application( + "point_cloud_outlier_removal", + cfg=used_conf.get( + "point_cloud_outlier_removal.1", + {"method": "small_components"}, + ), + ) + used_conf["point_cloud_outlier_removal.1"] = ( + self.pc_outlier_removal_1_app.get_conf() + ) + + # Points cloud statistical outlier removal + self.pc_outlier_removal_2_app = Application( + "point_cloud_outlier_removal", + cfg=used_conf.get( + "point_cloud_outlier_removal.2", + {"method": "statistical"}, + ), + ) + used_conf["point_cloud_outlier_removal.2"] = ( + self.pc_outlier_removal_2_app.get_conf() + ) + if self.save_output_dsm or self.save_output_point_cloud: # Point cloud denoising @@ -622,30 +650,6 @@ def check_applications( self.pc_fusion_application.get_conf() ) - # Points cloud outlier removing small components - self.pc_outliers_removing_1_app = Application( - "point_cloud_outliers_removing", - cfg=used_conf.get( - "point_cloud_outliers_removing.1", - {"method": "small_components"}, - ), - ) - used_conf["point_cloud_outliers_removing.1"] = ( - self.pc_outliers_removing_1_app.get_conf() - ) - - # Points cloud outlier removing statistical - self.pc_outliers_removing_2_app = Application( - "point_cloud_outliers_removing", - cfg=used_conf.get( - "point_cloud_outliers_removing.2", - {"method": "statistical"}, - ), - ) - used_conf["point_cloud_outliers_removing.2"] = ( - self.pc_outliers_removing_2_app.get_conf() - ) - return used_conf def check_applications_with_inputs(self, inputs_conf, application_conf): @@ -1652,12 +1656,32 @@ def sensor_to_depth_maps(self): # noqa: C901 output_geoid_path = None depth_map_dir = None + last_depth_map_application = None if self.save_output_depth_map: depth_map_dir = os.path.join( self.out_dir, "depth_map", pair_key ) safe_makedirs(depth_map_dir) + if ( + self.pc_outlier_removal_2_app.used_config.get( + "activated", False + ) + is True + and self.merging is False + ): + last_depth_map_application = "pc_outlier_removal_2" + elif ( + self.pc_outlier_removal_1_app.used_config.get( + "activated", False + ) + is True + and self.merging is False + ): + last_depth_map_application = "pc_outlier_removal_1" + else: + last_depth_map_application = "triangulation" + # Run epipolar triangulation application epipolar_points_cloud = self.triangulation_application.run( self.pairs[pair_key]["sensor_image_left"], @@ -1680,6 +1704,8 @@ def sensor_to_depth_maps(self): # noqa: C901 cloud_id=cloud_id, intervals=intervals, pair_output_dir=depth_map_dir, + save_output_coordinates=last_depth_map_application + == "triangulation", save_output_color=bool(depth_map_dir) and self.auxiliary[out_cst.AUX_COLOR], save_output_classification=bool(depth_map_dir) @@ -1697,25 +1723,70 @@ def sensor_to_depth_maps(self): # noqa: C901 if self.merging: self.list_epipolar_points_cloud.append(epipolar_points_cloud) - # denoising available only if we'll go further in the pipeline - elif self.save_output_dsm or self.save_output_point_cloud: - denoised_epipolar_points_cloud = ( - self.pc_denoising_application.run( + else: + filtering_out_dir = ( + depth_map_dir + if ( + depth_map_dir + and last_depth_map_application == "pc_outlier_removal_1" + ) + else None + ) + filtered_epipolar_points_cloud_1 = ( + self.pc_outlier_removal_1_app.run( epipolar_points_cloud, + output_dir=filtering_out_dir, + dump_dir=os.path.join( + self.dump_dir, "pc_outlier_removal_1", pair_key + ), + epsg=self.epsg, orchestrator=self.cars_orchestrator, - pair_folder=os.path.join( - self.dump_dir, "denoising", pair_key + ) + ) + if self.quit_on_app("point_cloud_outlier_removal.1"): + continue # keep iterating over pairs, but don't go further + filtering_out_dir = ( + depth_map_dir + if ( + depth_map_dir + and last_depth_map_application == "pc_outlier_removal_2" + ) + else None + ) + filtered_epipolar_points_cloud_2 = ( + self.pc_outlier_removal_2_app.run( + filtered_epipolar_points_cloud_1, + output_dir=filtering_out_dir, + dump_dir=os.path.join( + self.dump_dir, "pc_outlier_removal_2", pair_key ), - pair_key=pair_key, + epsg=self.epsg, + orchestrator=self.cars_orchestrator, ) ) + if self.quit_on_app("point_cloud_outlier_removal.2"): + continue # keep iterating over pairs, but don't go further - self.list_epipolar_points_cloud.append( - denoised_epipolar_points_cloud - ) + # denoising available only if we'll go further in the pipeline + if self.save_output_dsm or self.save_output_point_cloud: + denoised_epipolar_points_cloud = ( + self.pc_denoising_application.run( + filtered_epipolar_points_cloud_2, + orchestrator=self.cars_orchestrator, + pair_folder=os.path.join( + self.dump_dir, "denoising", pair_key + ), + pair_key=pair_key, + ) + ) - if self.quit_on_app("pc_denoising"): - continue # keep iterating over pairs, but don't go further + self.list_epipolar_points_cloud.append( + denoised_epipolar_points_cloud + ) + + if self.quit_on_app("pc_denoising"): + # keep iterating over pairs, but don't go further + continue if self.save_output_dsm or self.save_output_point_cloud: # Compute terrain bounding box /roi related to @@ -1761,11 +1832,14 @@ def sensor_to_depth_maps(self): # noqa: C901 ) # quit if any app in the loop over the pairs was the last one + # pylint:disable=too-many-boolean-expressions if ( self.quit_on_app("dense_matching") or self.quit_on_app("dense_matches_filling.1") or self.quit_on_app("dense_matches_filling.2") or self.quit_on_app("triangulation") + or self.quit_on_app("point_cloud_outlier_removal.1") + or self.quit_on_app("point_cloud_outlier_removal.2") or self.quit_on_app("pc_denoising") ): return True @@ -1917,7 +1991,7 @@ def rasterize_point_cloud(self): def preprocess_depth_maps(self): """ Adds multiple processing steps to the depth maps : - Merging, filtering, denoising, outliers removing. + Merging, filtering, denoising, outlier removal. Creates the point cloud that will be rasterized in the last step of the pipeline. """ @@ -1932,13 +2006,13 @@ def preprocess_depth_maps(self): ].attributes.get("color_type", None) else: # Merge point clouds - pc_outliers_removing_1_margins = ( - self.pc_outliers_removing_1_app.get_on_ground_margin( + pc_outlier_removal_1_margins = ( + self.pc_outlier_removal_1_app.get_on_ground_margin( resolution=self.resolution ) ) - pc_outliers_removing_2_margins = ( - self.pc_outliers_removing_2_app.get_on_ground_margin( + pc_outlier_removal_2_margins = ( + self.pc_outlier_removal_2_app.get_on_ground_margin( resolution=self.resolution ) ) @@ -1954,19 +2028,19 @@ def preprocess_depth_maps(self): if self.pc_denoising_application.used_method != "none": last_pc_application = "denoising" elif ( - self.pc_outliers_removing_2_app.used_config.get( + self.pc_outlier_removal_2_app.used_config.get( "activated", False ) is True ): - last_pc_application = "pc_outliers_removing_2" + last_pc_application = "pc_outlier_removal_2" elif ( - self.pc_outliers_removing_1_app.used_config.get( + self.pc_outlier_removal_1_app.used_config.get( "activated", False ) is True ): - last_pc_application = "pc_outliers_removing_1" + last_pc_application = "pc_outlier_removal_1" else: last_pc_application = "fusion" @@ -1985,8 +2059,8 @@ def preprocess_depth_maps(self): ), orchestrator=self.cars_orchestrator, margins=( - pc_outliers_removing_1_margins - + pc_outliers_removing_2_margins + pc_outlier_removal_1_margins + + pc_outlier_removal_2_margins + raster_app_margin ), optimal_terrain_tile_width=self.optimal_terrain_tile_width, @@ -1999,29 +2073,39 @@ def preprocess_depth_maps(self): return True # Remove outliers with small components method - filtered_1_merged_points_clouds = ( - self.pc_outliers_removing_1_app.run( - merged_points_clouds, - orchestrator=self.cars_orchestrator, - save_laz_output=self.save_output_point_cloud - and last_pc_application == "pc_outliers_removing_1", - ) + filtered_1_merged_points_clouds = self.pc_outlier_removal_1_app.run( + merged_points_clouds, + orchestrator=self.cars_orchestrator, + output_dir=( + os.path.join(self.out_dir, "point_cloud") + if self.save_output_point_cloud + and last_pc_application == "pc_outlier_removal_1" + else None + ), + dump_dir=os.path.join( + self.dump_dir, "point_cloud_outlier_removal_1" + ), ) - if self.quit_on_app("point_cloud_outliers_removing.1"): + if self.quit_on_app("point_cloud_outlier_removal.1"): return True # Remove outliers with statistical components method - filtered_2_merged_points_clouds = ( - self.pc_outliers_removing_2_app.run( - filtered_1_merged_points_clouds, - orchestrator=self.cars_orchestrator, - save_laz_output=self.save_output_point_cloud - and last_pc_application == "pc_outliers_removing_2", - ) + filtered_2_merged_points_clouds = self.pc_outlier_removal_2_app.run( + filtered_1_merged_points_clouds, + orchestrator=self.cars_orchestrator, + output_dir=( + os.path.join(self.out_dir, "point_cloud") + if self.save_output_point_cloud + and last_pc_application == "pc_outlier_removal_2" + else None + ), + dump_dir=os.path.join( + self.dump_dir, "point_cloud_outlier_removal_2" + ), ) - if self.quit_on_app("point_cloud_outliers_removing.2"): + if self.quit_on_app("point_cloud_outlier_removal.2"): return True # denoise point cloud @@ -2113,7 +2197,7 @@ def load_input_depth_maps(self): ) ) self.optimal_terrain_tile_width = min( - self.pc_outliers_removing_1_app.get_optimal_tile_size( + self.pc_outlier_removal_1_app.get_optimal_tile_size( self.cars_orchestrator.cluster.checked_conf_cluster[ "max_ram_per_worker" ], @@ -2122,7 +2206,7 @@ def load_input_depth_maps(self): ), point_cloud_resolution=average_distance_point_cloud, ), - self.pc_outliers_removing_2_app.get_optimal_tile_size( + self.pc_outlier_removal_2_app.get_optimal_tile_size( self.cars_orchestrator.cluster.checked_conf_cluster[ "max_ram_per_worker" ], diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 18ca45d5..1adf84b1 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -688,7 +688,7 @@ The structure follows this organisation: .. warning:: CARS will only compute a point cloud when the key ``merging`` in ``advanced`` is set to `True`, which means setting ``output_level`` as containing `point_cloud` will effectively force ``merging`` to `True`. - This behavior will have the side-effect of running the point cloud denoising and outliers removing applications. + This behavior will have the side-effect of running the point cloud denoising and outlier removal applications. .. note:: If you wish to save an individual point cloud for each input given, the key ``save_by_pair`` of ``output`` will need to be set to `True`. @@ -1257,20 +1257,20 @@ The structure follows this organisation: Please, see the section :ref:`merge_laz_files` to merge them into one single file. `save_by_pair` parameter enables saving by input pair. The csv/laz name aggregates row, col and corresponding pair key. - .. tab:: Point Cloud outliers removing + .. tab:: Point Cloud outlier removal - **Name**: "point_cloud_outliers_removing" + **Name**: "point_cloud_outlier_removal" **Description** - Point cloud outliers removing + Point cloud outlier removal **Configuration** +------------------------------+------------------------------------------+---------+-----------------------------------+---------------+----------+ | Name | Description | Type | Available value | Default value | Required | +==============================+==========================================+=========+===================================+===============+==========+ - | method | Method for point cloud outliers removing | string | "statistical", "small_components" | "statistical" | No | + | method | Method for point cloud outlier removal | string | "statistical", "small_components" | "statistical" | No | +------------------------------+------------------------------------------+---------+-----------------------------------+---------------+----------+ | save_intermediate_data | Save points clouds as laz and csv format | boolean | | false | No | +------------------------------+------------------------------------------+---------+-----------------------------------+---------------+----------+ @@ -1279,15 +1279,19 @@ The structure follows this organisation: If method is *statistical*: - +----------------+-------------+---------+-----------------+---------------+----------+ - | Name | Description | Type | Available value | Default value | Required | - +================+=============+=========+=================+===============+==========+ - | activated | | boolean | | false | No | - +----------------+-------------+---------+-----------------+---------------+----------+ - | k | | int | should be > 0 | 50 | No | - +----------------+-------------+---------+-----------------+---------------+----------+ - | std_dev_factor | | float | should be > 0 | 5.0 | No | - +----------------+-------------+---------+-----------------+---------------+----------+ + +--------------------+-------------+---------+-----------------+---------------+----------+ + | Name | Description | Type | Available value | Default value | Required | + +====================+=============+=========+=================+===============+==========+ + | activated | | boolean | | false | No | + +--------------------+-------------+---------+-----------------+---------------+----------+ + | k | | int | should be > 0 | 50 | No | + +--------------------+-------------+---------+-----------------+---------------+----------+ + | std_dev_factor | | float | should be > 0 | 5.0 | No | + +--------------------+-------------+---------+-----------------+---------------+----------+ + | use_median | | bool | | True | No | + +--------------------+-------------+---------+-----------------+---------------+----------+ + | half_epipolar_size | | int | | 5 | No | + +--------------------+-------------+---------+-----------------+---------------+----------+ If method is *small_components* @@ -1304,31 +1308,33 @@ The structure follows this organisation: +-----------------------------+-------------+---------+-----------------+---------------+----------+ | clusters_distance_threshold | | float | | None | No | +-----------------------------+-------------+---------+-----------------+---------------+----------+ + | half_epipolar_size | | int | | 5 | No | + +-----------------------------+-------------+---------+-----------------+---------------+----------+ .. warning:: - There is a particular case with the *Point Cloud outliers removing* application because it is called twice. + There is a particular case with the *Point Cloud outlier removal* application because it is called twice. The ninth step consists of Filter the 3D points cloud via two consecutive filters. So you can configure the application twice , once for the *small component filters*, the other for *statistical* filter. Because it is not possible to define twice the *application_name* on your json configuration file, we have decided to configure those two applications with : - * *point_cloud_outliers_removing.1* - * *point_cloud_outliers_removing.2* + * *point_cloud_outlier_removal.1* + * *point_cloud_outlier_removal.2* - Each one is associated to a particular *point_cloud_outliers_removing* method* + Each one is associated to a particular *point_cloud_outlier_removal* method* **Example** .. code-block:: json "applications": { - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "on_ground_margin": 10, "save_intermediate_data": true }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "k": 10, "save_intermediate_data": true, @@ -1608,7 +1614,7 @@ The structure follows this organisation: The point cloud output product consists of a collection of laz files, each containing a tile of the point cloud. If the `save_by_pair` option is set, laz will be produced for each sensor pair defined in input pairing. - The point cloud found in the product the highest level point cloud produced by cars. For exemple, if outlier removing and point cloud denoising are deactivated, the point cloud will correspond to the output of point cloud fusion. If only the first application of outlier removing is activated, this will be the output point cloud. + The point cloud found in the product the highest level point cloud produced by cars. For exemple, if outlier removal and point cloud denoising are deactivated, the point cloud will correspond to the output of point cloud fusion. If only the first application of outlier removal is activated, this will be the output point cloud. **Geoid** diff --git a/setup.cfg b/setup.cfg index 21ae7b5c..c82e701c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -81,6 +81,7 @@ install_requires = libsgm==0.4.6 cars-rasterize==0.2.* cars-resample==0.1.* + cars-filter==0.1.* vlsift==0.1.* shareloc==0.2.3 bulldozer-dtm diff --git a/tests/applications/point_cloud_outliers_removing/__init__.py b/tests/applications/point_cloud_outlier_removal/__init__.py similarity index 93% rename from tests/applications/point_cloud_outliers_removing/__init__.py rename to tests/applications/point_cloud_outlier_removal/__init__.py index e60b0692..5f7ca385 100644 --- a/tests/applications/point_cloud_outliers_removing/__init__.py +++ b/tests/applications/point_cloud_outlier_removal/__init__.py @@ -19,5 +19,5 @@ # limitations under the License. # """ -Cars tests/points_cloud_outliers_removing init file +Cars tests/point_cloud_outlier_removal init file """ diff --git a/tests/applications/point_cloud_outlier_removal/test_outlier_removal.py b/tests/applications/point_cloud_outlier_removal/test_outlier_removal.py new file mode 100644 index 00000000..795188fa --- /dev/null +++ b/tests/applications/point_cloud_outlier_removal/test_outlier_removal.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python +# coding: utf8 +# +# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). +# +# This file is part of CARS +# (see https://github.com/CNES/cars). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Cars tests/points_cloud_outlier_removal file +""" + +import datetime + +import laspy + +# Third party imports +import numpy as np +import outlier_filter # pylint:disable=E0401 +import pyproj +import pytest +import rasterio + +# CARS imports +from cars.applications.point_cloud_outlier_removal import outlier_removal_tools + +# CARS Tests imports +from tests.helpers import absolute_data_path + + +@pytest.mark.unit_tests +def test_detect_small_components(): + """ + Create fake cloud to process and test detect_small_components + """ + x_coord = np.zeros((5, 5)) + x_coord[4, 4] = 20 + x_coord[0, 4] = 19.55 + x_coord[0, 3] = 19.10 + y_coord = np.zeros((5, 5)) + + z_coord = np.zeros((5, 5)) + z_coord[0:2, 0:2] = 10 + z_coord[1, 1] = 12 + + cloud_arr = np.concatenate( + [ + np.stack((x_coord, y_coord, z_coord), axis=-1).reshape(-1, 3) + for x_coord, y_coord, z_coord in zip( # noqa: B905 + x_coord, y_coord, z_coord + ) + ], + axis=0, + ) + + indexes_to_filter = outlier_filter.pc_small_component_outlier_filtering( + cloud_arr[:, 0], cloud_arr[:, 1], cloud_arr[:, 2], 0.5, 10, 2 + ) + assert sorted(indexes_to_filter) == [3, 4, 24] + + # test without the second level of filtering + indexes_to_filter = outlier_filter.pc_small_component_outlier_filtering( + cloud_arr[:, 0], cloud_arr[:, 1], cloud_arr[:, 2], 0.5, 10, np.nan + ) + assert sorted(indexes_to_filter) == [0, 1, 3, 4, 5, 6, 24] + + +@pytest.mark.unit_tests +def test_detect_statistical_outliers(): + """ + Create fake cloud to process and test detect_statistical_outliers + """ + x_coord = np.zeros((5, 6)) + off = 0 + for line in range(5): + # x[line,:] = np.arange(off, off+(line+1)*5, line+1) + last_val = off + 5 + x_coord[line, :5] = np.arange(off, last_val) + off += (line + 2 + 1) * 5 + + # outlier + x_coord[line, 5] = (off + last_val - 1) / 2 + + y_coord = np.zeros((5, 6)) + z_coord = np.zeros((5, 6)) + + ref_cloud = np.concatenate( + [ + np.stack((x_coord, y_coord, z_coord), axis=-1).reshape(-1, 3) + for x_coord, y_coord, z_coord in zip( # noqa: B905 + x_coord, y_coord, z_coord + ) + ], + axis=0, + ) + + removed_elt_pos = outlier_filter.pc_statistical_outlier_filtering( + ref_cloud[:, 0], + ref_cloud[:, 1], + ref_cloud[:, 2], + k=4, + dev_factor=0.0, + use_median=False, + ) + assert sorted(removed_elt_pos) == [5, 11, 17, 23, 29] + + removed_elt_pos = outlier_filter.pc_statistical_outlier_filtering( + ref_cloud[:, 0], + ref_cloud[:, 1], + ref_cloud[:, 2], + k=4, + dev_factor=1.0, + use_median=False, + ) + assert sorted(removed_elt_pos) == [11, 17, 23, 29] + + removed_elt_pos = outlier_filter.pc_statistical_outlier_filtering( + ref_cloud[:, 0], + ref_cloud[:, 1], + ref_cloud[:, 2], + k=4, + dev_factor=2.0, + use_median=False, + ) + assert sorted(removed_elt_pos) == [23, 29] + + removed_elt_pos = outlier_filter.pc_statistical_outlier_filtering( + ref_cloud[:, 0], + ref_cloud[:, 1], + ref_cloud[:, 2], + k=4, + dev_factor=1.0, + use_median=True, + ) + assert sorted(removed_elt_pos) == [5, 11, 17, 23, 29] + + removed_elt_pos = outlier_filter.pc_statistical_outlier_filtering( + ref_cloud[:, 0], + ref_cloud[:, 1], + ref_cloud[:, 2], + k=4, + dev_factor=7.0, + use_median=True, + ) + assert sorted(removed_elt_pos) == [11, 17, 23, 29] + + removed_elt_pos = outlier_filter.pc_statistical_outlier_filtering( + ref_cloud[:, 0], + ref_cloud[:, 1], + ref_cloud[:, 2], + k=4, + dev_factor=15.0, + use_median=True, + ) + # Note: This is the expected result if median computation was exact, but it + # is not the case in this implementation. + # assert sorted(removed_elt_pos) == [23, 29] + assert sorted(removed_elt_pos) == [29] + + +@pytest.mark.unit_tests +@pytest.mark.parametrize("use_median", [True, False]) +def test_outlier_removal_point_cloud_statistical(use_median): + """ + Outlier filtering test from laz, using statistical method. + + The test verifies that cars-filter produces the same results as a Python + equivalent using scipy ckdtrees + """ + k = 50 + dev_factor = 1 + + with laspy.open( + absolute_data_path("input/nimes_laz/subsampled_nimes.laz") + ) as creader: + las = creader.read() + points = np.vstack((las.x, las.y, las.z)) + + start_time = datetime.datetime.now() + result_cpp = outlier_filter.pc_statistical_outlier_filtering( + las.x, las.y, las.z, dev_factor, k, use_median + ) + end_time = datetime.datetime.now() + cpp_duration = end_time - start_time + + print(f"Statistical filtering total duration (cpp): {cpp_duration}") + + # Perform the same filtering Scipy and compare the results + transposed_points = np.transpose(points) + + scipy_start = datetime.datetime.now() + detected_points = outlier_removal_tools.detect_statistical_outliers( + transposed_points, k, dev_factor, use_median + ) + + scipy_end = datetime.datetime.now() + scipy_duration = scipy_end - scipy_start + print(f"Statistical filtering total duration (Python): {scipy_duration}") + is_same_result = detected_points == result_cpp + assert is_same_result + + +@pytest.mark.unit_tests +@pytest.mark.parametrize("clusters_distance_threshold", [float("nan"), 4]) +def test_outlier_removal_point_cloud_small_components( + clusters_distance_threshold, +): + """ + Outlier filtering test from laz, using small components method. + + The test verifies that cars-filter produces the same results as a Python + equivalent using scipy ckdtrees + """ + + connection_val = 3 + nb_pts_threshold = 15 + + with laspy.open( + absolute_data_path("input/nimes_laz/subsampled_nimes.laz") + ) as creader: + las = creader.read() + points = np.vstack((las.x, las.y, las.z)) + + start_time = datetime.datetime.now() + result_cpp = outlier_filter.pc_small_component_outlier_filtering( + las.x, + las.y, + las.z, + connection_val, + nb_pts_threshold, + clusters_distance_threshold, + ) + end_time = datetime.datetime.now() + cpp_duration = end_time - start_time + + print(f"Small Component filtering total duration (cpp): {cpp_duration}") + print(f"result_cpp: {result_cpp}") + + transposed_points = np.transpose(points) + + scipy_start = datetime.datetime.now() + + cluster_to_remove = outlier_removal_tools.detect_small_components( + transposed_points, + connection_val, + nb_pts_threshold, + clusters_distance_threshold, + ) + + scipy_end = datetime.datetime.now() + scipy_duration = scipy_end - scipy_start + print( + f"Small Component filtering total duration (Python): {scipy_duration}" + ) + + cluster_to_remove.sort() + result_cpp.sort() + + is_same_result = cluster_to_remove == result_cpp + + assert is_same_result + print(f"Scipy and cars filter results are the same ? {is_same_result}") + + +@pytest.mark.unit_tests +@pytest.mark.parametrize("use_median", [True, False]) +def test_outlier_removal_epipolar_statistical(use_median): + """ + Outlier filtering test from depth map in epipolar geometry, using + statistical method + """ + k = 15 + half_window_size = 15 + dev_factor = 1 + + with rasterio.open( + absolute_data_path("input/depth_map_gizeh/X.tif") + ) as x_ds, rasterio.open( + absolute_data_path("input/depth_map_gizeh/Y.tif") + ) as y_ds, rasterio.open( + absolute_data_path("input/depth_map_gizeh/Z.tif") + ) as z_ds: + x_values = x_ds.read(1) + y_values = y_ds.read(1) + z_values = z_ds.read(1) + + input_shape = x_values.shape + + transformer = pyproj.Transformer.from_crs(4326, 32636) + # X-Y inversion required because WGS84 is lat first ? + # pylint: disable-next=unpacking-non-sequence + x_utm, y_utm = transformer.transform(x_values, y_values) + + # Make copies for reprocessing with kdtree + x_utm_flat = np.copy(x_utm).reshape(input_shape[0] * input_shape[1]) + y_utm_flat = np.copy(y_utm).reshape(input_shape[0] * input_shape[1]) + z_flat = np.copy(z_values).reshape(input_shape[0] * input_shape[1]) + + start_time = datetime.datetime.now() + + outlier_array = outlier_filter.epipolar_statistical_outlier_filtering( + x_utm, y_utm, z_values, k, half_window_size, dev_factor, use_median + ) + + end_time = datetime.datetime.now() + epipolar_processing_duration = end_time - start_time + + print(f"Epipolar filtering duration: {epipolar_processing_duration}") + + # filter NaNs + nan_pos = np.isnan(x_utm_flat) + x_utm_flat = x_utm_flat[~nan_pos] + y_utm_flat = y_utm_flat[~nan_pos] + z_flat = z_flat[~nan_pos] + + start_time = datetime.datetime.now() + + result_kdtree = np.array( + outlier_filter.pc_statistical_outlier_filtering( + x_utm_flat, y_utm_flat, z_flat, dev_factor, k, use_median + ) + ) + + end_time = datetime.datetime.now() + kdtree_processing_duration = end_time - start_time + print(f"KDTree filtering duration: {kdtree_processing_duration}") + + print(outlier_array.shape) + + outlier_array = outlier_array.reshape(input_shape[0] * input_shape[1]) + print(outlier_array.shape) + outlier_array = np.argwhere(outlier_array[~nan_pos]).flatten() + print(outlier_array.shape) + + # Find common outliers between the two methods + # common_outliers = np.intersect1d(result_kdtree, outlier_array) + # print(common_outliers) + + if use_median: + # Note that k and half_window_size have been chosed for this assertion + # to succeed. + # The two algorithms does not produce the same results if the epipolar + # neighborhood is too small. + assert (np.sort(outlier_array) == np.sort(result_kdtree)).all() + else: + # in mean/stddev mode, the results are differents because some outliers + # are not found in epipolar neighborhood + assert len(outlier_array) == 27612 + assert len(result_kdtree) == 39074 + + +@pytest.mark.unit_tests +@pytest.mark.parametrize("clusters_distance_threshold", [float("nan"), 2]) +def test_outlier_removal_epipolar_small_components( + clusters_distance_threshold, +): + """ + Outlier filtering test from depth map in epipolar geometry, using small + components method + """ + min_cluster_size = 15 + radius = 1 + half_window_size = 7 + + with rasterio.open( + absolute_data_path("input/depth_map_gizeh/X.tif") + ) as x_ds, rasterio.open( + absolute_data_path("input/depth_map_gizeh/Y.tif") + ) as y_ds, rasterio.open( + absolute_data_path("input/depth_map_gizeh/Z.tif") + ) as z_ds: + x_values = x_ds.read(1) + y_values = y_ds.read(1) + z_values = z_ds.read(1) + + input_shape = x_values.shape + + transformer = pyproj.Transformer.from_crs(4326, 32636) + # X-Y inversion required because WGS84 is lat first ? + # pylint: disable-next=unpacking-non-sequence + x_utm, y_utm = transformer.transform(x_values, y_values) + + # Make copies for reprocessing with kdtree + x_utm_flat = np.copy(x_utm).reshape(input_shape[0] * input_shape[1]) + y_utm_flat = np.copy(y_utm).reshape(input_shape[0] * input_shape[1]) + z_flat = np.copy(z_values).reshape(input_shape[0] * input_shape[1]) + + start_time = datetime.datetime.now() + + outlier_array = outlier_filter.epipolar_small_component_outlier_filtering( + x_utm, + y_utm, + z_values, + min_cluster_size, + radius, + half_window_size, + clusters_distance_threshold, + ) + + end_time = datetime.datetime.now() + epipolar_processing_duration = end_time - start_time + + print(f"Epipolar filtering duration: {epipolar_processing_duration}") + + # Test with KDTree + + # filter NaNs + nan_pos = np.isnan(x_utm_flat) + x_utm_flat = x_utm_flat[~nan_pos] + y_utm_flat = y_utm_flat[~nan_pos] + z_flat = z_flat[~nan_pos] + + start_time = datetime.datetime.now() + + result_kdtree = np.array( + outlier_filter.pc_small_component_outlier_filtering( + x_utm_flat, + y_utm_flat, + z_flat, + radius, + min_cluster_size, + clusters_distance_threshold, + ) + ) + + end_time = datetime.datetime.now() + kdtree_processing_duration = end_time - start_time + + print(f"KDTree filtering duration: {kdtree_processing_duration}") + + outlier_array = outlier_array.reshape(input_shape[0] * input_shape[1]) + print(outlier_array.shape) + outlier_array = np.argwhere(outlier_array[~nan_pos]).flatten() + print(outlier_array.shape) + # Find common outliers between the two methods + # common_outliers = np.intersect1d(result_kdtree, outlier_array) + # print(common_outliers) + + assert (np.sort(outlier_array) == np.sort(result_kdtree)).all() diff --git a/tests/applications/point_cloud_outliers_removing/test_outliers_removing_config.py b/tests/applications/point_cloud_outlier_removal/test_outlier_removal_config.py similarity index 89% rename from tests/applications/point_cloud_outliers_removing/test_outliers_removing_config.py rename to tests/applications/point_cloud_outlier_removal/test_outlier_removal_config.py index 9a037c5f..1a56ed41 100644 --- a/tests/applications/point_cloud_outliers_removing/test_outliers_removing_config.py +++ b/tests/applications/point_cloud_outlier_removal/test_outlier_removal_config.py @@ -28,12 +28,12 @@ # Third party imports import pytest -from cars.applications.point_cloud_outliers_removing.small_components import ( +from cars.applications.point_cloud_outlier_removal.small_components import ( SmallComponents, ) # CARS imports -from cars.applications.point_cloud_outliers_removing.statistical import ( +from cars.applications.point_cloud_outlier_removal.statistical import ( Statistical, ) @@ -52,6 +52,7 @@ def test_check_full_conf_small_components(): "connection_distance": 3.0, "nb_points_threshold": 50, "clusters_distance_threshold": None, + "half_epipolar_size": 5, } _ = SmallComponents(conf) @@ -68,6 +69,8 @@ def test_check_full_conf_statistical(): "activated": False, "k": 50, "std_dev_factor": 5.0, + "use_median": False, + "half_epipolar_size": 5, } _ = Statistical(conf) diff --git a/tests/applications/point_cloud_outliers_removing/test_outliers_removing.py b/tests/applications/point_cloud_outliers_removing/test_outliers_removing.py deleted file mode 100644 index 07a1e248..00000000 --- a/tests/applications/point_cloud_outliers_removing/test_outliers_removing.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python -# coding: utf8 -# -# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). -# -# This file is part of CARS -# (see https://github.com/CNES/cars). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -""" -Cars tests/points_cloud_outliers_removing file -""" - -# Third party imports -import numpy as np -import pytest - -# CARS imports -from cars.applications.point_cloud_outliers_removing import ( - outlier_removing_tools, -) - -# CARS Tests imports - - -@pytest.mark.unit_tests -def test_detect_small_components(): - """ - Create fake cloud to process and test detect_small_components - """ - x_coord = np.zeros((5, 5)) - x_coord[4, 4] = 20 - x_coord[0, 4] = 19.55 - x_coord[0, 3] = 19.10 - y_coord = np.zeros((5, 5)) - - z_coord = np.zeros((5, 5)) - z_coord[0:2, 0:2] = 10 - z_coord[1, 1] = 12 - - cloud_arr = np.concatenate( - [ - np.stack((x_coord, y_coord, z_coord), axis=-1).reshape(-1, 3) - for x_coord, y_coord, z_coord in zip( # noqa: B905 - x_coord, y_coord, z_coord - ) - ], - axis=0, - ) - - indexes_to_filter = outlier_removing_tools.detect_small_components( - cloud_arr, 0.5, 10, 2 - ) - assert sorted(indexes_to_filter) == [3, 4, 24] - - # test without the second level of filtering - indexes_to_filter = outlier_removing_tools.detect_small_components( - cloud_arr, 0.5, 10, None - ) - assert sorted(indexes_to_filter) == [0, 1, 3, 4, 5, 6, 24] - - -@pytest.mark.unit_tests -def test_detect_statistical_outliers(): - """ - Create fake cloud to process and test detect_statistical_outliers - """ - x_coord = np.zeros((5, 6)) - off = 0 - for line in range(5): - # x[line,:] = np.arange(off, off+(line+1)*5, line+1) - last_val = off + 5 - x_coord[line, :5] = np.arange(off, last_val) - off += (line + 2 + 1) * 5 - - # outlier - x_coord[line, 5] = (off + last_val - 1) / 2 - - y_coord = np.zeros((5, 6)) - z_coord = np.zeros((5, 6)) - - ref_cloud = np.concatenate( - [ - np.stack((x_coord, y_coord, z_coord), axis=-1).reshape(-1, 3) - for x_coord, y_coord, z_coord in zip( # noqa: B905 - x_coord, y_coord, z_coord - ) - ], - axis=0, - ) - - removed_elt_pos = outlier_removing_tools.detect_statistical_outliers( - ref_cloud, 4, 0.0, use_median=False - ) - assert sorted(removed_elt_pos) == [5, 11, 17, 23, 29] - - removed_elt_pos = outlier_removing_tools.detect_statistical_outliers( - ref_cloud, 4, 1.0, use_median=False - ) - assert sorted(removed_elt_pos) == [11, 17, 23, 29] - - removed_elt_pos = outlier_removing_tools.detect_statistical_outliers( - ref_cloud, 4, 2.0, use_median=False - ) - assert sorted(removed_elt_pos) == [23, 29] - - removed_elt_pos = outlier_removing_tools.detect_statistical_outliers( - ref_cloud, 4, 1.0, use_median=True - ) - assert sorted(removed_elt_pos) == [5, 11, 17, 23, 29] - - removed_elt_pos = outlier_removing_tools.detect_statistical_outliers( - ref_cloud, 4, 7.0, use_median=True - ) - assert sorted(removed_elt_pos) == [11, 17, 23, 29] - - removed_elt_pos = outlier_removing_tools.detect_statistical_outliers( - ref_cloud, 4, 15.0, use_median=True - ) - assert sorted(removed_elt_pos) == [23, 29] diff --git a/tests/data/input/nimes_laz/subsampled_nimes.laz b/tests/data/input/nimes_laz/subsampled_nimes.laz new file mode 100644 index 00000000..ab1ff38e Binary files /dev/null and b/tests/data/input/nimes_laz/subsampled_nimes.laz differ diff --git a/tests/test_end2end.py b/tests/test_end2end.py index 12dcd980..8ae64e43 100644 --- a/tests/test_end2end.py +++ b/tests/test_end2end.py @@ -624,13 +624,14 @@ def test_end2end_ventoux_unique(): input_config_dense_dsm = input_config_sparse_dsm.copy() # update applications dense_dsm_applications = { - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, + "use_median": False, }, "point_cloud_rasterization": { "method": "simple_gaussian", @@ -1099,13 +1100,14 @@ def test_end2end_ventoux_unique(): }, }, }, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, + "use_median": False, }, "point_cloud_rasterization": { "method": "simple_gaussian", @@ -1200,13 +1202,14 @@ def test_end2end_ventoux_unique(): "use_cross_validation": True, "use_global_disp_range": False, }, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, + "use_median": False, }, "point_cloud_rasterization": { "method": "simple_gaussian", @@ -1623,16 +1626,17 @@ def test_end2end_ventoux_unique_split(): "output": {"directory": output_path, "resolution": 0.5}, "pipeline": "dense_depth_maps_to_dense_dsm", "applications": { - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, "save_intermediate_data": True, "save_by_pair": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, "save_intermediate_data": True, + "use_median": False, }, "point_cloud_rasterization": { "method": "simple_gaussian", @@ -1659,7 +1663,7 @@ def test_end2end_ventoux_unique_split(): os.path.join( out_dir_dsm, "dump_dir", - "small_components", + "point_cloud_outlier_removal_1", "laz", "675292.3110543193_4897140.457149682_one.laz", ) @@ -1892,10 +1896,10 @@ def test_end2end_ventoux_unique_split(): ) del input_dsm_config["applications"][ - "point_cloud_outliers_removing.1" + "point_cloud_outlier_removal.1" ] del input_dsm_config["applications"][ - "point_cloud_outliers_removing.2" + "point_cloud_outlier_removal.2" ] input_dsm_config["pipeline"] = ( "dense_depth_maps_to_dense_dsm_no_merging" @@ -2959,6 +2963,7 @@ def test_end2end_ventoux_with_color(): input_json = absolute_data_path( "input/phr_ventoux/input_with_color.json" ) + # Run sparse dsm pipeline _, input_config_sparse_res = generate_input_json( input_json, @@ -3062,16 +3067,17 @@ def test_end2end_ventoux_with_color(): "save_intermediate_data": True, "save_by_pair": True, }, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, "save_intermediate_data": True, "save_by_pair": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, "save_intermediate_data": True, + "use_median": False, }, "triangulation": { "method": "line_of_sight_intersection", @@ -3140,7 +3146,7 @@ def test_end2end_ventoux_with_color(): os.path.join( out_dir, "dump_dir", - "small_components", + "point_cloud_outlier_removal_1", "laz", pc2 + "_left_right.laz", ) @@ -3152,7 +3158,7 @@ def test_end2end_ventoux_with_color(): os.path.join( out_dir, "dump_dir", - "small_components", + "point_cloud_outlier_removal_1", "csv", pc2 + "_left_right.csv", ) @@ -3164,7 +3170,11 @@ def test_end2end_ventoux_with_color(): assert ( os.path.exists( os.path.join( - out_dir, "dump_dir", "statistical", "laz", pc1 + ".laz" + out_dir, + "dump_dir", + "point_cloud_outlier_removal_2", + "laz", + pc1 + ".laz", ) ) is True @@ -3172,7 +3182,11 @@ def test_end2end_ventoux_with_color(): assert ( os.path.exists( os.path.join( - out_dir, "dump_dir", "statistical", "csv", pc1 + ".csv" + out_dir, + "dump_dir", + "point_cloud_outlier_removal_2", + "csv", + pc1 + ".csv", ) ) is True @@ -3330,15 +3344,16 @@ def test_end2end_ventoux_with_classif(): "method": "mapping_to_terrain_tiles", "save_intermediate_data": True, }, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, "save_intermediate_data": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, "save_intermediate_data": True, + "use_median": False, }, "triangulation": { "method": "line_of_sight_intersection", @@ -3392,7 +3407,11 @@ def test_end2end_ventoux_with_classif(): assert ( os.path.exists( os.path.join( - out_dir, "dump_dir", "small_components", "laz", pc1 + ".laz" + out_dir, + "dump_dir", + "point_cloud_outlier_removal_1", + "laz", + pc1 + ".laz", ) ) is True @@ -3400,7 +3419,11 @@ def test_end2end_ventoux_with_classif(): assert ( os.path.exists( os.path.join( - out_dir, "dump_dir", "small_components", "csv", pc1 + ".csv" + out_dir, + "dump_dir", + "point_cloud_outlier_removal_1", + "csv", + pc1 + ".csv", ) ) is True @@ -3409,7 +3432,11 @@ def test_end2end_ventoux_with_classif(): assert ( os.path.exists( os.path.join( - out_dir, "dump_dir", "statistical", "laz", pc1 + ".laz" + out_dir, + "dump_dir", + "point_cloud_outlier_removal_2", + "laz", + pc1 + ".laz", ) ) is True @@ -3417,7 +3444,11 @@ def test_end2end_ventoux_with_classif(): assert ( os.path.exists( os.path.join( - out_dir, "dump_dir", "statistical", "csv", pc1 + ".csv" + out_dir, + "dump_dir", + "point_cloud_outlier_removal_2", + "csv", + pc1 + ".csv", ) ) is True @@ -3659,13 +3690,14 @@ def test_compute_dsm_with_snap_to_img1(): "method": "line_of_sight_intersection", "snap_to_img1": True, }, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, + "use_median": False, }, "point_cloud_rasterization": { "method": "simple_gaussian", @@ -3773,13 +3805,14 @@ def test_end2end_quality_stats(): "use_cross_validation": True, "use_global_disp_range": False, }, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, + "use_median": False, }, "point_cloud_rasterization": { "method": "simple_gaussian", @@ -4068,13 +4101,14 @@ def test_end2end_ventoux_egm96_geoid(): "use_global_disp_range": False, }, "triangulation": {"method": "line_of_sight_intersection"}, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, + "use_median": False, }, "point_cloud_rasterization": { "method": "simple_gaussian", @@ -4199,13 +4233,14 @@ def test_end2end_ventoux_egm96_geoid(): "use_global_disp_range": False, }, "triangulation": {"method": "line_of_sight_intersection"}, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, + "use_median": False, }, "point_cloud_rasterization": { "method": "simple_gaussian", @@ -4288,13 +4323,14 @@ def test_end2end_ventoux_egm96_geoid(): "use_global_disp_range": False, }, "triangulation": {"method": "line_of_sight_intersection"}, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, + "use_median": False, }, "point_cloud_rasterization": { "method": "simple_gaussian", @@ -4429,13 +4465,14 @@ def test_end2end_paca_with_mask(): "use_cross_validation": True, "use_global_disp_range": False, }, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, + "use_median": False, }, "point_cloud_rasterization": { "method": "simple_gaussian", @@ -4628,13 +4665,14 @@ def test_end2end_disparity_filling(): "save_intermediate_data": True, "classification": ["shadow"], }, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, + "use_median": False, }, "point_cloud_rasterization": { "method": "simple_gaussian", @@ -4758,13 +4796,14 @@ def test_end2end_disparity_filling_with_zeros(): "save_intermediate_data": True, "classification": ["bat"], }, - "point_cloud_outliers_removing.1": { + "point_cloud_outlier_removal.1": { "method": "small_components", "activated": True, }, - "point_cloud_outliers_removing.2": { + "point_cloud_outlier_removal.2": { "method": "statistical", "activated": True, + "use_median": False, }, } input_config_dense_dsm["applications"].update(dense_dsm_applications) diff --git a/tutorials/sensor_to_dense_dsm_matching_methods_comparison.ipynb b/tutorials/sensor_to_dense_dsm_matching_methods_comparison.ipynb index 1edd5bea..669682a7 100644 --- a/tutorials/sensor_to_dense_dsm_matching_methods_comparison.ipynb +++ b/tutorials/sensor_to_dense_dsm_matching_methods_comparison.ipynb @@ -888,8 +888,8 @@ "metadata": {}, "outputs": [], "source": [ - "conf_outlier_removing_small_components = {\"method\": \"small_components\", \"activated\": True}\n", - "pc_outlier_removing_small_comp_application = Application(\"point_cloud_outliers_removing\", cfg=conf_outlier_removing_small_components)" + "conf_outlier_removal_small_components = {\"method\": \"small_components\", \"activated\": True}\n", + "pc_outlier_removal_small_comp_application = Application(\"point_cloud_outlier_removal\", cfg=conf_outlier_removal_small_components)" ] }, { @@ -899,8 +899,8 @@ "metadata": {}, "outputs": [], "source": [ - "conf_outlier_removing_small_statistical = {\"method\": \"statistical\", \"activated\": True}\n", - "pc_outlier_removing_stats_application = Application(\"point_cloud_outliers_removing\", cfg=conf_outlier_removing_small_statistical)" + "conf_outlier_removal_statistical = {\"method\": \"statistical\", \"activated\": True}\n", + "pc_outlier_removal_stats_application = Application(\"point_cloud_outlier_removal\", cfg=conf_outlier_removal_statistical)" ] }, { @@ -1119,7 +1119,7 @@ "id": "5afb8b4f", "metadata": {}, "source": [ - "### Point Cloud Outlier Removing : remove points with small components removing method" + "### Point Cloud Outlier Removal : remove points with the small component removal method" ] }, { @@ -1137,7 +1137,7 @@ "metadata": {}, "outputs": [], "source": [ - "filtered_sc_merged_points_clouds_census = pc_outlier_removing_small_comp_application.run(\n", + "filtered_sc_merged_points_clouds_census = pc_outlier_removal_small_comp_application.run(\n", " merged_points_clouds_census,\n", " orchestrator=cars_orchestrator,\n", ")" @@ -1158,7 +1158,7 @@ "metadata": {}, "outputs": [], "source": [ - "filtered_sc_merged_points_clouds_mccnn = pc_outlier_removing_small_comp_application.run(\n", + "filtered_sc_merged_points_clouds_mccnn = pc_outlier_removal_small_comp_application.run(\n", " merged_points_clouds_mccnn,\n", " orchestrator=cars_orchestrator,\n", ")" @@ -1169,7 +1169,7 @@ "id": "dcdacc34", "metadata": {}, "source": [ - "### Point Cloud Outlier Removing: remove points with statistical removing method" + "### Point Cloud Outlier Removal: remove points with statistical removal method" ] }, { @@ -1187,7 +1187,7 @@ "metadata": {}, "outputs": [], "source": [ - "filtered_stats_merged_points_clouds_census = pc_outlier_removing_stats_application.run(\n", + "filtered_stats_merged_points_clouds_census = pc_outlier_removal_stats_application.run(\n", " filtered_sc_merged_points_clouds_census,\n", " orchestrator=cars_orchestrator,\n", ")" @@ -1208,7 +1208,7 @@ "metadata": {}, "outputs": [], "source": [ - "filtered_stats_merged_points_clouds_mccnn = pc_outlier_removing_stats_application.run(\n", + "filtered_stats_merged_points_clouds_mccnn = pc_outlier_removal_stats_application.run(\n", " filtered_sc_merged_points_clouds_mccnn,\n", " orchestrator=cars_orchestrator,\n", ")" diff --git a/tutorials/sensor_to_dense_dsm_step_by_step.ipynb b/tutorials/sensor_to_dense_dsm_step_by_step.ipynb index bc463d14..2e8bef0a 100644 --- a/tutorials/sensor_to_dense_dsm_step_by_step.ipynb +++ b/tutorials/sensor_to_dense_dsm_step_by_step.ipynb @@ -377,9 +377,9 @@ "id": "2f4b68fe-f0f5-4e58-9a7c-06f976cf0235", "metadata": {}, "source": [ - "#### PointCloudOutliersRemoving : small components\n", + "#### PointCloudOutlierRemoval : small components\n", "\n", - "This application removes outliers points. The method used is the \"small components removing\"" + "This application removes outlier points. The method used is \"small component removal\"" ] }, { @@ -389,8 +389,8 @@ "metadata": {}, "outputs": [], "source": [ - "conf_outlier_removing_small_components = {\"method\": \"small_components\", \"activated\": True}\n", - "pc_outlier_removing_small_comp_application = Application(\"point_cloud_outliers_removing\", cfg=conf_outlier_removing_small_components)" + "conf_outlier_removal_small_components = {\"method\": \"small_components\", \"activated\": True}\n", + "pc_outlier_removal_small_comp_application = Application(\"point_cloud_outlier_removal\", cfg=conf_outlier_removal_small_components)" ] }, { @@ -398,9 +398,9 @@ "id": "51682146-3b56-471e-9530-44fdc53332cd", "metadata": {}, "source": [ - "#### PointCloudOutliersRemoving : statistical\n", + "#### PointCloudOutlierRemoval : statistical\n", "\n", - "This application removes outliers points. The method used is the \"statistical removing\"" + "This application removes outlier points. The method used is \"statistical removal\"" ] }, { @@ -410,8 +410,8 @@ "metadata": {}, "outputs": [], "source": [ - "conf_outlier_removing_small_statistical = {\"method\": \"statistical\", \"activated\": True}\n", - "pc_outlier_removing_stats_application = Application(\"point_cloud_outliers_removing\", cfg=conf_outlier_removing_small_statistical)" + "conf_outlier_removal_small_statistical = {\"method\": \"statistical\", \"activated\": True}\n", + "pc_outlier_removal_stats_application = Application(\"point_cloud_outlier_removal\", cfg=conf_outlier_removal_small_statistical)" ] }, { @@ -1168,7 +1168,7 @@ "metadata": {}, "outputs": [], "source": [ - "filtered_sc_merged_points_clouds = pc_outlier_removing_small_comp_application.run(\n", + "filtered_sc_merged_points_clouds = pc_outlier_removal_small_comp_application.run(\n", " merged_points_clouds,\n", " orchestrator=cars_orchestrator,\n", ") " @@ -1189,7 +1189,7 @@ "metadata": {}, "outputs": [], "source": [ - "filtered_stats_merged_points_clouds = pc_outlier_removing_stats_application.run(\n", + "filtered_stats_merged_points_clouds = pc_outlier_removal_stats_application.run(\n", " filtered_sc_merged_points_clouds,\n", " orchestrator=cars_orchestrator,\n", ")" @@ -1277,9 +1277,9 @@ ], "metadata": { "kernelspec": { - "display_name": "CARS venv", + "display_name": "cars_env", "language": "python", - "name": "cars_venv" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1291,12 +1291,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" - }, - "vscode": { - "interpreter": { - "hash": "0041c8a096542147678ce172f28dc37755329eb6be210e24ce9840eb2f3e2525" - } + "version": "3.10.14" } }, "nbformat": 4, diff --git a/tutorials/sensor_to_dsm_from_a_priori.ipynb b/tutorials/sensor_to_dsm_from_a_priori.ipynb index 5d51e962..d280be30 100644 --- a/tutorials/sensor_to_dsm_from_a_priori.ipynb +++ b/tutorials/sensor_to_dsm_from_a_priori.ipynb @@ -472,9 +472,9 @@ "id": "2f4b68fe-f0f5-4e58-9a7c-06f976cf0235", "metadata": {}, "source": [ - "#### PointCloudOutliersRemoving : small components\n", + "#### PointCloudOutlierRemoval : small components\n", "\n", - "This application removes outliers points. The method used is the \"small components removing\"" + "This application removes outliers points. The method used is \"small component removal\"" ] }, { @@ -484,8 +484,8 @@ "metadata": {}, "outputs": [], "source": [ - "conf_outlier_removing_small_components = {\"method\": \"small_components\", \"activated\": True}\n", - "pc_outlier_removing_small_comp_application = Application(\"point_cloud_outliers_removing\", cfg=conf_outlier_removing_small_components)" + "conf_outlier_removal_small_components = {\"method\": \"small_components\", \"activated\": True}\n", + "pc_outlier_removal_small_comp_application = Application(\"point_cloud_outlier_removal\", cfg=conf_outlier_removal_small_components)" ] }, { @@ -493,9 +493,9 @@ "id": "51682146-3b56-471e-9530-44fdc53332cd", "metadata": {}, "source": [ - "#### PointCloudOutliersRemoving : statistical\n", + "#### PointCloudOutlierRemoval : statistical\n", "\n", - "This application removes outliers points. The method used is the \"statistical removing\"" + "This application removes outliers points. The method used is \"statistical removal\"" ] }, { @@ -505,8 +505,8 @@ "metadata": {}, "outputs": [], "source": [ - "conf_outlier_removing_small_statistical = {\"method\": \"statistical\", \"activated\": True}\n", - "pc_outlier_removing_stats_application = Application(\"point_cloud_outliers_removing\", cfg=conf_outlier_removing_small_statistical)" + "conf_outlier_removal_small_statistical = {\"method\": \"statistical\", \"activated\": True}\n", + "pc_outlier_removal_stats_application = Application(\"point_cloud_outlier_removal\", cfg=conf_outlier_removal_small_statistical)" ] }, { @@ -1068,7 +1068,7 @@ "metadata": {}, "outputs": [], "source": [ - "filtered_sc_merged_points_clouds = pc_outlier_removing_small_comp_application.run(\n", + "filtered_sc_merged_points_clouds = pc_outlier_removal_small_comp_application.run(\n", " merged_points_clouds,\n", " orchestrator=cars_orchestrator,\n", ") \n", @@ -1091,7 +1091,7 @@ "metadata": {}, "outputs": [], "source": [ - "filtered_stats_merged_points_clouds = pc_outlier_removing_stats_application.run(\n", + "filtered_stats_merged_points_clouds = pc_outlier_removal_stats_application.run(\n", " filtered_sc_merged_points_clouds,\n", " orchestrator=cars_orchestrator,\n", ")\n",