diff --git a/cars/applications/rasterization/rasterization_tools.py b/cars/applications/rasterization/rasterization_tools.py index b3a83ac2..5d7882ab 100644 --- a/cars/applications/rasterization/rasterization_tools.py +++ b/cars/applications/rasterization/rasterization_tools.py @@ -65,7 +65,6 @@ def compute_xy_starts_and_sizes( # Clamp to a regular grid x_start = np.floor(xmin / resolution) * resolution - x_size = int(1 + np.floor((xmax - x_start) / resolution)) # Derive ystart ymin = np.nanmin(cloud[cst.Y].values) @@ -74,6 +73,8 @@ def compute_xy_starts_and_sizes( # Clamp to a regular grid y_start = np.ceil(ymax / resolution) * resolution + + x_size = int(1 + np.floor((xmax - x_start) / resolution)) y_size = int(1 + np.floor((y_start - ymin) / resolution)) return x_start, y_start, x_size, y_size @@ -202,6 +203,25 @@ def substring_in_list(src_list, substring): return len(res) > 0 +def phased_dsm(start: float, phase: float, resolution: float): + """ + Phased the dsm + + :param start: start of the roi + :param phase: the point for phasing + :param resolution: resolution of the dsm + """ + + div = np.abs(start - phase) / resolution + + if phase > start: + start = phase - resolution * np.floor(div) + else: + start = resolution * np.floor(div) + phase + + return start + + def find_indexes_in_point_cloud( cloud: pandas.DataFrame, tag: str, list_computed_layers: List[str] = None ) -> List[str]: @@ -845,19 +865,21 @@ def update_data( :return: updated current data """ + new_data = current_data + data = old_data if old_data is not None: old_data = np.squeeze(old_data) old_weights = np.squeeze(old_weights) shape = old_data.shape - if len(old_data.shape) == 3: + if len(data.shape) == 3 and data.shape[0] > 1: old_weights = np.repeat( np.expand_dims(old_weights, axis=0), old_data.shape[0], axis=0 ) current_data = np.squeeze(current_data) weights = np.squeeze(weights) - if len(current_data.shape) == 3: + if len(new_data.shape) == 3 and new_data.shape[0] > 1: weights = np.repeat( np.expand_dims(weights, axis=0), current_data.shape[0], axis=0 ) @@ -867,7 +889,9 @@ def update_data( old_valid = old_weights != 0 both_valid = np.logical_and(current_valid, old_valid) + total_weights = np.zeros(shape) + total_weights[both_valid] = ( weights[both_valid] + old_weights[both_valid] ) diff --git a/cars/applications/rasterization/simple_gaussian.py b/cars/applications/rasterization/simple_gaussian.py index 1d597a3e..53b7a777 100644 --- a/cars/applications/rasterization/simple_gaussian.py +++ b/cars/applications/rasterization/simple_gaussian.py @@ -217,6 +217,7 @@ def run( # noqa: C901 function is too complex filling_file_name=None, color_dtype=None, dump_dir=None, + phasing=None, ): """ Run PointCloudRasterisation application. @@ -271,6 +272,8 @@ def run( # noqa: C901 function is too complex :type color_dtype: str (numpy type) :param dump_dir: directory used for outputs with no associated filename :type dump_dir: str + :param phasing: if activated, we phase the dsm on this point + :type phasing: dict :return: raster DSM. CarsDataset contains: @@ -360,6 +363,21 @@ def run( # noqa: C901 function is too complex terrain_raster.generate_none_tiles() + if phasing is not None: + res = resolution + x_phase = phasing["point"][0] + y_phase = phasing["point"][1] + + for index, value in enumerate(bounds): + if index in (0, 2): + bounds[index] = rasterization_step.phased_dsm( + value, x_phase, res + ) + else: + bounds[index] = rasterization_step.phased_dsm( + value, y_phase, res + ) + # Derive output image files parameters to pass to rasterio xsize, ysize = tiling.roi_to_start_and_size(bounds, resolution)[2:] logging.info("DSM output image size: {}x{} pixels".format(xsize, ysize)) @@ -1000,6 +1018,7 @@ def rasterization_wrapper( xstart, ystart, xsize, ysize = tiling.roi_to_start_and_size( terrain_region, resolution ) + if window is None: transform = rio.Affine(*profile["transform"][0:6]) row_pix_pos, col_pix_pos = rio.transform.AffineTransformer( diff --git a/cars/core/constants.py b/cars/core/constants.py index 89f9f497..6c5db14e 100644 --- a/cars/core/constants.py +++ b/cars/core/constants.py @@ -130,3 +130,26 @@ INDEX_DEPTH_MAP_PERFORMANCE_MAP = "performance_map" INDEX_DEPTH_MAP_FILLING = "filling" INDEX_DEPTH_MAP_EPSG = "epsg" + +# dsms inputs index +DSM_CLASSIF = "classification" +DSM_ALT = "dsm" +DSM_ALT_INF = "dsm_inf" +DSM_ALT_SUP = "dsm_sup" +DSM_WEIGHTS_SUM = "weights" +DSM_MSK = "mask" +DSM_NB_PTS = "dsm_n_pts" +DSM_NB_PTS_IN_CELL = "dsm_pts_in_cell" +DSM_MEAN = "dsm_mean" +DSM_STD_DEV = "dsm_std" +DSM_INF_MEAN = "dsm_inf_mean" +DSM_INF_STD = "dsm_inf_std" +DSM_SUP_MEAN = "dsm_sup_mean" +DSM_SUP_STD = "dsm_sup_std" +DSM_CONFIDENCE_AMBIGUITY = "confidence_from_ambiguity" +DSM_CONFIDENCE_RISK_MIN = "confidence_from_risk_min" +DSM_CONFIDENCE_RISK_MAX = "confidence_from_risk_max" +DSM_PERFORMANCE_MAP = "performance_map" +DSM_SOURCE_PC = "source_pc" +DSM_FILLING = "filling" +DSM_COLOR = "color" diff --git a/cars/core/inputs.py b/cars/core/inputs.py index 4527c154..305242af 100644 --- a/cars/core/inputs.py +++ b/cars/core/inputs.py @@ -182,6 +182,28 @@ def rasterio_get_size(raster_file: str) -> Tuple[int, int]: return (descriptor.width, descriptor.height) +def rasterio_get_nodata(raster_file: str) -> Tuple[int, int]: + """ + Get the no data value + + :param raster_file: Image file + :return: the no data value + """ + with rio.open(raster_file, "r") as descriptor: + return descriptor.nodata + + +def rasterio_get_dtype(raster_file: str) -> Tuple[int, int]: + """ + Get the dtype of an image (file) + + :param raster_file: Image file + :return: The dtype + """ + with rio.open(raster_file, "r") as descriptor: + return descriptor.dtypes[0] + + def rasterio_get_pixel_points(raster_file: str, terrain_points) -> list: """ Get pixel point coordinates of terrain points diff --git a/cars/pipelines/default/default_pipeline.py b/cars/pipelines/default/default_pipeline.py index fd9d3497..06375f4c 100644 --- a/cars/pipelines/default/default_pipeline.py +++ b/cars/pipelines/default/default_pipeline.py @@ -50,7 +50,7 @@ from cars.core import constants_disparity as cst_disp from cars.core import preprocessing, roi_tools from cars.core.geometry.abstract_geometry import AbstractGeometry -from cars.core.inputs import get_descriptions_bands +from cars.core.inputs import get_descriptions_bands, rasterio_get_epsg from cars.core.utils import safe_makedirs from cars.data_structures import cars_dataset from cars.orchestrator import orchestrator @@ -59,6 +59,8 @@ from cars.pipelines.parameters import advanced_parameters_constants as adv_cst from cars.pipelines.parameters import depth_map_inputs from cars.pipelines.parameters import depth_map_inputs_constants as depth_cst +from cars.pipelines.parameters import dsm_inputs +from cars.pipelines.parameters import dsm_inputs_constants as dsm_cst from cars.pipelines.parameters import output_constants as out_cst from cars.pipelines.parameters import output_parameters, sensor_inputs from cars.pipelines.parameters import sensor_inputs_constants as sens_cst @@ -132,7 +134,7 @@ def __init__(self, conf, config_json_dir=None): ) self.used_conf[ADVANCED] = advanced - if sens_cst.SENSORS in self.used_conf[INPUTS]: + if self.used_conf[INPUTS][sens_cst.SENSORS] is not None: # Check geometry plugin and overwrite geomodel in conf inputs ( inputs, @@ -144,8 +146,10 @@ def __init__(self, conf, config_json_dir=None): inputs, advanced, conf.get(GEOMETRY_PLUGIN, None) ) self.used_conf[INPUTS] = inputs - - elif depth_cst.DEPTH_MAPS in self.used_conf[INPUTS]: + elif ( + depth_cst.DEPTH_MAPS in self.used_conf[INPUTS] + or dsm_cst.DSMS in self.used_conf[INPUTS] + ): # if there's an initial elevation with # point clouds as inputs, generate a plugin (used in dsm_filling) @@ -185,12 +189,16 @@ def __init__(self, conf, config_json_dir=None): self.depth_maps_in_inputs = ( depth_cst.DEPTH_MAPS in self.used_conf[INPUTS] ) + self.dsms_in_inputs = dsm_cst.DSMS in self.used_conf[INPUTS] self.merging = self.used_conf[ADVANCED][adv_cst.MERGING] + self.phasing = self.used_conf[ADVANCED][adv_cst.PHASING] + self.compute_depth_map = ( self.sensors_in_inputs + and (not self.output_level_none) + and not self.dsms_in_inputs and not self.depth_maps_in_inputs - and not self.output_level_none ) if self.output_level_none: @@ -206,7 +214,11 @@ def __init__(self, conf, config_json_dir=None): # Check conf application application_conf = self.check_applications(conf.get(APPLICATIONS, {})) - if self.sensors_in_inputs: + if ( + self.sensors_in_inputs + and not self.depth_maps_in_inputs + and not self.dsms_in_inputs + ): # Check conf application vs inputs application application_conf = self.check_applications_with_inputs( self.used_conf[INPUTS], application_conf @@ -294,7 +306,11 @@ def infer_conditions_from_applications(self, conf): ).format(key) logging.warning(warn_msg) - else: + elif ( + self.sensors_in_inputs + and not self.depth_maps_in_inputs + and not self.dsms_in_inputs + ): self.compute_depth_map = True self.last_application_to_run = max( self.last_application_to_run, self.app_values[key] @@ -302,7 +318,11 @@ def infer_conditions_from_applications(self, conf): elif key in depth_to_dsm_apps: - if not (self.sensors_in_inputs or self.depth_maps_in_inputs): + if not ( + self.sensors_in_inputs + or self.depth_maps_in_inputs + or self.dsms_in_inputs + ): warn_msg = ( "The application {} can only be used when sensor " "images or depth maps are given as an input. " @@ -311,7 +331,11 @@ def infer_conditions_from_applications(self, conf): logging.warning(warn_msg) else: - if self.sensors_in_inputs: + if ( + self.sensors_in_inputs + and not self.depth_maps_in_inputs + and not self.dsms_in_inputs + ): self.compute_depth_map = True # enabled to start the depth map to dsm process @@ -332,7 +356,11 @@ def infer_conditions_from_applications(self, conf): ).format(key) logging.warning(warn_msg) - elif not (self.sensors_in_inputs or self.depth_maps_in_inputs): + elif not ( + self.sensors_in_inputs + or self.depth_maps_in_inputs + or self.dsms_in_inputs + ): warn_msg = ( "The application {} can only be used when sensor " "images or depth maps are given as an input. " @@ -341,7 +369,11 @@ def infer_conditions_from_applications(self, conf): logging.warning(warn_msg) else: - if self.sensors_in_inputs: + if ( + self.sensors_in_inputs + and not self.depth_maps_in_inputs + and not self.dsms_in_inputs + ): self.compute_depth_map = True # enabled to start the depth map to dsm process @@ -392,19 +424,28 @@ def check_inputs(conf, config_json_dir=None): """ output_config = {} - if sens_cst.SENSORS in conf: + if ( + sens_cst.SENSORS in conf + and depth_cst.DEPTH_MAPS not in conf + and dsm_cst.DSMS not in conf + ): output_config = sensor_inputs.sensors_check_inputs( conf, config_json_dir=config_json_dir ) - - if depth_cst.DEPTH_MAPS in conf: + elif depth_cst.DEPTH_MAPS in conf: output_config = { **output_config, **depth_map_inputs.check_depth_maps_inputs( conf, config_json_dir=config_json_dir ), } - + else: + output_config = { + **output_config, + **dsm_inputs.check_dsm_inputs( + conf, config_json_dir=config_json_dir + ), + } return output_config @staticmethod @@ -487,6 +528,7 @@ def check_applications( # noqa: C901 "sparse_matching.pandora", "point_cloud_outlier_removal.1", "point_cloud_outlier_removal.2", + "auxiliary_filling", ]: if conf.get(app_key) is not None: config_app = conf.get(app_key) @@ -829,7 +871,13 @@ def sensor_to_depth_maps(self): # noqa: C901 output = self.used_conf[OUTPUT] # Initialize epsg for terrain tiles - self.epsg = output[out_cst.EPSG] + self.phasing = self.used_conf[ADVANCED][adv_cst.PHASING] + + if self.phasing is not None: + self.epsg = self.phasing["epsg"] + else: + self.epsg = output[out_cst.EPSG] + if self.epsg is not None: # Compute roi polygon, in output EPSG self.roi_poly = preprocessing.compute_roi_poly( @@ -2219,6 +2267,7 @@ def rasterize_point_cloud(self): filling_file_name=filling_file_name, color_dtype=self.color_type, dump_dir=rasterization_dump_dir, + phasing=self.phasing, ) # Cleaning: don't keep terrain bbox if save_intermediate_data @@ -2234,6 +2283,159 @@ def rasterize_point_cloud(self): # dsm needs to be saved before filling self.cars_orchestrator.breakpoint() + return False + + def filling(self): + """ + Fill the dsm + """ + + dsm_file_name = ( + os.path.join( + self.out_dir, + out_cst.DSM_DIRECTORY, + "dsm.tif", + ) + if self.save_output_dsm + else None + ) + + if self.dsms_in_inputs: + dsms_merging_dump_dir = os.path.join(self.dump_dir, "dsms_merging") + + dsm_dict = self.used_conf[INPUTS][dsm_cst.DSMS] + dict_path = {} + for key in dsm_dict.keys(): + for path_name in dsm_dict[key].keys(): + if dsm_dict[key][path_name] is not None: + if path_name not in dict_path: + dict_path[path_name] = [dsm_dict[key][path_name]] + else: + dict_path[path_name].append( + dsm_dict[key][path_name] + ) + + color_file_name = ( + os.path.join( + self.out_dir, + out_cst.DSM_DIRECTORY, + "color.tif", + ) + if "color" in dict_path + else None + ) + + mask_file_name = ( + os.path.join( + self.out_dir, + out_cst.DSM_DIRECTORY, + "mask.tif", + ) + if "mask" in dict_path + else None + ) + + performance_map_file_name = ( + os.path.join( + self.out_dir, + out_cst.DSM_DIRECTORY, + "performance_map.tif", + ) + if "performance_map" in dict_path + else None + ) + + classif_file_name = ( + os.path.join( + self.out_dir, + out_cst.DSM_DIRECTORY, + "classification.tif", + ) + if "classification" in dict_path + else None + ) + + contributing_all_pair_file_name = ( + os.path.join( + self.out_dir, + out_cst.DSM_DIRECTORY, + "contributing_pair.tif", + ) + if "source_pc" in dict_path + else None + ) + + filling_file_name = ( + os.path.join( + self.out_dir, + out_cst.DSM_DIRECTORY, + "filling.tif", + ) + if "filling" in dict_path + else None + ) + + self.epsg = rasterio_get_epsg(dict_path["dsm"][0]) + + # Compute roi polygon, in input EPSG + self.roi_poly = preprocessing.compute_roi_poly( + self.input_roi_poly, self.input_roi_epsg, self.epsg + ) + + _ = dsm_inputs.merge_dsm_infos( + dict_path, + self.cars_orchestrator, + self.roi_poly, + dsms_merging_dump_dir, + dsm_file_name, + color_file_name, + classif_file_name, + filling_file_name, + performance_map_file_name, + mask_file_name, + contributing_all_pair_file_name, + ) + + # dsm needs to be saved before filling + self.cars_orchestrator.breakpoint() + else: + filling_file_name = ( + os.path.join( + self.out_dir, + out_cst.DSM_DIRECTORY, + "filling.tif", + ) + if self.save_output_dsm + and self.used_conf[OUTPUT][out_cst.AUXILIARY][ + out_cst.AUX_FILLING + ] + else None + ) + + color_file_name = ( + os.path.join( + self.out_dir, + out_cst.DSM_DIRECTORY, + "color.tif", + ) + if self.save_output_dsm + and self.used_conf[OUTPUT][out_cst.AUXILIARY][out_cst.AUX_COLOR] + else None + ) + + classif_file_name = ( + os.path.join( + self.out_dir, + out_cst.DSM_DIRECTORY, + "classification.tif", + ) + if self.save_output_dsm + and self.used_conf[OUTPUT][out_cst.AUXILIARY][ + out_cst.AUX_CLASSIFICATION + ] + else None + ) + _ = self.dsm_filling_application.run( orchestrator=self.cars_orchestrator, # path to initial elevation file via geom plugin @@ -2446,9 +2648,12 @@ def final_cleanup(self): ) # Remove dump_dir if no intermediate data should be written - if not any( - app.get("save_intermediate_data", False) is True - for app in self.used_conf[APPLICATIONS].values() + if ( + not any( + app.get("save_intermediate_data", False) is True + for app in self.used_conf[APPLICATIONS].values() + ) + and not self.dsms_in_inputs ): self.cars_orchestrator.add_to_clean(self.dump_dir) @@ -2483,15 +2688,19 @@ def run(self): # noqa C901 # initialize out_json self.cars_orchestrator.update_out_info({"version": __version__}) - if self.compute_depth_map: - self.sensor_to_depth_maps() - else: - self.load_input_depth_maps() + if not self.dsms_in_inputs: + if self.compute_depth_map: + self.sensor_to_depth_maps() + else: + self.load_input_depth_maps() - if self.save_output_dsm or self.save_output_point_cloud: - end_pipeline = self.preprocess_depth_maps() + if self.save_output_dsm or self.save_output_point_cloud: + end_pipeline = self.preprocess_depth_maps() - if self.save_output_dsm and not end_pipeline: - self.rasterize_point_cloud() + if self.save_output_dsm and not end_pipeline: + self.rasterize_point_cloud() + self.filling() + else: + self.filling() self.final_cleanup() diff --git a/cars/pipelines/parameters/advanced_parameters.py b/cars/pipelines/parameters/advanced_parameters.py index f7fb326f..88920048 100644 --- a/cars/pipelines/parameters/advanced_parameters.py +++ b/cars/pipelines/parameters/advanced_parameters.py @@ -53,6 +53,8 @@ def check_advanced_parameters(conf, check_epipolar_a_priori=True): adv_cst.DEBUG_WITH_ROI, False ) + overloaded_conf[adv_cst.PHASING] = conf.get(adv_cst.PHASING, None) + overloaded_conf[adv_cst.MERGING] = conf.get(adv_cst.MERGING, False) if check_epipolar_a_priori: @@ -74,6 +76,7 @@ def check_advanced_parameters(conf, check_epipolar_a_priori=True): adv_cst.DEBUG_WITH_ROI: bool, adv_cst.MERGING: bool, adv_cst.SAVE_INTERMEDIATE_DATA: bool, + adv_cst.PHASING: Or(dict, None), } if check_epipolar_a_priori: schema[adv_cst.USE_EPIPOLAR_A_PRIORI] = bool diff --git a/cars/pipelines/parameters/advanced_parameters_constants.py b/cars/pipelines/parameters/advanced_parameters_constants.py index 2c8015f2..67360460 100644 --- a/cars/pipelines/parameters/advanced_parameters_constants.py +++ b/cars/pipelines/parameters/advanced_parameters_constants.py @@ -24,7 +24,7 @@ """ SAVE_INTERMEDIATE_DATA = "save_intermediate_data" - +PHASING = "phasing" DEBUG_WITH_ROI = "debug_with_roi" USE_EPIPOLAR_A_PRIORI = "use_epipolar_a_priori" diff --git a/cars/pipelines/parameters/depth_map_inputs.py b/cars/pipelines/parameters/depth_map_inputs.py index 0e270aef..2872bc05 100644 --- a/cars/pipelines/parameters/depth_map_inputs.py +++ b/cars/pipelines/parameters/depth_map_inputs.py @@ -63,11 +63,17 @@ def check_depth_maps_inputs(conf, config_json_dir=None): ) ) + overloaded_conf[sens_cst.SENSORS] = conf.get(sens_cst.SENSORS, None) + + overloaded_conf[sens_cst.PAIRING] = conf.get(sens_cst.PAIRING, None) + # Validate inputs inputs_schema = { depth_map_cst.DEPTH_MAPS: dict, sens_cst.ROI: Or(str, dict, None), sens_cst.INITIAL_ELEVATION: Or(dict, None), + sens_cst.SENSORS: Or(dict, None), + sens_cst.PAIRING: Or([[str]], None), } checker_inputs = Checker(inputs_schema) @@ -216,6 +222,9 @@ def check_depth_maps_inputs(conf, config_json_dir=None): overloaded_conf[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH] ) + if sens_cst.SENSORS in conf and conf[sens_cst.SENSORS] is not None: + sens_inp.check_sensors(conf, overloaded_conf, config_json_dir) + return overloaded_conf diff --git a/cars/pipelines/parameters/dsm_inputs.py b/cars/pipelines/parameters/dsm_inputs.py new file mode 100644 index 00000000..e60d53e7 --- /dev/null +++ b/cars/pipelines/parameters/dsm_inputs.py @@ -0,0 +1,859 @@ +#!/usr/bin/env python +# coding: utf8 +# +# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). +# +# This file is part of CARS +# (see https://github.com/CNES/cars). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=too-many-lines +""" +CARS dsm inputs +""" + +import collections +import logging +import os + +import numpy as np +import rasterio +import xarray as xr +from affine import Affine +from json_checker import Checker, Or +from rasterio.windows import from_bounds + +# CARS imports +import cars.orchestrator.orchestrator as ocht +import cars.pipelines.parameters.dsm_inputs_constants as dsm_cst +from cars.applications.rasterization.rasterization_tools import ( + update_data, + update_weights, +) +from cars.core import constants as cst +from cars.core import inputs, preprocessing, tiling +from cars.core.utils import make_relative_path_absolute, safe_makedirs +from cars.data_structures import cars_dataset +from cars.pipelines.parameters import sensor_inputs as sens_inp +from cars.pipelines.parameters import sensor_inputs_constants as sens_cst + + +def check_dsm_inputs(conf, config_json_dir=None): + """ + Check the inputs given + + :param conf: configuration of inputs + :type conf: dict + :param config_json_dir: directory of used json, if + user filled paths with relative paths + :type config_json_dir: str + + :return: overloader inputs + :rtype: dict + """ + + overloaded_conf = {} + + # Overload some optional parameters + overloaded_conf[dsm_cst.DSMS] = {} + + overloaded_conf[sens_cst.ROI] = conf.get(sens_cst.ROI, None) + + overloaded_conf[sens_cst.INITIAL_ELEVATION] = ( + sens_inp.get_initial_elevation( + conf.get(sens_cst.INITIAL_ELEVATION, None) + ) + ) + + overloaded_conf[sens_cst.SENSORS] = conf.get(sens_cst.SENSORS, None) + + overloaded_conf[sens_cst.PAIRING] = conf.get(sens_cst.PAIRING, None) + + # Validate inputs + inputs_schema = { + dsm_cst.DSMS: dict, + sens_cst.ROI: Or(str, dict, None), + sens_cst.INITIAL_ELEVATION: Or(dict, None), + sens_cst.SENSORS: Or(dict, None), + sens_cst.PAIRING: Or([[str]], None), + } + + checker_inputs = Checker(inputs_schema) + checker_inputs.validate(overloaded_conf) + + # Validate depth maps + + dsm_schema = { + cst.DSM_CLASSIF: Or(str, None), + cst.DSM_ALT: Or(str, None), + cst.DSM_ALT_INF: Or(str, None), + cst.DSM_ALT_SUP: Or(str, None), + cst.DSM_WEIGHTS_SUM: Or(str, None), + cst.DSM_MSK: Or(str, None), + cst.DSM_NB_PTS: Or(str, None), + cst.DSM_NB_PTS_IN_CELL: Or(str, None), + cst.DSM_MEAN: Or(str, None), + cst.DSM_STD_DEV: Or(str, None), + cst.DSM_INF_MEAN: Or(str, None), + cst.DSM_INF_STD: Or(str, None), + cst.DSM_SUP_MEAN: Or(str, None), + cst.DSM_SUP_STD: Or(str, None), + cst.DSM_CONFIDENCE_AMBIGUITY: Or(str, None), + cst.DSM_CONFIDENCE_RISK_MIN: Or(str, None), + cst.DSM_CONFIDENCE_RISK_MAX: Or(str, None), + cst.DSM_PERFORMANCE_MAP: Or(str, None), + cst.DSM_SOURCE_PC: Or(str, None), + cst.DSM_FILLING: Or(str, None), + cst.DSM_COLOR: Or(str, None), + } + + checker_pc = Checker(dsm_schema) + for dsm_key in conf[dsm_cst.DSMS]: + # Get depth maps with default + overloaded_conf[dsm_cst.DSMS][dsm_key] = {} + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_ALT] = conf[ + dsm_cst.DSMS + ][dsm_key].get("dsm", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_CLASSIF] = conf[ + dsm_cst.DSMS + ][dsm_key].get("classification", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_COLOR] = conf[ + dsm_cst.DSMS + ][dsm_key].get("color", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_MSK] = conf[ + dsm_cst.DSMS + ][dsm_key].get("mask", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_ALT_INF] = conf[ + dsm_cst.DSMS + ][dsm_key].get("dsm_inf", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_ALT_SUP] = conf[ + dsm_cst.DSMS + ][dsm_key].get("dsm_sup", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_WEIGHTS_SUM] = conf[ + dsm_cst.DSMS + ][dsm_key].get("weights", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_NB_PTS] = conf[ + dsm_cst.DSMS + ][dsm_key].get("dsm_n_pts", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_NB_PTS_IN_CELL] = conf[ + dsm_cst.DSMS + ][dsm_key].get("dsm_pts_in_cell", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_MEAN] = conf[ + dsm_cst.DSMS + ][dsm_key].get("dsm_mean", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_STD_DEV] = conf[ + dsm_cst.DSMS + ][dsm_key].get("dsm_std", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_INF_MEAN] = conf[ + dsm_cst.DSMS + ][dsm_key].get("dsm_inf_mean", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_INF_STD] = conf[ + dsm_cst.DSMS + ][dsm_key].get("dsm_inf_std", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_SUP_MEAN] = conf[ + dsm_cst.DSMS + ][dsm_key].get("dsm_sup_mean", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_SUP_STD] = conf[ + dsm_cst.DSMS + ][dsm_key].get("dsm_sup_std", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_CONFIDENCE_AMBIGUITY] = ( + conf[dsm_cst.DSMS][dsm_key].get("confidence_from_ambiguity", None) + ) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_CONFIDENCE_RISK_MIN] = ( + conf[dsm_cst.DSMS][dsm_key].get("confidence_from_risk_min", None) + ) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_CONFIDENCE_RISK_MAX] = ( + conf[dsm_cst.DSMS][dsm_key].get("confidence_from_risk_max", None) + ) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_PERFORMANCE_MAP] = conf[ + dsm_cst.DSMS + ][dsm_key].get("performance_map", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_SOURCE_PC] = conf[ + dsm_cst.DSMS + ][dsm_key].get("source_pc", None) + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.DSM_FILLING] = conf[ + dsm_cst.DSMS + ][dsm_key].get("filling", None) + + # validate + checker_pc.validate(overloaded_conf[dsm_cst.DSMS][dsm_key]) + + # Modify to absolute path + if config_json_dir is not None: + modify_to_absolute_path(config_json_dir, overloaded_conf) + else: + logging.debug( + "path of config file was not given," + "relative path are not transformed to absolute paths" + ) + + for dsm_key in conf[dsm_cst.DSMS]: + # check sizes + check_input_size( + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.INDEX_DSM_ALT], + overloaded_conf[dsm_cst.DSMS][dsm_key][ + cst.INDEX_DSM_CLASSIFICATION + ], + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.INDEX_DSM_COLOR], + overloaded_conf[dsm_cst.DSMS][dsm_key][cst.INDEX_DSM_MASK], + ) + + # Check srtm dir + sens_inp.check_srtm( + overloaded_conf[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH] + ) + + check_phasing(conf[dsm_cst.DSMS]) + + if sens_cst.SENSORS in conf and conf[sens_cst.SENSORS] is not None: + sens_inp.check_sensors(conf, overloaded_conf, config_json_dir) + + return overloaded_conf + + +def check_input_size(dsm, classif, color, mask): + """ + Check dsm, mask, color, classif given + + Images must have same size + + :param dsm: phased dsm path + :type dsm: str + :param classif: classif path + :type classif: str + :param color: color path + :type color: str + :param mask: mask path + :type mask: str + """ + + if inputs.rasterio_get_nb_bands(dsm) != 1: + raise RuntimeError("{} is not mono-band image".format(dsm)) + + for path in [mask, color, classif]: + if path is not None: + if inputs.rasterio_get_size(dsm) != inputs.rasterio_get_size(path): + raise RuntimeError( + "The image {} and {} " + "do not have the same size".format(dsm, path) + ) + + +def modify_to_absolute_path(config_json_dir, overloaded_conf): + """ + Modify input file path to absolute path + + :param config_json_dir: directory of the json configuration + :type config_json_dir: str + :param overloaded_conf: overloaded configuration json + :dict overloaded_conf: dict + """ + for dsm_key in overloaded_conf[dsm_cst.DSMS]: + dsms = overloaded_conf[dsm_cst.DSMS][dsm_key] + for tag in [ + cst.INDEX_DSM_ALT, + cst.INDEX_DSM_CLASSIFICATION, + cst.INDEX_DSM_COLOR, + cst.INDEX_DSM_MASK, + ]: + if dsms[tag] is not None: + dsms[tag] = make_relative_path_absolute( + dsms[tag], config_json_dir + ) + + if overloaded_conf[sens_cst.ROI] is not None: + if isinstance(overloaded_conf[sens_cst.ROI], str): + overloaded_conf[sens_cst.ROI] = make_relative_path_absolute( + overloaded_conf[sens_cst.ROI], config_json_dir + ) + + +def get_optimal_tile_size( + max_ram_per_worker, + global_bounds, + resolution=0.5, +): + """ + Get the optimal tile size to use, depending on memory available + + :param max_ram_per_worker: maximum ram available + :type max_ram_per_worker: int + :param global_bounds: bounds of the final dsm + :type global_bounds: list + :param resolution: resolution + :type resolution: float + + :return: optimal tile size in meter + :rtype: float + + """ + + tot_height = 7000 * (global_bounds[3] - global_bounds[1]) / resolution + tot_width = 7000 * (global_bounds[2] - global_bounds[0]) / resolution + + import_ = 200 # MiB + tile_size_height = int( + np.sqrt(float(((max_ram_per_worker - import_) * 2**23)) / tot_height) + ) + tile_size_width = int( + np.sqrt(float(((max_ram_per_worker - import_) * 2**23)) / tot_width) + ) + + logging.info( + "Estimated optimal tile size for dsms_merging: {} meters".format( + (tile_size_height, tile_size_width) + ) + ) + return tile_size_height, tile_size_width + + +def check_phasing(dsm_dict): + """ + Check if the dsm are phased, and if resolution and epsg code are equivalent + + :param dsm_dict: list of phased dsm + :type dsm_dict: dict + """ + + ref_key = next(iter(dsm_dict)) + ref_epsg = inputs.rasterio_get_epsg_code(dsm_dict[ref_key]["dsm"]) + ref_profile = inputs.rasterio_get_profile(dsm_dict[ref_key]["dsm"]) + ref_transform = list(ref_profile["transform"]) + ref_res_x = ref_transform[0] + ref_res_y = ref_transform[4] + ref_bounds = inputs.rasterio_get_bounds(dsm_dict[ref_key]["dsm"]) + + for dsm_key in dsm_dict: + if dsm_key == ref_key: + continue + + epsg = inputs.rasterio_get_epsg_code(dsm_dict[dsm_key]["dsm"]) + profile = inputs.rasterio_get_profile(dsm_dict[ref_key]["dsm"]) + transform = list(profile["transform"]) + res_x = transform[0] + res_y = transform[4] + bounds = inputs.rasterio_get_bounds(dsm_dict[dsm_key]["dsm"]) + + if epsg != ref_epsg: + raise RuntimeError( + f"EPSG mismatch: DSM {dsm_key} has EPSG {epsg}, " + f"expected {ref_epsg}." + ) + + if ref_res_x != res_x or ref_res_y != res_y: + raise RuntimeError( + f"Resolution mismatch: DSM {dsm_key} has resolution " + f"{(res_x, res_y)}, expected {(ref_res_x, ref_res_y)}." + ) + + # Compare the left_bottom corner + diff = ref_bounds[0:2] - bounds[0:2] + resolution = np.array([ref_res_x, -ref_res_y]) + res_ratio = diff / resolution + + if ~np.all(np.equal(res_ratio, res_ratio.astype(int))) and ~np.all( + np.equal(1 / res_ratio, (1 / res_ratio).astype(int)) + ): + raise RuntimeError(f"DSM {dsm_key} and {ref_key} are not phased") + + +def merge_dsm_infos( # noqa: C901 function is too complex + dict_path, + orchestrator, + roi_poly, + dump_dir=None, + dsm_file_name=None, + color_file_name=None, + classif_file_name=None, + filling_file_name=None, + performance_map_file_name=None, + mask_file_name=None, + contributing_pair_file_name=None, +): + """ + Merge all the dsms + + :param dict_path: path of all variables from all dsms + :type dict_path: dict + :param orchestrator: orchestrator used + :param dump_dir: output path + :type dump_dir: str + :param dsm_file_name: name of the dsm output file + :type dsm_file_name: str + :param color_file_name: name of the color output file + :type color_file_name: str + :param classif_file_name: name of the classif output file + :type classif_file_name: str + :param filling_file_name: name of the filling output file + :type filling_file_name: str + :param performance_map_file_name: name of the performance_map output file + :type performance_map_file_name: str + :param mask_file_name: name of the mask output file + :type mask_file_name: str + :param contributing_pair_file_name: name of contributing_pair output file + :type contributing_pair_file_name: str + + :return: raster DSM. CarsDataset contains: + + - Z x W Delayed tiles. \ + Each tile will be a future xarray Dataset containing: + + - data : with keys : "hgt", "img", "raster_msk",optional : \ + "n_pts", "pts_in_cell", "hgt_mean", "hgt_stdev",\ + "hgt_inf", "hgt_sup" + - attrs with keys: "epsg" + - attributes containing: None + + :rtype : CarsDataset filled with xr.Dataset + """ + + # Create CarsDataset + terrain_raster = cars_dataset.CarsDataset("arrays", name="rasterization") + + # find the global bounds of the dataset + for index, path in enumerate(dict_path["dsm"]): + with rasterio.open(path) as src: + if index == 0: + bounds = src.bounds + global_bounds = bounds + profile = src.profile + transform = list(profile["transform"]) + res_x = transform[0] + res_y = transform[4] + resolution = (res_y, res_x) + + epsg = src.crs + + dsm_nodata = src.nodata + else: + bounds = src.bounds + global_bounds = ( + min(bounds[0], global_bounds[0]), # xmin + min(bounds[1], global_bounds[1]), # ymin + max(bounds[2], global_bounds[2]), # xmax + max(bounds[3], global_bounds[3]), # ymax + ) + + if roi_poly is not None: + global_bounds = preprocessing.crop_terrain_bounds_with_roi( + roi_poly, + global_bounds[0], + global_bounds[1], + global_bounds[2], + global_bounds[3], + ) + + # Tiling of the dataset + [xmin, ymin, xmax, ymax] = global_bounds + terrain_tile_height, terrain_tile_width = get_optimal_tile_size( + orchestrator.cluster.checked_conf_cluster["max_ram_per_worker"], + global_bounds, + resolution[1], + ) + + terrain_raster.tiling_grid = tiling.generate_tiling_grid( + xmin, + ymin, + xmax, + ymax, + terrain_tile_height, + terrain_tile_width, + ) + + xsize, ysize = tiling.roi_to_start_and_size(global_bounds, resolution[1])[ + 2: + ] + + # build the tranform of the dataset + # Generate profile + geotransform = ( + global_bounds[0], + resolution[1], + 0.0, + global_bounds[3], + 0.0, + -resolution[1], + ) + + transform = Affine.from_gdal(*geotransform) + raster_profile = collections.OrderedDict( + { + "height": ysize, + "width": xsize, + "driver": "GTiff", + "dtype": "float32", + "transform": transform, + "crs": "EPSG:{}".format(epsg), + "tiled": True, + "no_data": dsm_nodata, + } + ) + + # Setup dump directory + if dump_dir is not None: + out_dump_dir = dump_dir + safe_makedirs(out_dump_dir) + else: + out_dump_dir = orchestrator.out_dir + + if dsm_file_name is not None: + safe_makedirs(os.path.dirname(dsm_file_name)) + + # Save all file that are in inputs + for key in dict_path.keys(): + if key in (cst.DSM_ALT, cst.DSM_COLOR, cst.DSM_WEIGHTS_SUM): + option = False + else: + option = True + + if key == cst.DSM_ALT and dsm_file_name is not None: + out_file_name = dsm_file_name + elif key == cst.DSM_COLOR and color_file_name is not None: + out_file_name = color_file_name + elif key == cst.DSM_CLASSIF and classif_file_name is not None: + out_file_name = classif_file_name + elif key == cst.DSM_FILLING and filling_file_name is not None: + out_file_name = filling_file_name + elif ( + key == cst.DSM_PERFORMANCE_MAP + and performance_map_file_name is not None + ): + out_file_name = performance_map_file_name + elif ( + key == cst.DSM_SOURCE_PC and contributing_pair_file_name is not None + ): + out_file_name = contributing_pair_file_name + elif key == cst.DSM_MSK and mask_file_name is not None: + out_file_name = mask_file_name + else: + out_file_name = os.path.join(out_dump_dir, key + ".tif") + + orchestrator.add_to_save_lists( + out_file_name, + key, + terrain_raster, + dtype=inputs.rasterio_get_dtype(dict_path[key][0]), + nodata=inputs.rasterio_get_nodata(dict_path[key][0]), + cars_ds_name=key, + optional_data=option, + ) + + [saving_info] = orchestrator.get_saving_infos([terrain_raster]) + for col in range(terrain_raster.shape[1]): + for row in range(terrain_raster.shape[0]): + # update saving infos for potential replacement + full_saving_info = ocht.update_saving_infos( + saving_info, row=row, col=col + ) + + # Delayed call to dsm merging operations using all + terrain_raster[row, col] = orchestrator.cluster.create_task( + dsm_merging_wrapper, nout=1 + )( + dict_path, + terrain_raster.tiling_grid[row, col], + resolution, + raster_profile, + full_saving_info, + ) + + return terrain_raster + + +def dsm_merging_wrapper( # noqa C901 + dict_path, + tile_bounds, + resolution, + profile, + saving_info=None, +): + """ + Merge all the variables + + :param dict_path: path of all variables from all dsms + :type dict_path: dict + :param tile_bounds: list of tiles coordinates + :type tile_bounds: list + :param resolution: resolution of the dsms + :type resolution: list + :param profile: profile of the global dsm + :type profile: OrderedDict + :saving_info: the saving infos + """ + + # create the tile dataset + x_value = np.arange(tile_bounds[0], tile_bounds[1], resolution[1]) + y_value = np.arange(tile_bounds[2], tile_bounds[3], resolution[1]) + height = len(y_value) + width = len(x_value) + + dataset = xr.Dataset( + data_vars={}, + coords={ + "y": y_value, + "x": x_value, + }, + ) + + # calculate the bounds intersection between each path + list_intersection = [] + + for path in dict_path["dsm"]: + with rasterio.open(path) as src: + intersect_bounds = ( + max(tile_bounds[0], src.bounds.left), # xmin + max(tile_bounds[2], src.bounds.bottom), # ymin + min(tile_bounds[1], src.bounds.right), # xmax + min(tile_bounds[3], src.bounds.top), # ymax + ) + + if ( + intersect_bounds[0] < intersect_bounds[2] + and intersect_bounds[1] < intersect_bounds[3] + ): + list_intersection.append(intersect_bounds) + else: + list_intersection.append("no intersection") + + # Update the data + for key in dict_path.keys(): + # Choose the method regarding the variable + if key in [cst.DSM_NB_PTS, cst.DSM_NB_PTS_IN_CELL]: + method = "sum" + elif key in [ + cst.DSM_FILLING, + cst.DSM_CLASSIF, + cst.DSM_SOURCE_PC, + ]: + method = "bool" + else: + method = "basic" + + # take band description information + band_description = inputs.get_descriptions_bands(dict_path[key][0]) + + if band_description[0] is not None: + if len(band_description) == 1: + band_description = np.array([band_description[0]]) + elif key != cst.DSM_CLASSIF: + band_description = np.array([band_description]) + else: + band_description = list(band_description) + + # Define the dimension of the data in the dataset + if key == cst.DSM_COLOR: + if len(band_description) == 3: + band_description = ["R", "G", "B"] + else: + band_description = ["R", "G", "B", "N"] + dataset.coords[cst.BAND_IM] = (cst.BAND_IM, band_description) + dim = [cst.BAND_IM, cst.Y, cst.X] + elif key == cst.DSM_SOURCE_PC: + dataset.coords[cst.BAND_SOURCE_PC] = ( + cst.BAND_SOURCE_PC, + band_description, + ) + dim = [cst.BAND_SOURCE_PC, cst.Y, cst.X] + elif key == cst.DSM_CLASSIF: + dataset.coords[cst.BAND_CLASSIF] = ( + cst.BAND_CLASSIF, + band_description, + ) + dim = [cst.BAND_CLASSIF, cst.Y, cst.X] + elif key == cst.DSM_FILLING: + dataset.coords[cst.BAND_FILLING] = ( + cst.BAND_FILLING, + band_description, + ) + dim = [cst.BAND_FILLING, cst.Y, cst.X] + else: + dim = [cst.Y, cst.X] + + # Update data + if key == cst.DSM_ALT: + # Update dsm_value et weights once + value, weights = assemblage( + dict_path[key], + dict_path[cst.DSM_WEIGHTS_SUM], + method, + list_intersection, + tile_bounds, + height, + width, + band_description, + ) + + dataset[key] = (dim, value) + dataset[cst.DSM_WEIGHTS_SUM] = (dim, weights) + elif key != cst.DSM_WEIGHTS_SUM: + # Update other variables + value, _ = assemblage( + dict_path[key], + dict_path[cst.DSM_WEIGHTS_SUM], + method, + list_intersection, + tile_bounds, + height, + width, + band_description, + ) + + dataset[key] = (dim, value) + + # Define the tile transform + bounds = [tile_bounds[0], tile_bounds[2], tile_bounds[1], tile_bounds[3]] + xstart, ystart, xsize, ysize = tiling.roi_to_start_and_size( + bounds, resolution[1] + ) + transform = rasterio.Affine(*profile["transform"][0:6]) + + row_pix_pos, col_pix_pos = rasterio.transform.AffineTransformer( + transform + ).rowcol(xstart, ystart) + window = [ + row_pix_pos, + row_pix_pos + ysize, + col_pix_pos, + col_pix_pos + xsize, + ] + + window = cars_dataset.window_array_to_dict(window) + + # Fill dataset + cars_dataset.fill_dataset( + dataset, + saving_info=saving_info, + window=window, + profile=profile, + overlaps=None, + ) + + return dataset + + +def assemblage( + out, + current_weights, + method, + intersect_bounds, + tile_bounds, + height, + width, + band_description=None, +): + """ + Update data + + :param out: the data to update + :type out: list of path + :param current_weights: the current weights of the data + :type current_weights: list of path + :param method: the method used to update the data + :type method: str + :param intersect_bounds: the bounds intersection + :type intersect_bounds: list of bounds + :param height: the height of the tile + :type height: int + :param width: the width of the tile + :type width: int + :param band_description: the band description of the data + :type band_description: str of list + + """ + # Initialize the tile + nb_bands = inputs.rasterio_get_nb_bands(out[0]) + + dtype = inputs.rasterio_get_dtype(out[0]) + nodata = inputs.rasterio_get_nodata(out[0]) + + if band_description[0] is not None: + tile = np.full((nb_bands, height, width), nodata, dtype=dtype) + else: + tile = np.full((height, width), nodata, dtype=dtype) + + # Initialize the weights + weights = np.full((height, width), 0, dtype=dtype) + + for idx, path in enumerate(out): + with rasterio.open(path) as src, rasterio.open( + current_weights[idx] + ) as drt: + if intersect_bounds[idx] != "no intersection": + # Build the window + window = from_bounds( + *intersect_bounds[idx], transform=src.transform + ) + + # Extract the data + if band_description[0] is not None: + data = src.read(window=window) + _, rows, cols = data.shape + else: + data = src.read(1, window=window) + rows, cols = data.shape + + current_weights_window = drt.read(1, window=window) + + # Calculate the x and y offset because the current_data + # doesn't equal to the entire tile + x_offset = int( + (intersect_bounds[idx][0] - tile_bounds[0]) + * np.abs(src.res[0]) + ) + y_offset = int( + (tile_bounds[3] - intersect_bounds[idx][3]) + * np.abs(src.res[1]) + ) + + if cols > 0 and rows > 0: + tab_x = np.arange(x_offset, x_offset + cols) + + tab_y = np.arange(y_offset, y_offset + rows) + + ind_y, ind_x = np.ix_(tab_y, tab_x) # pylint: disable=W0632 + + if rows == 1: + ind_y = np.full_like(ind_x, tab_y[0]) + + # Update data + if band_description[0] is not None: + tile[:, ind_y, ind_x] = np.reshape( + update_data( + tile[:, ind_y, ind_x], + data, + current_weights_window, + weights[ind_y, ind_x], + nodata, + method=method, + ), + tile[:, ind_y, ind_x].shape, + ) + else: + tile[ind_y, ind_x] = np.reshape( + update_data( + tile[ind_y, ind_x], + data, + current_weights_window, + weights[ind_y, ind_x], + nodata, + method=method, + ), + tile[ind_y, ind_x].shape, + ) + + # Update weights + weights[ind_y, ind_x] = update_weights( + weights[ind_y, ind_x], current_weights_window + ) + + return tile, weights diff --git a/cars/pipelines/parameters/dsm_inputs_constants.py b/cars/pipelines/parameters/dsm_inputs_constants.py new file mode 100644 index 00000000..a1a36132 --- /dev/null +++ b/cars/pipelines/parameters/dsm_inputs_constants.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# coding: utf8 +# +# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). +# +# This file is part of CARS +# (see https://github.com/CNES/cars). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +CARS phased dsms to full resolution dsm pipeline constants file +""" + +DSMS = "dsms" diff --git a/cars/pipelines/parameters/sensor_inputs.py b/cars/pipelines/parameters/sensor_inputs.py index 981d2f34..51826c89 100644 --- a/cars/pipelines/parameters/sensor_inputs.py +++ b/cars/pipelines/parameters/sensor_inputs.py @@ -74,6 +74,19 @@ def sensors_check_inputs(conf, config_json_dir=None): # noqa: C901 checker_inputs = Checker(inputs_schema) checker_inputs.validate(overloaded_conf) + check_sensors(conf, overloaded_conf, config_json_dir) + + # Check srtm dir + check_srtm(overloaded_conf[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH]) + + return overloaded_conf + + +def check_sensors(conf, overloaded_conf, config_json_dir=None): + """ + Check sensors + + """ # Validate each sensor image sensor_schema = { sens_cst.INPUT_IMG: str, @@ -83,6 +96,7 @@ def sensors_check_inputs(conf, config_json_dir=None): # noqa: C901 sens_cst.INPUT_MSK: Or(str, None), sens_cst.INPUT_CLASSIFICATION: Or(str, None), } + checker_sensor = Checker(sensor_schema) for sensor_image_key in conf[sens_cst.SENSORS]: @@ -139,6 +153,30 @@ def sensors_check_inputs(conf, config_json_dir=None): # noqa: C901 overloaded_conf[sens_cst.SENSORS][sensor_image_key] ) + # Modify to absolute path + if config_json_dir is not None: + modify_to_absolute_path(config_json_dir, overloaded_conf) + + # Check image, msk and color size compatibility + for sensor_image_key in overloaded_conf[sens_cst.SENSORS]: + sensor_image = overloaded_conf[sens_cst.SENSORS][sensor_image_key] + check_input_size( + sensor_image[sens_cst.INPUT_IMG], + sensor_image[sens_cst.INPUT_MSK], + sensor_image[sens_cst.INPUT_COLOR], + sensor_image[sens_cst.INPUT_CLASSIFICATION], + ) + # check band nbits of msk and classification + check_nbits( + sensor_image[sens_cst.INPUT_MSK], + sensor_image[sens_cst.INPUT_CLASSIFICATION], + ) + # check image and color data consistency + check_input_data( + sensor_image[sens_cst.INPUT_IMG], + sensor_image[sens_cst.INPUT_COLOR], + ) + # Validate pairs # If there is two inputs with no associated pairing, consider that the first # image is left and the second image is right @@ -172,7 +210,6 @@ def sensors_check_inputs(conf, config_json_dir=None): # noqa: C901 # Modify to absolute path if config_json_dir is not None: modify_to_absolute_path(config_json_dir, overloaded_conf) - else: logging.debug( "path of config file was not given," @@ -185,29 +222,6 @@ def sensors_check_inputs(conf, config_json_dir=None): # noqa: C901 overloaded_conf[sens_cst.SENSORS], sens_cst.INPUT_IMG, key1, key2 ) - # Check image, msk and color size compatibility - for sensor_image_key in overloaded_conf[sens_cst.SENSORS]: - sensor_image = overloaded_conf[sens_cst.SENSORS][sensor_image_key] - check_input_size( - sensor_image[sens_cst.INPUT_IMG], - sensor_image[sens_cst.INPUT_MSK], - sensor_image[sens_cst.INPUT_COLOR], - sensor_image[sens_cst.INPUT_CLASSIFICATION], - ) - # check band nbits of msk and classification - check_nbits( - sensor_image[sens_cst.INPUT_MSK], - sensor_image[sens_cst.INPUT_CLASSIFICATION], - ) - # check image and color data consistency - check_input_data( - sensor_image[sens_cst.INPUT_IMG], - sensor_image[sens_cst.INPUT_COLOR], - ) - - # Check srtm dir - check_srtm(overloaded_conf[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH]) - return overloaded_conf diff --git a/docs/source/usage/configuration.rst b/docs/source/usage/configuration.rst index f0fa8e3e..88affc85 100644 --- a/docs/source/usage/configuration.rst +++ b/docs/source/usage/configuration.rst @@ -87,6 +87,7 @@ The structure follows this organization: - *classification*: This image is a multiband binary file. Each band should have a specific name (Please, see the section :ref:`add_band_description_in_image` to add band name / description in order to be used in Applications). By using this file, a different process for each band is applied for the 1 values (Please, see the Applications section for details). - Please, see the section :ref:`convert_image_to_binary_image` to make binary *mask* image or binary *classification* image with 1 bit per band. - *geomodel*: If the geomodel file is not provided, CARS will try to use the RPC loaded with rasterio opening *image*. + - It is possible to add sensors inputs while using depth_maps or dsm inputs **Pairing** @@ -164,7 +165,9 @@ The structure follows this organization: To generate confidence maps, `z_inf` and `z_sup`, the parameter `save_intermediate_data` of `triangulation` should be activated. - To generate the performance map, the parameters `generate_performance_map` and `save_intermediate_data` of the `dense_matching` application must be activated. + To generate the performance map, the parameters `generate_performance_map` and `save_intermediate_data` of the `dense_matching` application must be activated. + + It is possible to add sensors inputs while using depth_maps inputs +------------------+-------------------------------------------------------------------+----------------+---------------+----------+ | Name | Description | Type | Default value | Required | @@ -192,6 +195,96 @@ The structure follows this organization: | *epsg* | Epsg code of depth map | int | 4326 | No | +------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + .. tab:: DSMS inputs + + +-------------------------+---------------------------------------------------------------------+-----------------------+----------------------+----------+ + | Name | Description | Type | Default value | Required | + +=========================+=====================================================================+=======================+======================+==========+ + | *dsm* | Dsms to merge | dict | No | Yes | + +-------------------------+---------------------------------------------------------------------+-----------------------+----------------------+----------+ + + + + **DSMS** + + For each DSMS, give a particular name (what you want): + + .. code-block:: json + + { + "inputs": { + "dsms": { + "my_name_for_this_dsm": { + "dsm" : "path_to_dsm.tif", + "classification" : "path_to_classif.tif", + "color" : "path_to_color.tif", + "performance_map" : "path_to_performance_map.tif", + "filling" : "path_to_filling.tif", + "mask" : "path_to_mask.tif", + "weights": "path_to_weights.tif", + "dsm_inf": "path_to_dsm_inf.tif", + "dsm_sup": "path_to_dsm_sup.tif" + } + } + } + } + + These input files can be generated by running CARS with `product_level: ["dsm"]` and `auxiliary` dictionary filled with desired auxiliary files + + .. note:: + + To generate confidence maps, `z_inf` and `z_sup`, the parameter `save_intermediate_data` of `triangulation` should be activated. + + To generate the performance map, the parameters `generate_performance_map` and `save_intermediate_data` of the `dense_matching` application must be activated. + + It is possible to add sensors inputs while using dsm inputs + + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | Name | Description | Type | Default value | Required | + +============================+===================================================================+================+===============+==========+ + | *dsm* | Path to the dsm file | string | | Yes | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *weights* | Path to the weights file | string | | Yes | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *color* | Path to the color file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *classification* | Path to the classification file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *mask* | Path to the mask file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *filling* | Path to the filling file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *performance_map* | Path to the performance_map file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *source_pc* | Path to the source_pc file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *dsm_inf* | Path to the dsm_inf file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *dsm_sup* | Path to the dsm_sup file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *dsm_mean* | Path to the dsm_mean file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *dsm_std* | Path to the dsm_std file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *dsm_inf_mean* | Path to the dsm_inf_mean file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *dsm_inf_std* | Path to the dsm_inf_std file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *dsm_sup_mean* | Path to the dsm_sup_mean file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *dsm_sup_std* | Path to the dsm_sup_std file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *dsm_n_pts* | Path to the dsm_n_pts file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *dsm_pts_in_cell* | Path to the dsm_pts_in_cell file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *confidence_from_ambiguity*| Path to the confidence_from_ambiguity file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *confidence_from_risk_min* | Path to the confidence_from_risk_min file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + | *confidence_from_risk_max* | Path to the confidence_from_risk_min file | string | | No | + +----------------------------+-------------------------------------------------------------------+----------------+---------------+----------+ + .. tab:: ROI +-------------------------+---------------------------------------------------------------------+-----------------------+----------------------+----------+ diff --git a/tests/applications/rasterization/test_rasterization_tools.py b/tests/applications/rasterization/test_rasterization_tools.py index 9be9621a..e299fbc9 100644 --- a/tests/applications/rasterization/test_rasterization_tools.py +++ b/tests/applications/rasterization/test_rasterization_tools.py @@ -39,6 +39,8 @@ # CARS imports from cars.core import constants as cst +from cars.core import inputs +from cars.pipelines.parameters import dsm_inputs # CARS Tests imports from tests.helpers import absolute_data_path, add_color, assert_same_datasets @@ -131,6 +133,69 @@ def test_simple_rasterization_synthetic_case(): ) +@pytest.mark.unit_tests +def test_phased_dsm(): + """ + Test phased_dsm function + """ + + dsm_one = absolute_data_path("ref_output/dsm_end2end_ventoux_split.tif") + + dsm_two = absolute_data_path("ref_output/dsm_end2end_paca_bulldozer.tif") + + input_test = { + "one": { + "dsm": dsm_one, + }, + "two": { + "dsm": dsm_two, + }, + } + + with pytest.raises(RuntimeError) as exception: + dsm_inputs.check_phasing(input_test) + + assert str(exception.value) == "DSM two and one are not phased" + + bounds_one = inputs.rasterio_get_bounds(dsm_one) + bounds_two = inputs.rasterio_get_bounds(dsm_two) + profile = inputs.rasterio_get_profile(dsm_one) + transform = list(profile["transform"]) + resolution = transform[4] + + x_phase = 675292.31105432 + y_phase = 4897082.95714968 + + for index, value in enumerate(bounds_one): + if index in (0, 2): + bounds_one[index] = rasterization_tools.phased_dsm( + value, x_phase, resolution + ) + else: + bounds_one[index] = rasterization_tools.phased_dsm( + value, y_phase, resolution + ) + + for index, value in enumerate(bounds_two): + if index in (0, 2): + bounds_two[index] = rasterization_tools.phased_dsm( + value, x_phase, resolution + ) + else: + bounds_two[index] = rasterization_tools.phased_dsm( + value, y_phase, resolution + ) + + diff = bounds_one[0:2] - bounds_two[0:2] + resolution = np.array([resolution, resolution]) + res_ratio = diff / resolution + + if ~np.all(np.equal(res_ratio, res_ratio.astype(int))) and ~np.all( + np.equal(1 / res_ratio, (1 / res_ratio).astype(int)) + ): + raise RuntimeError("DSM are not phased") + + @pytest.mark.unit_tests def test_simple_rasterization_single(): """ diff --git a/tests/data/input/phr_ventoux/input_with_color_and_classif.json b/tests/data/input/phr_ventoux/input_with_color_and_classif.json new file mode 100644 index 00000000..768cee0b --- /dev/null +++ b/tests/data/input/phr_ventoux/input_with_color_and_classif.json @@ -0,0 +1,23 @@ +{ + "inputs": { + "sensors" : { + "left": { + "image": "left_image.tif", + "color": "color_image.tif", + "geomodel": { + "path": "left_image.geom" + }, + "classification": "left_classif.tif" + }, + "right": { + "image": "right_image.tif", + "geomodel": { + "path": "right_image.geom" + }, + "classification": "right_classif.tif" + } + }, + "pairing": [["left", "right"]], + "initial_elevation": "srtm/N44E005.hgt" + } +} \ No newline at end of file diff --git a/tests/data/ref_output/classif_end2end_ventoux_lr.tif b/tests/data/ref_output/classif_end2end_ventoux_lr.tif new file mode 100644 index 00000000..63798efc Binary files /dev/null and b/tests/data/ref_output/classif_end2end_ventoux_lr.tif differ diff --git a/tests/data/ref_output/classif_end2end_ventoux_rl.tif b/tests/data/ref_output/classif_end2end_ventoux_rl.tif new file mode 100644 index 00000000..c90010a8 Binary files /dev/null and b/tests/data/ref_output/classif_end2end_ventoux_rl.tif differ diff --git a/tests/data/ref_output/color_end2end_ventoux_fusion.tif b/tests/data/ref_output/color_end2end_ventoux_fusion.tif new file mode 100644 index 00000000..eefeb653 Binary files /dev/null and b/tests/data/ref_output/color_end2end_ventoux_fusion.tif differ diff --git a/tests/data/ref_output/color_end2end_ventoux_lr.tif b/tests/data/ref_output/color_end2end_ventoux_lr.tif new file mode 100644 index 00000000..bce0c640 Binary files /dev/null and b/tests/data/ref_output/color_end2end_ventoux_lr.tif differ diff --git a/tests/data/ref_output/color_end2end_ventoux_split_4326.tif b/tests/data/ref_output/color_end2end_ventoux_split_4326.tif index 2d0dda98..35365813 100644 Binary files a/tests/data/ref_output/color_end2end_ventoux_split_4326.tif and b/tests/data/ref_output/color_end2end_ventoux_split_4326.tif differ diff --git a/tests/data/ref_output/colorisation_end2end_gizeh_reentrance.tif b/tests/data/ref_output/colorisation_end2end_gizeh_reentrance.tif new file mode 100644 index 00000000..04e2f72f Binary files /dev/null and b/tests/data/ref_output/colorisation_end2end_gizeh_reentrance.tif differ diff --git a/tests/data/ref_output/phased_dsm_end2end_ventoux_fusion.tif b/tests/data/ref_output/phased_dsm_end2end_ventoux_fusion.tif new file mode 100644 index 00000000..6700e411 Binary files /dev/null and b/tests/data/ref_output/phased_dsm_end2end_ventoux_fusion.tif differ diff --git a/tests/data/ref_output/phased_dsm_end2end_ventoux_lr.tif b/tests/data/ref_output/phased_dsm_end2end_ventoux_lr.tif new file mode 100644 index 00000000..d2073d65 Binary files /dev/null and b/tests/data/ref_output/phased_dsm_end2end_ventoux_lr.tif differ diff --git a/tests/data/ref_output/phased_dsm_end2end_ventoux_rl.tif b/tests/data/ref_output/phased_dsm_end2end_ventoux_rl.tif new file mode 100644 index 00000000..6700e411 Binary files /dev/null and b/tests/data/ref_output/phased_dsm_end2end_ventoux_rl.tif differ diff --git a/tests/data/ref_output/weights_end2end_ventoux_lr.tif b/tests/data/ref_output/weights_end2end_ventoux_lr.tif new file mode 100644 index 00000000..ad377d76 Binary files /dev/null and b/tests/data/ref_output/weights_end2end_ventoux_lr.tif differ diff --git a/tests/data/ref_output/weights_end2end_ventoux_rl.tif b/tests/data/ref_output/weights_end2end_ventoux_rl.tif new file mode 100644 index 00000000..6e28aa0d Binary files /dev/null and b/tests/data/ref_output/weights_end2end_ventoux_rl.tif differ diff --git a/tests/pipelines/test_dsm_inputs.py b/tests/pipelines/test_dsm_inputs.py new file mode 100644 index 00000000..a249f374 --- /dev/null +++ b/tests/pipelines/test_dsm_inputs.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# coding: utf8 +# +# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES). +# +# This file is part of CARS +# (see https://github.com/CNES/cars). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Test module for cars/pipelines/parameters/dsm_inputs.py +""" + +import pytest + +from cars.pipelines.parameters import dsm_inputs + +from ..helpers import absolute_data_path + + +@pytest.mark.unit_tests +def test_input_none_phased_dsm(): + """ + Test with non phased dsms + """ + + dsm_one = absolute_data_path("ref_output/dsm_end2end_ventoux_split.tif") + + dsm_two = absolute_data_path("ref_output/dsm_end2end_paca_bulldozer.tif") + + input_test = { + "one": { + "dsm": dsm_one, + }, + "two": { + "dsm": dsm_two, + }, + } + + with pytest.raises(RuntimeError) as exception: + dsm_inputs.check_phasing(input_test) + + assert str(exception.value) == "DSM two and one are not phased" + + +@pytest.mark.unit_tests +def test_input_same_dsm(): + """ + Test with same dsms + """ + + dsm_one = absolute_data_path("ref_output/dsm_end2end_ventoux_split.tif") + + dsm_two = absolute_data_path("ref_output/dsm_end2end_ventoux_split.tif") + + input_test = { + "one": { + "dsm": dsm_one, + }, + "two": { + "dsm": dsm_two, + }, + } + + dsm_inputs.check_phasing(input_test) diff --git a/tests/test_end2end.py b/tests/test_end2end.py index b72d1895..81479a7d 100644 --- a/tests/test_end2end.py +++ b/tests/test_end2end.py @@ -28,6 +28,7 @@ # Standard imports from __future__ import absolute_import +import copy import json import math import os @@ -56,6 +57,316 @@ NB_WORKERS = 2 +@pytest.mark.end2end_tests +def test_end2end_dsm_fusion(): + """ + End to end processing + + test the phased dsm re-entrance + """ + + with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: + input_json = absolute_data_path( + "input/phr_ventoux/input_with_color_and_classif.json" + ) + + # Run dense dsm pipeline + _, input_dense_dsm_lr = generate_input_json( + input_json, + directory, + "multiprocessing", + orchestrator_parameters={ + "nb_workers": NB_WORKERS, + "max_ram_per_worker": 500, + }, + ) + dense_dsm_applications = { + "grid_generation": {"method": "epipolar", "epi_step": 30}, + "dense_matching": { + "method": "census_sgm", + "use_cross_validation": True, + "use_global_disp_range": True, + }, + "point_cloud_rasterization": { + "method": "simple_gaussian", + "dsm_radius": 3, + "sigma": 0.3, + "dsm_no_data": -999, + "color_no_data": 0, + "msk_no_data": 254, + "save_intermediate_data": True, + }, + } + input_dense_dsm_lr["applications"].update(dense_dsm_applications) + + # update epsg + final_epsg = 32631 + input_dense_dsm_lr["output"]["epsg"] = final_epsg + resolution = 0.5 + input_dense_dsm_lr["output"]["resolution"] = resolution + input_dense_dsm_lr["output"]["auxiliary"] = { + "mask": True, + "performance_map": True, + } + + dense_dsm_pipeline = default.DefaultPipeline(input_dense_dsm_lr) + dense_dsm_pipeline.run() + + out_dir = input_dense_dsm_lr["output"]["directory"] + + # Ref output dir dependent from geometry plugin chosen + ref_output_dir = "ref_output" + + # copy2( + # os.path.join(out_dir, "dsm", "dsm.tif"), + # absolute_data_path( + # os.path.join( + # ref_output_dir, "phased_dsm_end2end_ventoux_lr.tif" + # ) + # ), + # ) + # copy2( + # os.path.join(out_dir, "dsm", "color.tif"), + # absolute_data_path( + # os.path.join( + # ref_output_dir, "color_end2end_ventoux_lr.tif" + # ) + # ), + # ) + # copy2( + # os.path.join(out_dir, "dump_dir", "rasterization", + # "classification.tif"), + # absolute_data_path( + # os.path.join( + # ref_output_dir, "classif_end2end_ventoux_lr.tif" + # ) + # ), + # ) + # + # copy2( + # os.path.join(out_dir, "dump_dir/rasterization/", "weights.tif"), + # absolute_data_path( + # os.path.join( + # ref_output_dir, "weights_end2end_ventoux_lr.tif" + # ) + # ), + # ) + + input_dense_dsm_rl = copy.deepcopy(input_dense_dsm_lr) + input_dense_dsm_rl["inputs"]["sensors"]["left"] = input_dense_dsm_lr[ + "inputs" + ]["sensors"]["right"] + input_dense_dsm_rl["inputs"]["sensors"]["right"] = input_dense_dsm_lr[ + "inputs" + ]["sensors"]["left"] + + dense_dsm_pipeline = default.DefaultPipeline(input_dense_dsm_rl) + dense_dsm_pipeline.run() + + # copy2( + # os.path.join(out_dir, "dsm", "dsm.tif"), + # absolute_data_path( + # os.path.join( + # ref_output_dir, "phased_dsm_end2end_ventoux_rl.tif" + # ) + # ), + # ) + # + # copy2( + # os.path.join(out_dir, "dump_dir", "rasterization", + # "classification.tif"), + # absolute_data_path( + # os.path.join( + # ref_output_dir, "classif_end2end_ventoux_rl.tif" + # ) + # ), + # ) + # copy2( + # os.path.join(out_dir, "dump_dir/rasterization/", "weights.tif"), + # absolute_data_path( + # os.path.join( + # ref_output_dir, "weights_end2end_ventoux_rl.tif" + # ) + # ), + # ) + + input_dsm_config = { + "inputs": { + "dsms": { + "one": { + "dsm": absolute_data_path( + "ref_output/phased_dsm_end2end_ventoux_lr.tif" + ), + "weights": absolute_data_path( + "ref_output/weights_end2end_ventoux_lr.tif" + ), + "color": absolute_data_path( + "ref_output/color_end2end_ventoux_lr.tif" + ), + "classification": absolute_data_path( + "ref_output/classif_end2end_ventoux_lr.tif" + ), + }, + "two": { + "dsm": absolute_data_path( + "ref_output/phased_dsm_end2end_ventoux_rl.tif" + ), + "weights": absolute_data_path( + "ref_output/weights_end2end_ventoux_rl.tif" + ), + "color": absolute_data_path( + "ref_output/color_end2end_ventoux_lr.tif" + ), + "classification": absolute_data_path( + "ref_output/classif_end2end_ventoux_rl.tif" + ), + }, + } + } + } + + input_dsm_config["output"] = {} + input_dsm_config["output"]["directory"] = directory + + dsm_merging_pipeline = default.DefaultPipeline(input_dsm_config) + dsm_merging_pipeline.run() + + # copy2( + # os.path.join(out_dir, "dsm", "dsm.tif"), + # absolute_data_path( + # os.path.join( + # ref_output_dir, "phased_dsm_end2end_ventoux_fusion.tif" + # ) + # ), + # ) + # copy2( + # os.path.join(out_dir, "dsm", "color.tif"), + # absolute_data_path( + # os.path.join( + # ref_output_dir, "color_end2end_ventoux_fusion.tif" + # ) + # ), + # ) + + assert_same_images( + os.path.join(out_dir, "dsm", "dsm.tif"), + absolute_data_path( + os.path.join( + ref_output_dir, "phased_dsm_end2end_ventoux_fusion.tif" + ) + ), + atol=0.0001, + rtol=1e-6, + ) + assert_same_images( + os.path.join(out_dir, "dsm", "color.tif"), + absolute_data_path( + os.path.join(ref_output_dir, "color_end2end_ventoux_fusion.tif") + ), + atol=0.0001, + rtol=1e-6, + ) + + +@pytest.mark.end2end_tests +def test_end2end_color_after_dsm_reentrance(): + """ + End to end processing + + test the colorisation after depth_map re entrance + """ + + with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: + input_dsm_config = { + "inputs": { + "dsms": { + "one": { + "dsm": absolute_data_path( + "ref_output/phased_dsm_end2end_ventoux_lr.tif" + ), + "weights": absolute_data_path( + "ref_output/weights_end2end_ventoux_lr.tif" + ), + "color": absolute_data_path( + "ref_output/color_end2end_ventoux_lr.tif" + ), + }, + "two": { + "dsm": absolute_data_path( + "ref_output/phased_dsm_end2end_ventoux_rl.tif" + ), + "weights": absolute_data_path( + "ref_output/weights_end2end_ventoux_rl.tif" + ), + "color": absolute_data_path( + "ref_output/color_end2end_ventoux_lr.tif" + ), + }, + }, + "sensors": { + "one": { + "image": absolute_data_path( + "input/phr_ventoux/left_image.tif" + ), + "geomodel": { + "path": absolute_data_path( + "input/phr_ventoux/left_image.geom" + ), + }, + }, + "two": { + "image": absolute_data_path( + "input/phr_ventoux/right_image.tif" + ), + "geomodel": { + "path": absolute_data_path( + "input/phr_ventoux/right_image.geom" + ), + }, + }, + }, + "pairing": [["one", "two"]], + "initial_elevation": absolute_data_path( + "input/phr_ventoux/srtm/N44E005.hgt" + ), + }, + "applications": { + "auxiliary_filling": {"save_intermediate_data": True} + }, + } + + input_dsm_config["output"] = {} + input_dsm_config["output"]["directory"] = directory + + out_dir = input_dsm_config["output"]["directory"] + + ref_output_dir = "ref_output" + + dsm_merging_pipeline = default.DefaultPipeline(input_dsm_config) + dsm_merging_pipeline.run() + + # copy2( + # os.path.join(out_dir, "dsm", "color.tif"), + # absolute_data_path( + # os.path.join( + # ref_output_dir, + # "colorisation_end2end_gizeh_reentrance.tif" + # ) + # ), + # ) + + assert_same_images( + os.path.join(out_dir, "dsm", "color.tif"), + absolute_data_path( + os.path.join( + ref_output_dir, "colorisation_end2end_gizeh_reentrance.tif" + ) + ), + atol=0.0001, + rtol=1e-6, + ) + + @pytest.mark.end2end_tests def test_end2end_gizeh_rectangle_epi_image_performance_map(): """ @@ -1263,6 +1574,35 @@ def test_end2end_ventoux_unique_split_epsg_4326(): } ], }, + "sensors": { + "left": { + "image": absolute_data_path( + "input/phr_ventoux/left_image.tif" + ), + "color": absolute_data_path( + "input/phr_ventoux/left_image.tif" + ), + "geomodel": { + "path": absolute_data_path( + "input/phr_ventoux/left_image.geom" + ) + }, + }, + "right": { + "image": absolute_data_path( + "input/phr_ventoux/right_image.tif" + ), + "geomodel": { + "path": absolute_data_path( + "input/phr_ventoux/left_image.geom" + ), + }, + }, + }, + "pairing": [["left", "right"]], + "initial_elevation": absolute_data_path( + "input/phr_ventoux/srtm/N44E005.hgt" + ), }, "geometry_plugin": geometry_plugin_name, "output": { @@ -1275,7 +1615,8 @@ def test_end2end_ventoux_unique_split_epsg_4326(): "point_cloud_rasterization": { "method": "simple_gaussian", "save_intermediate_data": True, - } + }, + "auxiliary_filling": {"activated": True}, }, }