From 45046ca9da54afe148c897b786535254ca5ae0e2 Mon Sep 17 00:00:00 2001 From: Roux Mathis Date: Thu, 6 Feb 2025 14:37:43 +0100 Subject: [PATCH 1/8] feat: move parameters pipeline eand geometry_plugin to advanced --- cars/pipelines/default/default_pipeline.py | 45 +++++------------ .../parameters/advanced_parameters.py | 49 ++++++++++++++++++- .../advanced_parameters_constants.py | 4 ++ cars/pipelines/parameters/sensor_inputs.py | 11 +---- 4 files changed, 64 insertions(+), 45 deletions(-) diff --git a/cars/pipelines/default/default_pipeline.py b/cars/pipelines/default/default_pipeline.py index b68b411b..c8e8d28e 100644 --- a/cars/pipelines/default/default_pipeline.py +++ b/cars/pipelines/default/default_pipeline.py @@ -68,11 +68,9 @@ from cars.pipelines.pipeline_constants import ( ADVANCED, APPLICATIONS, - GEOMETRY_PLUGIN, INPUTS, ORCHESTRATOR, OUTPUT, - PIPELINE, ) from cars.pipelines.pipeline_template import PipelineTemplate @@ -111,9 +109,9 @@ def __init__(self, conf, config_json_dir=None): """ # Used conf - self.used_conf = {PIPELINE: "default"} + self.used_conf = {} - # check global conf + # Check global conf self.check_global_schema(conf) # Check conf orchestrator @@ -129,37 +127,18 @@ def __init__(self, conf, config_json_dir=None): # Check advanced parameters # TODO static method in the base class - advanced = advanced_parameters.check_advanced_parameters( - conf.get(ADVANCED, {}), check_epipolar_a_priori=True + ( + inputs, + advanced, + self.geometry_plugin, + self.geom_plugin_without_dem_and_geoid, + self.geom_plugin_with_dem_and_geoid, + self.dem_generation_roi, + ) = advanced_parameters.check_advanced_parameters( + inputs, conf.get(ADVANCED, {}), check_epipolar_a_priori=True ) self.used_conf[ADVANCED] = advanced - if self.used_conf[INPUTS][sens_cst.SENSORS] is not None: - # Check geometry plugin and overwrite geomodel in conf inputs - ( - inputs, - self.used_conf[GEOMETRY_PLUGIN], - self.geom_plugin_without_dem_and_geoid, - self.geom_plugin_with_dem_and_geoid, - self.dem_generation_roi, - ) = sensor_inputs.check_geometry_plugin( - inputs, advanced, conf.get(GEOMETRY_PLUGIN, None) - ) - self.used_conf[INPUTS] = inputs - elif ( - depth_cst.DEPTH_MAPS in self.used_conf[INPUTS] - or dsm_cst.DSMS in self.used_conf[INPUTS] - ): - - # if there's an initial elevation with - # point clouds as inputs, generate a plugin (used in dsm_filling) - - self.geom_plugin_with_dem_and_geoid = ( - depth_map_inputs.check_geometry_plugin( - inputs, conf.get(GEOMETRY_PLUGIN, None) - ) - ) - # Get ROI ( self.input_roi_poly, @@ -1400,7 +1379,7 @@ def sensor_to_depth_maps(self): # noqa: C901 ): self.geom_plugin_with_dem_and_geoid = ( sensor_inputs.generate_geometry_plugin_with_dem( - self.used_conf[GEOMETRY_PLUGIN], + self.geometry_plugin, inputs, dem=dem_median, ) diff --git a/cars/pipelines/parameters/advanced_parameters.py b/cars/pipelines/parameters/advanced_parameters.py index fa46939a..c7d54499 100644 --- a/cars/pipelines/parameters/advanced_parameters.py +++ b/cars/pipelines/parameters/advanced_parameters.py @@ -31,10 +31,15 @@ from cars.pipelines.parameters import advanced_parameters_constants as adv_cst from cars.pipelines.parameters.sensor_inputs import CARS_GEOID_PATH +from cars.pipelines.parameters import depth_map_inputs +from cars.pipelines.parameters import depth_map_inputs_constants as depth_cst +from cars.pipelines.parameters import dsm_inputs_constants as dsm_cst +from cars.pipelines.parameters import sensor_inputs +from cars.pipelines.parameters import sensor_inputs_constants as sens_cst from cars.pipelines.pipeline_constants import ADVANCED -def check_advanced_parameters(conf, check_epipolar_a_priori=True): +def check_advanced_parameters(inputs, conf, check_epipolar_a_priori=True): """ Check the advanced parameters consistency @@ -99,6 +104,38 @@ def check_advanced_parameters(conf, check_epipolar_a_priori=True): adv_cst.TERRAIN_A_PRIORI, {} ) + # Check geometry plugin + geom_plugin_without_dem_and_geoid = None + geom_plugin_with_dem_and_geoid = None + dem_generation_roi = None + if inputs[sens_cst.SENSORS] is not None: + # Check geometry plugin and overwrite geomodel in conf inputs + ( + inputs, + overloaded_conf[adv_cst.GEOMETRY_PLUGIN], + geom_plugin_without_dem_and_geoid, + geom_plugin_with_dem_and_geoid, + dem_generation_roi, + ) = sensor_inputs.check_geometry_plugin( + inputs, conf.get(adv_cst.GEOMETRY_PLUGIN, None) + ) + elif depth_cst.DEPTH_MAPS in inputs or dsm_cst.DSMS in inputs: + # If there's an initial elevation with + # point clouds as inputs, generate a plugin (used in dsm_filling) + geom_plugin_with_dem_and_geoid = depth_map_inputs.check_geometry_plugin( + inputs, conf.get(adv_cst.GEOMETRY_PLUGIN, None) + ) + + # If use a priori, override initial elevation with dem_median + if adv_cst.USE_EPIPOLAR_A_PRIORI in overloaded_conf: + if overloaded_conf[adv_cst.USE_EPIPOLAR_A_PRIORI]: + if adv_cst.DEM_MEDIAN in overloaded_conf[adv_cst.TERRAIN_A_PRIORI]: + inputs[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH] = ( + overloaded_conf[adv_cst.TERRAIN_A_PRIORI][ + adv_cst.DEM_MEDIAN + ] + ) + # Validate inputs schema = { adv_cst.DEBUG_WITH_ROI: bool, @@ -107,6 +144,7 @@ def check_advanced_parameters(conf, check_epipolar_a_priori=True): adv_cst.GROUND_TRUTH_DSM: Or(dict, str), adv_cst.PHASING: Or(dict, None), adv_cst.PERFORMANCE_MAP_CLASSES: Or(None, list), + adv_cst.GEOMETRY_PLUGIN: Or(str, dict), } if check_epipolar_a_priori: schema[adv_cst.USE_EPIPOLAR_A_PRIORI] = bool @@ -163,7 +201,14 @@ def check_advanced_parameters(conf, check_epipolar_a_priori=True): ): validate_epipolar_a_priori(overloaded_conf, checker_epipolar) - return overloaded_conf + return ( + inputs, + overloaded_conf, + overloaded_conf[adv_cst.GEOMETRY_PLUGIN], + geom_plugin_without_dem_and_geoid, + geom_plugin_with_dem_and_geoid, + dem_generation_roi, + ) def check_performance_classes(performance_map_classes): diff --git a/cars/pipelines/parameters/advanced_parameters_constants.py b/cars/pipelines/parameters/advanced_parameters_constants.py index 18e490f5..bed5820c 100644 --- a/cars/pipelines/parameters/advanced_parameters_constants.py +++ b/cars/pipelines/parameters/advanced_parameters_constants.py @@ -55,3 +55,7 @@ INPUT_EPSG = "epsg" PERFORMANCE_MAP_CLASSES = "performance_map_classes" + +GEOMETRY_PLUGIN = "geometry_plugin" + +PIPELINE = "pipeline" diff --git a/cars/pipelines/parameters/sensor_inputs.py b/cars/pipelines/parameters/sensor_inputs.py index 0912a5aa..9b91227b 100644 --- a/cars/pipelines/parameters/sensor_inputs.py +++ b/cars/pipelines/parameters/sensor_inputs.py @@ -33,7 +33,6 @@ from cars.core import inputs, preprocessing, roi_tools from cars.core.geometry.abstract_geometry import AbstractGeometry from cars.core.utils import make_relative_path_absolute -from cars.pipelines.parameters import advanced_parameters_constants as adv_cst from cars.pipelines.parameters import ( depth_map_inputs_constants as depth_map_cst, ) @@ -225,7 +224,7 @@ def check_sensors(conf, overloaded_conf, config_json_dir=None): return overloaded_conf -def check_geometry_plugin(conf_inputs, conf_advanced, conf_geom_plugin): +def check_geometry_plugin(conf_inputs, conf_geom_plugin): """ Check the geometry plugin with inputs @@ -251,14 +250,6 @@ def check_geometry_plugin(conf_inputs, conf_advanced, conf_geom_plugin): ) ) - # If use a priori, override initial elevation with dem_median - if adv_cst.USE_EPIPOLAR_A_PRIORI in conf_advanced: - if conf_advanced[adv_cst.USE_EPIPOLAR_A_PRIORI]: - if adv_cst.DEM_MEDIAN in conf_advanced[adv_cst.TERRAIN_A_PRIORI]: - conf_inputs[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH] = ( - conf_advanced[adv_cst.TERRAIN_A_PRIORI][adv_cst.DEM_MEDIAN] - ) - # Check products consistency with this plugin overloaded_conf_inputs = conf_inputs.copy() for sensor_key, sensor_image in conf_inputs[sens_cst.SENSORS].items(): From f7075d7026181dec61127185d70b619006603f32 Mon Sep 17 00:00:00 2001 From: Roux Mathis Date: Wed, 12 Feb 2025 11:27:03 +0100 Subject: [PATCH 2/8] tests: adapt tests to new API --- cars/cars.py | 2 +- cars/pipelines/default/default_pipeline.py | 2 +- .../parameters/advanced_parameters.py | 8 ++- cars/pipelines/pipeline_template.py | 4 +- tests/helpers.py | 8 +-- tests/pipelines/test_advanced_parameters.py | 49 ++++++++++++++++--- tests/test_end2end.py | 15 +++--- 7 files changed, 64 insertions(+), 24 deletions(-) diff --git a/cars/cars.py b/cars/cars.py index 8f13cb50..e41da09c 100644 --- a/cars/cars.py +++ b/cars/cars.py @@ -110,7 +110,7 @@ def main_cli(args, dry_run=False): # noqa: C901 del config["output"]["out_dir"] config_json_dir = os.path.abspath(os.path.dirname(args.conf)) - pipeline_name = config.get("pipeline", "default") + pipeline_name = config.get("advanced", {}).get("pipeline", "default") old_pipelines = [ "sensors_to_dense_dsm", "sensors_to_dense_dsm_no_merging", diff --git a/cars/pipelines/default/default_pipeline.py b/cars/pipelines/default/default_pipeline.py index c8e8d28e..af49cf37 100644 --- a/cars/pipelines/default/default_pipeline.py +++ b/cars/pipelines/default/default_pipeline.py @@ -1786,7 +1786,7 @@ def sensor_to_depth_maps(self): # noqa: C901 "save_intermediate_data" ] = True new_geomplugin_dsm = AbstractGeometry( # pylint: disable=E0110 - self.used_conf[GEOMETRY_PLUGIN], + self.geometry_plugin, dem=self.used_conf[ADVANCED][adv_cst.GROUND_TRUTH_DSM][ adv_cst.INPUT_GROUND_TRUTH_DSM ], diff --git a/cars/pipelines/parameters/advanced_parameters.py b/cars/pipelines/parameters/advanced_parameters.py index c7d54499..67b4a033 100644 --- a/cars/pipelines/parameters/advanced_parameters.py +++ b/cars/pipelines/parameters/advanced_parameters.py @@ -30,12 +30,12 @@ from json_checker import Checker, OptionalKey, Or from cars.pipelines.parameters import advanced_parameters_constants as adv_cst -from cars.pipelines.parameters.sensor_inputs import CARS_GEOID_PATH from cars.pipelines.parameters import depth_map_inputs from cars.pipelines.parameters import depth_map_inputs_constants as depth_cst from cars.pipelines.parameters import dsm_inputs_constants as dsm_cst from cars.pipelines.parameters import sensor_inputs from cars.pipelines.parameters import sensor_inputs_constants as sens_cst +from cars.pipelines.parameters.sensor_inputs import CARS_GEOID_PATH from cars.pipelines.pipeline_constants import ADVANCED @@ -43,6 +43,8 @@ def check_advanced_parameters(inputs, conf, check_epipolar_a_priori=True): """ Check the advanced parameters consistency + :param inputs: configuration of inputs + :type inputs: dict :param conf: configuration of advanced parameters :type conf: dict :param check_epipolar_a_priori: use epipolar a priori parameters @@ -136,6 +138,9 @@ def check_advanced_parameters(inputs, conf, check_epipolar_a_priori=True): ] ) + # Check pipeline + overloaded_conf[adv_cst.PIPELINE] = conf.get(adv_cst.PIPELINE, "default") + # Validate inputs schema = { adv_cst.DEBUG_WITH_ROI: bool, @@ -145,6 +150,7 @@ def check_advanced_parameters(inputs, conf, check_epipolar_a_priori=True): adv_cst.PHASING: Or(dict, None), adv_cst.PERFORMANCE_MAP_CLASSES: Or(None, list), adv_cst.GEOMETRY_PLUGIN: Or(str, dict), + adv_cst.PIPELINE: str, } if check_epipolar_a_priori: schema[adv_cst.USE_EPIPOLAR_A_PRIORI] = bool diff --git a/cars/pipelines/pipeline_template.py b/cars/pipelines/pipeline_template.py index 2115d216..40962a2f 100644 --- a/cars/pipelines/pipeline_template.py +++ b/cars/pipelines/pipeline_template.py @@ -25,7 +25,7 @@ from abc import ABCMeta, abstractmethod -from json_checker import Checker, OptionalKey, Or +from json_checker import Checker, OptionalKey # CARS imports from cars.orchestrator import orchestrator @@ -68,9 +68,7 @@ def check_global_schema(self, conf): pipeline_constants.INPUTS: dict, pipeline_constants.OUTPUT: dict, OptionalKey(pipeline_constants.APPLICATIONS): dict, - OptionalKey(pipeline_constants.GEOMETRY_PLUGIN): Or(str, dict), OptionalKey(pipeline_constants.ORCHESTRATOR): dict, - OptionalKey(pipeline_constants.PIPELINE): str, OptionalKey(pipeline_constants.ADVANCED): dict, } diff --git a/tests/helpers.py b/tests/helpers.py index e93e44fc..f087688a 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -111,10 +111,10 @@ def generate_input_json( if geometry_plugin_name is None: geometry_plugin_name = get_geometry_plugin().plugin_name - config["geometry_plugin"] = geometry_plugin_name - - # overload pipeline - config["pipeline"] = "default" + config["advanced"] = { + "geometry_plugin": geometry_plugin_name, + "pipeline": "default", + } # Create keys if "applications" not in config: diff --git a/tests/pipelines/test_advanced_parameters.py b/tests/pipelines/test_advanced_parameters.py index b511dbbe..a479f22b 100644 --- a/tests/pipelines/test_advanced_parameters.py +++ b/tests/pipelines/test_advanced_parameters.py @@ -26,7 +26,9 @@ import pytest import rasterio as rio -from cars.pipelines.parameters import advanced_parameters +from cars.pipelines.parameters import advanced_parameters, sensor_inputs + +from ..helpers import absolute_data_path @pytest.mark.unit_tests @@ -62,7 +64,18 @@ def test_advanced_parameters_full_config(): }, } - advanced_parameters.check_advanced_parameters(config) + inputs_config = { + "sensors": { + "one": {"image": "img1_crop.tif", "geomodel": "img1_crop.geom"}, + "two": {"image": "img2_crop.tif", "geomodel": "img2_crop.geom"}, + } + } + inputs_config = sensor_inputs.sensors_check_inputs( + inputs_config, + config_json_dir=absolute_data_path("input/data_gizeh_crop/"), + ) + + advanced_parameters.check_advanced_parameters(inputs_config, config) @pytest.mark.unit_tests @@ -73,7 +86,18 @@ def test_advanced_parameters_minimal(): config = {"debug_with_roi": True} - advanced_parameters.check_advanced_parameters(config) + inputs_config = { + "sensors": { + "one": {"image": "img1_crop.tif", "geomodel": "img1_crop.geom"}, + "two": {"image": "img2_crop.tif", "geomodel": "img2_crop.geom"}, + } + } + inputs_config = sensor_inputs.sensors_check_inputs( + inputs_config, + config_json_dir=absolute_data_path("input/data_gizeh_crop/"), + ) + + advanced_parameters.check_advanced_parameters(inputs_config, config) @pytest.mark.unit_tests @@ -84,9 +108,22 @@ def test_advanced_parameters_update_conf(): config = {"debug_with_roi": True} + inputs_config = { + "sensors": { + "one": {"image": "img1_crop.tif", "geomodel": "img1_crop.geom"}, + "two": {"image": "img2_crop.tif", "geomodel": "img2_crop.geom"}, + } + } + inputs_config = sensor_inputs.sensors_check_inputs( + inputs_config, + config_json_dir=absolute_data_path("input/data_gizeh_crop/"), + ) + # First config check without epipolar a priori - updated_config = advanced_parameters.check_advanced_parameters( - config, check_epipolar_a_priori=False + _, updated_config, _, _, _, _ = ( + advanced_parameters.check_advanced_parameters( + inputs_config, config, check_epipolar_a_priori=False + ) ) # TODO: maybe move this inside update conf @@ -112,7 +149,7 @@ def test_advanced_parameters_update_conf(): # First config check without epipolar a priori _ = advanced_parameters.check_advanced_parameters( - full_config["advanced"], check_epipolar_a_priori=True + inputs_config, full_config["advanced"], check_epipolar_a_priori=True ) diff --git a/tests/test_end2end.py b/tests/test_end2end.py index 5e11d68b..5f824f16 100644 --- a/tests/test_end2end.py +++ b/tests/test_end2end.py @@ -1669,7 +1669,7 @@ def test_end2end_ventoux_unique_split_epsg_4326(): pc_pipeline.run() out_dir = input_config_pc["output"]["directory"] - geometry_plugin_name = input_config_pc["geometry_plugin"] + geometry_plugin_name = input_config_pc["advanced"]["geometry_plugin"] # Create input json for pc to dsm pipeline with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory2: @@ -1737,13 +1737,12 @@ def test_end2end_ventoux_unique_split_epsg_4326(): "input/phr_ventoux/srtm/N44E005.hgt" ), }, - "geometry_plugin": geometry_plugin_name, + "advanced": {"geometry_plugin": geometry_plugin_name}, "output": { "directory": output_path, "resolution": 0.000005, "epsg": 4326, }, - "pipeline": "dense_depth_maps_to_dense_dsm", "applications": { "point_cloud_rasterization": { "method": "simple_gaussian", @@ -1947,7 +1946,7 @@ def test_end2end_ventoux_unique_split(): pc_pipeline.run() out_dir = input_config_pc["output"]["directory"] - geometry_plugin_name = input_config_pc["geometry_plugin"] + geometry_plugin_name = input_config_pc["advanced"]["geometry_plugin"] # Create input json for pc to dsm pipeline with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory2: @@ -2008,9 +2007,7 @@ def test_end2end_ventoux_unique_split(): ], }, }, - "geometry_plugin": geometry_plugin_name, "output": {"directory": output_path, "resolution": 0.5}, - "pipeline": "dense_depth_maps_to_dense_dsm", "applications": { "point_cloud_rasterization": { "method": "simple_gaussian", @@ -2021,7 +2018,10 @@ def test_end2end_ventoux_unique_split(): "save_intermediate_data": True, }, }, - "advanced": {"merging": True}, + "advanced": { + "geometry_plugin": geometry_plugin_name, + "merging": True, + }, } dsm_pipeline = default.DefaultPipeline(input_dsm_config) @@ -3481,7 +3481,6 @@ def test_end2end_ventoux_with_color(): input_config_dense_dsm["output"]["resolution"] = 0.5 # update pipeline - input_config_dense_dsm["pipeline"] = "sensors_to_dense_dsm" input_config_dense_dsm["output"]["product_level"] = ["dsm"] dense_dsm_pipeline = default.DefaultPipeline(input_config_dense_dsm) From 51b641061edf88f121048182d3f5cb1dff0acf5c Mon Sep 17 00:00:00 2001 From: Roux Mathis Date: Thu, 13 Feb 2025 09:07:46 +0100 Subject: [PATCH 3/8] notebooks: update tutorials for new API --- .../pipelines/parameters/advanced_parameters.py | 5 ++++- cars/pipelines/parameters/depth_map_inputs.py | 4 ++-- cars/pipelines/pipeline_constants.py | 2 -- ..._dense_dsm_matching_methods_comparison.ipynb | 17 +++++++++-------- .../sensor_to_dense_dsm_step_by_step.ipynb | 15 +++++++-------- tutorials/sensor_to_dsm_from_a_priori.ipynb | 11 +++++------ 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/cars/pipelines/parameters/advanced_parameters.py b/cars/pipelines/parameters/advanced_parameters.py index 67b4a033..9d4a56eb 100644 --- a/cars/pipelines/parameters/advanced_parameters.py +++ b/cars/pipelines/parameters/advanced_parameters.py @@ -124,7 +124,10 @@ def check_advanced_parameters(inputs, conf, check_epipolar_a_priori=True): elif depth_cst.DEPTH_MAPS in inputs or dsm_cst.DSMS in inputs: # If there's an initial elevation with # point clouds as inputs, generate a plugin (used in dsm_filling) - geom_plugin_with_dem_and_geoid = depth_map_inputs.check_geometry_plugin( + ( + overloaded_conf[adv_cst.GEOMETRY_PLUGIN], + geom_plugin_with_dem_and_geoid, + ) = depth_map_inputs.check_geometry_plugin( inputs, conf.get(adv_cst.GEOMETRY_PLUGIN, None) ) diff --git a/cars/pipelines/parameters/depth_map_inputs.py b/cars/pipelines/parameters/depth_map_inputs.py index 2872bc05..f0e0f434 100644 --- a/cars/pipelines/parameters/depth_map_inputs.py +++ b/cars/pipelines/parameters/depth_map_inputs.py @@ -244,7 +244,7 @@ def check_geometry_plugin(conf_inputs, conf_geom_plugin): dem_path = conf_inputs[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH] if dem_path is None: - return None + return conf_geom_plugin, None # Initialize a geometry plugin with elevation information geom_plugin_with_dem_and_geoid = ( @@ -256,7 +256,7 @@ def check_geometry_plugin(conf_inputs, conf_geom_plugin): ) ) - return geom_plugin_with_dem_and_geoid + return conf_geom_plugin, geom_plugin_with_dem_and_geoid def check_input_size( diff --git a/cars/pipelines/pipeline_constants.py b/cars/pipelines/pipeline_constants.py index b3c18adf..317ec478 100644 --- a/cars/pipelines/pipeline_constants.py +++ b/cars/pipelines/pipeline_constants.py @@ -25,9 +25,7 @@ # Sensor input INPUTS = "inputs" -GEOMETRY_PLUGIN = "geometry_plugin" OUTPUT = "output" APPLICATIONS = "applications" ADVANCED = "advanced" ORCHESTRATOR = "orchestrator" -PIPELINE = "pipeline" diff --git a/tutorials/sensor_to_dense_dsm_matching_methods_comparison.ipynb b/tutorials/sensor_to_dense_dsm_matching_methods_comparison.ipynb index dfa4c1d8..012897c2 100644 --- a/tutorials/sensor_to_dense_dsm_matching_methods_comparison.ipynb +++ b/tutorials/sensor_to_dense_dsm_matching_methods_comparison.ipynb @@ -67,7 +67,8 @@ "import cars.pipelines.parameters.sensor_inputs_constants as sens_cst\n", "\n", "from cars.pipelines.parameters import sensor_inputs, output_parameters\n", - "from cars.pipelines.pipeline_constants import GEOMETRY_PLUGIN\n", + "from cars.pipelines.parameters import advanced_parameters_constants as adv_cst\n", + "from cars.pipelines.pipeline_constants import ADVANCED\n", "\n", "# Conf, core, orchestrator\n", "from cars.core import cars_logging\n", @@ -183,12 +184,12 @@ "# Get geometry plugin\n", "(\n", " _,\n", - " updated_inputs_conf[GEOMETRY_PLUGIN],\n", + " geometry_plugin_name,\n", " geom_plugin_without_dem_and_geoid,\n", " geom_plugin_with_dem_and_geoid,\n", " _\n", ") = sensor_inputs.check_geometry_plugin(\n", - " updated_inputs_conf, {}, updated_inputs_conf.get(GEOMETRY_PLUGIN, None)\n", + " updated_inputs_conf, None\n", ")" ] }, @@ -536,12 +537,12 @@ " updated_inputs_conf[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH] = dem_median\n", " (\n", " _,\n", - " _,\n", + " geometry_plugin_name,\n", " geom_plugin_without_dem_and_geoid,\n", " geom_plugin_with_dem_and_geoid,\n", " _,\n", " ) = sensor_inputs.check_geometry_plugin(\n", - " updated_inputs_conf, {}, None\n", + " updated_inputs_conf, None\n", " )\n", " \n", " # Generate new objects\n", @@ -645,14 +646,14 @@ "# Generate geometry loader with dem min and max and geoid\n", "geom_plugin_with_dem_min_and_geoid = (\n", " sensor_inputs.generate_geometry_plugin_with_dem(\n", - " updated_inputs_conf.get(GEOMETRY_PLUGIN, None),\n", + " geometry_plugin_name,\n", " updated_inputs_conf,\n", " dem=dem_min,\n", " )\n", ")\n", "geom_plugin_with_dem_max_and_geoid = (\n", " sensor_inputs.generate_geometry_plugin_with_dem(\n", - " updated_inputs_conf.get(GEOMETRY_PLUGIN, None),\n", + " geometry_plugin_name,\n", " updated_inputs_conf,\n", " dem=dem_max,\n", " )\n", @@ -1376,7 +1377,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.15" }, "vscode": { "interpreter": { diff --git a/tutorials/sensor_to_dense_dsm_step_by_step.ipynb b/tutorials/sensor_to_dense_dsm_step_by_step.ipynb index 105175b3..63694125 100644 --- a/tutorials/sensor_to_dense_dsm_step_by_step.ipynb +++ b/tutorials/sensor_to_dense_dsm_step_by_step.ipynb @@ -61,7 +61,6 @@ "# Pipelines\n", "import cars.pipelines.parameters.sensor_inputs_constants as sens_cst\n", "from cars.pipelines.parameters import sensor_inputs, output_parameters\n", - "from cars.pipelines.pipeline_constants import GEOMETRY_PLUGIN\n", "\n", "# Conf, core, orchestrator\n", "from cars.core import cars_logging\n", @@ -177,12 +176,12 @@ "# Get geometry plugin\n", "(\n", " _,\n", - " updated_inputs_conf[GEOMETRY_PLUGIN],\n", + " geometry_plugin_name,\n", " geom_plugin_without_dem_and_geoid,\n", " geom_plugin_with_dem_and_geoid,\n", " _\n", ") = sensor_inputs.check_geometry_plugin(\n", - " updated_inputs_conf, {}, updated_inputs_conf.get(GEOMETRY_PLUGIN, None)\n", + " updated_inputs_conf, None\n", ")" ] }, @@ -728,12 +727,12 @@ " updated_inputs_conf[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH] = dem_median\n", " (\n", " _,\n", - " _,\n", + " geometry_plugin_name,\n", " geom_plugin_without_dem_and_geoid,\n", " geom_plugin_with_dem_and_geoid,\n", " _\n", " ) = sensor_inputs.check_geometry_plugin(\n", - " updated_inputs_conf, {}, updated_inputs_conf.get(GEOMETRY_PLUGIN, None)\n", + " updated_inputs_conf, None\n", " )\n", " \n", " # Generate new objects\n", @@ -873,14 +872,14 @@ "# Generate geometry loader with dem min and max and geoid\n", "geom_plugin_with_dem_min_and_geoid = (\n", " sensor_inputs.generate_geometry_plugin_with_dem(\n", - " updated_inputs_conf.get(GEOMETRY_PLUGIN, None),\n", + " geometry_plugin_name,\n", " updated_inputs_conf,\n", " dem=dem_min,\n", " )\n", ")\n", "geom_plugin_with_dem_max_and_geoid = (\n", " sensor_inputs.generate_geometry_plugin_with_dem(\n", - " updated_inputs_conf.get(GEOMETRY_PLUGIN, None),\n", + " geometry_plugin_name,\n", " updated_inputs_conf,\n", " dem=dem_max,\n", " )\n", @@ -1290,7 +1289,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.15" } }, "nbformat": 4, diff --git a/tutorials/sensor_to_dsm_from_a_priori.ipynb b/tutorials/sensor_to_dsm_from_a_priori.ipynb index a031a84c..d11fa271 100644 --- a/tutorials/sensor_to_dsm_from_a_priori.ipynb +++ b/tutorials/sensor_to_dsm_from_a_priori.ipynb @@ -75,7 +75,6 @@ "# Pipelines\n", "import cars.pipelines.parameters.sensor_inputs_constants as sens_cst\n", "from cars.pipelines.parameters import sensor_inputs, output_parameters\n", - "from cars.pipelines.pipeline_constants import GEOMETRY_PLUGIN\n", "\n", "# Conf, core, orchestrator\n", "from cars.core import cars_logging\n", @@ -269,12 +268,12 @@ "updated_inputs_conf[\"initial_elevation\"][\"dem\"] = dem_median\n", "(\n", " _,\n", - " updated_inputs_conf[GEOMETRY_PLUGIN],\n", + " geometry_plugin_name,\n", " geom_plugin_without_dem_and_geoid,\n", " geom_plugin_with_dem_and_geoid,\n", " _\n", ") = sensor_inputs.check_geometry_plugin(\n", - " updated_inputs_conf, {}, updated_inputs_conf.get(GEOMETRY_PLUGIN, None)\n", + " updated_inputs_conf, None\n", ")" ] }, @@ -704,14 +703,14 @@ "# Generate geometry loader with dem min and max and geoid\n", "geom_plugin_with_dem_min_and_geoid = (\n", " sensor_inputs.generate_geometry_plugin_with_dem(\n", - " updated_inputs_conf.get(GEOMETRY_PLUGIN, None),\n", + " geometry_plugin_name,\n", " updated_inputs_conf,\n", " dem=dem_min,\n", " )\n", ")\n", "geom_plugin_with_dem_max_and_geoid = (\n", " sensor_inputs.generate_geometry_plugin_with_dem(\n", - " updated_inputs_conf.get(GEOMETRY_PLUGIN, None),\n", + " geometry_plugin_name,\n", " updated_inputs_conf,\n", " dem=dem_max,\n", " )\n", @@ -1198,7 +1197,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.15" } }, "nbformat": 4, From 8e8e26d35e767b3307671f828eab187de59dced4 Mon Sep 17 00:00:00 2001 From: Roux Mathis Date: Thu, 13 Feb 2025 17:20:22 +0100 Subject: [PATCH 4/8] fix: override initial_elevation before geom plugin init --- .../parameters/advanced_parameters.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/cars/pipelines/parameters/advanced_parameters.py b/cars/pipelines/parameters/advanced_parameters.py index 9d4a56eb..5642f4e8 100644 --- a/cars/pipelines/parameters/advanced_parameters.py +++ b/cars/pipelines/parameters/advanced_parameters.py @@ -110,6 +110,17 @@ def check_advanced_parameters(inputs, conf, check_epipolar_a_priori=True): geom_plugin_without_dem_and_geoid = None geom_plugin_with_dem_and_geoid = None dem_generation_roi = None + + # If use a priori, override initial elevation with dem_median + if adv_cst.USE_EPIPOLAR_A_PRIORI in overloaded_conf: + if overloaded_conf[adv_cst.USE_EPIPOLAR_A_PRIORI]: + if adv_cst.DEM_MEDIAN in overloaded_conf[adv_cst.TERRAIN_A_PRIORI]: + inputs[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH] = ( + overloaded_conf[adv_cst.TERRAIN_A_PRIORI][ + adv_cst.DEM_MEDIAN + ] + ) + if inputs[sens_cst.SENSORS] is not None: # Check geometry plugin and overwrite geomodel in conf inputs ( @@ -131,16 +142,6 @@ def check_advanced_parameters(inputs, conf, check_epipolar_a_priori=True): inputs, conf.get(adv_cst.GEOMETRY_PLUGIN, None) ) - # If use a priori, override initial elevation with dem_median - if adv_cst.USE_EPIPOLAR_A_PRIORI in overloaded_conf: - if overloaded_conf[adv_cst.USE_EPIPOLAR_A_PRIORI]: - if adv_cst.DEM_MEDIAN in overloaded_conf[adv_cst.TERRAIN_A_PRIORI]: - inputs[sens_cst.INITIAL_ELEVATION][sens_cst.DEM_PATH] = ( - overloaded_conf[adv_cst.TERRAIN_A_PRIORI][ - adv_cst.DEM_MEDIAN - ] - ) - # Check pipeline overloaded_conf[adv_cst.PIPELINE] = conf.get(adv_cst.PIPELINE, "default") From cb19e5a6f111558883ec814ae72bd4b38ce5aa8e Mon Sep 17 00:00:00 2001 From: Roux Mathis Date: Mon, 17 Feb 2025 09:17:03 +0100 Subject: [PATCH 5/8] fix: add real paths for dem in tests --- .../dump_dir/dem_generation/dem_max.tif | Bin 0 -> 262541 bytes .../dump_dir/dem_generation/dem_median.tif | Bin 0 -> 262541 bytes .../dump_dir/dem_generation/dem_min.tif | Bin 0 -> 262541 bytes tests/pipelines/test_advanced_parameters.py | 24 +++++++++++++----- 4 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 tests/data/input/data_gizeh_crop/dump_dir/dem_generation/dem_max.tif create mode 100644 tests/data/input/data_gizeh_crop/dump_dir/dem_generation/dem_median.tif create mode 100644 tests/data/input/data_gizeh_crop/dump_dir/dem_generation/dem_min.tif diff --git a/tests/data/input/data_gizeh_crop/dump_dir/dem_generation/dem_max.tif b/tests/data/input/data_gizeh_crop/dump_dir/dem_generation/dem_max.tif new file mode 100644 index 0000000000000000000000000000000000000000..c0db0fa7d6f6a7fcac2cd5e7d415a99a6ca415f8 GIT binary patch literal 262541 zcmeI%yNjMh6bIn<+YK>dydc`C1nfjDR5XJHK4$lXZck=CIXU;R{$CqqQpT1)`wk*qy%d**C zHZgl5#%6#1WX!kr=O<#my+3d3j;`i6ujaR`<`3`B+c$n>HGgz-WN)u`;{C_==gV?r zIkUfJE584AZyaB?<9RX0>u(-kj>hw!7@zHKFDK%8cl_D&yT_Nd_QjZA*xlNz>NjrXKWILo8{sSn`0Mt+rCHle*f<6S06jS+rECy-g8;re&OZ!Z$0-?`+lcB ze)`#mo_YMtiIexevfOp=JrCS}a=9tqH?C|DOkM=a3CRAHv)_!o2vii1fBCQ2*J>IQ zkbn6f>lCXh3dq0wSL|ywjS0xV{Eu~tRTTx~U;ZohwVK8RoAr0`f2a75iFE zV*>Io|6`qERYd{$m;Z`=t)?*n`IrB(PO++@fc(pU#lBY4n1KAt|5&G3RZ&3x<-cNI zt7%L?{^ftHQ>>~eApi1Tv9HxMCLsUvKh`N$RTPka`LEd5Y8n%efB7Hl6ssx<$iMto z>}xfR3CO?vk9CSw6$Rv9{wwyin#Kg=U;f8B#j1(|@-P1t`&vz70`f2aW1V7EMFIJj z|B8LBrZEBem;bR&v8tkg{L6pEzE;zifc(q`?wJE?_#w4uSfPd<2YUf$_ZrWs^`8xTmHX|vCe*u#1!l>A_Bkuwy_yJQ=c?`B|IOb&|NCeD`k8%B zokF0ZKz6SB{ypp82XdeHXJ3=~ta$foB9NJ>%kABAf4Q$^y{}UUR20a{(`@@zyn8hf zm@S7{=hZ0$Dhg!fX|{bU-o2U#%$CEf^Xe1=6$P^LG~2!v?_Ny=X3JsLd36ebiUL`A znr+{TcdsS_v*j@BygG$IMS-k5&9-mFyH^u|*>ae5UY$aqqCi%jX4|*o-K&YfY&pz2 zuTCLQQ6MW%v+Y~)?$tzKwj5@iSEmrDD3Fz>+4ik?_i7?ATMo0%t5XP66v)ccZ2MNc zdo>Z5Er(g>)hPrj3S{MJwtXw!y_yKjmcy*`>J$PM1+wxq+rAa=UQGmM%VE}ebqaxs z0$F*QZQqJ_uOWSD#pPR_}lm%DtA!@Mjk98OayrM)RN?bJkdIil&;SE6q9 z>zSzA{W{+_S=IZhdVf`)>(_bW^HqJJndq(cCdM!J>y#$aV*j32%)jgTbZSRlk9hWC zI!#9Y6Y;IvPBW2@#ow-Wr&GRnIqJ1;t9OzoQ9oU2rEM|(D(ZY4xzqXm^V984$^S+- z{`vCY_Ya?Ucjn*y`1}@95=}UN1d*e_-W7o^SrkwVP*eTw0u&J^45tJ#qZ>soAoMOn?9Z0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfWQa}oQXZ<=#1drY>5DY;R}?WFL^GO$o*f(@K-e> z0Ro#XPk}BhK;8GH*8jtIHY0&i6evAkXZ=5lPs*MM3|FA;`Ty(s zfB(83?hedGfWT%6vh9{}|&BBbFnUBJM`yF&6;>;~`LbeNfK*p8Nkn#QlhS z5iSA*#!FyO@7H7byFb4JFRRL zM*IBxuI)~nZM@Z<&%fB6w%hnmdwxCKoc7xIO#8Pt4tJ+{?T2lC^Kj$%B%ikVXV*8T zyW0D&+I;@=)cgB?{qf^_`!W9a_UymEeB`$ue!74E;Je>{`SY!NUp$DFi{q->CvP>Q zwsR}Jjct=gxR`D|u-LhIIPbfC{PnjN@7}p_IDh}1<8hk4{@~;9AHM$4{CNjIzy0n@ z@4S9>@A9Qjrl+5K_J!v!Pk$dB9eM6-5Z7TOuucN9f9LGaM(cFv&O#sru0`f2a(F?}$1ms`-$Lo404gvX>|L6tdcmnb-|KoMN6NiBO%YXEOaXbO} zm;dp)-ibp%{^dVp=_ysmfR5RiZQk6tj2Cm{dw zKVH{6aR|u2{6{Ys#}kl$`5&+Aoj3&KU;d*PjN=K&zxv|^+0r{8z=mq0=0`f2a<8{3ghk*RcfAoTJJOTNa|M9xsi9|MDNbU>r|C{^ftXu6N=Pkbn7)UNDX)aNhF&XuE>>dAvR6skZSt!-+=_I1d?% z%*XgxgP-%i`efHRLN7SucmmnEip;N$r{zEU=YjD$!-+=_kbl4bkI?7Nc)Ec6%m3+} z;q3?lnfYJ+ng8T`^>_c(c~-q6^nx>vCy}>iroYyUt!u>j+d5$j(*lzxsRr$$6~F94pL0h=oP4z!>oFHJ*^{9Ngyjv)%LA)r)nk8D^M+mS@rgM zT1TLgKvtfr?OW+i)k>gOpjr;I>h1NkjzA@WtUOiQx6+-el|ZjRwH#*E+v{l^fl2~d zd8)Q=r8`wCfnI@XIn1iJ*V8%zl?1Z#RBhi%cdAwby#m#8m{o7Dr*#A>31sD|+P;j+d5$jVc-eJkCmS_$+DRLfyjy}h2+5vU}Pm8WX^R=QKQ66h7E zmcy)idp)fqP)Q&wPu2FVbf;=1&?`_ahgtRZdRj-Il0a6Ts_k3pPSr}FSD;!Bv+C{j zw2nX}fvh}L+qcr4s+B;mK(!oZ)!XZ79f3*$S$V3qZ>2j`D}i2tYB|iRx7X7;0+j@^ z@>Ff#N_VPO0=)v&a+p Date: Mon, 17 Feb 2025 15:25:18 +0100 Subject: [PATCH 6/8] docs: some fixs on advanced config documentation --- docs/source/how_to_use_CARS/advanced_configuration.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/how_to_use_CARS/advanced_configuration.rst b/docs/source/how_to_use_CARS/advanced_configuration.rst index fdf99a11..66c49c9b 100644 --- a/docs/source/how_to_use_CARS/advanced_configuration.rst +++ b/docs/source/how_to_use_CARS/advanced_configuration.rst @@ -928,7 +928,7 @@ The structure follows this organization: - Active epipolar a priori - bool - False - - Yes[Michel J. et al, 2020] + - Yes * - epipolar_a_priori - Provide epipolar a priori information (see section below) - dict @@ -960,12 +960,12 @@ The structure follows this organization: - - No * - geometry_plugin - - The plugin to use - - str + - Name of the geometry plugin to use and optional parameters + - str or dict - - No * - pipeline - - The pipeline to use + - Name of the pipeline to use - str - - No From 2df3722e0d87fc895bd2fcab03727992873e8827 Mon Sep 17 00:00:00 2001 From: Roux Mathis Date: Wed, 19 Feb 2025 17:25:42 +0100 Subject: [PATCH 7/8] fix: remove retrocompatibility with old pipelines --- cars/cars.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/cars/cars.py b/cars/cars.py index e41da09c..d57f33a6 100644 --- a/cars/cars.py +++ b/cars/cars.py @@ -111,23 +111,6 @@ def main_cli(args, dry_run=False): # noqa: C901 config_json_dir = os.path.abspath(os.path.dirname(args.conf)) pipeline_name = config.get("advanced", {}).get("pipeline", "default") - old_pipelines = [ - "sensors_to_dense_dsm", - "sensors_to_dense_dsm_no_merging", - "sensors_to_dense_depth_maps", - "sensors_to_dense_point_clouds", - "dense_point_clouds_to_dense_dsm", - "dense_point_clouds_to_dense_dsm_no_merging", - ] - if pipeline_name in old_pipelines: - warnings.warn( - f"The 'pipeline' value {pipeline_name} corresponds to" - " an old pipeline." - "Using the default CARS pipeline instead.", - FutureWarning, - stacklevel=2, - ) - pipeline_name = "default" # Logging configuration with args Loglevel loglevel = getattr(args, "loglevel", "PROGRESS").upper() From c6dc2976fdeaeb6404b27f6915ad2a92ffafd14e Mon Sep 17 00:00:00 2001 From: Roux Mathis Date: Thu, 20 Feb 2025 14:50:51 +0100 Subject: [PATCH 8/8] docs: add default values for geometry plugin and pipeline --- docs/source/how_to_use_CARS/advanced_configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/how_to_use_CARS/advanced_configuration.rst b/docs/source/how_to_use_CARS/advanced_configuration.rst index 66c49c9b..777f92f5 100644 --- a/docs/source/how_to_use_CARS/advanced_configuration.rst +++ b/docs/source/how_to_use_CARS/advanced_configuration.rst @@ -962,12 +962,12 @@ The structure follows this organization: * - geometry_plugin - Name of the geometry plugin to use and optional parameters - str or dict - - + - "SharelocGeometry" - No * - pipeline - Name of the pipeline to use - str - - + - "default" - No