diff --git a/eodag/api/product/metadata_mapping.py b/eodag/api/product/metadata_mapping.py index cdc9086aa..d6a0ee909 100644 --- a/eodag/api/product/metadata_mapping.py +++ b/eodag/api/product/metadata_mapping.py @@ -151,6 +151,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str: The currently understood converters are: - ``ceda_collection_name``: generate a CEDA collection name from a string + - ``wekeo_to_cop_collection``: converts the name of a collection from the WEkEO format to the Copernicus format - ``csv_list``: convert to a comma separated list - ``datetime_to_timestamp_milliseconds``: converts a utc date string to a timestamp in milliseconds - ``dict_filter_and_sub``: filter dict items using jsonpath and then apply recursive_sub_str @@ -251,6 +252,9 @@ def get_field(self, field_name: str, args: Any, kwargs: Any) -> Any: field_name = conversion_func_spec.groupdict()["field_name"] converter = conversion_func_spec.groupdict()["converter"] self.custom_args = conversion_func_spec.groupdict()["args"] + # converts back "_COLON_" to ":" + if self.custom_args is not None and "_COLON_" in self.custom_args: + self.custom_args = self.custom_args.replace("_COLON_", ":") self.custom_converter = getattr(self, "convert_{}".format(converter)) return super(MetadataFormatter, self).get_field(field_name, args, kwargs) @@ -1051,6 +1055,11 @@ def convert_assets_list_to_dict( assets_dict[asset_basename] = assets_dict.pop(asset_name) return assets_dict + @staticmethod + def convert_wekeo_to_cop_collection(val: str, prefix: str) -> str: + """Converts the name of a collection from the WEkEO format to the Copernicus format.""" + return val.removeprefix(prefix).lower().replace("_", "-") + # if stac extension colon separator `:` is in search params, parse it to prevent issues with vformat if re.search(r"{[\w-]*:[\w#-]*\(?.*}", search_param): search_param = re.sub( @@ -1059,6 +1068,16 @@ def convert_assets_list_to_dict( search_param, ) kwargs = {k.replace(":", "_COLON_"): v for k, v in kwargs.items()} + # convert colons `:` in the parameters passed to the converter (e.g. 'foo#boo(fun:with:colons)') + if re.search(r"{[\w-]*#[\w-]*\([^)]*:.*}", search_param): + search_param = re.sub( + r"({[\w-]*#[\w-]*)\(([^)]*)(.*})", + lambda m: m.group(1) + + "(" + + m.group(2).replace(":", "_COLON_") + + m.group(3), + search_param, + ) return MetadataFormatter().vformat(search_param, args, kwargs) diff --git a/eodag/plugins/apis/ecmwf.py b/eodag/plugins/apis/ecmwf.py index a02bb726d..aeb591c9c 100644 --- a/eodag/plugins/apis/ecmwf.py +++ b/eodag/plugins/apis/ecmwf.py @@ -286,10 +286,13 @@ def clear(self) -> None: pass def discover_queryables( - self, **kwargs: Any + self, + queryables_config: Optional[dict[str, Any]] = None, + **kwargs: Any, ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]: """Fetch queryables list from provider using metadata mapping + :param queryables_config discover queryables configuration :param kwargs: additional filters for queryables (`collection` and other search arguments) :returns: fetched queryable parameters dict diff --git a/eodag/plugins/search/build_search_result.py b/eodag/plugins/search/build_search_result.py index 7253a199f..12b2268c8 100644 --- a/eodag/plugins/search/build_search_result.py +++ b/eodag/plugins/search/build_search_result.py @@ -686,10 +686,13 @@ def _get_collection_queryables( return QueryablesDict(additional_properties=False, **queryables) def discover_queryables( - self, **kwargs: Any + self, + queryables_config: Optional[dict[str, Any]] = None, + **kwargs: Any, ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]: """Fetch queryables list from provider using its constraints file + :param queryables_config discover queryables configuration :param kwargs: additional filters for queryables (`collection` and other search arguments) :returns: fetched queryable parameters dict @@ -700,6 +703,10 @@ def discover_queryables( default_values = deepcopy(pt_config) default_values.pop("metadata_mapping", None) + + if queryables_config is None: + queryables_config = getattr(self.config, "discover_queryables", {}) + filters = {**default_values, **kwargs} if "start" in filters: @@ -713,13 +720,13 @@ def discover_queryables( ) constraints_url = format_metadata( - getattr(self.config, "discover_queryables", {}).get("constraints_url", ""), + queryables_config.get("constraints_url", ""), **filters, ) constraints: list[dict[str, Any]] = self._fetch_data(constraints_url) form_url = format_metadata( - getattr(self.config, "discover_queryables", {}).get("form_url", ""), + queryables_config.get("form_url", ""), **filters, ) form: list[dict[str, Any]] = self._fetch_data(form_url) @@ -1565,3 +1572,29 @@ def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]: return [{}] else: return QueryStringSearch.do_search(self, *args, **kwargs) + + def discover_queryables( + self, + queryables_config: Optional[dict[str, Any]] = None, + **kwargs: Any, + ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]: + """Fetch queryables list from provider using its constraints file + + Dynamically get the discover queryables configuration if no one is given. + + :param queryables_config discover queryables configuration + :param kwargs: additional filters for queryables (`collection` and other search + arguments) + :returns: fetched queryable parameters dict + """ + if queryables_config is None: + dynamic_config = getattr(self.config, "dynamic_discover_queryables", []) + for dc in dynamic_config: + for cs in dc["collection_selector"]: + field = cs["field"] + if kwargs[field].startswith(cs["prefix"]): + queryables_config = dc["discover_queryables"] + break + if queryables_config: + break + return super().discover_queryables(queryables_config, **kwargs) diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml index ad1d4a7df..adf942a05 100644 --- a/eodag/resources/providers.yml +++ b/eodag/resources/providers.yml @@ -3528,7 +3528,41 @@ max_items_per_page: 200 discover_collections: fetch_url: null - available_values_url: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/queryable/{dataset}' + dynamic_discover_queryables: + - collection_selector: # cop_ads + - field: dataset + prefix: EO:ECMWF:DAT:CAMS + discover_queryables: + fetch_url: null + product_type_fetch_url: null + constraints_url: https://ads.atmosphere.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/constraints.json + form_url: https://ads.atmosphere.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/form.json + - collection_selector: # cop_cds + - field: dataset + prefix: EO:ECMWF:DAT:SATELLITE + - field: dataset + prefix: EO:ECMWF:DAT:SEASONAL + - field: dataset + prefix: EO:ECMWF:DAT:INSITU + - field: dataset + prefix: EO:ECMWF:DAT:DERIVED + - field: dataset + prefix: EO:ECMWF:DAT:REANALYSIS + - field: dataset + prefix: EO:ECMWF:DAT:SIS + discover_queryables: + fetch_url: null + collection_fetch_url: null + constraints_url: https://cds.climate.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/constraints.json + form_url: https://cds.climate.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/form.json + - collection_selector: # cop_ewds + - field: dataset + prefix: EO:ECMWF:DAT:CEMS + discover_queryables: + fetch_url: null + product_type_fetch_url: null + constraints_url: https://ewds.climate.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/constraints.json + form_url: https://ewds.climate.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/form.json metadata_mapping: geometry: - '{{"bbox": {geometry#to_bounds}}}' diff --git a/tests/resources/constraints.json b/tests/resources/constraints.json index e2be90705..a1626f97c 100644 --- a/tests/resources/constraints.json +++ b/tests/resources/constraints.json @@ -1,7 +1,7 @@ [ { "year": [ - "1940","2000", "2001", "2002", "2003", "2004", "2005" + "1940","2000", "2001", "2002", "2003", "2004", "2005", "2015" ], "month": [ "01", "02", "03" @@ -10,7 +10,7 @@ "01", "02", "10", "20" ], "time": [ - "01:00", "09:00", "12:00", "18:00" + "00:00", "01:00", "09:00", "12:00", "18:00" ], "variable": [ "a", "b" diff --git a/tests/resources/form.json b/tests/resources/form.json index 738a35545..dff3721d9 100644 --- a/tests/resources/form.json +++ b/tests/resources/form.json @@ -17,7 +17,7 @@ "name": "year", "type": "StringListWidget", "details": { - "values": ["1940","2000", "2001", "2002", "2003", "2004", "2005"] + "values": ["1940","2000", "2001", "2002", "2003", "2004", "2005", "2015"] }, "id": 2 }, diff --git a/tests/units/test_core.py b/tests/units/test_core.py index 3d4b05743..183e52a92 100644 --- a/tests/units/test_core.py +++ b/tests/units/test_core.py @@ -1691,6 +1691,45 @@ def test_list_queryables_additional( self.assertEqual(queryables.additional_properties, True) + @mock.patch( + "eodag.plugins.manager.PluginManager.get_auth_plugin", + autospec=True, + ) + @mock.patch( + "eodag.plugins.search.build_search_result.ECMWFSearch.discover_queryables", + autospec=True, + ) + def test_list_queryables_dynamic_discover_queryables( + self, + mock_discover_queryables: mock.Mock, + mock_auth_plugin: mock.Mock, + ): + """WekeoECMWFSearch must dynamically get the discover queryables configuration""" + provider = "wekeo_ecmwf" + # get the original cop_* provider for each wekeo_ecmwf product + original_cop_providers = { + pt: next(p for p in providers if p.startswith("cop_")) + for pt, providers in self.SUPPORTED_COLLECTIONS.items() + if provider in providers + } + copernicus_urls = { + "cop_ads": "ads.atmosphere.copernicus.eu", + "cop_cds": "cds.climate.copernicus.eu", + "cop_ewds": "ewds.climate.copernicus.eu", + } + for product, original_provider in original_cop_providers.items(): + self.dag.list_queryables(provider=provider, collection=product) + mock_discover_queryables.assert_called_once() + args, kwargs = mock_discover_queryables.call_args + queryables_config = args[1] + # check if URLs in queryables_config are the copernicus ones + original_url = copernicus_urls[original_provider] + self.assertIn("constraints_url", queryables_config) + self.assertIn(original_url, queryables_config["constraints_url"]) + self.assertIn("form_url", queryables_config) + self.assertIn(original_url, queryables_config["form_url"]) + mock_discover_queryables.reset_mock() + def test_queryables_repr(self): queryables = self.dag.list_queryables(provider="peps", collection="S1_SAR_GRD") self.assertIsInstance(queryables, QueryablesDict) diff --git a/tests/units/test_search_plugins.py b/tests/units/test_search_plugins.py index d47f8adb0..b5f0bf911 100644 --- a/tests/units/test_search_plugins.py +++ b/tests/units/test_search_plugins.py @@ -2965,17 +2965,24 @@ def test_plugins_search_ecmwf_search_wekeo_discover_queryables( search_plugin = self.get_search_plugin(provider="wekeo_ecmwf") self.assertEqual("WekeoECMWFSearch", search_plugin.__class__.__name__) self.assertEqual( - "ECMWFSearch", + "WekeoECMWFSearch", search_plugin.discover_queryables.__func__.__qualname__.split(".")[0], ) constraints_path = os.path.join(TEST_RESOURCES_PATH, "constraints.json") with open(constraints_path) as f: constraints = json.load(f) - wekeo_ecmwf_constraints = {"constraints": constraints[0]} - mock_requests_get.return_value = MockResponse( - wekeo_ecmwf_constraints, status_code=200 - ) + wekeo_ecmwf_constraints = [constraints[0]] + + form_path = os.path.join(TEST_RESOURCES_PATH, "form.json") + with open(form_path) as f: + form = json.load(f) + wekeo_ecmwf_form = form + + mock_requests_get.side_effect = [ + MockResponse(wekeo_ecmwf_constraints, status_code=200), + MockResponse(wekeo_ecmwf_form, status_code=200), + ] provider_queryables_from_constraints_file = [ "ecmwf_year", @@ -2993,10 +3000,19 @@ def test_plugins_search_ecmwf_search_wekeo_discover_queryables( ) self.assertIsNotNone(queryables) - mock_requests_get.assert_called_once_with( + self.assertEqual(len(mock_requests_get.call_args), 2) + mock_requests_get.assert_any_call( mock.ANY, - "https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/queryable/" - "EO:ECMWF:DAT:REANALYSIS_ERA5_SINGLE_LEVELS_MONTHLY_MEANS", + "https://cds.climate.copernicus.eu/api/catalogue/v1/collections/" + "reanalysis-era5-single-levels-monthly-means/constraints.json", + headers=USER_AGENT, + auth=None, + timeout=60, + ) + mock_requests_get.assert_any_call( + mock.ANY, + "https://cds.climate.copernicus.eu/api/catalogue/v1/collections/" + "reanalysis-era5-single-levels-monthly-means/form.json", headers=USER_AGENT, auth=None, timeout=60, @@ -3048,11 +3064,12 @@ def test_plugins_search_ecmwf_search_wekeo_discover_queryables( # with additional param queryables = search_plugin.discover_queryables( collection="ERA5_SL_MONTHLY", + dataset="EO:ECMWF:DAT:REANALYSIS_ERA5_SINGLE_LEVELS_MONTHLY_MEANS", **{"ecmwf:variable": "a"}, ) self.assertIsNotNone(queryables) - self.assertEqual(10, len(queryables)) + self.assertEqual(12, len(queryables)) # default properties called in function arguments are added and must be default values of the queryables queryable = queryables.get("ecmwf:variable") if queryable is not None: