feat: use original Copernicus provider to list queryables of wekeo_ecmwf

eodag/api/product/metadata_mapping.py

@@ -151,6 +151,7 @@ def format_metadata(search_param: str, *args: Any, **kwargs: Any) -> str:
 
     The currently understood converters are:
         - ``ceda_collection_name``: generate a CEDA collection name from a string
+        - ``wekeo_to_cop_collection``: converts the name of a collection from the WEkEO format to the Copernicus format
         - ``csv_list``: convert to a comma separated list
         - ``datetime_to_timestamp_milliseconds``: converts a utc date string to a timestamp in milliseconds
         - ``dict_filter_and_sub``: filter dict items using jsonpath and then apply recursive_sub_str
@@ -251,6 +252,9 @@ def get_field(self, field_name: str, args: Any, kwargs: Any) -> Any:
                 field_name = conversion_func_spec.groupdict()["field_name"]
                 converter = conversion_func_spec.groupdict()["converter"]
                 self.custom_args = conversion_func_spec.groupdict()["args"]
+                # converts back "_COLON_" to ":"
+                if self.custom_args is not None and "_COLON_" in self.custom_args:
+                    self.custom_args = self.custom_args.replace("_COLON_", ":")
                 self.custom_converter = getattr(self, "convert_{}".format(converter))
 
             return super(MetadataFormatter, self).get_field(field_name, args, kwargs)
@@ -1051,6 +1055,11 @@ def convert_assets_list_to_dict(
                     assets_dict[asset_basename] = assets_dict.pop(asset_name)
             return assets_dict
 
+        @staticmethod
+        def convert_wekeo_to_cop_collection(val: str, prefix: str) -> str:
+            """Converts the name of a collection from the WEkEO format to the Copernicus format."""
+            return val.removeprefix(prefix).lower().replace("_", "-")
+
     # if stac extension colon separator `:` is in search params, parse it to prevent issues with vformat
     if re.search(r"{[\w-]*:[\w#-]*\(?.*}", search_param):
         search_param = re.sub(
@@ -1059,6 +1068,16 @@ def convert_assets_list_to_dict(
             search_param,
         )
         kwargs = {k.replace(":", "_COLON_"): v for k, v in kwargs.items()}
+    # convert colons `:` in the parameters passed to the converter (e.g. 'foo#boo(fun:with:colons)')
+    if re.search(r"{[\w-]*#[\w-]*\([^)]*:.*}", search_param):
+        search_param = re.sub(
+            r"({[\w-]*#[\w-]*)\(([^)]*)(.*})",
+            lambda m: m.group(1)
+            + "("
+            + m.group(2).replace(":", "_COLON_")
+            + m.group(3),
+            search_param,
+        )
 
     return MetadataFormatter().vformat(search_param, args, kwargs)
 

eodag/plugins/apis/ecmwf.py

@@ -286,10 +286,13 @@ def clear(self) -> None:
         pass
 
     def discover_queryables(
-        self, **kwargs: Any
+        self,
+        queryables_config: Optional[dict[str, Any]] = None,
+        **kwargs: Any,
     ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
         """Fetch queryables list from provider using metadata mapping
 
+        :param queryables_config discover queryables configuration
         :param kwargs: additional filters for queryables (`collection` and other search
                        arguments)
         :returns: fetched queryable parameters dict

eodag/plugins/search/build_search_result.py

@@ -686,10 +686,13 @@ def _get_collection_queryables(
         return QueryablesDict(additional_properties=False, **queryables)
 
     def discover_queryables(
-        self, **kwargs: Any
+        self,
+        queryables_config: Optional[dict[str, Any]] = None,
+        **kwargs: Any,
     ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
         """Fetch queryables list from provider using its constraints file
 
+        :param queryables_config discover queryables configuration
         :param kwargs: additional filters for queryables (`collection` and other search
                        arguments)
         :returns: fetched queryable parameters dict
@@ -700,6 +703,10 @@ def discover_queryables(
 
         default_values = deepcopy(pt_config)
         default_values.pop("metadata_mapping", None)
+
+        if queryables_config is None:
+            queryables_config = getattr(self.config, "discover_queryables", {})
+
         filters = {**default_values, **kwargs}
 
         if "start" in filters:
@@ -713,13 +720,13 @@ def discover_queryables(
         )
 
         constraints_url = format_metadata(
-            getattr(self.config, "discover_queryables", {}).get("constraints_url", ""),
+            queryables_config.get("constraints_url", ""),
             **filters,
         )
         constraints: list[dict[str, Any]] = self._fetch_data(constraints_url)
 
         form_url = format_metadata(
-            getattr(self.config, "discover_queryables", {}).get("form_url", ""),
+            queryables_config.get("form_url", ""),
             **filters,
         )
         form: list[dict[str, Any]] = self._fetch_data(form_url)
@@ -1565,3 +1572,29 @@ def do_search(self, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
             return [{}]
         else:
             return QueryStringSearch.do_search(self, *args, **kwargs)
+
+    def discover_queryables(
+        self,
+        queryables_config: Optional[dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]:
+        """Fetch queryables list from provider using its constraints file
+
+        Dynamically get the discover queryables configuration if no one is given.
+
+        :param queryables_config discover queryables configuration
+        :param kwargs: additional filters for queryables (`collection` and other search
+                       arguments)
+        :returns: fetched queryable parameters dict
+        """
+        if queryables_config is None:
+            dynamic_config = getattr(self.config, "dynamic_discover_queryables", [])
+            for dc in dynamic_config:
+                for cs in dc["collection_selector"]:
+                    field = cs["field"]
+                    if kwargs[field].startswith(cs["prefix"]):
+                        queryables_config = dc["discover_queryables"]
+                        break
+                if queryables_config:
+                    break
+        return super().discover_queryables(queryables_config, **kwargs)

eodag/resources/providers.yml

@@ -3528,7 +3528,41 @@
       max_items_per_page: 200
     discover_collections:
       fetch_url: null
-    available_values_url: 'https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/queryable/{dataset}'
+    dynamic_discover_queryables:
+      - collection_selector: # cop_ads
+          - field: dataset
+            prefix: EO:ECMWF:DAT:CAMS
+        discover_queryables:
+          fetch_url: null
+          product_type_fetch_url: null
+          constraints_url: https://ads.atmosphere.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/constraints.json
+          form_url: https://ads.atmosphere.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/form.json
+      - collection_selector: # cop_cds
+          - field: dataset
+            prefix: EO:ECMWF:DAT:SATELLITE
+          - field: dataset
+            prefix: EO:ECMWF:DAT:SEASONAL
+          - field: dataset
+            prefix: EO:ECMWF:DAT:INSITU
+          - field: dataset
+            prefix: EO:ECMWF:DAT:DERIVED
+          - field: dataset
+            prefix: EO:ECMWF:DAT:REANALYSIS
+          - field: dataset
+            prefix: EO:ECMWF:DAT:SIS
+        discover_queryables:
+          fetch_url: null
+          collection_fetch_url: null
+          constraints_url: https://cds.climate.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/constraints.json
+          form_url: https://cds.climate.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/form.json
+      - collection_selector: # cop_ewds
+          - field: dataset
+            prefix: EO:ECMWF:DAT:CEMS
+        discover_queryables:
+          fetch_url: null
+          product_type_fetch_url: null
+          constraints_url: https://ewds.climate.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/constraints.json
+          form_url: https://ewds.climate.copernicus.eu/api/catalogue/v1/collections/{dataset#wekeo_to_cop_collection(EO:ECMWF:DAT:)}/form.json
     metadata_mapping:
       geometry:
         - '{{"bbox": {geometry#to_bounds}}}'

tests/resources/constraints.json

@@ -1,7 +1,7 @@
 [
   {
     "year": [
-      "1940","2000", "2001", "2002", "2003", "2004", "2005"
+      "1940","2000", "2001", "2002", "2003", "2004", "2005", "2015"
     ],
     "month": [
       "01", "02", "03"
@@ -10,7 +10,7 @@
       "01", "02", "10", "20"
     ],
     "time": [
-      "01:00", "09:00", "12:00", "18:00"
+      "00:00", "01:00", "09:00", "12:00", "18:00"
     ],
     "variable": [
       "a", "b"

tests/resources/form.json

@@ -17,7 +17,7 @@
     "name": "year",
     "type": "StringListWidget",
     "details": {
-      "values": ["1940","2000", "2001", "2002", "2003", "2004", "2005"]
+      "values": ["1940","2000", "2001", "2002", "2003", "2004", "2005", "2015"]
     },
     "id": 2
   },

tests/units/test_core.py

@@ -1691,6 +1691,45 @@ def test_list_queryables_additional(
 
         self.assertEqual(queryables.additional_properties, True)
 
+    @mock.patch(
+        "eodag.plugins.manager.PluginManager.get_auth_plugin",
+        autospec=True,
+    )
+    @mock.patch(
+        "eodag.plugins.search.build_search_result.ECMWFSearch.discover_queryables",
+        autospec=True,
+    )
+    def test_list_queryables_dynamic_discover_queryables(
+        self,
+        mock_discover_queryables: mock.Mock,
+        mock_auth_plugin: mock.Mock,
+    ):
+        """WekeoECMWFSearch must dynamically get the discover queryables configuration"""
+        provider = "wekeo_ecmwf"
+        # get the original cop_* provider for each wekeo_ecmwf product
+        original_cop_providers = {
+            pt: next(p for p in providers if p.startswith("cop_"))
+            for pt, providers in self.SUPPORTED_COLLECTIONS.items()
+            if provider in providers
+        }
+        copernicus_urls = {
+            "cop_ads": "ads.atmosphere.copernicus.eu",
+            "cop_cds": "cds.climate.copernicus.eu",
+            "cop_ewds": "ewds.climate.copernicus.eu",
+        }
+        for product, original_provider in original_cop_providers.items():
+            self.dag.list_queryables(provider=provider, collection=product)
+            mock_discover_queryables.assert_called_once()
+            args, kwargs = mock_discover_queryables.call_args
+            queryables_config = args[1]
+            # check if URLs in queryables_config are the copernicus ones
+            original_url = copernicus_urls[original_provider]
+            self.assertIn("constraints_url", queryables_config)
+            self.assertIn(original_url, queryables_config["constraints_url"])
+            self.assertIn("form_url", queryables_config)
+            self.assertIn(original_url, queryables_config["form_url"])
+            mock_discover_queryables.reset_mock()
+
     def test_queryables_repr(self):
         queryables = self.dag.list_queryables(provider="peps", collection="S1_SAR_GRD")
         self.assertIsInstance(queryables, QueryablesDict)

tests/units/test_search_plugins.py

@@ -2965,17 +2965,24 @@ def test_plugins_search_ecmwf_search_wekeo_discover_queryables(
         search_plugin = self.get_search_plugin(provider="wekeo_ecmwf")
         self.assertEqual("WekeoECMWFSearch", search_plugin.__class__.__name__)
         self.assertEqual(
-            "ECMWFSearch",
+            "WekeoECMWFSearch",
             search_plugin.discover_queryables.__func__.__qualname__.split(".")[0],
         )
 
         constraints_path = os.path.join(TEST_RESOURCES_PATH, "constraints.json")
         with open(constraints_path) as f:
             constraints = json.load(f)
-        wekeo_ecmwf_constraints = {"constraints": constraints[0]}
-        mock_requests_get.return_value = MockResponse(
-            wekeo_ecmwf_constraints, status_code=200
-        )
+        wekeo_ecmwf_constraints = [constraints[0]]
+
+        form_path = os.path.join(TEST_RESOURCES_PATH, "form.json")
+        with open(form_path) as f:
+            form = json.load(f)
+        wekeo_ecmwf_form = form
+
+        mock_requests_get.side_effect = [
+            MockResponse(wekeo_ecmwf_constraints, status_code=200),
+            MockResponse(wekeo_ecmwf_form, status_code=200),
+        ]
 
         provider_queryables_from_constraints_file = [
             "ecmwf_year",
@@ -2993,10 +3000,19 @@ def test_plugins_search_ecmwf_search_wekeo_discover_queryables(
         )
         self.assertIsNotNone(queryables)
 
-        mock_requests_get.assert_called_once_with(
+        self.assertEqual(len(mock_requests_get.call_args), 2)
+        mock_requests_get.assert_any_call(
             mock.ANY,
-            "https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/queryable/"
-            "EO:ECMWF:DAT:REANALYSIS_ERA5_SINGLE_LEVELS_MONTHLY_MEANS",
+            "https://cds.climate.copernicus.eu/api/catalogue/v1/collections/"
+            "reanalysis-era5-single-levels-monthly-means/constraints.json",
+            headers=USER_AGENT,
+            auth=None,
+            timeout=60,
+        )
+        mock_requests_get.assert_any_call(
+            mock.ANY,
+            "https://cds.climate.copernicus.eu/api/catalogue/v1/collections/"
+            "reanalysis-era5-single-levels-monthly-means/form.json",
             headers=USER_AGENT,
             auth=None,
             timeout=60,
@@ -3048,11 +3064,12 @@ def test_plugins_search_ecmwf_search_wekeo_discover_queryables(
         # with additional param
         queryables = search_plugin.discover_queryables(
             collection="ERA5_SL_MONTHLY",
+            dataset="EO:ECMWF:DAT:REANALYSIS_ERA5_SINGLE_LEVELS_MONTHLY_MEANS",
             **{"ecmwf:variable": "a"},
         )
         self.assertIsNotNone(queryables)
 
-        self.assertEqual(10, len(queryables))
+        self.assertEqual(12, len(queryables))
         # default properties called in function arguments are added and must be default values of the queryables
         queryable = queryables.get("ecmwf:variable")
         if queryable is not None: