feat(plugins): allow multiple download or auth plugins for a single provider

docs/add_provider.rst

@@ -36,22 +36,22 @@ provide the new provider's configuration in a ``YAML`` format. The following exa
          GENERIC_PRODUCT_TYPE:
             productType: '{productType}'
       download:
-         type: AwsDownload
+         - type: AwsDownload
       auth:
-         type: AwsAuth
-         credentials:
-            aws_access_key_id: PLEASE_CHANGE_ME
-            aws_secret_access_key: PLEASE_CHANGE_ME
+         - type: AwsAuth
+           credentials:
+              aws_access_key_id: PLEASE_CHANGE_ME
+              aws_secret_access_key: PLEASE_CHANGE_ME
 
 It configures the following existing plugins: :class:`~eodag.plugins.search.qssearch.StacSearch` (search),
 :class:`~eodag.plugins.authentication.aws_auth.AwsAuth` (authentication) and :class:`~eodag.plugins.download.aws.AwsDownload` (download).
 
 Each plugin configuration is inserted following the appropriate plugin topic key:
 
-- ``search`` for `search plugins <plugins_reference/search.rst>`_
-- ``download`` for `download plugins <plugins_reference/download.rst>`_
-- ``auth``, ``search_auth``, or ``download_auth`` for `authentication plugins <plugins_reference/auth.rst>`_
-- ``api`` for `api plugins <plugins_reference/api.rst>`_
+- ``search`` for `search plugins <plugins_reference/search.rst>`_ - one single plugin
+- ``download`` for `download plugins <plugins_reference/download.rst>`_ - list of plugings
+- ``auth`` for `authentication plugins <plugins_reference/auth.rst>`_ - list of plugings
+- ``api`` for `api plugins <plugins_reference/api.rst>`_ - one single plugin
 
 Of course, it is also necessary to know how to configure these plugins (which parameters they take, what values they can have, etc.).
 You can get some inspiration from the *Providers pre-configuration* section by analysing how ``eodag`` configures the providers it comes installed with.
@@ -118,13 +118,15 @@ The plugin structure is reflected in the internal providers configuration file.
             ...
          ...
       download:
-         plugin: CustomDownloadPlugin
-         # Same as with search for random config keys as needed by the plugin class
-         ...
+         # list of download plugins
+         - plugin: CustomDownloadPlugin
+           # Same as with search for random config keys as needed by the plugin class
+           ...
       auth:
-         plugin: CustomAuthPlugin
-         # Same as with search for random config keys as needed by the plugin class
-         ...
+         # list of auth plugins
+         - plugin: CustomAuthPlugin
+           # Same as with search for random config keys as needed by the plugin class
+           ...
 
 Note however, that for a provider which already has a Python library for accessing its products, the configuration
 varies a little bit. It does not have the 'search' and 'download' keys. Instead, there is a single 'api' key like this::

docs/getting_started_guide/register.rst

@@ -35,8 +35,8 @@ You need credentials for both EOS (search) and AWS (download):
 * In access keys, click on create access key.
 * Add these credentials to the user configuration file:
 
-  * ``search_auth.credentials.api_key``
-  * ``download_auth.credentials.aws_access_key_id`` and ``download_auth.credentials.aws_secret_access_key`` or ``download_auth.credentials.aws_profile``
+  * ``auth.credentials.api_key``
+  * ``auth.credentials.aws_access_key_id`` and ``auth.credentials.aws_secret_access_key`` or ``auth.credentials.aws_profile``
 
 .. note::
 

docs/notebooks/api_user_guide/3_configuration.ipynb

@@ -95,7 +95,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -108,7 +108,7 @@
     "            GENERIC_PRODUCT_TYPE:\n",
     "                productType: '{productType}'\n",
     "        download:\n",
-    "            type: HTTPDownload\n",
+    "            - type: HTTPDownload\n",
     "\"\"\")"
    ]
   },

docs/plugins_reference/auth.rst

@@ -4,10 +4,11 @@
 Authentication Plugins
 ======================
 
-Multiple authentication plugins can be defined per provider under ``auth``, ``search_auth``, or ``download_auth``
+Multiple authentication plugins can be defined per provider under ``auth``
 `provider configuration entries <../add_provider.rst>`_. They are shared across all providers, and can be identified
-using *matching settings* :attr:`~eodag.config.PluginConfig.matching_url` and
-:attr:`~eodag.config.PluginConfig.matching_conf` as configuration parameters.
+using *matching settings* with :attr:`~eodag.config.PluginConfig.match`. The parameter `href` can be used to define a url
+to match the authentication plugin with  the download/order link of a product or the `api_endpoint` of a search or download plugin.
+Other parameters can be added to match the authentication plugin with the configuration of a search or download plugin.
 Credentials are automatically shared between plugins having the same *matching settings*.
 Authentication plugins without *matching settings* configured will not be shared and will automatically match their
 provider.

eodag/api/core.py

@@ -29,7 +29,7 @@
 from typing import TYPE_CHECKING, Any, Iterator, Optional, Union
 
 import geojson
-import yaml.parser
+import yaml
 from whoosh import analysis, fields
 from whoosh.fields import Schema
 from whoosh.index import exists_in, open_dir
@@ -463,7 +463,7 @@ def add_provider(
         products: dict[str, Any] = {
             GENERIC_PRODUCT_TYPE: {"productType": "{productType}"}
         },
-        download: dict[str, Any] = {"type": "HTTPDownload", "auth_error_code": 401},
+        download: list[dict[str, Any]] = [],
         **kwargs: dict[str, Any],
     ):
         """Adds a new provider.
@@ -480,9 +480,11 @@ def add_provider(
         :param priority: Provider priority. If None, provider will be set as preferred (highest priority)
         :param search: Search :class:`~eodag.config.PluginConfig` mapping
         :param products: Provider product types mapping
-        :param download: Download :class:`~eodag.config.PluginConfig` mapping
+        :param download: list of Download :class:`~eodag.config.PluginConfig` mapping
         :param kwargs: Additional :class:`~eodag.config.ProviderConfig` mapping
         """
+        if not download:
+            download.extend([{"type": "HTTPDownload", "auth_error_code": 401}])
         conf_dict: dict[str, Any] = {
             name: {
                 "url": url,
@@ -491,11 +493,7 @@ def add_provider(
                     GENERIC_PRODUCT_TYPE: {"productType": "{productType}"},
                     **products,
                 },
-                "download": {
-                    "type": "HTTPDownload",
-                    "auth_error_code": 401,
-                    **download,
-                },
+                "download": download,
                 **kwargs,
             }
         }
@@ -552,36 +550,27 @@ def _prune_providers_list(self) -> None:
                         "%s: provider needing auth for search has been pruned because no credentials could be found",
                         provider,
                     )
-            elif hasattr(conf, "search") and getattr(conf.search, "need_auth", False):
-                if not hasattr(conf, "auth") and not hasattr(conf, "search_auth"):
-                    # credentials needed but no auth plugin was found
-                    self._pruned_providers_config[provider] = self.providers_config.pop(
-                        provider
-                    )
-                    update_needed = True
-                    logger.info(
-                        "%s: provider needing auth for search has been pruned because no auth plugin could be found",
-                        provider,
-                    )
+            elif hasattr(conf, "search"):
+                if not hasattr(conf, "auth"):
+                    # no auth plugin found -> we assume that no auth is necessary
                     continue
-                credentials_exist = (
-                    hasattr(conf, "search_auth")
-                    and credentials_in_auth(conf.search_auth)
-                ) or (
-                    not hasattr(conf, "search_auth")
-                    and hasattr(conf, "auth")
-                    and credentials_in_auth(conf.auth)
-                )
-                if not credentials_exist:
-                    # credentials needed but not found
-                    self._pruned_providers_config[provider] = self.providers_config.pop(
-                        provider
-                    )
-                    update_needed = True
-                    logger.info(
-                        "%s: provider needing auth for search has been pruned because no credentials could be found",
-                        provider,
-                    )
+                for auth_config in conf.auth:
+                    if "search" not in getattr(auth_config, "required_for", []):
+                        # plugin not required for search
+                        continue
+                    credentials_exist = credentials_in_auth(auth_config)
+                    if not credentials_exist:
+                        update_needed = True
+                        logger.info(
+                            "%s: provider needing auth for search has been pruned because "
+                            "no credentials could be found",
+                            provider,
+                        )
+                        self._pruned_providers_config[
+                            provider
+                        ] = self.providers_config.pop(provider)
+                        break
+
             elif not hasattr(conf, "api") and not hasattr(conf, "search"):
                 # provider should have at least an api or search plugin
                 self._pruned_providers_config[provider] = self.providers_config.pop(
@@ -891,20 +880,18 @@ def discover_product_types(
                     "fetch_url"
                 ):
                     continue
-                # append auth to search plugin if needed
-                if getattr(search_plugin.config, "need_auth", False):
-                    if auth := self._plugins_manager.get_auth(
-                        search_plugin.provider,
-                        getattr(search_plugin.config, "api_endpoint", None),
-                        search_plugin.config,
-                    ):
-                        kwargs["auth"] = auth
-                    else:
-                        logger.debug(
-                            f"Could not authenticate on {provider} for product types discovery"
-                        )
-                        ext_product_types_conf[provider] = None
-                        continue
+                # append auth to search plugin if available
+                if auth := self._plugins_manager.get_auth(
+                    search_plugin.provider,
+                    "search",
+                    getattr(search_plugin.config, "api_endpoint", None),
+                    search_plugin.config,
+                ):
+                    kwargs["auth"] = auth
+                else:
+                    logger.debug(
+                        f"No authentication plugin for {provider} for product types discovery found"
+                    )
 
                 ext_product_types_conf[provider] = search_plugin.discover_product_types(
                     **kwargs
@@ -1708,14 +1695,14 @@ def _fetch_external_product_type(self, provider: str, product_type: str):
 
         kwargs: dict[str, Any] = {"productType": product_type}
 
-        # append auth if needed
-        if getattr(plugin.config, "need_auth", False):
-            if auth := self._plugins_manager.get_auth(
-                plugin.provider,
-                getattr(plugin.config, "api_endpoint", None),
-                plugin.config,
-            ):
-                kwargs["auth"] = auth
+        # append auth if available
+        if auth := self._plugins_manager.get_auth(
+            plugin.provider,
+            "search",
+            getattr(plugin.config, "api_endpoint", None),
+            plugin.config,
+        ):
+            kwargs["auth"] = auth
 
         product_type_config = plugin.discover_product_types(**kwargs)
         self.update_product_types_list({provider: product_type_config})
@@ -1914,15 +1901,14 @@ def _do_search(
 
         try:
             prep = PreparedSearch(count=count)
-
-            # append auth if needed
-            if getattr(search_plugin.config, "need_auth", False):
-                if auth := self._plugins_manager.get_auth(
-                    search_plugin.provider,
-                    getattr(search_plugin.config, "api_endpoint", None),
-                    search_plugin.config,
-                ):
-                    prep.auth = auth
+            # append auth if available
+            if auth := self._plugins_manager.get_auth(
+                search_plugin.provider,
+                "search",
+                getattr(search_plugin.config, "api_endpoint", None),
+                search_plugin.config,
+            ):
+                prep.auth = auth
 
             prep.page = kwargs.pop("page", None)
             prep.items_per_page = kwargs.pop("items_per_page", None)
@@ -2372,9 +2358,8 @@ def list_queryables(
                     product_type_configs[pt] = plugin.config.product_type_config
 
             # authenticate if required
-            if getattr(plugin.config, "need_auth", False) and (
-                auth := self._plugins_manager.get_auth_plugin(plugin)
-            ):
+            auth = self._plugins_manager.get_auth_plugin(plugin)
+            if auth and "search" in getattr(auth, "required_for", []):
                 try:
                     plugin.auth = auth.authenticate()
                 except AuthenticationError:

eodag/api/product/_product.py

@@ -43,7 +43,6 @@
     DEFAULT_GEOMETRY,
     NOT_AVAILABLE,
     NOT_MAPPED,
-    ONLINE_STATUS,
 )
 from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
@@ -250,23 +249,8 @@ def _register_downloader_from_manager(self, plugins_manager: PluginManager) -> N
                                 the download and authentication plugins.
         """
         download_plugin = plugins_manager.get_download_plugin(self)
-        if len(self.assets) > 0:
-            matching_url = next(iter(self.assets.values()))["href"]
-        elif self.properties.get("storageStatus") != ONLINE_STATUS:
-            matching_url = self.properties.get("orderLink") or self.properties.get(
-                "downloadLink"
-            )
-        else:
-            matching_url = self.properties.get("downloadLink")
-
         try:
-            auth_plugin = next(
-                plugins_manager.get_auth_plugins(
-                    self.provider,
-                    matching_url=matching_url,
-                    matching_conf=download_plugin.config,
-                )
-            )
+            auth_plugin = plugins_manager.get_auth_plugin(download_plugin, self)
         except StopIteration:
             auth_plugin = None
         self.register_downloader(download_plugin, auth_plugin)