feat(search): Remove ECMWF search default dates #1904

~ fix default date bevaviour
~ re-set option plugin_config.dates_required behaviour
~ fix ECMWFSearch.available_values_from_contraints: must not fill default values outside from contraints set
~ fix build_search_result: sort json list parameters value to make tests easier
~ fix build_search_result _fetch_data unconsistent type
- remove plugin_config "dates_required"
- core.py:1825 fix v2 code not pudated
+ add discover_queryables_defaults, defined minimal parameters required to discover_queryables
+ steadify date parsing
+ implement indirect date computed from parmezeters
+ update tests
~ update integration tests: skip on misconfigured provider
+ core: force internal define date on search_by_id to not makes paramaters required

Author: pdavid-cssopra

eodag/api/core.py

@@ -63,6 +63,7 @@
     DEFAULT_DOWNLOAD_WAIT,
     DEFAULT_ITEMS_PER_PAGE,
     DEFAULT_MAX_ITEMS_PER_PAGE,
+    DEFAULT_MISSION_START_DATE,
     DEFAULT_PAGE,
     GENERIC_COLLECTION,
     GENERIC_STAC_PROVIDER,
@@ -1163,6 +1164,7 @@ def search(
             provider=provider,
             **kwargs,
         )
+
         if search_kwargs.get("id"):
             # Don't validate requests by ID. "id" is not queryable.
             return self._search_by_id(
@@ -1453,6 +1455,15 @@ def _search_by_id(
         :returns: A search result with one EO product or None at all
         """
         collection = kwargs.get("collection")
+        # Search by id no need date restrict, set largest time definition
+        if "start_datetime" not in kwargs:
+            kwargs["start_datetime"] = DEFAULT_MISSION_START_DATE
+        if "end_datetime" not in kwargs:
+            now_utc = datetime.datetime.now(datetime.timezone.utc).replace(
+                microsecond=0
+            )
+            kwargs["end_datetime"] = now_utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+
         if collection is not None:
             try:
                 collection = self.get_collection_from_alias(collection)
@@ -1462,6 +1473,7 @@ def _search_by_id(
         search_plugins = self._plugins_manager.get_search_plugins(
             **get_search_plugins_kwargs
         )
+
         # datacube query string
         _dc_qs = kwargs.pop("_dc_qs", None)
 
@@ -1813,10 +1825,10 @@ def _do_search(
 
             search_result = search_plugin.query(prep, **search_params)
 
-            if not isinstance(search_result.data, list):
+            if not isinstance(search_result, SearchResult):
                 raise PluginImplementationError(
                     "The query function of a Search plugin must return a list of "
-                    "results, got {} instead".format(type(search_result.data))
+                    "results, got {} instead".format(type(search_result))
                 )
             # Filter and attach to each eoproduct in the result the plugin capable of
             # downloading it (this is done to enable the eo_product to download itself
@@ -1827,7 +1839,7 @@ def _do_search(
             # WARNING: this means an eo_product that has an invalid geometry can still
             # be returned as a search result if there was no search extent (because we
             # will not try to do an intersection)
-            for eo_product in search_result.data:
+            for eo_product in search_result:
                 # if collection is not defined, try to guess using properties
                 if eo_product.collection is None:
                     pattern = re.compile(r"[^\w,]+")

eodag/api/product/metadata_mapping.py

@@ -241,8 +241,13 @@ def get_value(
             """
             if isinstance(key, str):
                 original_key = key.replace("__", ":")
+                if original_key in kwargs:
+                    return kwargs.get(original_key)
+
                 key_with_COLON = key.replace("__", "_COLON_")
-                return kwargs.get(original_key) or kwargs.get(key_with_COLON)
+                if key_with_COLON in kwargs:
+                    return kwargs.get(original_key)
+
             return super().get_value(key, args, kwargs)
 
         def get_field(self, field_name: str, args: Any, kwargs: Any) -> Any:

eodag/config.py

@@ -397,9 +397,6 @@ class MetadataPreMapping(TypedDict, total=False):
     max_connections: int
     #: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
     #: if date parameters are mandatory in the request
-    dates_required: bool
-    #: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
-    #: Whether end date should be excluded from search request or not
     end_date_excluded: bool
     #: :class:`~eodag.plugins.search.build_search_result.ECMWFSearch`
     #: List of parameters used to parse metadata but that must not be included to the query

eodag/plugins/apis/ecmwf.py

@@ -19,7 +19,6 @@
 
 import logging
 import os
-from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Annotated
 
 import geojson
@@ -35,8 +34,6 @@
 from eodag.utils import (
     DEFAULT_DOWNLOAD_TIMEOUT,
     DEFAULT_DOWNLOAD_WAIT,
-    DEFAULT_MISSION_START_DATE,
-    get_collection_dates,
     get_geometry_from_various,
     path_to_uri,
     sanitize,
@@ -135,18 +132,6 @@ def query(
                 kwargs.get("ecmwf:levtype", ""),
             )
 
-        col_start, col_end = get_collection_dates(
-            getattr(self.config, "collection_config", {})
-        )
-        # start date
-        if "start_datetime" not in kwargs:
-            kwargs["start_datetime"] = col_start or DEFAULT_MISSION_START_DATE
-        # end date
-        if "end_datetime" not in kwargs:
-            kwargs["end_datetime"] = col_end or datetime.now(timezone.utc).isoformat(
-                timespec="seconds"
-            )
-
         # geometry
         if "geometry" in kwargs:
             kwargs["geometry"] = get_geometry_from_various(geometry=kwargs["geometry"])
@@ -285,5 +270,6 @@ def discover_queryables(
                        arguments)
         :returns: fetched queryable parameters dict
         """
+        kwargs = self.discover_queryables_defaults(kwargs)
         collection = kwargs.get("collection")
         return self.queryables_from_metadata_mapping(collection)

eodag/plugins/search/base.py

@@ -17,6 +17,7 @@
 # limitations under the License.
 from __future__ import annotations
 
+import datetime
 import logging
 from typing import TYPE_CHECKING, Annotated, get_args
 
@@ -37,6 +38,7 @@
 from eodag.types.search_args import SortByList
 from eodag.types.stac_metadata import CommonStacMetadata, create_stac_metadata_model
 from eodag.utils import (
+    DEFAULT_MISSION_START_DATE,
     GENERIC_COLLECTION,
     copy_deepcopy,
     deepcopy,
@@ -111,6 +113,38 @@ def discover_collections(self, **kwargs: Any) -> Optional[dict[str, Any]]:
         """Fetch collections list from provider using `discover_collections` conf"""
         return None
 
+    def discover_queryables_defaults(self, kwargs: dict[str, Any]) -> dict[str, Any]:
+        """Set queryables default parameters."""
+        if "start_datetime" not in kwargs or "end_datetime" not in kwargs:
+            # Alias
+            if "start_datetime" not in kwargs and "start" in kwargs:
+                kwargs["start_datetime"] = kwargs["start"]
+            if "end_datetime" not in kwargs and "end" in kwargs:
+                kwargs["end_datetime"] = kwargs["end"]
+
+        if "start_datetime" not in kwargs or "end_datetime" not in kwargs:
+            # Collection limit
+            (collection_start_datetime, collection_end_datetime) = get_collection_dates(
+                getattr(self.config, "collection_config", {})
+            )
+            if "start_datetime" not in kwargs and collection_start_datetime is not None:
+                kwargs["start_datetime"] = collection_start_datetime
+            if "end_datetime" not in kwargs and collection_end_datetime is not None:
+                kwargs["end_datetime"] = collection_end_datetime
+
+        if "start_datetime" not in kwargs or "end_datetime" not in kwargs:
+            # Hardcode
+            if "start_datetime" not in kwargs:
+                kwargs["start_datetime"] = "{}Z".format(
+                    DEFAULT_MISSION_START_DATE[0:19]
+                )
+            if "end_datetime" not in kwargs:
+                kwargs["end_datetime"] = datetime.datetime.now(
+                    datetime.timezone.utc
+                ).strftime("%Y-%m-%dT%H:%I:%SZ")
+
+        return kwargs
+
     def discover_queryables(
         self, **kwargs: Any
     ) -> Optional[dict[str, Annotated[Any, FieldInfo]]]: