Milestone v0.11.0

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "eopf-stac"
-version = "0.10.0"
+version = "0.11.0"
 authors = [
     { name="Mario Winkler", email="mario.winkler@dlr.de" }
 ]

src/eopf_stac/common/constants.py

@@ -91,11 +91,13 @@
 
 MEDIA_TYPE_ZARR = "application/vnd+zarr"
 MEDIA_TYPE_JSON = "application/json"
+MEDIA_TYPE_ZIP = "application/zip"
 
 ROLE_DATA = "data"
 ROLE_METADATA = "metadata"
 ROLE_VISUAL = "visual"
 ROLE_DATASET = "dataset"
+ROLE_ARCHIVE = "archive"
 
 SENTINEL_LICENSE: Final[Link] = Link(
     rel="license",
@@ -141,6 +143,9 @@
     "xarray:open_dataset_kwargs": EopfXarrayBackendConfig(mode=OpMode.NATIVE).to_dict()
 }
 
+ZIPPED_PRODUCT_ASSET_KEY: Final[str] = "zipped_product"
+ZIPPED_PRODUCT_HREF_BASE = "https://download.user.eopf.eodc.eu/zip"
+
 
 def get_item_asset_metadata() -> ItemAssetDefinition:
     return ItemAssetDefinition.create(
@@ -154,18 +159,28 @@ def get_item_asset_metadata() -> ItemAssetDefinition:
 def get_item_asset_product():
     return ItemAssetDefinition.create(
         title="EOPF Product",
-        description="The full Zarr hierarchy of the EOPF product",
+        description="The full Zarr store of the EOPF product",
         media_type=MEDIA_TYPE_ZARR,
         roles=[ROLE_DATA, ROLE_METADATA],
         extra_fields=deepcopy(PRODUCT_ASSET_EXTRA_FIELDS),
     )
 
 
+def get_item_asset_zipped_product() -> ItemAssetDefinition:
+    return ItemAssetDefinition.create(
+        title="Zipped EOPF Product",
+        description="The full EOPF Zarr store as zip archive",
+        media_type=MEDIA_TYPE_ZIP,
+        roles=[ROLE_DATA, ROLE_METADATA, ROLE_ARCHIVE],
+    )
+
+
 PRODUCT_EXTENSION_SCHEMA_URI = "https://stac-extensions.github.io/product/v0.1.0/schema.json"
 PROCESSING_EXTENSION_SCHEMA_URI = "https://stac-extensions.github.io/processing/v1.2.0/schema.json"
 EOPF_EXTENSION_SCHEMA_URI = "https://cs-si.github.io/eopf-stac-extension/v1.2.0/schema.json"
 VERSION_EXTENSION_SCHEMA_URI = "https://stac-extensions.github.io/version/v1.2.0/schema.json"
 RASTER_EXTENSION_SCHEMA_URI = "https://stac-extensions.github.io/raster/v2.0.0/schema.json"
+EO_EXTENSION_SCHEMA_URI = "https://stac-extensions.github.io/eo/v2.0.0/schema.json"
 
 S2_MGRS_PATTERN: Final[re.Pattern[str]] = re.compile(
     r"_T(\d{1,2})([CDEFGHJKLMNPQRSTUVWX])([ABCDEFGHJKLMNPQRSTUVWXYZ][ABCDEFGHJKLMNPQRSTUV])"

src/eopf_stac/common/stac.py

@@ -7,20 +7,22 @@
 import pystac
 import shapely
 from footprint_facility import rework_to_polygon_geometry
-from pystac import Link
-from pystac.extensions.eo import EOExtension
+from pystac import Asset, Link
 from pystac.extensions.grid import GridExtension
 from pystac.extensions.sat import OrbitState, SatExtension
 from pystac.extensions.timestamps import TimestampsExtension
 from pystac.utils import now_in_utc, str_to_datetime
 from stactools.sentinel2.mgrs import MgrsExtension
 
 from eopf_stac.common.constants import (
+    EO_EXTENSION_SCHEMA_URI,
     EOPF_EXTENSION_SCHEMA_URI,
     PROCESSING_EXTENSION_SCHEMA_URI,
     PRODUCT_EXTENSION_SCHEMA_URI,
     S2_MGRS_PATTERN,
     VERSION_EXTENSION_SCHEMA_URI,
+    ZIPPED_PRODUCT_HREF_BASE,
+    get_item_asset_zipped_product,
 )
 
 logger = logging.getLogger(__name__)
@@ -166,12 +168,12 @@ def fill_eo_properties(item: pystac.Item, properties: dict) -> None:
     cloud_cover = properties.get("eo:cloud_cover")
     snow_cover = properties.get("eo:snow_cover")
 
-    if any_not_none([cloud_cover, snow_cover]):
-        eo = EOExtension.ext(item, add_if_missing=True)
-        if cloud_cover is not None:
-            eo.cloud_cover = cloud_cover
-        if snow_cover is not None:
-            eo.snow_cover = snow_cover
+    if cloud_cover is not None:
+        item.properties["eo:cloud_cover"] = cloud_cover
+    if snow_cover is not None:
+        item.properties["eo:snow_cover"] = snow_cover
+
+    item.stac_extensions.append(EO_EXTENSION_SCHEMA_URI)
 
 
 def fill_processing_properties(
@@ -313,3 +315,11 @@ def create_cdse_link(cdse_scene_href: str) -> Link:
         target=cdse_scene_href,
         media_type="application/geo+json",
     )
+
+
+def create_zipped_product_asset(collection_id: str, item_id: str) -> Asset:
+    if is_valid_string(collection_id) and is_valid_string(item_id):
+        href = os.path.join(ZIPPED_PRODUCT_HREF_BASE, "collections", collection_id, "items", item_id + ".zip")
+        return get_item_asset_zipped_product().create_asset(href=href)
+    else:
+        raise ValueError(f"Unable to create zip product asset for collection={collection_id} and item={item_id}")

src/eopf_stac/io.py

@@ -81,6 +81,10 @@ def create_item(metadata: dict, eopf_href: str, source_uri: str | None) -> pysta
     if cdse_scene_href is None:
         logger.warning("Unable to determine link to the original scene at CSDE STAC API!")
 
+    collection = PRODUCT_TYPE_TO_COLLECTION.get(product_type)
+    if collection is None:
+        raise ValueError(f"No collection defined for product type '{product_type}'")
+
     item = None
     if product_type in SUPPORTED_PRODUCT_TYPES_S1:
         item = create_item_s1(
@@ -90,6 +94,7 @@ def create_item(metadata: dict, eopf_href: str, source_uri: str | None) -> pysta
             cpm_version=cpm_version,
             cdse_scene_id=cdse_scene_id,
             cdse_scene_href=cdse_scene_href,
+            collection_id=collection,
         )
     elif product_type in SUPPORTED_PRODUCT_TYPES_S2:
         item = create_item_s2(
@@ -99,6 +104,7 @@ def create_item(metadata: dict, eopf_href: str, source_uri: str | None) -> pysta
             cpm_version=cpm_version,
             cdse_scene_id=cdse_scene_id,
             cdse_scene_href=cdse_scene_href,
+            collection_id=collection,
         )
     elif product_type in SUPPORTED_PRODUCT_TYPES_S3:
         item = create_item_s3(
@@ -108,24 +114,20 @@ def create_item(metadata: dict, eopf_href: str, source_uri: str | None) -> pysta
             cpm_version=cpm_version,
             cdse_scene_id=cdse_scene_id,
             cdse_scene_href=cdse_scene_href,
+            collection_id=collection,
         )
     else:
         raise ValueError(f"The product type '{product_type}' is not supported")
 
+    item.collection_id = collection
+
     logger.info("Sucessfully created STAC item")
     return item
 
 
 def register_item(item: pystac.Item, stac_api_url: str) -> pystac.Item:
     logger.info(f"Inserting STAC item into catalog {stac_api_url} ...")
 
-    product_type = item.properties["product:type"]
-    collection = PRODUCT_TYPE_TO_COLLECTION.get(product_type)
-    if collection is None:
-        raise ValueError(f"No collection defined for product type '{product_type}'")
-    else:
-        item.collection_id = collection
-
     item.remove_links("self")
     session = requests.Session()
     if "STAC_INGEST_USER" in os.environ and "STAC_INGEST_PASS" in os.environ:

src/eopf_stac/sentinel1/stac.py

@@ -10,9 +10,11 @@
     LICENSE_PROVIDER,
     SENTINEL_LICENSE,
     SENTINEL_PROVIDER,
+    ZIPPED_PRODUCT_ASSET_KEY,
 )
 from eopf_stac.common.stac import (
     create_cdse_link,
+    create_zipped_product_asset,
     fill_eopf_properties,
     fill_processing_properties,
     fill_product_properties,
@@ -42,6 +44,7 @@ def create_item(
     cpm_version: str = None,
     cdse_scene_id: str | None = None,
     cdse_scene_href: str | None = None,
+    collection_id: str | None = None,
 ) -> pystac.Item:
     stac_discovery = metadata[".zattrs"]["stac_discovery"]
     other_metadata = metadata[".zattrs"]["other_metadata"]
@@ -218,6 +221,8 @@ def create_item(
     else:
         raise ValueError(f"Unsupported Sentinel-1 product type '{product_type}'")
 
+    assets[ZIPPED_PRODUCT_ASSET_KEY] = create_zipped_product_asset(collection_id=collection_id, item_id=item.id)
+
     for key, asset in assets.items():
         assert key not in item.assets
         item.add_asset(key, asset)

src/eopf_stac/sentinel2/assets.py

@@ -4,10 +4,6 @@
 import numpy as np
 import pystac
 from pystac.extensions.eo import Band
-from stactools.sentinel2.constants import (
-    BANDS_TO_ASSET_NAME,
-    SENTINEL_BANDS,
-)
 
 from eopf_stac.common.constants import (
     DATASET_ASSET_EXTRA_FIELDS,
@@ -18,15 +14,19 @@
     RASTER_EXTENSION_SCHEMA_URI,
     ROLE_DATA,
     ROLE_DATASET,
+    ZIPPED_PRODUCT_ASSET_KEY,
     get_item_asset_metadata,
     get_item_asset_product,
 )
+from eopf_stac.common.stac import create_zipped_product_asset
 from eopf_stac.sentinel2.constants import (
     ASSET_TO_DESCRIPTION,
     DATASET_PATHS_TO_ASSET,
     L2A_AOT_WVP_ASSETS_TO_PATH,
     L2A_SCL_ASSETS_TO_PATH,
     ROLE_REFLECTANCE,
+    SENTINEL2_BANDS_DICT,
+    SENTINEL2_BANDS_TO_ASSET_NAME,
 )
 
 
@@ -50,7 +50,6 @@ def get_band_assets(
     for key, item_asset in item_assets.items():
         href_suffix = band_asset_defs[key]
         asset = item_asset.create_asset(os.path.join(asset_href, href_suffix))
-        update_alternate_xarray_asset(asset=asset, asset_href=asset_href, href_suffix=href_suffix)
 
         attrs = metadata.get(f"{href_suffix}/.zattrs")
         if attrs:
@@ -68,19 +67,13 @@ def get_aot_wvp_item_assets() -> dict[str, pystac.ItemAssetDefinition]:
             asset_key=key,
             roles=[ROLE_DATA],
             band_keys=[],
+            extra_fields={},
             title_with_resolution=False,
         )
         item_assets[key] = item_asset
     return item_assets
 
 
-def update_alternate_xarray_asset(asset: pystac.Asset, asset_href: str, href_suffix: str):
-    # set href of alternate xarray asset pointing to the dataset
-    alternate_asset = asset.extra_fields.get("alternate", {}).get("xarray", None)
-    if alternate_asset is not None:
-        alternate_asset["href"] = os.path.dirname(os.path.join(asset_href, href_suffix))
-
-
 def get_aot_wvp_assets(
     aot_wvp_asset_defs: dict, asset_href: str, metadata: dict, item: pystac.Item
 ) -> dict[str, pystac.Asset]:
@@ -89,7 +82,6 @@ def get_aot_wvp_assets(
     for key, item_asset in item_assets.items():
         href_suffix = aot_wvp_asset_defs[key]
         asset = item_asset.create_asset(os.path.join(asset_href, href_suffix))
-        update_alternate_xarray_asset(asset=asset, asset_href=asset_href, href_suffix=href_suffix)
 
         attrs = metadata.get(f"{href_suffix}/.zattrs")
         if attrs:
@@ -112,6 +104,7 @@ def get_scl_item_assets() -> dict[str, pystac.ItemAssetDefinition]:
             asset_key=key,
             roles=[ROLE_DATA],
             band_keys=[],
+            extra_fields={},
             title_with_resolution=False,
         )
     return item_assets
@@ -125,7 +118,6 @@ def get_scl_assets(scl_asset_defs: dict, asset_href: str, metadata: dict, item:
         # SCL can be opened as zarr group / xarray dataset -> remove the 'scl' part of the path
         # asset = item_asset.create_asset(os.path.dirname(os.path.join(asset_href, href_suffix)))
         asset = item_asset.create_asset(os.path.join(asset_href, href_suffix))
-        update_alternate_xarray_asset(asset=asset, asset_href=asset_href, href_suffix=href_suffix)
 
         attrs = metadata.get(f"{href_suffix}/.zattrs")
         if attrs:
@@ -147,6 +139,7 @@ def get_tci_item_assets(tci_asset_defs: dict) -> dict[str, pystac.ItemAssetDefin
             asset_key=key,
             roles=[ROLE_DATA],
             band_keys=["B04", "B03", "B02"],
+            extra_fields={},
             title_with_resolution=False,
         )
     return item_assets
@@ -158,7 +151,6 @@ def get_tci_assets(tci_asset_defs: dict, asset_href: str, metadata: dict, item:
     for key, item_asset in item_assets.items():
         href_suffix = tci_asset_defs[key]
         asset = item_asset.create_asset(os.path.join(asset_href, href_suffix))
-        update_alternate_xarray_asset(asset=asset, asset_href=asset_href, href_suffix=href_suffix)
 
         attrs = metadata.get(f"{href_suffix}/.zattrs")
         if attrs:
@@ -213,14 +205,14 @@ def get_dataset_assets(
     return assets
 
 
-def get_extra_assets(asset_href: str, item: pystac.Item) -> dict[str, pystac.Asset]:
+def get_extra_assets(asset_href: str, item: pystac.Item, collection_id: str) -> dict[str, pystac.Asset]:
     metadata = get_item_asset_metadata().create_asset(os.path.join(asset_href, PRODUCT_METADATA_PATH))
     product = get_item_asset_product().create_asset(asset_href)
+    zip_product = create_zipped_product_asset(collection_id=collection_id, item_id=item.id)
+    metadata.set_owner(item)
     product.set_owner(item)
-    return {
-        PRODUCT_METADATA_ASSET_KEY: metadata,
-        PRODUCT_ASSET_KEY: product,
-    }
+    zip_product.set_owner(item)
+    return {PRODUCT_METADATA_ASSET_KEY: metadata, PRODUCT_ASSET_KEY: product, ZIPPED_PRODUCT_ASSET_KEY: zip_product}
 
 
 def create_item_asset(
@@ -254,9 +246,9 @@ def create_item_asset(
 def get_bands_for_band_keys(keys: list[str]) -> list[Band]:
     bands = []
     for band_key in keys:
-        band = SENTINEL_BANDS[BANDS_TO_ASSET_NAME[band_key]]
-        band.description = f"{ASSET_TO_DESCRIPTION[band_key]}"
-        bands.append(band.to_dict())
+        band = SENTINEL2_BANDS_DICT[SENTINEL2_BANDS_TO_ASSET_NAME[band_key]]
+        band["description"] = f"{ASSET_TO_DESCRIPTION[band_key]}"
+        bands.append(band)
     return bands