From 33d0be981981e511bab778f673267f7e27bbdbc8 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 16 Jul 2025 18:57:17 -0400 Subject: [PATCH 1/6] add 'validate_extensions' option and run stac-pydantic extension validation on STAC objects if enabled --- stac_fastapi/pgstac/config.py | 7 +++++++ stac_fastapi/pgstac/transactions.py | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/stac_fastapi/pgstac/config.py b/stac_fastapi/pgstac/config.py index c8741812..26f58c05 100644 --- a/stac_fastapi/pgstac/config.py +++ b/stac_fastapi/pgstac/config.py @@ -168,6 +168,13 @@ class Settings(ApiSettings): invalid_id_chars: List[str] = DEFAULT_INVALID_ID_CHARS base_item_cache: Type[BaseItemCache] = DefaultBaseItemCache + validate_extensions: bool = False + """ + Validate `stac_extensions` schemas against submitted data when creating or updated STAC objects. + + Implies that the `Transactions` extension is enabled. + """ + cors_origins: str = "*" cors_methods: str = "GET,POST,OPTIONS" diff --git a/stac_fastapi/pgstac/transactions.py b/stac_fastapi/pgstac/transactions.py index f4ed11c9..2e095869 100644 --- a/stac_fastapi/pgstac/transactions.py +++ b/stac_fastapi/pgstac/transactions.py @@ -20,6 +20,7 @@ ) from stac_fastapi.types import stac as stac_types from stac_pydantic import Collection, Item, ItemCollection +from stac_pydantic.extensions import validate_extensions from starlette.responses import JSONResponse, Response from stac_fastapi.pgstac.config import Settings @@ -41,8 +42,29 @@ def _validate_id(self, id: str, settings: Settings): detail=f"ID ({id}) cannot contain the following characters: {' '.join(invalid_chars)}", ) + def _validate_extensions( + self, + stac_object: stac_types.Item | stac_types.Collection | stac_types.Catalog, + settings: Settings, + ): + """Validate extensions of the STAC object data.""" + if not settings.validate_extensions or not stac_object.stac_extensions: + return + + try: + validate_extensions( + stac_object, + reraise_exceptions=True, + ) + except Exception as err: + raise HTTPException( + status_code=422, + detail=f"STAC Extensions failed validation: {str(err)}", + ) from err + def _validate_collection(self, request: Request, collection: stac_types.Collection): self._validate_id(collection["id"], request.app.state.settings) + self._validate_extensions(collection, request.app.state.settings) def _validate_item( self, @@ -56,6 +78,7 @@ def _validate_item( body_item_id = item.get("id") self._validate_id(body_item_id, request.app.state.settings) + self._validate_extensions(item, request.app.state.settings) if item.get("geometry", None) is None: raise HTTPException( @@ -177,6 +200,7 @@ async def update_collection( """Update collection.""" col = collection.model_dump(mode="json") + self._validate_collection(request, col) async with request.app.state.get_connection(request, "w") as conn: await dbfunc(conn, "update_collection", col) From 2ecb30c3bb09e4995f3c1724d352bc73d84ec2cb Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 16 Jul 2025 22:49:48 -0400 Subject: [PATCH 2/6] adjust missing dependencies and handle literal JSON for extension validation --- CHANGES.md | 4 ++++ setup.py | 4 ++++ stac_fastapi/pgstac/transactions.py | 18 ++++++++++++------ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b838437b..d98a003b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -41,6 +41,10 @@ ### Added +- add `validate_extensions` setting that enables validation of `stac_extensions` from submitted STAC objects + using the `stac_pydantic.extensions.validate_extensions` utility. Applicable only when `TransactionExtension` + is active. +- add `validate` extra requirement to install additional dependencies required by extension validation - add `write_connection_pool` option in `stac_fastapi.pgstac.db.connect_to_db` function - add `write_postgres_settings` option in `stac_fastapi.pgstac.db.connect_to_db` function to set specific settings for the `writer` DB connection pool - add specific error message when trying to create `Item` with null geometry (not supported by PgSTAC) diff --git a/setup.py b/setup.py index 68ca0da8..3b3ead6b 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,10 @@ ] extra_reqs = { + "validate": [ + "jsonschema", + "requests" + ], "dev": [ "pystac[validation]", "pypgstac[psycopg]==0.9.*", diff --git a/stac_fastapi/pgstac/transactions.py b/stac_fastapi/pgstac/transactions.py index 2e095869..2e65eafc 100644 --- a/stac_fastapi/pgstac/transactions.py +++ b/stac_fastapi/pgstac/transactions.py @@ -2,7 +2,7 @@ import logging import re -from typing import List, Optional, Union +from typing import Any, Dict, List, Optional, Union import attr from buildpg import render @@ -44,22 +44,28 @@ def _validate_id(self, id: str, settings: Settings): def _validate_extensions( self, - stac_object: stac_types.Item | stac_types.Collection | stac_types.Catalog, + stac_object: stac_types.Item | stac_types.Collection | stac_types.Catalog | Dict[str, Any], settings: Settings, - ): + ) -> None: """Validate extensions of the STAC object data.""" - if not settings.validate_extensions or not stac_object.stac_extensions: + if not settings.validate_extensions: return + if isinstance(stac_object, dict): + if not stac_object.get("stac_extensions"): + return + else: + if not stac_object.stac_extensions: + return try: validate_extensions( stac_object, - reraise_exceptions=True, + reraise_exception=True, ) except Exception as err: raise HTTPException( status_code=422, - detail=f"STAC Extensions failed validation: {str(err)}", + detail=f"STAC Extensions failed validation: {err!s}", ) from err def _validate_collection(self, request: Request, collection: stac_types.Collection): From 86ef9c329fe2725424d95cf63ac65c7d7f251a99 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 16 Jul 2025 23:02:33 -0400 Subject: [PATCH 3/6] use proper dependency 'stac_pydantic[validation]' --- CHANGES.md | 2 +- setup.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d98a003b..51baaa73 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -44,7 +44,7 @@ - add `validate_extensions` setting that enables validation of `stac_extensions` from submitted STAC objects using the `stac_pydantic.extensions.validate_extensions` utility. Applicable only when `TransactionExtension` is active. -- add `validate` extra requirement to install additional dependencies required by extension validation +- add `validation` extra requirement to install dependencies of `stac_pydantic` required for extension validation - add `write_connection_pool` option in `stac_fastapi.pgstac.db.connect_to_db` function - add `write_postgres_settings` option in `stac_fastapi.pgstac.db.connect_to_db` function to set specific settings for the `writer` DB connection pool - add specific error message when trying to create `Item` with null geometry (not supported by PgSTAC) diff --git a/setup.py b/setup.py index 3b3ead6b..fc91400b 100644 --- a/setup.py +++ b/setup.py @@ -21,10 +21,6 @@ ] extra_reqs = { - "validate": [ - "jsonschema", - "requests" - ], "dev": [ "pystac[validation]", "pypgstac[psycopg]==0.9.*", @@ -49,6 +45,9 @@ ], "server": ["uvicorn[standard]==0.35.0"], "awslambda": ["mangum"], + "validation": [ + "stac_pydantic[validation]", + ], } From 1457e446d427cbdf5fc7572ea0b079d1c83724ff Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 8 Aug 2025 14:38:22 +0200 Subject: [PATCH 4/6] add tests --- stac_fastapi/pgstac/transactions.py | 6 +++- tests/api/test_api.py | 36 +++++++++++++++++++++++ tests/conftest.py | 45 +++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/pgstac/transactions.py b/stac_fastapi/pgstac/transactions.py index 10a57b2c..c241fc01 100644 --- a/stac_fastapi/pgstac/transactions.py +++ b/stac_fastapi/pgstac/transactions.py @@ -47,12 +47,16 @@ def _validate_id(self, id: str, settings: Settings): def _validate_extensions( self, - stac_object: stac_types.Item | stac_types.Collection | stac_types.Catalog | Dict[str, Any], + stac_object: stac_types.Item + | stac_types.Collection + | stac_types.Catalog + | Dict[str, Any], settings: Settings, ) -> None: """Validate extensions of the STAC object data.""" if not settings.validate_extensions: return + if isinstance(stac_object, dict): if not stac_object.get("stac_extensions"): return diff --git a/tests/api/test_api.py b/tests/api/test_api.py index bd2a3641..7f67e28d 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -948,3 +948,39 @@ async def test_default_app(default_client, default_app, load_test_data): assert "https://api.stacspec.org/v1.0.0/collections" in conf assert "https://api.stacspec.org/v1.0.0/ogcapi-features#query" in conf assert "https://api.stacspec.org/v1.0.0/ogcapi-features#sort" in conf + + +async def test_app_transactions_validate_extension( + app_client_validate_ext, load_test_data +): + coll = load_test_data("test_collection.json") + # Add attribution extension + # https://github.com/stac-extensions/attribution + coll["stac_extensions"] = [ + "https://stac-extensions.github.io/attribution/v0.1.0/schema.json", + ] + + resp = await app_client_validate_ext.post("/collections", json=coll) + assert resp.status_code == 422 + assert "STAC Extensions failed validation:" in resp.json()["detail"] + + # add attribution + coll["attribution"] = "something" + resp = await app_client_validate_ext.post("/collections", json=coll) + assert resp.status_code == 201 + + item = load_test_data("test_item.json") + item["stac_extensions"].append( + "https://stac-extensions.github.io/attribution/v0.1.0/schema.json", + ) + resp = await app_client_validate_ext.post( + f"/collections/{coll['id']}/items", json=item + ) + assert resp.status_code == 422 + assert "STAC Extensions failed validation:" in resp.json()["detail"] + + item["properties"]["attribution"] = "something" + resp = await app_client_validate_ext.post( + f"/collections/{coll['id']}/items", json=item + ) + assert resp.status_code == 201 diff --git a/tests/conftest.py b/tests/conftest.py index d3495936..dbbb5979 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -454,3 +454,48 @@ async def app_client_advanced_freetext(app_advanced_freetext): transport=ASGITransport(app=app_advanced_freetext), base_url="http://test" ) as c: yield c + + +@pytest.fixture(scope="function") +async def app_transaction_validation_ext(database): + """Default stac-fastapi-pgstac application with extension validation in transaction.""" + api_settings = Settings(testing=True, validate_extensions=True) + api = StacApi( + settings=api_settings, + extensions=[ + TransactionExtension( + client=TransactionsClient(), + settings=api_settings, + ) + ], + client=CoreCrudClient(), + health_check=health_check, + ) + + postgres_settings = PostgresSettings( + pguser=database.user, + pgpassword=database.password, + pghost=database.host, + pgport=database.port, + pgdatabase=database.dbname, + ) + logger.info("Creating app Fixture") + await connect_to_db( + api.app, + postgres_settings=postgres_settings, + add_write_connection_pool=True, + ) + yield api.app + await close_db_connection(api.app) + + logger.info("Closed Pools.") + + +@pytest.fixture(scope="function") +async def app_client_validate_ext(app_transaction_validation_ext): + logger.info("creating app_client") + async with AsyncClient( + transport=ASGITransport(app=app_transaction_validation_ext), + base_url="http://test", + ) as c: + yield c From 43b979e1ff73012f6e01887ccd8984843675fe77 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 8 Aug 2025 14:42:00 +0200 Subject: [PATCH 5/6] install validation modules --- .github/workflows/cicd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 0cdf3653..30b73f75 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -44,7 +44,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install .[dev,server] + python -m pip install .[dev,server,validation] python -m pip install "pypgstac==${{ matrix.pypgstac }}" - name: Run test suite From 3de37fd888b3a15c327abc2bf5388b54f08ce49d Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 8 Aug 2025 14:44:33 +0200 Subject: [PATCH 6/6] adjust type for python 3.9 --- stac_fastapi/pgstac/transactions.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/stac_fastapi/pgstac/transactions.py b/stac_fastapi/pgstac/transactions.py index c241fc01..bb2588e9 100644 --- a/stac_fastapi/pgstac/transactions.py +++ b/stac_fastapi/pgstac/transactions.py @@ -47,10 +47,9 @@ def _validate_id(self, id: str, settings: Settings): def _validate_extensions( self, - stac_object: stac_types.Item - | stac_types.Collection - | stac_types.Catalog - | Dict[str, Any], + stac_object: Union[ + stac_types.Item, stac_types.Collection, stac_types.Catalog, Dict[str, Any] + ], settings: Settings, ) -> None: """Validate extensions of the STAC object data."""