Skip to content

Commit

Permalink
adding test for glue endpoint override
Browse files Browse the repository at this point in the history
  • Loading branch information
sebpretzer committed Mar 18, 2024
1 parent c311dac commit d77d6c9
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 1 deletion.
6 changes: 5 additions & 1 deletion pyiceberg/catalog/glue.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,10 @@
GLUE_SKIP_ARCHIVE = "glue.skip-archive"
GLUE_SKIP_ARCHIVE_DEFAULT = True

# Configure an alternative endpoint of the Glue service for GlueCatalog to access.
# This could be used to use GlueCatalog with any glue-compatible metastore service that has a different endpoint
GLUE_CATALOG_ENDPOINT = "glue.endpoint"

ICEBERG_FIELD_ID = "iceberg.field.id"
ICEBERG_FIELD_OPTIONAL = "iceberg.field.optional"
ICEBERG_FIELD_CURRENT = "iceberg.field.current"
Expand Down Expand Up @@ -285,7 +289,7 @@ def __init__(self, name: str, **properties: Any):
aws_secret_access_key=properties.get("aws_secret_access_key"),
aws_session_token=properties.get("aws_session_token"),
)
self.glue: GlueClient = session.client("glue")
self.glue: GlueClient = session.client("glue", endpoint_url=properties.get(GLUE_CATALOG_ENDPOINT))

if glue_catalog_id := properties.get(GLUE_ID):
_register_glue_catalog_id_with_glue_client(self.glue, glue_catalog_id)
Expand Down
16 changes: 16 additions & 0 deletions tests/catalog/test_glue.py
Original file line number Diff line number Diff line change
Expand Up @@ -692,3 +692,19 @@ def test_commit_table_properties(
updated_table_metadata = table.metadata
assert test_catalog._parse_metadata_version(table.metadata_location) == 1
assert updated_table_metadata.properties == {"test_a": "test_aa", "test_c": "test_c"}


def test_glue_endpoint_override(moto_endpoint_url: str, database_name: str) -> None:
catalog_name = "glue"
test_catalog = GlueCatalog(
catalog_name, **{"s3.endpoint": moto_endpoint_url, "warehouse": f"s3://{BUCKET_NAME}", "glue.endpoint": moto_endpoint_url}
)
assert test_catalog.glue.meta.endpoint_url == moto_endpoint_url

test_catalog.create_namespace(namespace=database_name)
assert (database_name,) in test_catalog.list_namespaces()

with mock_aws():
other_catalog = GlueCatalog(catalog_name, **{"s3.endpoint": moto_endpoint_url, "warehouse": f"s3://{BUCKET_NAME}"})
assert other_catalog.glue.meta.endpoint_url != moto_endpoint_url
assert (database_name,) not in other_catalog.list_namespaces()

0 comments on commit d77d6c9

Please sign in to comment.