From 9f8c746747176dbe558aea3ffc4f66d36c13dbf4 Mon Sep 17 00:00:00 2001 From: Honah J Date: Sun, 24 Mar 2024 22:19:38 -0700 Subject: [PATCH] Correctly convert `partition_spec` and `sort_order` (#544) (cherry picked from commit cc8552dec490b342d0ebb8159deeaf0c8f35ab40) --- pyiceberg/table/metadata.py | 7 ++++++- tests/table/test_metadata.py | 17 +++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index a5dfb6ce4c..d5e08cf9db 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -306,6 +306,11 @@ def construct_partition_specs(cls, data: Dict[str, Any]) -> Dict[str, Any]: fields = data[PARTITION_SPEC] data[PARTITION_SPECS] = [{SPEC_ID: INITIAL_SPEC_ID, FIELDS: fields}] data[DEFAULT_SPEC_ID] = INITIAL_SPEC_ID + elif data.get("partition_spec") is not None: + # Promote the spec from partition_spec to partition-specs + fields = data["partition_spec"] + data[PARTITION_SPECS] = [{SPEC_ID: INITIAL_SPEC_ID, FIELDS: fields}] + data[DEFAULT_SPEC_ID] = INITIAL_SPEC_ID else: data[PARTITION_SPECS] = [{"field-id": 0, "fields": ()}] @@ -328,7 +333,7 @@ def set_sort_orders(cls, data: Dict[str, Any]) -> Dict[str, Any]: Returns: The TableMetadata with the sort_orders set, if not provided. """ - if not data.get(SORT_ORDERS): + if not data.get(SORT_ORDERS) and not data.get("sort_orders"): data[SORT_ORDERS] = [UNSORTED_SORT_ORDER] return data diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index 97a7931cbb..19c9c267c1 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -235,6 +235,11 @@ def test_new_table_metadata_with_explicit_v1_format() -> None: expected_spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="bar")) + expected_sort_order = SortOrder( + SortField(source_id=1, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_LAST), + order_id=1, + ) + expected = TableMetadataV1( location="s3://some_v1_location/", table_uuid=actual.table_uuid, @@ -252,20 +257,16 @@ def test_new_table_metadata_with_explicit_v1_format() -> None: snapshots=[], snapshot_log=[], metadata_log=[], - sort_orders=[ - SortOrder( - SortField( - source_id=1, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_LAST - ), - order_id=1, - ) - ], + sort_orders=[expected_sort_order], default_sort_order_id=1, refs={}, format_version=1, ) assert actual.model_dump() == expected.model_dump() + assert actual.schemas == [expected_schema] + assert actual.partition_specs == [expected_spec] + assert actual.sort_orders == [expected_sort_order] def test_invalid_format_version(example_table_metadata_v1: Dict[str, Any]) -> None: