From c52adcafd1a2f5ca92309e8d312e51d88f65cd8f Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 24 Jan 2025 11:08:52 -0800 Subject: [PATCH 1/2] add all transforms --- tests/table/test_partitioning.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index bdd68ea7a2..73bb520d11 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -23,7 +23,15 @@ from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.transforms import BucketTransform, IdentityTransform, TruncateTransform +from pyiceberg.transforms import ( + BucketTransform, + DayTransform, + HourTransform, + IdentityTransform, + MonthTransform, + TruncateTransform, + YearTransform, +) from pyiceberg.typedef import Record from pyiceberg.types import ( BinaryType, @@ -186,11 +194,27 @@ def test_partition_type(table_schema_simple: Schema) -> None: (BinaryType(), b"\x8e\xd1\x87\x01"), ], ) -def test_bucketing_function(source_type: PrimitiveType, value: Any) -> None: - bucket = BucketTransform(2) # type: ignore +def test_transform_consistency_with_pyarrow_transform(source_type: PrimitiveType, value: Any) -> None: import pyarrow as pa - assert bucket.transform(source_type)(value) == bucket.pyarrow_transform(source_type)(pa.array([value])).to_pylist()[0] + all_transforms = [ # type: ignore + IdentityTransform(), + BucketTransform(10), + TruncateTransform(10), + YearTransform(), + MonthTransform(), + DayTransform(), + HourTransform(), + ] + for t in all_transforms: + if t.can_transform(source_type): + try: + assert t.transform(source_type)(value) == t.pyarrow_transform(source_type)(pa.array([value])).to_pylist()[0] + except ValueError as e: + # Skip for unsupported feature + if "FeatureUnsupported => Unsupported data type for truncate transform" in str(e): + continue + raise def test_deserialize_partition_field_v2() -> None: From e207f5125a44f529e11c00c1526a9302644d8f15 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 24 Jan 2025 11:10:09 -0800 Subject: [PATCH 2/2] comment --- tests/table/test_partitioning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index 73bb520d11..edda6d3aa8 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -211,7 +211,7 @@ def test_transform_consistency_with_pyarrow_transform(source_type: PrimitiveType try: assert t.transform(source_type)(value) == t.pyarrow_transform(source_type)(pa.array([value])).to_pylist()[0] except ValueError as e: - # Skip for unsupported feature + # Skipping unsupported feature if "FeatureUnsupported => Unsupported data type for truncate transform" in str(e): continue raise