Skip to content

Commit c409678

Browse files
smaheshwar-pltrSreesh Maheshwar
andauthored
Use ObjectStoreLocationProvider by default (#1509)
* Make object storage the default location provider * Nit: Remove comment beside property to prefer docs - Removed table proper * Nit: Add asserts for table properties defaults as well as comment in test --------- Co-authored-by: Sreesh Maheshwar <[email protected]>
1 parent aface46 commit c409678

File tree

3 files changed

+8
-7
lines changed

3 files changed

+8
-7
lines changed

pyiceberg/table/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ class TableProperties:
190190
WRITE_PY_LOCATION_PROVIDER_IMPL = "write.py-location-provider.impl"
191191

192192
OBJECT_STORE_ENABLED = "write.object-storage.enabled"
193-
OBJECT_STORE_ENABLED_DEFAULT = False
193+
OBJECT_STORE_ENABLED_DEFAULT = True
194194

195195
WRITE_OBJECT_STORE_PARTITIONED_PATHS = "write.object-storage.partitioned-paths"
196196
WRITE_OBJECT_STORE_PARTITIONED_PATHS_DEFAULT = True

tests/integration/test_writes/test_partitioned_writes.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -294,11 +294,13 @@ def test_object_storage_location_provider_excludes_partition_path(
294294
PartitionField(source_id=nested_field.field_id, field_id=1001, transform=IdentityTransform(), name=part_col)
295295
)
296296

297+
# write.object-storage.enabled and write.object-storage.partitioned-paths don't need to be specified as they're on by default
298+
assert TableProperties.OBJECT_STORE_ENABLED_DEFAULT
299+
assert TableProperties.WRITE_OBJECT_STORE_PARTITIONED_PATHS_DEFAULT
297300
tbl = _create_table(
298301
session_catalog=session_catalog,
299302
identifier=f"default.arrow_table_v{format_version}_with_null_partitioned_on_col_{part_col}",
300-
# write.object-storage.partitioned-paths defaults to True
301-
properties={"format-version": str(format_version), TableProperties.OBJECT_STORE_ENABLED: True},
303+
properties={"format-version": str(format_version)},
302304
data=[arrow_table_with_null],
303305
partition_spec=partition_spec,
304306
)

tests/table/test_locations.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def new_data_location(self, data_file_name: str, partition_key: Optional[Partiti
3939

4040

4141
def test_default_location_provider() -> None:
42-
provider = load_location_provider(table_location="table_location", table_properties=EMPTY_DICT)
42+
provider = load_location_provider(table_location="table_location", table_properties={"write.object-storage.enabled": "false"})
4343

4444
assert provider.new_data_location("my_file") == "table_location/data/my_file"
4545

@@ -66,7 +66,7 @@ def test_custom_location_provider_not_found() -> None:
6666

6767

6868
def test_object_storage_injects_entropy() -> None:
69-
provider = load_location_provider(table_location="table_location", table_properties={"write.object-storage.enabled": "true"})
69+
provider = load_location_provider(table_location="table_location", table_properties=EMPTY_DICT)
7070

7171
location = provider.new_data_location("test.parquet")
7272
parts = location.split("/")
@@ -104,7 +104,6 @@ def test_object_storage_partitioned_paths_disabled(partition_key: Optional[Parti
104104
provider = load_location_provider(
105105
table_location="table_location",
106106
table_properties={
107-
"write.object-storage.enabled": "true",
108107
"write.object-storage.partitioned-paths": "false",
109108
},
110109
)
@@ -125,6 +124,6 @@ def test_object_storage_partitioned_paths_disabled(partition_key: Optional[Parti
125124
],
126125
)
127126
def test_hash_injection(data_file_name: str, expected_hash: str) -> None:
128-
provider = load_location_provider(table_location="table_location", table_properties={"write.object-storage.enabled": "true"})
127+
provider = load_location_provider(table_location="table_location", table_properties=EMPTY_DICT)
129128

130129
assert provider.new_data_location(data_file_name) == f"table_location/data/{expected_hash}/{data_file_name}"

0 commit comments

Comments
 (0)