Skip to content

Commit

Permalink
rewrite set_properties
Browse files Browse the repository at this point in the history
  • Loading branch information
kevinjqliu committed Mar 8, 2024
1 parent 548ca39 commit 5ceb80d
Showing 1 changed file with 3 additions and 8 deletions.
11 changes: 3 additions & 8 deletions tests/integration/test_writes.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@
from pyiceberg.catalog.sql import SqlCatalog
from pyiceberg.exceptions import NoSuchTableError
from pyiceberg.schema import Schema
from pyiceberg.table import Table, TableProperties, _dataframe_to_data_files
from pyiceberg.typedef import Properties
from pyiceberg.table import SetPropertiesUpdate, Table, TableProperties, _dataframe_to_data_files
from pyiceberg.types import (
BinaryType,
BooleanType,
Expand Down Expand Up @@ -391,11 +391,6 @@ def get_data_files_count(identifier: str) -> int:
"""
).count()

def set_table_properties(tbl: Table, properties: Properties) -> Table:
with tbl.transaction() as transaction:
transaction._apply((SetPropertiesUpdate(updates=properties),))
return tbl

# writes 1 data file since the table is smaller than default target file size
assert arrow_table_with_null.nbytes < TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT
tbl.overwrite(arrow_table_with_null)
Expand All @@ -409,15 +404,15 @@ def set_table_properties(tbl: Table, properties: Properties) -> Table:

# writes multiple data files once target file size is overridden
target_file_size = arrow_table_with_null.nbytes
tbl = set_table_properties(tbl, {TableProperties.WRITE_TARGET_FILE_SIZE_BYTES: str(target_file_size)})
tbl = tbl.transaction().set_properties({TableProperties.WRITE_TARGET_FILE_SIZE_BYTES: str(target_file_size)}).commit_transaction()
assert str(target_file_size) == tbl.properties.get(TableProperties.WRITE_TARGET_FILE_SIZE_BYTES)
assert target_file_size < bigger_arrow_tbl.nbytes
tbl.overwrite(bigger_arrow_tbl)
assert get_data_files_count(identifier) == 10

# writes half the number of data files when target file size doubles
target_file_size = arrow_table_with_null.nbytes * 2
tbl = set_table_properties(tbl, {TableProperties.WRITE_TARGET_FILE_SIZE_BYTES: str(target_file_size)})
tbl = tbl.transaction().set_properties({TableProperties.WRITE_TARGET_FILE_SIZE_BYTES: str(target_file_size)}).commit_transaction()
assert str(target_file_size) == tbl.properties.get(TableProperties.WRITE_TARGET_FILE_SIZE_BYTES)
assert target_file_size < bigger_arrow_tbl.nbytes
tbl.overwrite(bigger_arrow_tbl)
Expand Down

0 comments on commit 5ceb80d

Please sign in to comment.