Skip to content

Commit

Permalink
update API
Browse files Browse the repository at this point in the history
  • Loading branch information
chinmay-bhat committed Sep 10, 2024
1 parent 5b85d1c commit fc98648
Show file tree
Hide file tree
Showing 6 changed files with 14 additions and 14 deletions.
2 changes: 1 addition & 1 deletion pyiceberg/catalog/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -835,7 +835,7 @@ def purge_table(self, identifier: Union[str, Identifier]) -> None:
manifest_lists_to_delete = set()
manifests_to_delete: List[ManifestFile] = []
for snapshot in metadata.snapshots:
manifests_to_delete += snapshot.manifests(io, snapshot.manifest_list)
manifests_to_delete += snapshot.manifests(io)
if snapshot.manifest_list is not None:
manifest_lists_to_delete.add(snapshot.manifest_list)

Expand Down
2 changes: 1 addition & 1 deletion pyiceberg/cli/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ def files(self, table: Table, history: bool) -> None:
manifest_list_str = f": {snapshot.manifest_list}" if snapshot.manifest_list else ""
list_tree = snapshot_tree.add(f"Snapshot {snapshot.snapshot_id}, schema {snapshot.schema_id}{manifest_list_str}")

manifest_list = snapshot.manifests(io, manifest_list_str)
manifest_list = snapshot.manifests(io)
for manifest in manifest_list:
manifest_tree = list_tree.add(f"Manifest: {manifest.manifest_path}")
for manifest_entry in manifest.fetch_manifest_entry(io, discard_deleted=False):
Expand Down
6 changes: 5 additions & 1 deletion pyiceberg/table/snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,13 +252,17 @@ def __str__(self) -> str:

@staticmethod
@lru_cache
def manifests(io: FileIO, manifest_list: str) -> List[ManifestFile]:
def _manifests(io: FileIO, manifest_list: str) -> List[ManifestFile]:
"""Return the manifests for the given snapshot."""
if manifest_list not in (None, ""):
file = io.new_input(manifest_list)
return list(read_manifest_list(file))
return []

def manifests(self, io: FileIO) -> List[ManifestFile]:
"""Return the manifests for the given snapshot."""
return Snapshot._manifests(io, self.manifest_list)


class MetadataLogEntry(IcebergBaseModel):
metadata_file: str = Field(alias="metadata-file")
Expand Down
8 changes: 2 additions & 6 deletions tests/integration/test_partitioning_key.py
Original file line number Diff line number Diff line change
Expand Up @@ -763,14 +763,10 @@ def test_partition_key(
snapshot = iceberg_table.current_snapshot()
assert snapshot
spark_partition_for_justification = (
snapshot.manifests(iceberg_table.io, snapshot.manifest_list)[0]
.fetch_manifest_entry(iceberg_table.io)[0]
.data_file.partition
snapshot.manifests(iceberg_table.io)[0].fetch_manifest_entry(iceberg_table.io)[0].data_file.partition
)
spark_path_for_justification = (
snapshot.manifests(iceberg_table.io, snapshot.manifest_list)[0]
.fetch_manifest_entry(iceberg_table.io)[0]
.data_file.file_path
snapshot.manifests(iceberg_table.io)[0].fetch_manifest_entry(iceberg_table.io)[0].data_file.file_path
)
assert spark_partition_for_justification == expected_partition_record
assert expected_hive_partition_path_slice in spark_path_for_justification
2 changes: 1 addition & 1 deletion tests/integration/test_rest_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def test_write_sample_manifest(table_test_all_types: Table) -> None:
if test_snapshot is None:
raise ValueError("Table has no current snapshot, check the docker environment")
io = table_test_all_types.io
test_manifest_file = test_snapshot.manifests(io, test_snapshot.manifest_list)[0]
test_manifest_file = test_snapshot.manifests(io)[0]
test_manifest_entries = test_manifest_file.fetch_manifest_entry(io)
entry = test_manifest_entries[0]
test_schema = table_test_all_types.schema()
Expand Down
8 changes: 4 additions & 4 deletions tests/utils/test_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ def test_read_manifest_v1(generated_manifest_file_file_v1: str) -> None:
summary=Summary(Operation.APPEND),
schema_id=3,
)
manifest_list = snapshot.manifests(io, snapshot.manifest_list)[0]
manifest_list = snapshot.manifests(io)[0]

assert manifest_list.manifest_length == 7989
assert manifest_list.partition_spec_id == 0
Expand Down Expand Up @@ -267,7 +267,7 @@ def test_read_manifest_v2(generated_manifest_file_file_v2: str) -> None:
summary=Summary(Operation.APPEND),
schema_id=3,
)
manifest_list = snapshot.manifests(io, manifest_list=snapshot.manifest_list)[0]
manifest_list = snapshot.manifests(io)[0]

assert manifest_list.manifest_length == 7989
assert manifest_list.partition_spec_id == 0
Expand Down Expand Up @@ -336,7 +336,7 @@ def test_write_manifest(
summary=Summary(Operation.APPEND),
schema_id=3,
)
demo_manifest_file = snapshot.manifests(io, snapshot.manifest_list)[0]
demo_manifest_file = snapshot.manifests(io)[0]
manifest_entries = demo_manifest_file.fetch_manifest_entry(io)
test_schema = Schema(
NestedField(1, "VendorID", IntegerType(), False), NestedField(2, "tpep_pickup_datetime", IntegerType(), False)
Expand Down Expand Up @@ -508,7 +508,7 @@ def test_write_manifest_list(
schema_id=3,
)

demo_manifest_list = snapshot.manifests(io, snapshot.manifest_list)
demo_manifest_list = snapshot.manifests(io)
with TemporaryDirectory() as tmp_dir:
path = tmp_dir + "/manifest-list.avro"
output = io.new_output(path)
Expand Down

0 comments on commit fc98648

Please sign in to comment.