From c95a86bf9c2347fa871448c4f8fc095d1fc9790b Mon Sep 17 00:00:00 2001 From: Fokko Date: Mon, 20 Jan 2025 16:01:00 +0100 Subject: [PATCH] Add table with deletion vector --- dev/Dockerfile | 4 ++-- tests/integration/test_reads.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index baa3e09747..1b4758a339 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -47,10 +47,10 @@ RUN curl --retry 5 -s -C - https://dlcdn.apache.org/spark/spark-${SPARK_VERSION} && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz # Download iceberg spark runtime -RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.8.0-SNAPSHOT/iceberg-spark-runtime-3.5_2.12-1.8.0-20250115.001733-68.jar -Lo /opt/spark/jars/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar +RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.8.0-SNAPSHOT/iceberg-spark-runtime-3.5_2.12-1.8.0-20250120.001750-73.jar -Lo /opt/spark/jars/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar # Download AWS bundle -RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-aws-bundle/1.8.0-SNAPSHOT/iceberg-aws-bundle-1.8.0-20250115.002654-137.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar +RUN curl --retry 5 -s https://repository.apache.org/content/groups/snapshots/org/apache/iceberg/iceberg-aws-bundle/1.8.0-SNAPSHOT/iceberg-aws-bundle-1.8.0-20250120.002709-147.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar COPY spark-defaults.conf /opt/spark/conf ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}" diff --git a/tests/integration/test_reads.py b/tests/integration/test_reads.py index f2e79bae60..e4e6c2da12 100644 --- a/tests/integration/test_reads.py +++ b/tests/integration/test_reads.py @@ -950,3 +950,12 @@ def test_read_from_s3_and_local_fs(catalog: Catalog, tmp_path: PosixPath) -> Non result_table = tbl.scan().to_arrow() assert result_table["colA"].to_pylist() == ["one", "one"] + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")]) +def test_read_table_with_deletion_vector(catalog: Catalog) -> None: + tbl = catalog.load_table("default.test_deletion_vectors") + + result_table = tbl.scan().to_arrow() + result_table