Skip to content

Commit 95a63cb

Browse files
soumya-ghoshsoumya-ghosh
committed
Merge branch 'main' into all_files_metadata_tables
2 parents fb10185 + 201057e commit 95a63cb

File tree

122 files changed

+9000
-4047
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

122 files changed

+9000
-4047
lines changed

‎.asf.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ github:
3434
rebase: true
3535
protected_branches:
3636
main:
37+
required_status_checks:
38+
# strict means "Require branches to be up to date before merging".
39+
strict: true
40+
3741
required_pull_request_reviews:
3842
required_approving_review_count: 1
3943

‎.github/ISSUE_TEMPLATE/iceberg_bug_report.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ body:
2828
description: What Apache Iceberg version are you using?
2929
multiple: false
3030
options:
31-
- "0.8.1 (latest release)"
31+
- "0.9.0 (latest release)"
32+
- "0.8.1"
3233
- "0.8.0"
3334
- "0.7.1"
3435
- "0.7.0"

‎.github/dependabot.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ updates:
2222
- package-ecosystem: "pip"
2323
directory: "/"
2424
schedule:
25-
interval: "daily"
25+
interval: "weekly"
2626
open-pull-requests-limit: 50
2727
- package-ecosystem: "github-actions"
2828
directory: "/"
2929
schedule:
30-
interval: "daily"
30+
interval: "weekly"

‎.github/pull_request_template.md

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<!--
2+
Thanks for opening a pull request!
3+
-->
4+
5+
<!-- In the case this PR will resolve an issue, please replace ${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
6+
<!-- Closes #${GITHUB_ISSUE_ID} -->
7+
8+
# Rationale for this change
9+
10+
# Are these changes tested?
11+
12+
# Are there any user-facing changes?
13+
14+
<!-- In the case of user-facing changes, please add the changelog label. -->

‎.github/workflows/pypi-build-artifacts.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ jobs:
6262
if: startsWith(matrix.os, 'ubuntu')
6363

6464
- name: Build wheels
65-
uses: pypa/cibuildwheel@v2.22.0
65+
uses: pypa/cibuildwheel@v2.23.3
6666
with:
6767
output-dir: wheelhouse
6868
config-file: "pyproject.toml"

‎.github/workflows/python-ci.yml

+21-1
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,29 @@ jobs:
5858
python-version: ${{ matrix.python }}
5959
cache: poetry
6060
cache-dependency-path: ./poetry.lock
61+
- name: Install system dependencies
62+
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
6163
- name: Install
6264
run: make install-dependencies
6365
- name: Linters
6466
run: make lint
6567
- name: Tests
66-
run: make test-coverage
68+
run: make test-coverage-unit
69+
70+
integration-test:
71+
runs-on: ubuntu-22.04
72+
strategy:
73+
matrix:
74+
python: ['3.9', '3.10', '3.11', '3.12']
75+
76+
steps:
77+
- uses: actions/checkout@v4
78+
- name: Install system dependencies
79+
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
80+
- name: Install
81+
run: make install
82+
- name: Run integration tests
83+
run: make test-coverage-integration
84+
- name: Show debug logs
85+
if: ${{ failure() }}
86+
run: docker compose -f dev/docker-compose.yml logs

‎.github/workflows/python-integration.yml

-59
This file was deleted.

‎.github/workflows/svn-build-artifacts.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ jobs:
5757
if: startsWith(matrix.os, 'ubuntu')
5858

5959
- name: Build wheels
60-
uses: pypa/cibuildwheel@v2.22.0
60+
uses: pypa/cibuildwheel@v2.23.3
6161
with:
6262
output-dir: wheelhouse
6363
config-file: "pyproject.toml"

‎Makefile

+14-7
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,20 @@
1919
help: ## Display this help
2020
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
2121

22-
install-poetry: ## Install poetry if the user has not done that yet.
23-
@if ! command -v poetry &> /dev/null; then \
24-
echo "Poetry could not be found. Installing..."; \
25-
pip install --user poetry==2.0.1; \
26-
else \
27-
echo "Poetry is already installed."; \
28-
fi
22+
POETRY_VERSION = 2.1.1
23+
install-poetry: ## Ensure Poetry is installed and the correct version is being used.
24+
@if ! command -v poetry &> /dev/null; then \
25+
echo "Poetry could not be found. Installing..."; \
26+
pip install --user poetry==$(POETRY_VERSION); \
27+
else \
28+
INSTALLED_VERSION=$$(pip show poetry | grep Version | awk '{print $$2}'); \
29+
if [ "$$INSTALLED_VERSION" != "$(POETRY_VERSION)" ]; then \
30+
echo "Poetry version $$INSTALLED_VERSION does not match required version $(POETRY_VERSION). Updating..."; \
31+
pip install --user --upgrade poetry==$(POETRY_VERSION); \
32+
else \
33+
echo "Poetry version $$INSTALLED_VERSION is already installed."; \
34+
fi \
35+
fi
2936

3037
install-dependencies: ## Install dependencies including dev, docs, and all extras
3138
poetry install --all-extras

‎dev/.rat-excludes

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
.github/*
12
.rat-excludes
23
build
34
.git

‎dev/Dockerfile

+8-6
Original file line numberDiff line numberDiff line change
@@ -37,21 +37,23 @@ RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/
3737
WORKDIR ${SPARK_HOME}
3838

3939
# Remember to also update `tests/conftest`'s spark setting
40-
ENV SPARK_VERSION=3.5.3
40+
ENV SPARK_VERSION=3.5.4
4141
ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
42-
ENV ICEBERG_VERSION=1.6.0
43-
ENV PYICEBERG_VERSION=0.8.1
42+
ENV ICEBERG_VERSION=1.9.0
43+
ENV PYICEBERG_VERSION=0.9.0
4444

4545
RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
4646
&& tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \
4747
&& rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz
4848

4949
# Download iceberg spark runtime
50-
RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar -Lo iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
51-
&& mv iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar /opt/spark/jars
50+
RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
51+
-Lo /opt/spark/jars/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar
52+
5253

5354
# Download AWS bundle
54-
RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
55+
RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
56+
-Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
5557

5658
COPY spark-defaults.conf /opt/spark/conf
5759
ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}"

0 commit comments

Comments
 (0)