Merge branch 'apache:main' into python-3.12

Author: steinsgateted

.github/workflows/python-ci-docs.yml

@@ -31,7 +31,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
       - name: Install

.github/workflows/python-ci.yml

@@ -40,7 +40,7 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install poetry
       run: make install-poetry
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python }}
         cache: poetry

.github/workflows/python-release.yml

@@ -41,7 +41,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.8'
 

dev/Dockerfile

@@ -36,7 +36,7 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$
 RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events
 WORKDIR ${SPARK_HOME}
 
-ENV SPARK_VERSION=3.4.1
+ENV SPARK_VERSION=3.4.2
 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.4_2.12
 ENV ICEBERG_VERSION=1.4.0
 ENV AWS_SDK_VERSION=2.20.18

mkdocs/docs/api.md

@@ -54,12 +54,6 @@ catalog = load_catalog(
 )
 ```
 
-If the catalog has not been initialized before, you need to run:
-
-```python
-catalog.create_tables()
-```
-
 Let's create a namespace:
 
 ```python

mkdocs/docs/configuration.md

@@ -32,13 +32,19 @@ There are three ways to pass in configuration:
 - Through environment variables
 - By passing in credentials through the CLI or the Python API
 
-The configuration file is recommended since that's the most transparent way. If you prefer environment configuration:
+The configuration file is recommended since that's the easiest way to manage the credentials.
+
+Another option is through environment variables:
 
 ```sh
 export PYICEBERG_CATALOG__DEFAULT__URI=thrift://localhost:9083
+export PYICEBERG_CATALOG__DEFAULT__S3__ACCESS_KEY_ID=username
+export PYICEBERG_CATALOG__DEFAULT__S3__SECRET_ACCESS_KEY=password
 ```
 
-The environment variable picked up by Iceberg starts with `PYICEBERG_` and then follows the yaml structure below, where a double underscore `__` represents a nested field.
+The environment variable picked up by Iceberg starts with `PYICEBERG_` and then follows the yaml structure below, where a double underscore `__` represents a nested field, and the underscore `_` is converted into a dash `-`.
+
+For example, `PYICEBERG_CATALOG__DEFAULT__S3__ACCESS_KEY_ID`, sets `s3.access-key-id` on the `default` catalog.
 
 ## FileIO
 
@@ -80,8 +86,6 @@ For the FileIO there are several configuration options available:
 
 ### Azure Data lake
 
-### Azure Data lake
-
 | Key                     | Example                                                                                   | Description                                                                                                                                                                                                                                                                            |
 | ----------------------- | ----------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | adlfs.connection-string | AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqF...;BlobEndpoint=http://localhost/ | A [connection string](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string). This could be used to use FileIO with any adlfs-compatible object storage service that has a different endpoint (like [azurite](https://github.com/azure/azurite)). |

mkdocs/requirements.txt

@@ -16,13 +16,13 @@
 # under the License.
 
 mkdocs==1.5.3
-griffe==0.38.0
+griffe==0.38.1
 jinja2==3.1.2
 mkdocstrings==0.24.0
 mkdocstrings-python==1.7.5
 mkdocs-literate-nav==0.6.1
 mkdocs-autorefs==0.5.0
 mkdocs-gen-files==0.5.0
-mkdocs-material==9.4.12
+mkdocs-material==9.4.14
 mkdocs-material-extensions==1.3.1
 mkdocs-section-index==0.3.8

poetry.lock

@@ -31,7 +31,7 @@
     union,
     update,
 )
-from sqlalchemy.exc import IntegrityError
+from sqlalchemy.exc import IntegrityError, OperationalError
 from sqlalchemy.orm import (
     DeclarativeBase,
     Mapped,
@@ -98,6 +98,18 @@ def __init__(self, name: str, **properties: str):
             raise NoSuchPropertyException("SQL connection URI is required")
         self.engine = create_engine(uri_prop, echo=True)
 
+        self._ensure_tables_exist()
+
+    def _ensure_tables_exist(self) -> None:
+        with Session(self.engine) as session:
+            for table in [IcebergTables, IcebergNamespaceProperties]:
+                stmt = select(1).select_from(table)
+                try:
+                    session.scalar(stmt)
+                except OperationalError:
+                    self.create_tables()
+                    return
+
     def create_tables(self) -> None:
         SqlCatalogBaseTable.metadata.create_all(self.engine)
 

pyiceberg/catalog/sql.py

@@ -369,10 +369,6 @@ def table(ctx: Context, identifier: str, property_name: str) -> None:  # noqa: F
     catalog, output = _catalog_and_output(ctx)
     table = catalog.load_table(identifier)
     if property_name in table.metadata.properties:
-        # We should think of the process here
-        # Do we want something similar as in Java:
-        # https://github.com/apache/iceberg/blob/master/api/src/main/java/org/apache/iceberg/Table.java#L178
-        del table.metadata.properties
         output.exception(NotImplementedError("Writing is WIP"))
         ctx.exit(1)
     else: