diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 7541360bf2..65eb42d779 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -17,21 +17,184 @@ # under the License. # -name: "Python Release" +name: "Python Build Release Candidate" on: + push: + tags: + # Trigger this workflow when tag follows the versioning format: pyiceberg-..rc + # Example valid tags: pyiceberg-0.8.0rc2, pyiceberg-1.0.0rc1 + - 'pyiceberg-[0-9]+.[0-9]+.[0-9]+rc[0-9]+' workflow_dispatch: inputs: version: - description: 'Version' + description: 'Version (e.g., 0.8.0)' type: string - default: 'main' - + required: true + rc: + description: 'Release Candidate (RC) (e.g., 1)' + type: number + required: true jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} + validate-inputs: + runs-on: ubuntu-latest + outputs: + VERSION: ${{ steps.validate-inputs.outputs.VERSION }} # e.g. 0.8.0 + RC: ${{ steps.validate-inputs.outputs.RC }} # e.g. 1 + steps: + - name: Validate and Extract Version and RC + id: validate-inputs + run: | + if [ "$GITHUB_EVENT_NAME" = "push" ]; then + echo "Workflow triggered by tag push." + TAG=${GITHUB_REF#refs/tags/} # Extract the tag name + VERSION_AND_RC=${TAG#pyiceberg-} # Remove the 'pyiceberg-' prefix + VERSION=${VERSION_AND_RC%rc*} # Extract VERSION by removing everything after 'rc' + RC=${VERSION_AND_RC#*rc} # Extract RC by keeping everything after 'rc' + + if [[ -z "$VERSION" || -z "$RC" ]]; then + echo "Error: Unable to parse VERSION or RC from tag ($TAG). Ensure the tag format is correct." + exit 1 + fi + else + echo "Workflow triggered manually via workflow_dispatch." + VERSION="${{ github.event.inputs.version }}" + RC="${{ github.event.inputs.rc }}" + + # Validate version (e.g., 1.0.0) + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Error: version ($VERSION) must be in the format: .." + exit 1 + fi + + # Validate rc (e.g., 1) + if [[ ! "$RC" =~ ^[0-9]+$ ]]; then + echo "Error: rc ($RC) must be in the format: " + exit 1 + fi + fi + + # Export variables for future steps + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + echo "RC=$RC" >> $GITHUB_OUTPUT + + - name: Display Extracted Version and RC + run: | + echo "Using Version: ${{ steps.validate-inputs.outputs.VERSION }}" + echo "Using RC: ${{ steps.validate-inputs.outputs.RC }}" + + validate-library-version: + runs-on: ubuntu-latest + needs: + - validate-inputs + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install Poetry + run: | + pip install poetry + + - name: Validate current pyiceberg version + env: + VERSION: ${{ needs.validate-inputs.outputs.VERSION }} + run: | + # Extract the current version from Poetry + current_pyiceberg_version=$(poetry version --short) + echo "Detected Poetry version: $current_pyiceberg_version" + + # Compare the input version with the Poetry version + if [[ "$VERSION" != "$current_pyiceberg_version" ]]; then + echo "Error: Input version ($VERSION) does not match the Poetry version ($current_pyiceberg_version)" + exit 1 + fi + + # SVN + svn_build_artifacts: + name: Build artifacts for SVN on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: + - validate-inputs + - validate-library-version + strategy: + matrix: + os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - uses: actions/setup-python@v5 + with: + python-version: | + 3.9 + 3.10 + 3.11 + 3.12 + + - name: Install poetry + run: pip install poetry + + # Publish the source distribution with the version that's in + # the repository, otherwise the tests will fail + - name: Compile source distribution + run: python3 -m poetry build --format=sdist + if: startsWith(matrix.os, 'ubuntu') + + - name: Build wheels + uses: pypa/cibuildwheel@v2.22.0 + with: + output-dir: wheelhouse + config-file: "pyproject.toml" + env: + # Ignore 32 bit architectures + CIBW_ARCHS: "auto64" + CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13" + CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1" + CIBW_TEST_EXTRAS: "s3fs,glue" + CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py" + # There is an upstream issue with installing on MacOSX + # https://github.com/pypa/cibuildwheel/issues/1603 + # Ignore tests for pypy since not all dependencies are compiled for it + # and would require a local rust build chain + CIBW_TEST_SKIP: "pp* *macosx*" + + - name: Add source distribution + if: startsWith(matrix.os, 'ubuntu') + run: ls -lah dist/* && cp dist/* wheelhouse/ + + - uses: actions/upload-artifact@v4 + with: + name: "svn-release-candidate-${{ matrix.os }}" + path: ./wheelhouse/* + + svn_merge_artifacts: + runs-on: ubuntu-latest + needs: + - validate-inputs + - svn_build_artifacts + steps: + - name: Merge Artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: "svn-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}" + pattern: svn-release-candidate* + delete-merged: true + + # PyPi + pypi_build_artifacts: + name: Build artifacts for PyPi on ${{ matrix.os }} runs-on: ${{ matrix.os }} + needs: + - validate-inputs + - validate-library-version strategy: matrix: os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ] @@ -39,7 +202,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 + fetch-depth: 1 - uses: actions/setup-python@v5 with: @@ -52,9 +215,11 @@ jobs: - name: Install poetry run: pip install poetry - - name: Set version - run: python -m poetry version "${{ inputs.version }}" - if: "${{ github.event.inputs.version != 'main' }}" + - name: Set version with RC + env: + VERSION: ${{ needs.validate-inputs.outputs.VERSION }} + RC: ${{ needs.validate-inputs.outputs.RC }} + run: python -m poetry version "${{ env.VERSION }}rc${{ env.RC }}" # e.g., 0.8.0rc1 # Publish the source distribution with the version that's in # the repository, otherwise the tests will fail @@ -86,15 +251,18 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: "release-${{ matrix.os }}" + name: "pypi-release-candidate-${{ matrix.os }}" path: ./wheelhouse/* - merge: + + pypi_merge_artifacts: runs-on: ubuntu-latest - needs: build_wheels + needs: + - validate-inputs + - pypi_build_artifacts steps: - name: Merge Artifacts uses: actions/upload-artifact/merge@v4 with: - name: "release-${{ github.event.inputs.version }}" - pattern: release-* + name: "pypi-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}" + pattern: pypi-release-candidate* delete-merged: true diff --git a/mkdocs/docs/assets/images/ghactions-run-workflow-pypi-upload.png b/mkdocs/docs/assets/images/ghactions-run-workflow-pypi-upload.png deleted file mode 100644 index 3e1476b5dc..0000000000 Binary files a/mkdocs/docs/assets/images/ghactions-run-workflow-pypi-upload.png and /dev/null differ diff --git a/mkdocs/docs/assets/images/ghactions-run-workflow-svn-upload.png b/mkdocs/docs/assets/images/ghactions-run-workflow-svn-upload.png deleted file mode 100644 index 4577a2dd30..0000000000 Binary files a/mkdocs/docs/assets/images/ghactions-run-workflow-svn-upload.png and /dev/null differ diff --git a/mkdocs/docs/how-to-release.md b/mkdocs/docs/how-to-release.md index bea5548748..5b93e1bb85 100644 --- a/mkdocs/docs/how-to-release.md +++ b/mkdocs/docs/how-to-release.md @@ -109,66 +109,81 @@ Create a signed tag: Replace `VERSION` and `RC` with the appropriate values for the release. ```bash -export RC=rc1 -export VERSION=0.7.0${RC} -export VERSION_WITHOUT_RC=${VERSION/rc?/} -export VERSION_BRANCH=${VERSION_WITHOUT_RC//./-} -export GIT_TAG=pyiceberg-${VERSION} +export VERSION=0.7.0 +export RC=1 -git tag -s ${GIT_TAG} -m "PyIceberg ${VERSION}" +export VERSION_WITH_RC=${VERSION}rc${RC} +export GIT_TAG=pyiceberg-${VERSION_WITH_RC} + +git tag -s ${GIT_TAG} -m "PyIceberg ${VERSION_WITH_RC}" git push git@github.com:apache/iceberg-python.git ${GIT_TAG} ``` -### Publish Release Candidate (RC) +### Create Artifacts -#### Upload to Apache Dev SVN +The [`Python Build Release Candidate` Github Action](https://github.com/apache/iceberg-python/actions/workflows/python-release.yml) will run automatically upon tag push. -##### Create Artifacts for SVN +This action will generate artifacts that will include both source distribution (`sdist`) and binary distributions (`wheels` using [`cibuildwheel`](https://github.com/pypa/cibuildwheel)) for each architectures. -Run the [`Python release` Github Action](https://github.com/apache/iceberg-python/actions/workflows/python-release.yml). +This action will generate two final artifacts: -* Tag: Use the newly created tag. -* Version: Set the `version` to `main`, as the source cannot be modified. +* `svn-release-candidate-${VERSION}rc${RC}` for SVN +* `pypi-release-candidate-${VERSION}rc${RC}` for PyPi -![Github Actions Run Workflow for SVN Upload](assets/images/ghactions-run-workflow-svn-upload.png) +If `gh` is available, watch the GitHub Action progress using: + +```bash +RUN_ID=$(gh run list --repo apache/iceberg-python --workflow "Python Build Release Candidate" --branch "${GIT_TAG}" --event push --json databaseId -q '.[0].databaseId') +echo "Waiting for workflow to complete, this will take several minutes..." +gh run watch $RUN_ID --repo apache/iceberg-python +``` -This action will generate: +and download the artifacts using: + +```bash +gh run download $RUN_ID --repo apache/iceberg-python +``` + +### Publish Release Candidate (RC) -* Source distribution (`sdist`) -* Binary distributions (`wheels`) for each architectures. These are created using [`cibuildwheel`](https://github.com/pypa/cibuildwheel) +#### Upload to Apache Dev SVN ##### Download Artifacts, Sign, and Generate Checksums -Download the ZIP file containing the artifacts from the GitHub Actions run and unzip it. +Download the SVN artifact from the GitHub Action and unzip it. -Navigate to the release directory. Sign the files and generate checksums: +Navigate to the artifact directory. Generate signature and checksum files: * `.asc` files: GPG-signed versions of each artifact to ensure authenticity. * `.sha512` files: SHA-512 checksums for verifying file integrity. ```bash -cd release-main/ - -for name in $(ls pyiceberg-*.whl pyiceberg-*.tar.gz) -do - gpg --yes --armor --output "${name}.asc" --detach-sig "${name}" - shasum -a 512 "${name}" > "${name}.sha512" -done +( + cd svn-release-candidate-${VERSION}rc${RC} + + for name in $(ls pyiceberg-*.whl pyiceberg-*.tar.gz) + do + gpg --yes --armor --output "${name}.asc" --detach-sig "${name}" + shasum -a 512 "${name}" > "${name}.sha512" + done +) ``` +The parentheses `()` create a subshell. Any changes to the directory (`cd`) are limited to this subshell, so the current directory in the parent shell remains unchanged. + ##### Upload Artifacts to Apache Dev SVN Now, upload the files from the same directory: ```bash -export SVN_TMP_DIR=/tmp/iceberg-${VERSION_BRANCH}/ +export SVN_TMP_DIR=/tmp/iceberg-${VERSION}/ svn checkout https://dist.apache.org/repos/dist/dev/iceberg $SVN_TMP_DIR -export SVN_TMP_DIR_VERSIONED=${SVN_TMP_DIR}pyiceberg-$VERSION/ +export SVN_TMP_DIR_VERSIONED=${SVN_TMP_DIR}pyiceberg-$VERSION_WITH_RC/ mkdir -p $SVN_TMP_DIR_VERSIONED -cp * $SVN_TMP_DIR_VERSIONED +cp svn-release-candidate-${VERSION}rc${RC}/* $SVN_TMP_DIR_VERSIONED svn add $SVN_TMP_DIR_VERSIONED -svn ci -m "PyIceberg ${VERSION}" ${SVN_TMP_DIR_VERSIONED} +svn ci -m "PyIceberg ${VERSION_WITH_RC}" ${SVN_TMP_DIR_VERSIONED} ``` Verify the artifact is uploaded to [https://dist.apache.org/repos/dist/dev/iceberg](https://dist.apache.org/repos/dist/dev/iceberg/). @@ -183,22 +198,13 @@ svn delete https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg- @@ -208,7 +214,7 @@ Upload release candidate to PyPi. This **won't** bump the version for everyone t ```bash -twine upload release-${VERSION}/* +twine upload pypi-release-candidate-${VERSION}rc${RC}/* ``` Verify the artifact is uploaded to [PyPi](https://pypi.org/project/pyiceberg/#history). @@ -226,10 +232,10 @@ export LAST_COMMIT_ID=$(git rev-list ${GIT_TAG} 2> /dev/null | head -n 1) cat << EOF > release-announcement-email.txt To: dev@iceberg.apache.org -Subject: [VOTE] Release Apache PyIceberg $VERSION +Subject: [VOTE] Release Apache PyIceberg $VERSION_WITH_RC Hi Everyone, -I propose that we release the following RC as the official PyIceberg $VERSION_WITHOUT_RC release. +I propose that we release the following RC as the official PyIceberg $VERSION release. A summary of the high level features: @@ -243,7 +249,7 @@ The commit ID is $LAST_COMMIT_ID The release tarball, signature, and checksums are here: -* https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-$VERSION/ +* https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-$VERSION_WITH_RC/ You can find the KEYS file here: @@ -251,9 +257,9 @@ You can find the KEYS file here: Convenience binary artifacts are staged on pypi: -https://pypi.org/project/pyiceberg/$VERSION/ +https://pypi.org/project/pyiceberg/$VERSION_WITH_RC/ -And can be installed using: pip3 install pyiceberg==$VERSION +And can be installed using: pip3 install pyiceberg==$VERSION_WITH_RC Instructions for verifying a release can be found here: @@ -262,7 +268,7 @@ Instructions for verifying a release can be found here: Please download, verify, and test. Please vote in the next 72 hours. -[ ] +1 Release this as PyIceberg $VERSION_WITHOUT_RC +[ ] +1 Release this as PyIceberg $VERSION [ ] +0 [ ] -1 Do not release this because... EOF @@ -302,10 +308,10 @@ Kind regards, ```bash -export SVN_DEV_DIR_VERSIONED="https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-${VERSION}" -export SVN_RELEASE_DIR_VERSIONED="https://dist.apache.org/repos/dist/release/iceberg/pyiceberg-${VERSION_WITHOUT_RC}" +export SVN_DEV_DIR_VERSIONED="https://dist.apache.org/repos/dist/dev/iceberg/pyiceberg-${VERSION_WITH_RC}" +export SVN_RELEASE_DIR_VERSIONED="https://dist.apache.org/repos/dist/release/iceberg/pyiceberg-${VERSION}" -svn mv ${SVN_DEV_DIR_VERSIONED} ${SVN_RELEASE_DIR_VERSIONED} -m "PyIceberg: Add release ${VERSION_WITHOUT_RC}" +svn mv ${SVN_DEV_DIR_VERSIONED} ${SVN_RELEASE_DIR_VERSIONED} -m "PyIceberg: Add release ${VERSION}" ``` Verify the artifact is uploaded to [https://dist.apache.org/repos/dist/release/iceberg](https://dist.apache.org/repos/dist/release/iceberg/). @@ -331,7 +337,7 @@ The latest version can be pushed to PyPi. Check out the Apache SVN and make sure ```bash svn checkout https://dist.apache.org/repos/dist/release/iceberg /tmp/iceberg-dist-release/ -cd /tmp/iceberg-dist-release/pyiceberg-${VERSION_WITHOUT_RC} +cd /tmp/iceberg-dist-release/pyiceberg-${VERSION} twine upload pyiceberg-*.whl pyiceberg-*.tar.gz ``` @@ -359,18 +365,6 @@ This Python release can be downloaded from: https://pypi.org/project/pyiceberg/< Thanks to everyone for contributing! ``` -### Release the docs - -Run the [`Release Docs` Github Action](https://github.com/apache/iceberg-python/actions/workflows/python-release-docs.yml). - -### Update the Github template - -Make sure to create a PR to update the [GitHub issues template](https://github.com/apache/iceberg-python/blob/main/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml) with the latest version. - -### Update the integration tests - -Ensure to update the `PYICEBERG_VERSION` in the [Dockerfile](https://github.com/apache/iceberg-python/blob/main/dev/Dockerfile). - ### Create a Github Release Note Create a [new Release Note](https://github.com/apache/iceberg-python/releases/new) on the iceberg-python Github repository. @@ -385,6 +379,18 @@ Then, select the previous release version as the **Previous tag** to use the dif **Set as the latest release** and **Publish**. +### Release the docs + +Run the [`Release Docs` Github Action](https://github.com/apache/iceberg-python/actions/workflows/python-release-docs.yml). + +### Update the Github template + +Make sure to create a PR to update the [GitHub issues template](https://github.com/apache/iceberg-python/blob/main/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml) with the latest version. + +### Update the integration tests + +Ensure to update the `PYICEBERG_VERSION` in the [Dockerfile](https://github.com/apache/iceberg-python/blob/main/dev/Dockerfile). + ## Misc ### Set up GPG key and Upload to Apache Iceberg KEYS file