Skip to content

Commit 3e91bcf

Browse files
evertlammertsevertlammerts
committed
Adjust release workflow
1 parent a132b75 commit 3e91bcf

File tree

4 files changed

+254
-29
lines changed

4 files changed

+254
-29
lines changed

‎.github/workflows/packaging_sdist.yml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: Sdist packaging
2+
on:
3+
workflow_call:
4+
inputs:
5+
testsuite:
6+
type: string
7+
description: Testsuite to run (none, fast, all)
8+
required: true
9+
default: all
10+
git-ref:
11+
type: string
12+
description: Git ref of the DuckDB python package
13+
required: false
14+
duckdb-git-ref:
15+
type: string
16+
description: Git ref of DuckDB
17+
required: false
18+
set-version:
19+
description: Force version (vX.Y.Z-((rc|post)N))
20+
required: false
21+
type: string
22+
outputs:
23+
package-version:
24+
description: The version of the DuckDB Python package
25+
value: ${{ jobs.build_sdist.outputs.pkg_version }}
26+
duckdb-version:
27+
description: The version of DuckDB that was packaged
28+
value: ${{ jobs.build_sdist.outputs.duckdb_version }}
29+
30+
jobs:
31+
build_sdist:
32+
name: Build sdist
33+
runs-on: ubuntu-24.04
34+
outputs:
35+
pkg_version: ${{ steps.versioning.outputs.pkg_version }}
36+
duckdb_version: ${{ steps.versioning.outputs.duckdb_version }}
37+
steps:
38+
39+
- name: Checkout DuckDB Python
40+
uses: actions/checkout@v4
41+
with:
42+
ref: ${{ inputs.git-ref }}
43+
fetch-depth: 0
44+
submodules: true
45+
46+
- name: Checkout DuckDB
47+
shell: bash
48+
run: |
49+
cd external/duckdb
50+
git fetch origin
51+
git checkout ${{ inputs.duckdb-git-ref }}
52+
53+
- name: Set OVERRIDE_GIT_DESCRIBE
54+
if: ${{ inputs.set-version != '' }}
55+
run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
56+
57+
- name: Install Astral UV
58+
uses: astral-sh/setup-uv@v6
59+
with:
60+
version: "0.7.14"
61+
python-version: 3.11
62+
63+
- name: Build sdist
64+
run: uv build --sdist
65+
66+
- name: Install sdist
67+
run: |
68+
cd ${{ runner.temp }}
69+
uv venv
70+
uv pip install ${{ github.workspace }}/dist/duckdb-*.tar.gz
71+
72+
- name: Test sdist
73+
if: ${{ inputs.testsuite != 'none' }}
74+
run: |
75+
# install the test requirements
76+
uv export --only-group test --no-emit-project --output-file ${{ runner.temp }}/pylock.toml --quiet
77+
cd ${{ runner.temp }}
78+
uv pip install -r pylock.toml
79+
# run tests
80+
tests_root="${{ github.workspace }}/tests"
81+
tests_dir="${tests_root}${{ inputs.testsuite == 'fast' && '/fast' || '/' }}"
82+
uv run --verbose pytest $tests_dir --verbose --ignore=${tests_root}/stubs
83+
84+
- id: versioning
85+
run: |
86+
cd ${{ runner.temp }}
87+
echo "pkg_version=$( .venv/bin/python -c 'import duckdb; print(duckdb.__version__)' )" >> $GITHUB_OUTPUT
88+
echo "duckdb_version=$( .venv/bin/python -c 'import duckdb; print(duckdb.duckdb_version)' )" >> $GITHUB_OUTPUT
89+
90+
- uses: actions/upload-artifact@v4
91+
with:
92+
name: sdist
93+
path: dist/*.tar.gz
94+
compression-level: 0

‎.github/workflows/packaging_wheels.yml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: Wheels packaging
2+
on:
3+
workflow_call:
4+
inputs:
5+
minimal:
6+
type: boolean
7+
description: Build a minimal set of wheels to do a sanity check
8+
default: false
9+
testsuite:
10+
type: string
11+
description: Testsuite to run (none, fast, all)
12+
required: true
13+
default: all
14+
git-ref:
15+
type: string
16+
description: Git ref of the DuckDB python package
17+
required: false
18+
duckdb-git-ref:
19+
type: string
20+
description: Git ref of DuckDB
21+
required: false
22+
set-version:
23+
description: Force version (vX.Y.Z-((rc|post)N))
24+
required: false
25+
type: string
26+
27+
jobs:
28+
build_wheels:
29+
name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
30+
strategy:
31+
fail-fast: false
32+
matrix:
33+
python: [ cp39, cp310, cp311, cp312, cp313 ]
34+
platform:
35+
- { os: windows-2025, arch: amd64, cibw_system: win }
36+
- { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux }
37+
- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux }
38+
- { os: macos-15, arch: arm64, cibw_system: macosx }
39+
- { os: macos-15, arch: universal2, cibw_system: macosx }
40+
- { os: macos-13, arch: x86_64, cibw_system: macosx }
41+
minimal:
42+
- ${{ inputs.minimal }}
43+
exclude:
44+
- { minimal: true, python: cp310 }
45+
- { minimal: true, python: cp311 }
46+
- { minimal: true, python: cp312 }
47+
- { minimal: true, platform: { arch: universal2 } }
48+
runs-on: ${{ matrix.platform.os }}
49+
env:
50+
CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
51+
CIBW_TEST_SOURCES: tests
52+
CIBW_BEFORE_TEST: >
53+
uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} &&
54+
uv pip install -r pylock.toml
55+
CIBW_TEST_COMMAND: >
56+
uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs
57+
58+
steps:
59+
- name: Checkout DuckDB Python
60+
uses: actions/checkout@v4
61+
with:
62+
ref: ${{ inputs.git-ref }}
63+
fetch-depth: 0
64+
submodules: true
65+
66+
- name: Checkout DuckDB
67+
shell: bash
68+
run: |
69+
cd external/duckdb
70+
git fetch origin
71+
git checkout ${{ inputs.duckdb-git-ref }}
72+
73+
# Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
74+
- name: Set OVERRIDE_GIT_DESCRIBE
75+
if: ${{ inputs.set-version != '' }}
76+
run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
77+
78+
# Install Astral UV, which will be used as build-frontend for cibuildwheel
79+
- uses: astral-sh/setup-uv@v6
80+
with:
81+
version: "0.7.14"
82+
enable-cache: false
83+
cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
84+
85+
- name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
86+
uses: pypa/[email protected]
87+
env:
88+
CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
89+
CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
90+
91+
- name: Upload wheel
92+
uses: actions/upload-artifact@v4
93+
with:
94+
name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
95+
path: wheelhouse/*.whl
96+
compression-level: 0

‎.github/workflows/release.yml‎

Lines changed: 61 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,41 @@ defaults:
2828
shell: bash
2929

3030
jobs:
31-
build_and_test:
31+
build_sdist:
32+
name: Build an sdist and determine versions
33+
uses: ./.github/workflows/packaging_sdist.yml
34+
with:
35+
testsuite: all
36+
git-ref: ${{ github.ref }}
37+
duckdb-git-ref: ${{ inputs.duckdb-sha }}
38+
set-version: ${{ inputs.stable-version }}
39+
40+
check_pypi:
41+
name: Check whether this version was already released
42+
needs: build_sdist
43+
outputs:
44+
can_upload: ${{ steps.index_check.outputs.can_upload }}
45+
runs-on: ubuntu-latest
46+
steps:
47+
- id: index_check
48+
run: |
49+
# Check PyPI whether the release we're building is already present
50+
pypi_hostname="${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org"
51+
pkg_version=${{ needs.build_sdist.outputs.package-version }}
52+
http_status=$( curl -s -o /dev/null -w "%{http_code}" https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json )
53+
if [ "$http_status" == "200" ]; then
54+
echo "::warning::DuckDB ${pkg_version} is already present on ${pypi_hostname}, will not release"
55+
echo "can_upload=true" >> $GITHUB_GITHUB_OUTPUT
56+
else
57+
echo "can_upload=false" >> $GITHUB_GITHUB_OUTPUT
58+
fi
59+
60+
61+
build_wheels:
3262
name: Build and test releases
33-
uses: ./.github/workflows/packaging.yml
63+
needs: check_pypi
64+
if: ${{ needs.check_pypi.outputs.can_upload }}
65+
uses: ./.github/workflows/packaging_wheels.yml
3466
with:
3567
minimal: false
3668
testsuite: all
@@ -41,8 +73,10 @@ jobs:
4173
upload_s3:
4274
name: Upload Artifacts to S3
4375
runs-on: ubuntu-latest
44-
needs: [build_and_test]
76+
needs: [build_sdist, build_wheels]
4577
if: ${{ github.repository_owner == 'duckdb' && ( inputs.pypi-index == 'prod' || inputs.store-s3 ) }}
78+
outputs:
79+
s3-upload-url: ${{ steps.input.outputs.s3_upload_url }}
4680
steps:
4781
- name: Fetch artifacts
4882
uses: actions/download-artifact@v4
@@ -56,11 +90,8 @@ jobs:
5690
run: |
5791
sha=${{ github.sha }}
5892
dsha=${{ inputs.duckdb-sha }}
59-
version=$(basename artifacts/*.tar.gz | sed 's/duckdb-\(.*\).tar.gz/\1/g')
93+
version=${{ needs.build_sdist.outputs.package-version }}
6094
url="s3://duckdb-staging/python/${version}/${sha:0:10}-duckdb-${dsha:0:10}/"
61-
echo "short_sha=${sha:0:10}" >> $GITHUB_OUTPUT
62-
echo "short_dsha=${dsha:0:10}" >> $GITHUB_OUTPUT
63-
echo "version=${version}" >> $GITHUB_OUTPUT
6495
echo "s3_upload_url=${url}" >> $GITHUB_OUTPUT
6596
6697
- name: Authenticate with AWS
@@ -74,18 +105,10 @@ jobs:
74105
run: |
75106
aws s3 cp artifacts ${{ steps.input.outputs.s3_upload_url }} --recursive
76107
77-
- name: S3 Upload Summary
78-
run : |
79-
echo "## S3 Upload Summary" >> $GITHUB_STEP_SUMMARY
80-
echo "* Version: ${{ steps.input.outputs.version }}" >> $GITHUB_STEP_SUMMARY
81-
echo "* SHA: ${{ steps.input.outputs.short_sha }}" >> $GITHUB_STEP_SUMMARY
82-
echo "* DuckDB SHA: ${{ steps.input.outputs.short_dsha }}" >> $GITHUB_STEP_SUMMARY
83-
echo "* S3 URL: ${{ steps.input.outputs.s3_upload_url }}" >> $GITHUB_STEP_SUMMARY
84-
85108
determine_environment:
86109
name: Determine the Github Actions environment to use
87110
runs-on: ubuntu-latest
88-
needs: build_and_test
111+
needs: [build_sdist, build_wheels]
89112
outputs:
90113
env_name: ${{ steps.set-env.outputs.env_name }}
91114
steps:
@@ -105,7 +128,7 @@ jobs:
105128
fi
106129
;;
107130
*)
108-
echo "Error: invalid combination of inputs.pypi-index='${{ inputs.pypi-index }}' and inputs.stable-version='${{ inputs.stable-version }}'" >&2
131+
echo "::error::invalid combination of inputs.pypi-index='${{ inputs.pypi-index }}' and inputs.stable-version='${{ inputs.stable-version }}'"
109132
exit 1
110133
;;
111134
esac
@@ -139,17 +162,9 @@ jobs:
139162
packages-dir: packages
140163
verbose: 'true'
141164

142-
- name: PyPI Upload Summary
143-
run : |
144-
version=$(basename packages/*.tar.gz | sed 's/duckdb-\(.*\).tar.gz/\1/g')
145-
echo "## PyPI Upload Summary" >> $GITHUB_STEP_SUMMARY
146-
echo "* Version: ${version}" >> $GITHUB_STEP_SUMMARY
147-
echo "* PyPI Host: ${{ vars.PYPI_HOST }}" >> $GITHUB_STEP_SUMMARY
148-
echo "* CI Environment: ${{ needs.determine_environment.outputs.env_name }}" >> $GITHUB_STEP_SUMMARY
149-
150165
cleanup_nightlies:
151166
name: Remove Nightlies from PyPI
152-
needs: [determine_environment, publish_pypi]
167+
needs: [ determine_environment, publish_pypi ]
153168
if: ${{ inputs.stable-version == '' }}
154169
uses: ./.github/workflows/cleanup_pypi.yml
155170
with:
@@ -158,3 +173,23 @@ jobs:
158173
# reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206
159174
PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}}
160175
PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}}
176+
177+
summary:
178+
name: Release summary
179+
runs-on: ubuntu-latest
180+
needs: [build_sdist, check_pypi, upload_s3, determine_environment]
181+
if: true
182+
steps:
183+
- run: |
184+
sha=${{ github.sha }}
185+
dsha=${{ inputs.duckdb-sha }}
186+
echo "## Release Summary" >> $GITHUB_STEP_SUMMARY
187+
echo "* Package Version: ${{ needs.build_sdist.outputs.package-version }} (${sha:0:10})" >> $GITHUB_STEP_SUMMARY
188+
echo "* DuckDB Version: ${{ needs.build_sdist.outputs.duckdb-version }} (${dsha:0:10})" >> $GITHUB_STEP_SUMMARY
189+
echo "* PyPI: ${{ vars.PYPI_HOST }}" >> $GITHUB_STEP_SUMMARY
190+
if [ "${{ needs.check_pypi.outputs.can_upload }}" == "false" ]; then
191+
echo " * Package already exists on this PyPI, nothing was uploaded" >> $GITHUB_STEP_SUMMARY
192+
exit 0
193+
fi
194+
echo "* CI Environment: ${{ needs.determine_environment.outputs.env_name }}" >> $GITHUB_STEP_SUMMARY
195+
echo "* S3 URL: ${{ needs.upload_s3.outputs.s3_upload_url }}" >> $GITHUB_STEP_SUMMARY

‎duckdb_packaging/setuptools_scm_version.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ def forced_version_from_env():
8686

8787
if override_value:
8888
print(f"[versioning] Found {OVERRIDE_GIT_DESCRIBE_ENV_VAR}={override_value}")
89-
pep440_version = _git_describe_override_to_pep_440(override_value)
89+
pep440_version = git_describe_output_to_pep_440(override_value)
9090
os.environ[SCM_PRETEND_ENV_VAR] = pep440_version
9191
print(f"[versioning] Injected {SCM_PRETEND_ENV_VAR}={pep440_version}")
9292
elif SCM_PRETEND_ENV_VAR in os.environ:
@@ -99,8 +99,8 @@ def forced_version_from_env():
9999
return pep440_version
100100

101101

102-
def _git_describe_override_to_pep_440(override_value: str) -> str:
103-
"""Process the OVERRIDE_GIT_DESCRIBE value."""
102+
def git_describe_output_to_pep_440(override_value: str) -> str:
103+
"""Process the git describe output."""
104104
describe_pattern = re.compile(
105105
r"""
106106
^v(?P<tag>\d+\.\d+\.\d+(?:-post\d+|-rc\d+)?) # vX.Y.Z or vX.Y.Z-postN or vX.Y.Z-rcN

0 commit comments

Comments
 (0)