release.yml

# Release is called by duckdb's InvokeCI -> NotifyExternalRepositories job
name: Release
on:
  workflow_dispatch:
    inputs:
      duckdb-python-sha:
        type: string
        description: The commit to build against (defaults to latest commit of current ref)
        required: false
      duckdb-sha:
        type: string
        description: The DuckDB submodule commit or ref to build against
        required: true
      stable-version:
        type: string
        description: Release a stable version (vX.Y.Z-((rc|post)N))
        required: false
      pypi-index:
        type: choice
        description: Which PyPI to use
        required: true
        options:
          - test
          - prod
      store-s3:
        type: boolean
        description: Also store test packages in S3 (always true for prod)
        default: false

defaults:
  run:
    shell: bash

jobs:
  build_sdist:
    name: Build an sdist and determine versions
    uses: ./.github/workflows/packaging_sdist.yml
    with:
      testsuite: all
      duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }}
      duckdb-sha: ${{ inputs.duckdb-sha }}
      set-version: ${{ inputs.stable-version }}

  workflow_state:
    name: Set state for the release workflow
    needs: build_sdist
    outputs:
      pypi_state: ${{ steps.index_check.outputs.pypi_state }}
      ci_env: ${{ steps.ci_env_check.outputs.ci_env }}
      s3_url: ${{ steps.s3_check.outputs.s3_url }}
    runs-on: ubuntu-latest
    steps:
      - id: index_check
        name: Check ${{ needs.build_sdist.outputs.package-version }} on PyPI
        run: |
          set -eu
          # Check PyPI whether the release we're building is already present
          pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org
          pkg_version=${{ needs.build_sdist.outputs.package-version }}
          url=https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json
          http_status=$( curl -s -o /dev/null -w "%{http_code}" $url || echo $? )
          if [[ $http_status == "200" ]]; then
            echo "::warning::Package version ${pkg_version} is already present on ${pypi_hostname}"
            pypi_state=VERSION_FOUND
          elif [[ $http_status == 000* ]]; then
            echo "::error::Error checking PyPI at ${url}: curl exit code ${http_status#'000'}"
            pypi_state=UNKNOWN
          else
            echo "::notice::Package version ${pkg_version} not found on ${pypi_hostname} (http status: ${http_status})"
            pypi_state=VERSION_NOT_FOUND
          fi
          echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT

      - id: ci_env_check
        name: Determine CI environment
        run: |
          set -eu
          if [[ test == "${{ inputs.pypi-index }}" ]]; then
            ci_env=pypi-test
          elif [[ prod == "${{ inputs.pypi-index }}" ]]; then
            ci_env=pypi-prod${{ inputs.stable-version == '' && '-nightly' || '' }}
          else
            echo "::error::Invalid value for inputs.pypi-index: ${{ inputs.pypi-index }}"
            exit 1
          fi
          echo "ci_env=${ci_env}" >> "$GITHUB_OUTPUT"
          echo "::notice::Using CI environment ${ci_env}"

      - id: s3_check
        name: Generate S3 upload URL
        if: github.repository_owner == 'duckdb'
        run: |
          set -eu
          should_store=${{ (inputs.pypi-index == 'prod' || inputs.store-s3) && '1' || '0' }}
          if [[ $should_store == 0 ]]; then
            echo "::notice::S3 upload disabled in inputs, not generating S3 URL"
            exit 0
          fi
          if [[ VERSION_FOUND == "${{ steps.index_check.outputs.pypi_state }}" ]]; then
            echo "::warning::S3 upload disabled because package version already uploaded to PyPI"
            exit 0
          fi
          sha=${{ github.sha }}
          dsha=${{ inputs.duckdb-sha }}
          version=${{ needs.build_sdist.outputs.package-version }}
          s3_url="s3://duckdb-staging/python/${version}/${sha:0:10}-duckdb-${dsha:0:10}/"
          echo "::notice::Generated S3 URL: ${s3_url}"
          echo "s3_url=${s3_url}" >> $GITHUB_OUTPUT

  build_wheels:
    name: Build and test releases
    needs: workflow_state
    if: ${{ needs.workflow_state.outputs.pypi_state != 'VERSION_FOUND' }}
    uses: ./.github/workflows/packaging_wheels.yml
    with:
      minimal: false
      testsuite: all
      duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }}
      duckdb-sha: ${{ inputs.duckdb-sha }}
      set-version: ${{ inputs.stable-version }}

  upload_s3:
    name: Upload Artifacts to S3
    runs-on: ubuntu-latest
    needs: [build_sdist, build_wheels, workflow_state]
    if: ${{ needs.workflow_state.outputs.s3_url }}
    steps:
      - name: Fetch artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: '{sdist,wheel}*'
          path: artifacts/
          merge-multiple: true

      - name: Authenticate with AWS
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: 'us-east-2'
          aws-access-key-id: ${{ secrets.S3_DUCKDB_STAGING_ID }}
          aws-secret-access-key: ${{ secrets.S3_DUCKDB_STAGING_KEY }}

      - name: Upload Artifacts
        run: |
          aws s3 cp artifacts ${{ needs.workflow_state.outputs.s3_url }} --recursive

  publish_pypi:
    name: Publish Artifacts to PyPI
    runs-on: ubuntu-latest
    needs: [workflow_state, build_sdist, build_wheels]
    environment:
      name: ${{ needs.workflow_state.outputs.ci_env }}
    permissions:
      # this is needed for the OIDC flow that is used with trusted publishing on PyPI
      id-token: write
    steps:
      - if: ${{ vars.PYPI_HOST == '' }}
        run: |
          echo "Error: PYPI_HOST is not set in CI environment '${{ needs.workflow_state.outputs.ci_env }}'"
          exit 1

      - name: Fetch artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: '{sdist,wheel}*'
          path: packages/
          merge-multiple: true

      - name: Upload artifacts to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository-url: 'https://${{ vars.PYPI_HOST }}/legacy/'
          packages-dir: packages
          verbose: 'true'

  cleanup_nightlies:
    name: Remove Nightlies from PyPI
    needs: [workflow_state, publish_pypi]
    if: ${{ inputs.stable-version == '' }}
    uses: ./.github/workflows/cleanup_pypi.yml
    with:
      environment: ${{ needs.workflow_state.outputs.ci_env }}
    secrets:
      # reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206
      PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}}
      PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}}

  summary:
    name: Release summary
    runs-on: ubuntu-latest
    needs: [build_sdist, workflow_state, build_wheels, upload_s3, publish_pypi, cleanup_nightlies]
    if: always()
    steps:
      - run: |
          sha=${{ github.sha }}
          dsha=${{ inputs.duckdb-sha }}
          pversion=${{ needs.build_sdist.outputs.package-version }}
          long_pversion="${pversion} (${sha:0:10})"
          pypi_host=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org
          pypi_duckdb_url=https://${pypi_host}/project/duckdb/${pversion}/
          was_released=${{ needs.publish_pypi.result == 'success' && '1' || '0' }}
          if [[ $was_released == 1 ]]; then
            echo "## Version ${long_pversion} successfully released" >> $GITHUB_STEP_SUMMARY
            echo "* Package URL: [${pypi_duckdb_url}](${pypi_duckdb_url})" >> $GITHUB_STEP_SUMMARY
          else
            echo "## Version ${long_pversion} was not released" >> $GITHUB_STEP_SUMMARY
            echo "* Package index state before release: ${{ needs.workflow_state.outputs.pypi_state }}" >> $GITHUB_STEP_SUMMARY
          fi
          echo "* Package index: ${pypi_host}" >> $GITHUB_STEP_SUMMARY
          echo "* Vendored DuckDB Version: ${{ needs.build_sdist.outputs.duckdb-version }} (${dsha:0:10})" >> $GITHUB_STEP_SUMMARY
          echo "* S3 upload status: ${{ needs.upload_s3.result == 'success' && needs.workflow_state.outputs.s3_url || needs.upload_s3.result }}" >> $GITHUB_STEP_SUMMARY
          echo "* CI Environment: ${{ needs.workflow_state.outputs.ci_env }}" >> $GITHUB_STEP_SUMMARY