Merge branch 'v1.3-ossivalis'

evertlammerts · evertlammerts · commit c7bef3a5b5c3 · 2025-07-25T08:07:24.000+02:00
diff --git a/.github/workflows/cleanup_pypi.yml b/.github/workflows/cleanup_pypi.yml
@@ -34,11 +34,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-
-      - if: ${{ vars.PYPI_HOST == '' }}
-        run: |
-          echo "Error: PYPI_HOST is not set in CI environment '${{ inputs.environment }}'"
-          exit 1
       - if: ${{ vars.PYPI_CLEANUP_USERNAME == '' }}
         run: |
           echo "Error: PYPI_CLEANUP_USERNAME is not set in CI environment '${{ inputs.environment }}'"
@@ -56,8 +51,20 @@ jobs:
       - name: Run Cleanup
         run: |
           set -x
+          pypi_index_flag=${{ inputs.environment == 'production.pypi' && '--prod' || '--test' }}
           uv sync --only-group pypi --no-install-project
           uv run --no-sync -s scripts/pypi_cleanup.py ${{ inputs.dry-run && '--dry' || '' }} \
-            --index-hostname "${{ vars.PYPI_HOST }}" \
+            ${pypi_index_flag} \
             --username "${{ vars.PYPI_CLEANUP_USERNAME }}" \
-            --max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }}
+            --max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }} > cleanup_output 2>&1
+
+      - name: PyPI Cleanup Summary
+        run : |
+          echo "## PyPI Cleanup Summary" >> $GITHUB_STEP_SUMMARY
+          echo "* Dry run: ${{ inputs.dry-run }}" >> $GITHUB_STEP_SUMMARY
+          echo "* PyPI Host: ${{ vars.PYPI_HOST }}" >> $GITHUB_STEP_SUMMARY
+          echo "* CI Environment: ${{ inputs.environment }}" >> $GITHUB_STEP_SUMMARY
+          echo "* Output:" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          cat cleanup_output >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/on_external_dispatch.yml b/.github/workflows/on_external_dispatch.yml
@@ -80,7 +80,7 @@ jobs:
   upload_s3:
     name: Upload Artifacts to the S3 Staging Bucket
     runs-on: ubuntu-latest
-    needs: externally_triggered_build
+    needs: [commit_submodule, externally_triggered_build]
     outputs:
       version: ${{ steps.s3_upload.outputs.version }}
     if: ${{ github.repository_owner == 'duckdb' && inputs.publish-packages }}
@@ -102,15 +102,24 @@ jobs:
       - name: Upload Artifacts
         id: s3_upload
         run: |
-          version=$(basename artifacts/*.tar.gz | sed 's/duckdb-\(.*\).tar.gz/\1/g')
-          aws s3 cp artifacts s3://duckdb-staging/${{ github.repository }}/${version}/ --recursive
+          sha=${{ needs.commit_submodule.outputs.sha-after-commit }}
+          aws s3 cp artifacts s3://duckdb-staging/${{ github.repository }}/${sha:0:10}/ --recursive
           echo "version=${version}" >> $GITHUB_OUTPUT
 
+      - name: S3 Upload Summary
+        run : |
+          sha=${{ needs.commit_submodule.outputs.sha-after-commit }}
+          version=$(basename artifacts/*.tar.gz | sed 's/duckdb-\(.*\).tar.gz/\1/g')
+          echo "## S3 Upload Summary" >> $GITHUB_STEP_SUMMARY
+          echo "* Version: ${version}" >> $GITHUB_STEP_SUMMARY
+          echo "* SHA: ${sha:0:10}" >> $GITHUB_STEP_SUMMARY
+          echo "* S3 URL: s3://duckdb-staging/${{ github.repository }}/${sha:0:10}/" >> $GITHUB_STEP_SUMMARY
+
   publish_to_pypi:
     name: Upload Artifacts to PyPI
-    needs: upload_s3
-    if: ${{ force-version == '' }}
+    needs: [ commit_submodule, upload_s3 ]
+    if: ${{ inputs.force-version == '' }}
     uses: ./.github/workflows/upload_to_pypi.yml
     with:
-      version: ${{ needs.upload_s3.outputs.version }}
-      environment: pypi.production
+      sha: ${{ needs.commit_submodule.outputs.sha-after-commit }}
+      environment: pypi.production
diff --git a/.github/workflows/upload_to_pypi.yml b/.github/workflows/upload_to_pypi.yml
@@ -6,8 +6,8 @@ on:
         description: CI environment to run in (test.pypi or production.pypi)
         type: string
         required: true
-      version:
-        description: The version to upload (must be present in the S3 staging bucket)
+      sha:
+        description: The SHA of the commit that the packages were built from
         type: string
         required: true
   workflow_dispatch:
@@ -20,13 +20,13 @@ on:
         options:
           - test.pypi
           - production.pypi
-      version:
-        description: The version to upload (must be present in the S3 staging bucket)
+      sha:
+        description: The SHA of the commit that the packages were built from
         type: string
         required: true
 
 concurrency:
-  group: ${{ inputs.version }}
+  group: ${{ inputs.sha }}
   cancel-in-progress: true
 
 jobs:
@@ -53,7 +53,7 @@ jobs:
 
       - name: Download Artifacts From S3
         env:
-          S3_URL: 's3://duckdb-staging/${{ github.repository }}/${{ inputs.version }}/'
+          S3_URL: 's3://duckdb-staging/${{ github.repository }}/${{ inputs.sha }}/'
           AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
         run: |
@@ -66,6 +66,14 @@ jobs:
           repository-url: 'https://${{ vars.PYPI_HOST }}/legacy/'
           packages-dir: packages
 
+      - name: PyPI Upload Summary
+        run : |
+          version=$(basename packages/*.tar.gz | sed 's/duckdb-\(.*\).tar.gz/\1/g')
+          echo "## PyPI Upload Summary" >> $GITHUB_STEP_SUMMARY
+          echo "* Version: ${version}" >> $GITHUB_STEP_SUMMARY
+          echo "* PyPI Host: ${{ vars.PYPI_HOST }}" >> $GITHUB_STEP_SUMMARY
+          echo "* CI Environment: ${{ inputs.environment }}" >> $GITHUB_STEP_SUMMARY
+
   cleanup_nightlies:
     name: Remove Nightlies from PyPI
     needs: publish-pypi
diff --git a/scripts/pypi_cleanup.py b/scripts/pypi_cleanup.py
@@ -14,9 +14,8 @@
 import requests
 from requests.exceptions import RequestException
 
-import argparse
-import re
-import os
+_PYPI_URL_PROD = 'https://pypi.org/'
+_PYPI_URL_TEST = 'https://test.pypi.org/'
 
 def valid_hostname(hostname):
     """Validate hostname format"""
@@ -41,7 +40,9 @@ def non_empty_string(value):
     epilog="Environment variables required (unless --dry): PYPI_CLEANUP_PASSWORD, PYPI_CLEANUP_OTP"
 )
 parser.add_argument("--dry", action="store_true", help="Show what would be deleted but don't actually do it")
-parser.add_argument("-i", "--index-hostname", type=valid_hostname, required=True, help="Index hostname (required)")
+host_group = parser.add_mutually_exclusive_group(required=True)
+host_group.add_argument("--prod", action="store_true", help="Use production PyPI (pypi.org)")
+host_group.add_argument("--test", action="store_true", help="Use test PyPI (test.pypi.org)")
 parser.add_argument("-m", "--max-nightlies", type=int, default=2, help="Max number of nightlies of unreleased versions (default=2)")
 parser.add_argument("-u", "--username", type=non_empty_string, help="Username (required unless --dry)")
 args = parser.parse_args()
@@ -62,21 +63,23 @@ def non_empty_string(value):
     if not otp:
         parser.error("PYPI_CLEANUP_OTP environment variable is required when not in dry-run mode")
 
-print(f"Dry run: {args.dry}")
-print(f"Max nightlies: {args.max_nightlies}")
+dry_run = args.dry
+pypi_username = args.username
+max_dev_releases = args.max_nightlies
+if args.prod:
+    pypi_url = _PYPI_URL_PROD
+else:
+    pypi_url = _PYPI_URL_TEST
+pypi_password = password
+pypi_otp = otp
+
+print(f"Dry run: {dry_run}")
+print(f"Max nightlies: {max_dev_releases}")
 if not args.dry:
-    print(f"Hostname: {args.index_hostname}")
-    print(f"Username: {args.username}")
+    print(f"URL: {pypi_url}")
+    print(f"Username: {pypi_username}")
     print("Password and OTP loaded from environment variables")
 
-# deletes old dev wheels from pypi. evil hack.
-actually_delete = not args.dry
-pypi_username = args.username or "user"
-max_dev_releases = args.max_nightlies
-host = 'https://{}/'.format(args.index_hostname)
-pypi_password = password or "password"
-pypi_otp = otp or "otp"
-
 patterns = [re.compile(r".*\.dev\d+$")]
 ###### NOTE: This code is taken from the pypi-cleanup package (https://github.com/arcivanov/pypi-cleanup/tree/master)
 class CsfrParser(HTMLParser):
@@ -185,8 +188,8 @@ def run(self):
                     sorted_versions = sorted(versions, key=lambda x: int(x.split('dev')[-1]))
                     pkg_vers.extend(sorted_versions[:-self.max_dev_releases])
 
+            print("Following pkg_vers can be deleted: ", pkg_vers)
             if not self.do_it:
-                print("Following pkg_vers can be deleted: ", pkg_vers)
                 return
 
             if not pkg_vers:
@@ -231,9 +234,9 @@ def run(self):
 
             two_factor = False
             with s.post(
-                f"{self.url}/account/login/",
-                data={"csrf_token": csrf, "username": self.username, "password": self.password},
-                headers={"referer": f"{self.url}/account/login/"},
+                    f"{self.url}/account/login/",
+                    data={"csrf_token": csrf, "username": self.username, "password": self.password},
+                    headers={"referer": f"{self.url}/account/login/"},
             ) as r:
                 r.raise_for_status()
                 if r.url == f"{self.url}/account/login/":
@@ -255,9 +258,9 @@ def run(self):
                 for i in range(3):
                     auth_code = pyotp.TOTP(self.otp).now()
                     with s.post(
-                        two_factor_url,
-                        data={"csrf_token": csrf, "method": "totp", "totp_value": auth_code},
-                        headers={"referer": two_factor_url},
+                            two_factor_url,
+                            data={"csrf_token": csrf, "method": "totp", "totp_value": auth_code},
+                            headers={"referer": two_factor_url},
                     ) as r:
                         r.raise_for_status()
                         if r.url == two_factor_url:
@@ -286,12 +289,12 @@ def run(self):
                         referer = r.url
 
                     with s.post(
-                        form_url,
-                        data={
-                            "csrf_token": csrf,
-                            "confirm_delete_version": pkg_ver,
-                        },
-                        headers={"referer": referer},
+                            form_url,
+                            data={
+                                "csrf_token": csrf,
+                                "confirm_delete_version": pkg_ver,
+                            },
+                            headers={"referer": referer},
                     ) as r:
                         r.raise_for_status()
 
@@ -300,4 +303,4 @@ def run(self):
                     logging.info(f"Would be deleting {self.package} version {pkg_ver}, but not doing it!")
 
 
-PypiCleanup(host, pypi_username, 'duckdb', pypi_password, pypi_otp, patterns, actually_delete, max_dev_releases).run()
+PypiCleanup(pypi_url, pypi_username, 'duckdb', pypi_password, pypi_otp, patterns, not dry_run, max_dev_releases).run()
