chore: revamp CI dist build

jansegre · Feb 27, 2025 · c50f279 · c50f279
1 parent f92701a
commit c50f279
Show file tree

Hide file tree

Showing 3 changed files with 157 additions and 120 deletions.
diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -60,7 +60,7 @@ jobs:
           sudo make install-headers PREFIX=${{ env.LIBROCKSDB_PATH }}
 
   test-linux:
-    name: Test on Python ${{ matrix.python-version}} with librocksdb-${{ matrix.rocksdb-version }}
+    name: Test (Python ${{ matrix.python-version }} with librocksdb-${{ matrix.rocksdb-version }})
     needs: librocksdb
     runs-on: ubuntu-latest
     env:
@@ -107,11 +107,14 @@ jobs:
           uvx pytest
 
   test-macos:
-    name: Test on Python ${{ matrix.python-version}} with librocksdb from homebrew
-    runs-on: macos-latest
+    name: Test (Python ${{ matrix.python-version }} with macOS ${{ matrix.os.arch }})
+    runs-on: ${{ matrix.os.runner }}
     strategy:
       fail-fast: false
       matrix:
+        runner:
+          - { arch: 'Intel', runner: 'macos-13' }
+          - { arch: 'Arm', runner: 'macos-latest' }
         python-version: ['3.10', '3.11', '3.12', '3.13']
 
     steps:
@@ -137,15 +140,19 @@ jobs:
           uvx pytest
 
   test-debian:
-    name: Test on Python ${{ matrix.python-version}} with Debian ${{ matrix.debian-dist }} librocksdb
-    runs-on: ubuntu-latest
+    name: Test (Python ${{ matrix.python-version }} with Debian ${{ matrix.debian-dist }} ${{ matrix.os.arch }} librocksdb)
+    runs-on: ${{ matrix.os.runner }}
     container: debian:${{ matrix.debian-dist }}-slim
     strategy:
       fail-fast: false
       matrix:
+        os:
+          - { arch: 'x64', runner: 'ubuntu-latest' }
+          - { arch: 'arm', runner: 'ubuntu-24.04-arm' }
         python-version: ['3.10', '3.11', '3.12', '3.13']
         debian-dist: [bullseye, bookworm]
     steps:
+      - run: uname -a
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -0,0 +1,145 @@
+# yamllint disable rule:line-length
+name: Build distribution
+
+on: # yamllint disable-line rule:truthy
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+      - main
+      - dev
+      - release*
+    tags:
+      - v*
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  sdist:
+    runs-on: ubuntu-latest
+    name: Build sdists
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - uses: astral-sh/setup-uv@v5
+
+      - name: Make sdist
+        run: uv build --sdist
+      - run: ls dist
+
+      - uses: actions/upload-artifact@v4
+        name: Upload build artifacts
+        with:
+          name: sdist
+          path: dist/rocksdb*
+
+  manylinux:
+    needs: [sdist]
+    name: Build wheels
+    runs-on: ${{ matrix.manylinux.runner }}
+    # env:
+    #   LIBROCKSDB_PATH: /opt/rocksdb-${{ matrix.rocksdb_ver }}
+    strategy:
+      matrix:
+        # rocksdb_ver: ['8.11.fb']
+        python:
+          - { version: "3.13" }
+          # - { version: "cp311-cp311" }
+          # - { version: "cp311-cp311" }
+        manylinux:
+          - { name: "manylinux2014_x86_64",   container: "cryptography-manylinux2014:x86_64",   runner: "ubuntu-latest" }
+          # - { name: "manylinux_2_28_x86_64",  container: "cryptography-manylinux_2_28:x86_64",  runner: "ubuntu-latest" }
+          # - { name: "manylinux_2_34_x86_64",  container: "cryptography-manylinux_2_34:x86_64",  runner: "ubuntu-latest" }
+          # - { name: "musllinux_1_2_x86_64",   container: "cryptography-musllinux_1_2:x86_64",   runner: "ubuntu-latest" }
+          # - { name: "manylinux2014_aarch64",  container: "cryptography-manylinux2014_aarch64",  runner: "ubuntu-24.04-arm" }
+          # - { name: "manylinux_2_28_aarch64", container: "cryptography-manylinux_2_28:aarch64", runner: "ubuntu-24.04-arm" }
+          # - { name: "manylinux_2_34_aarch64", container: "cryptography-manylinux_2_34:aarch64", runner: "ubuntu-24.04-arm" }
+          # - { name: "musllinux_1_2_aarch64",  container: "cryptography-musllinux_1_2:aarch64",  runner: "ubuntu-24.04-arm" }
+          # - { name: "manylinux_2_31_armv7l",  container: "cryptography-manylinux_2_31:armv7l",  runner: "ubuntu-24.04-arm" }
+
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: sdist
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python.version }}
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: false
+
+      - name: Install libraries
+        run: sudo apt install -y build-essential librocksdb-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+
+      - run: mkdir tmpwheelhouse
+
+      - name: Build the wheel
+        run: uv build --wheel rocksdb*.tar.gz -o tmpwheelhouse/
+
+      - run: auditwheel repair --plat ${{ matrix.manylinux.name }} tmpwheelhouse/rocksdb*.whl -w wheelhouse/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: "rocksdb-v0.0.0-${{ matrix.manylinux.name }}-${{ matrix.python.version }}"
+          path: wheelhouse/
+
+      # - name: Install cibuildwheel
+      #   run: |
+      #     python3 -m pip install cibuildwheel==1.7.1
+
+      # - name: Build wheels
+      #   run: |
+      #     python3 -m cibuildwheel --output-dir dist
+      #   env:
+      #     CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+      #     CIBW_BUILD: 'cp31*'
+      #     CIBW_SKIP: '*-manylinux_i686'
+      #     # Install python package and test-deps.
+      #     CIBW_TEST_REQUIRES: '.[test] pytest'
+      #     # Use `--pyargs` to interpret parameter as module to import, not as a
+      #     # path, and do not use `python3 -m pytest`. This way we prevent
+      #     # importing the module from the current directory instead of the
+      #     # installed package, and failing when it cannot find the shared
+      #     # library.
+      #     CIBW_TEST_COMMAND: 'pytest --pyargs rocksdb'
+      #     # Avoid re-building the C library in every iteration by testing for
+      #     # the build directory.
+      #     CIBW_BEFORE_BUILD: >
+      #       yum install -y snappy snappy-devel zlib zlib-devel bzip2 bzip2-devel lz4-devel libzstd-devel &&
+      #       test -d ${{ env.LIBROCKSDB_PATH }} || (
+      #       git clone https://github.com/facebook/rocksdb --depth 1
+      #       --branch ${{ matrix.rocksdb_ver }} ${{ env.LIBROCKSDB_PATH }} &&
+      #       cd ${{ env.LIBROCKSDB_PATH }} &&
+      #       CXXFLAGS='-flto -Os -s' PORTABLE=1 make shared_lib -j 4
+      #       ) &&
+      #       pushd ${{ env.LIBROCKSDB_PATH }} &&
+      #       make install-shared &&
+      #       ldconfig &&
+      #       popd
+      # - uses: actions/upload-artifact@v4
+      #   name: Upload build artifacts
+      #   with:
+      #     path: 'dist/*.whl'
+
+#  publish:
+#    name: Upload packages
+#    needs: [manylinux, sdist]
+#    runs-on: 'ubuntu-latest'
+#    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
+#    steps:
+#      - uses: actions/download-artifact@v4
+#        name: 'Download artifacts'
+#        with:
+#          name: 'artifact'
+#          path: 'dist'
+#
+#      - uses: pypa/gh-action-pypi-publish@master
+#        name: 'Publish built packages'
+#        with:
+#          user: '__token__'
+#          password: '${{ secrets.PYPI_API_TOKEN }}'