Add CI to build python wheels

.github/workflows/build.yml

@@ -0,0 +1,293 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Python Release Build
+on:
+  pull_request:
+    branches: ["main"]
+  push:
+    tags: ["*-rc*"]
+    branches: ["branch-*"]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          python_version: "3.12"
+
+      # Use the --no-install-package to only install the dependencies
+      # but do not yet build the rust library
+      - name: Install dependencies
+        run: uv sync --dev --no-install-package datafusion-ray
+
+  generate-license:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Generate license file
+        run: uv run --no-project python ./dev/create_license.py
+      - uses: actions/upload-artifact@v4
+        with:
+          name: python-wheel-license
+          path: LICENSE.txt
+
+  build-python-mac-win:
+    needs: [generate-license]
+    name: Mac/Win
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.13"]
+        #os: [macos-latest, windows-latest]
+        os: [macos-latest]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - run: rm LICENSE.txt
+      - name: Download LICENSE.txt
+        uses: actions/download-artifact@v4
+        with:
+          name: python-wheel-license
+          path: .
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: "27.4"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Build Python package
+        run: |
+          uv sync --dev --no-install-package datafusion-ray
+          uv run --no-project maturin build --release --strip
+
+      - name: List Windows wheels
+        if: matrix.os == 'windows-latest'
+        run: dir target\wheels\
+        # since the runner is dynamic shellcheck (from actionlint) can't infer this is powershell
+        # so we specify it explicitly
+        shell: powershell
+
+      - name: List Mac wheels
+        if: matrix.os != 'windows-latest'
+        run: find target/wheels/
+
+      - name: Archive wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-${{ matrix.os  }}
+          path: target/wheels/*
+
+  build-macos-x86_64:
+    needs: [generate-license]
+    name: Mac x86_64
+    runs-on: macos-13
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - run: rm LICENSE.txt
+      - name: Download LICENSE.txt
+        uses: actions/download-artifact@v4
+        with:
+          name: python-wheel-license
+          path: .
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: "27.4"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Build Python package
+        run: |
+          uv sync --dev --no-install-package datafusion-ray
+          uv run --no-project maturin build --release --strip
+
+      - name: List Mac wheels
+        run: find target/wheels/
+
+      - name: Archive wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-macos-aarch64
+          path: target/wheels/*
+
+  build-manylinux-x86_64:
+    needs: [generate-license]
+    name: Manylinux x86_64
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rm LICENSE.txt
+      - name: Download LICENSE.txt
+        uses: actions/download-artifact@v4
+        with:
+          name: python-wheel-license
+          path: .
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        env:
+          RUST_BACKTRACE: 1
+        with:
+          rust-toolchain: nightly
+          target: x86_64
+          manylinux: auto
+          rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
+          args: --release --manylinux 2014
+          before-script-linux: |
+            yum install -y wget
+            cd /
+            wget https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-linux-x86_64.zip
+            unzip -o proto*zip
+            cd -
+            which protoc
+            protoc --version
+
+      - name: Archive wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-manylinux-x86_64
+          path: target/wheels/*
+
+  build-manylinux-aarch64:
+    needs: [generate-license]
+    name: Manylinux arm64
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rm LICENSE.txt
+      - name: Download LICENSE.txt
+        uses: actions/download-artifact@v4
+        with:
+          name: python-wheel-license
+          path: .
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        env:
+          RUST_BACKTRACE: 1
+        with:
+          rust-toolchain: nightly
+          target: aarch64
+          # Use manylinux_2_28-cross because the manylinux2014-cross has GCC 4.8.5, which causes the build to fail
+          manylinux: 2_28
+          rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
+          args: --release
+          before-script-linux: |
+            apt-get install -y unzip
+            cd /
+            wget https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-linux-x86_64.zip
+            unzip -o proto*zip
+            cd -
+            which protoc
+            protoc --version
+
+      - name: Archive wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-manylinux-aarch64
+          path: target/wheels/*
+
+  build-sdist:
+    needs: [generate-license]
+    name: Source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rm LICENSE.txt
+      - name: Download LICENSE.txt
+        uses: actions/download-artifact@v4
+        with:
+          name: python-wheel-license
+          path: .
+
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          rust-toolchain: stable
+          manylinux: auto
+          rustup-components: rust-std rustfmt
+          args: --release --sdist --out dist
+          before-script-linux: |
+            yum install -y wget
+            cd /
+            wget https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-linux-x86_64.zip
+            unzip -o proto*zip
+            cd -
+            which protoc
+            protoc --version
+
+      - name: Assert sdist build does not generate wheels
+        run: |
+          if [ "$(ls -A target/wheels)" ]; then
+            echo "Error: Sdist build generated wheels"
+            exit 1
+          else
+            echo "Directory is clean"
+          fi
+        shell: bash
+
+  merge-build-artifacts:
+    runs-on: ubuntu-latest
+    needs:
+      - build-python-mac-win
+      - build-macos-x86_64
+      - build-manylinux-x86_64
+      - build-manylinux-aarch64
+      - build-sdist
+    steps:
+      - name: Merge Build Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: dist
+          pattern: dist-*

Cargo.toml

@@ -85,6 +85,7 @@ tonic-build = { version = "0.8", default-features = false, features = [
   "prost",
 ] }
 url = "2"
+protobuf-src = "2.1"
 
 [dev-dependencies]
 tempfile = "3.17"