refactor wheel builder to use workflows and download/build separately (#5412)

* refactor wheel builder to use workflows and download/build separately

* Update .github/workflows/wheel-builder.yml

Co-authored-by: Alex Gaynor <[email protected]>

Co-authored-by: Alex Gaynor <[email protected]>

Author: reaperhulk

.github/workflows/wheel-builder.yml

@@ -1,7 +1,9 @@
 name: Wheel Builder
 on:
-  repository_dispatch:
-    types: [wheel-builder]
+  workflow_dispatch:
+    inputs:
+      version:
+        required: true
 
 jobs:
   manylinux:
@@ -15,21 +17,21 @@ jobs:
             CONTAINER: "pyca/cryptography-manylinux1:x86_64"
           - NAME: manylinux2010_x86_64
             CONTAINER: "pyca/cryptography-manylinux2010:x86_64"
-    name: "Python ${{ matrix.PYTHON }} for ${{ matrix.MANYLINUX.NAME }}"
+    name: "${{ matrix.PYTHON }} for ${{ matrix.MANYLINUX.NAME }}"
     steps:
       - run: /opt/python/${{ matrix.PYTHON }}/bin/python -m virtualenv .venv
-      - name: Downgrade pip, can't remember why
-        run: .venv/bin/pip install -U pip==10.0.1
       - name: Install Python dependencies
-        run: .venv/bin/pip install cffi six ipaddress "enum34; python_version < '3'"
+        run: .venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'"
+      - run: .venv/bin/pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir tmpwheelhouse
       - run: |
           REGEX="cp3([0-9])*"
           if [[ "${{ matrix.PYTHON }}" =~ $REGEX ]]; then
-              PY_LIMITED_API="--build-option --py-limited-api=cp3${BASH_REMATCH[1]}"
+              PY_LIMITED_API="--py-limited-api=cp3${BASH_REMATCH[1]}"
           fi
+          cd cryptography*
           LDFLAGS="-L/opt/pyca/cryptography/openssl/lib" \
               CFLAGS="-I/opt/pyca/cryptography/openssl/include -Wl,--exclude-libs,ALL" \
-              .venv/bin/pip wheel cryptography==${{ github.event.client_payload.BUILD_VERSION }} --no-binary cryptography --no-deps --wheel-dir=tmpwheelhouse $PY_LIMITED_API
+              ../.venv/bin/python setup.py bdist_wheel $PY_LIMITED_API && mv dist/cryptography*.whl ../tmpwheelhouse
       - run: auditwheel repair --plat ${{ matrix.MANYLINUX.NAME }} tmpwheelhouse/cryptograph*.whl -w wheelhouse/
       - run: unzip wheelhouse/*.whl -d execstack.check
       - run: |
@@ -40,16 +42,14 @@ jobs:
           else
             exit 0
           fi
-      - name: Upgrade pip again so we can actually use manylinux2010
-        run: .venv/bin/pip install -U pip
       - run: .venv/bin/pip install cryptography --no-index -f wheelhouse/
       - run: |
           .venv/bin/python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))"
       - run: mkdir cryptography-wheelhouse
       - run: mv wheelhouse/cryptography*.whl cryptography-wheelhouse/
       - uses: actions/upload-artifact@v1
         with:
-          name: "cryptography-${{ github.event.client_payload.BUILD_VERSION }}-${{ matrix.MANYLINUX.NAME }}-${{ matrix.PYTHON }}"
+          name: "cryptography-${{ github.event.inputs.version }}-${{ matrix.MANYLINUX.NAME }}-${{ matrix.PYTHON }}"
           path: cryptography-wheelhouse/
 
   macos:
@@ -65,7 +65,7 @@ jobs:
             ABI_VERSION: '3.5'
             DOWNLOAD_URL: 'https://www.python.org/ftp/python/3.8.2/python-3.8.2-macosx10.9.pkg'
             BIN_PATH: '/Library/Frameworks/Python.framework/Versions/3.8/bin/python3'
-    name: "Python ${{ matrix.PYTHON.VERSION }} for ABI ${{ matrix.PYTHON.ABI_VERSION }} on macOS"
+    name: "${{ matrix.PYTHON.VERSION }} ABI ${{ matrix.PYTHON.ABI_VERSION }} macOS"
     steps:
       - uses: actions/checkout@master
       - run: |
@@ -81,24 +81,20 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - run: ${{ matrix.PYTHON.BIN_PATH }} -m virtualenv venv
-      # Downgrade pip, I can't remember why
-      - run: venv/bin/pip install -U pip==10.0.1
-      - run: venv/bin/pip install -U wheel
-      - run: venv/bin/pip install cffi six ipaddress "enum34; python_version < '3'"
-
+      - run: venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'"
+      - run: venv/bin/pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir wheelhouse
       - name: Build the wheel
         run: |
           REGEX="3\.([0-9])*"
-          if [[ "$PYTHON_VERSION" =~ $REGEX ]]; then
-              PY_LIMITED_API="--build-option --py-limited-api=cp3${BASH_REMATCH[1]}"
+          if [[ "${{ matrix.PYTHON.ABI_VERSION }}" =~ $REGEX ]]; then
+              PY_LIMITED_API="--py-limited-api=cp3${BASH_REMATCH[1]}"
           fi
 
+          cd cryptography*
           CRYPTOGRAPHY_SUPPRESS_LINK_FLAGS="1" \
               LDFLAGS="${HOME}/openssl-macos/lib/libcrypto.a ${HOME}/openssl-macos/lib/libssl.a" \
               CFLAGS="-I${HOME}/openssl-macos/include -mmacosx-version-min=10.10 -march=core2" \
-              venv/bin/pip wheel cryptography==${{ github.event.client_payload.BUILD_VERSION }} --wheel-dir=wheelhouse --no-binary cryptography --no-deps $PY_LIMITED_API
-        env:
-          PYTHON_VERSION: ${{ matrix.PYTHON.ABI_VERSION }}
+              ../venv/bin/python setup.py bdist_wheel $PY_LIMITED_API && mv dist/cryptography*.whl ../wheelhouse
       - run: venv/bin/pip install -f wheelhouse --no-index cryptography
       - run: |
           venv/bin/python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))"
@@ -107,7 +103,7 @@ jobs:
       - run: mv wheelhouse/cryptography*.whl cryptography-wheelhouse/
       - uses: actions/upload-artifact@v1
         with:
-          name: "cryptography-${{ github.event.client_payload.BUILD_VERSION }}-macOS-${{ matrix.PYTHON.ABI_VERSION }}"
+          name: "cryptography-${{ github.event.inputs.version }}-macOS-${{ matrix.PYTHON.ABI_VERSION }}"
           path: cryptography-wheelhouse/
 
   windows:
@@ -118,16 +114,17 @@ jobs:
           - {ARCH: 'x86', WINDOWS: 'win32'}
           - {ARCH: 'x64', WINDOWS: 'win64'}
         PYTHON:
-          - {VERSION: "2.7", TOXENV: "py27", MSVC_VERSION: "2010"}
-          - {VERSION: "3.5", TOXENV: "py35", MSVC_VERSION: "2019"}
-          - {VERSION: "3.6", TOXENV: "py36", MSVC_VERSION: "2019"}
-          - {VERSION: "3.7", TOXENV: "py37", MSVC_VERSION: "2019"}
-          - {VERSION: "3.8", TOXENV: "py38", MSVC_VERSION: "2019"}
-    name: "Python ${{ matrix.PYTHON.VERSION }} on ${{ matrix.WINDOWS.WINDOWS }}"
+          - {VERSION: "2.7", MSVC_VERSION: "2010"}
+          - {VERSION: "3.5", MSVC_VERSION: "2019"}
+          - {VERSION: "3.6", MSVC_VERSION: "2019"}
+          - {VERSION: "3.7", MSVC_VERSION: "2019"}
+          - {VERSION: "3.8", MSVC_VERSION: "2019"}
+          - {VERSION: "3.8", MSVC_VERSION: "2019", "USE_ABI3": "true", "ABI_VERSION": "cp36"}
+    name: "${{ matrix.PYTHON.VERSION }} ${{ matrix.WINDOWS.WINDOWS }} ${{ matrix.PYTHON.ABI_VERSION }}"
     steps:
       - uses: actions/checkout@master
       - name: Setup python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.PYTHON.VERSION }}
           architecture: ${{ matrix.WINDOWS.ARCH }}
@@ -147,8 +144,13 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - run: pip install cffi six ipaddress "enum34; python_version < '3'"
-      - run: pip wheel cryptography==${{ github.event.client_payload.BUILD_VERSION }} --wheel-dir=wheelhouse --no-binary cryptography
+      - run: python -m pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'"
+      - run: pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir wheelhouse
+        shell: bash
+      - run: cd cryptography* && python setup.py bdist_wheel && mv dist/cryptography*.whl ../wheelhouse
+        if: matrix.PYTHON.USE_ABI3 != 'true'
+      - run: cd cryptography* && python setup.py bdist_wheel --py-limited-api=${{ matrix.PYTHON.ABI_VERSION }} && mv dist/cryptography*.whl ../wheelhouse
+        if: matrix.PYTHON.USE_ABI3 == 'true'
       - run: pip install -f wheelhouse --no-index cryptography
       - name: Print the OpenSSL we built and linked against
         run: |
@@ -158,5 +160,5 @@ jobs:
       - run: move wheelhouse\cryptography*.whl cryptography-wheelhouse\
       - uses: actions/upload-artifact@v1
         with:
-          name: "cryptography-${{ github.event.client_payload.BUILD_VERSION }}-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.VERSION }}"
+          name: "cryptography-${{ github.event.inputs.version }}-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.VERSION }}-${{ matrix.PYTHON.ABI_VERSION}}"
           path: cryptography-wheelhouse\

release.py

@@ -83,18 +83,14 @@ def build_github_actions_wheels(token, version):
     session = requests.Session()
 
     response = session.post(
-        "https://api.github.com/repos/pyca/cryptography/dispatches",
+        "https://api.github.com/repos/pyca/cryptography/actions/workflows/"
+        "wheel-builder.yml/dispatches",
         headers={
             "Content-Type": "application/json",
-            "Accept": "application/vnd.github.everest-preview+json",
+            "Accept": "application/vnd.github.v3+json",
             "Authorization": "token {}".format(token),
         },
-        data=json.dumps(
-            {
-                "event_type": "wheel-builder",
-                "client_payload": {"BUILD_VERSION": version},
-            }
-        ),
+        data=json.dumps({"ref": "master", "inputs": {"version": version}}),
     )
     response.raise_for_status()
 
@@ -103,7 +99,7 @@ def build_github_actions_wheels(token, version):
     response = session.get(
         (
             "https://api.github.com/repos/pyca/cryptography/actions/workflows/"
-            "wheel-builder.yml/runs?event=repository_dispatch"
+            "wheel-builder.yml/runs?event=workflow_dispatch"
         ),
         headers={
             "Content-Type": "application/json",