Integrate Rust into the build process properly

Author: alex

.github/workflows/ci.yml

@@ -15,16 +15,23 @@ jobs:
     strategy:
       matrix:
         PYTHON:
-          - {VERSION: "2.7", TOXENV: "py27", EXTRA_CFLAGS: ""}
           - {VERSION: "3.5", TOXENV: "py35", EXTRA_CFLAGS: ""}
           - {VERSION: "3.8", TOXENV: "py38", EXTRA_CFLAGS: "-DUSE_OSRANDOM_RNG_FOR_TESTING"}
+        RUST:
+          - stable
     name: "Python ${{ matrix.PYTHON.VERSION }} on macOS"
     steps:
       - uses: actions/checkout@master
       - name: Setup python
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.PYTHON.VERSION }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.RUST }}
+          override: true
+          default: true
 
       - run: python -m pip install tox requests coverage
 
@@ -55,14 +62,15 @@ jobs:
     strategy:
       matrix:
         WINDOWS:
-          - {ARCH: 'x86', WINDOWS: 'win32'}
-          - {ARCH: 'x64', WINDOWS: 'win64'}
+          - {ARCH: 'x86', WINDOWS: 'win32', RUST_TRIPLE: 'i686-pc-windows-msvc'}
+          - {ARCH: 'x64', WINDOWS: 'win64', RUST_TRIPLE: 'x86_64-pc-windows-msvc'}
         PYTHON:
-          - {VERSION: "2.7", TOXENV: "py27", MSVC_VERSION: "2010", CL_FLAGS: ""}
           - {VERSION: "3.5", TOXENV: "py35", MSVC_VERSION: "2019", CL_FLAGS: ""}
           - {VERSION: "3.6", TOXENV: "py36", MSVC_VERSION: "2019", CL_FLAGS: ""}
           - {VERSION: "3.7", TOXENV: "py37", MSVC_VERSION: "2019", CL_FLAGS: ""}
           - {VERSION: "3.8", TOXENV: "py38", MSVC_VERSION: "2019", CL_FLAGS: "/D USE_OSRANDOM_RNG_FOR_TESTING"}
+        RUST:
+          - stable
     name: "Python ${{ matrix.PYTHON.VERSION }} on ${{ matrix.WINDOWS.WINDOWS }}"
     steps:
       - uses: actions/checkout@master
@@ -71,14 +79,14 @@ jobs:
         with:
           python-version: ${{ matrix.PYTHON.VERSION }}
           architecture: ${{ matrix.WINDOWS.ARCH }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.RUST }}
+          override: true
+          default: true
+          target: ${{ matrix.WINDOWS.RUST_TRIPLE }}
 
-      - name: Install MSVC for Python 2.7
-        run: |
-            Invoke-WebRequest -Uri https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -OutFile VCForPython27.msi
-            Start-Process msiexec -Wait -ArgumentList @('/i', 'VCForPython27.msi', '/qn', 'ALLUSERS=1')
-            Remove-Item VCForPython27.msi -Force
-        shell: powershell
-        if: matrix.PYTHON.VERSION == '2.7'
       - run: python -m pip install tox requests coverage
       - name: Download OpenSSL
         run: |

.github/workflows/wheel-builder.yml

@@ -11,7 +11,7 @@ jobs:
     container: ${{ matrix.MANYLINUX.CONTAINER }}
     strategy:
       matrix:
-        PYTHON: ["cp27-cp27m", "cp27-cp27mu", "cp35-cp35m"]
+        PYTHON: ["cp35-cp35m"]
         MANYLINUX:
           - NAME: manylinux1_x86_64
             CONTAINER: "pyca/cryptography-manylinux1:x86_64"
@@ -21,7 +21,7 @@ jobs:
     steps:
       - run: /opt/python/${{ matrix.PYTHON }}/bin/python -m virtualenv .venv
       - name: Install Python dependencies
-        run: .venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'"
+        run: .venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3' setuptools-rust"
       - run: .venv/bin/pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir tmpwheelhouse
       - run: |
           REGEX="cp3([0-9])*"
@@ -57,10 +57,6 @@ jobs:
     strategy:
       matrix:
         PYTHON:
-          - VERSION: '2.7'
-            ABI_VERSION: '2.7'
-            DOWNLOAD_URL: 'https://www.python.org/ftp/python/2.7.17/python-2.7.17-macosx10.9.pkg'
-            BIN_PATH: '/Library/Frameworks/Python.framework/Versions/2.7/bin/python'
           - VERSION: '3.8'
             ABI_VERSION: '3.5'
             DOWNLOAD_URL: 'https://www.python.org/ftp/python/3.8.2/python-3.8.2-macosx10.9.pkg'
@@ -79,9 +75,15 @@ jobs:
             ${{ matrix.PYTHON.BIN_PATH }} .github/workflows/download_openssl.py macos openssl-macos
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          default: true
 
       - run: ${{ matrix.PYTHON.BIN_PATH }} -m virtualenv venv
-      - run: venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'"
+      - run: venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3' setuptools-rust"
       - run: venv/bin/pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir wheelhouse
       - name: Build the wheel
         run: |
@@ -111,10 +113,9 @@ jobs:
     strategy:
       matrix:
         WINDOWS:
-          - {ARCH: 'x86', WINDOWS: 'win32'}
-          - {ARCH: 'x64', WINDOWS: 'win64'}
+          - {ARCH: 'x86', WINDOWS: 'win32', RUST_TRIPLE: 'i686-pc-windows-msvc'}
+          - {ARCH: 'x64', WINDOWS: 'win64', RUST_TRIPLE: 'x86_64-pc-windows-msvc'}
         PYTHON:
-          - {VERSION: "2.7", MSVC_VERSION: "2010"}
           - {VERSION: "3.5", MSVC_VERSION: "2019"}
           - {VERSION: "3.6", MSVC_VERSION: "2019"}
           - {VERSION: "3.7", MSVC_VERSION: "2019"}
@@ -128,13 +129,13 @@ jobs:
         with:
           python-version: ${{ matrix.PYTHON.VERSION }}
           architecture: ${{ matrix.WINDOWS.ARCH }}
-      - name: Install MSVC for Python 2.7
-        run: |
-            Invoke-WebRequest -Uri https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -OutFile VCForPython27.msi
-            Start-Process msiexec -Wait -ArgumentList @('/i', 'VCForPython27.msi', '/qn', 'ALLUSERS=1')
-            Remove-Item VCForPython27.msi -Force
-        shell: powershell
-        if: matrix.PYTHON.VERSION == '2.7'
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          default: true
+          target: ${{ matrix.WINDOWS.RUST_TRIPLE }}
       - run: pip install requests
       - name: Download OpenSSL
         run: |
@@ -144,7 +145,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - run: python -m pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'"
+      - run: python -m pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3' setuptools-rust"
       - run: pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir wheelhouse
         shell: bash
       - run: cd cryptography* && python setup.py bdist_wheel && mv dist/cryptography*.whl ../wheelhouse

.gitignore

@@ -12,3 +12,5 @@ htmlcov/
 .eggs/
 *.py[cdo]
 .hypothesis/
+target/
+Cargo.lock

.travis.yml

@@ -19,26 +19,18 @@ branches:
 matrix:
     include:
         - python: 3.8
-          env: TOXENV=pep8,packaging
+          env: TOXENV=pep8,rust,packaging
         # Setting 'python' is just to make travis's UI a bit prettier
         - python: 3.6
           env: TOXENV=py36
           # Travis lists available Pythons (including PyPy) by arch and distro here:
           # https://docs.travis-ci.com/user/languages/python/#python-versions
-        - python: pypy2.7-7.3.1
-          env: TOXENV=pypy-nocoverage
         - python: pypy3.6-7.3.1
           env: TOXENV=pypy3-nocoverage
         - python: 3.8
           env: TOXENV=py38 OPENSSL=1.0.2u
-        - python: 2.7
-          env: TOXENV=py27 OPENSSL=1.1.0l
-        - python: 2.7
-          env: TOXENV=py27-ssh OPENSSL=1.1.0l
         - python: 3.8
           env: TOXENV=py38 OPENSSL=1.1.0l
-        - python: 2.7
-          env: TOXENV=py27 OPENSSL=1.1.1g
         - python: 3.8
           env: TOXENV=py38 OPENSSL=1.1.1g
         - python: 3.8
@@ -54,21 +46,12 @@ matrix:
         - python: 3.8
           env: TOXENV=py38 LIBRESSL=3.2.0
 
-        - python: 2.7
-          services: docker
-          env: TOXENV=py27 DOCKER=pyca/cryptography-runner-centos7
-        - python: 2.7
-          services: docker
-          env: TOXENV=py27 DOCKER=pyca/cryptography-runner-centos8
         - python: 3.6
           services: docker
           env: TOXENV=py36 DOCKER=pyca/cryptography-runner-centos8
         - python: 3.6
           services: docker
           env: TOXENV=py36 OPENSSL_FORCE_FIPS_MODE=1 DOCKER=pyca/cryptography-runner-centos8-fips
-        - python: 2.7
-          services: docker
-          env: TOXENV=py27 DOCKER=pyca/cryptography-runner-stretch
         - python: 3.5
           services: docker
           env: TOXENV=py35 DOCKER=pyca/cryptography-runner-stretch
@@ -87,9 +70,6 @@ matrix:
         - python: 3.8
           services: docker
           env: TOXENV=py38 DOCKER=pyca/cryptography-runner-ubuntu-focal
-        - python: 2.7
-          services: docker
-          env: TOXENV=py27 DOCKER=pyca/cryptography-runner-ubuntu-rolling
         - python: 3.8
           services: docker
           env: TOXENV=py38 DOCKER=pyca/cryptography-runner-ubuntu-rolling
@@ -138,9 +118,11 @@ matrix:
           dist: xenial
 
 install:
+    - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --component rustmft
     - ./.travis/install.sh
 
 script:
+    - source $HOME/.cargo/env
     - ./.travis/run.sh
 
 after_success:

.zuul.d/jobs.yaml

@@ -25,13 +25,6 @@
     vars:
       tox_envlist: py36
 
-- job:
-    name: pyca-cryptography-centos-8-py27-arm64
-    parent: pyca-cryptography-base
-    nodeset: centos-8-arm64
-    vars:
-      tox_envlist: py27
-
 - job:
     name: pyca-cryptography-build-wheel
     abstract: true
@@ -57,12 +50,8 @@
         - platform: manylinux1_x86_64
           image: pyca/cryptography-manylinux1:x86_64
           pythons:
-            - cp27-cp27m
-            - cp27-cp27mu
             - cp35-cp35m
         - platform: manylinux2010_x86_64
           image: pyca/cryptography-manylinux2010:x86_64
           pythons:
-            - cp27-cp27m
-            - cp27-cp27mu
             - cp35-cp35m

.zuul.d/project.yaml

@@ -6,7 +6,6 @@
         - pyca-cryptography-ubuntu-focal-py38-arm64
         - pyca-cryptography-ubuntu-bionic-py36-arm64
         - pyca-cryptography-centos-8-py36-arm64
-        - pyca-cryptography-centos-8-py27-arm64
     release:
       jobs:
         - pyca-cryptography-build-wheel-arm64

.zuul.playbooks/playbooks/tox/main.yaml

@@ -27,6 +27,10 @@
       become: yes
       when: ansible_distribution == 'CentOS'
 
+    - name: Install rust
+      include_role:
+        name: ensure-rust
+
     - name: Clone wycheproof
       git:
         repo: https://github.com/google/wycheproof

.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/files/build-wheels.sh

@@ -11,7 +11,7 @@ for P in ${PYTHONS}; do
 
     "${PYBIN}"/python -m virtualenv .venv
 
-    .venv/bin/pip install cffi six ipaddress "enum34; python_version < '3'"
+    .venv/bin/pip install cffi six ipaddress "enum34; python_version < '3'" setuptools-rust
 
     REGEX="cp3([0-9])*"
     if [[ "${PYBIN}" =~ $REGEX ]]; then

.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/tasks/main.yaml

@@ -53,6 +53,15 @@
   become: yes
   when: ansible_distribution in ['Debian', 'Ubuntu']
 
+- name: Install rust
+  include_role:
+    name: ensure-rust
+
+- name: Install setuptools-rust
+  pip:
+    name: setuptools-rust
+  become: yes
+
 - name: Create sdist
   command: |
     python3 setup.py sdist

CHANGELOG.rst

@@ -1,6 +1,8 @@
 Changelog
 =========
 
+TODO: Add Rust to the current release when this is ready to land.
+
 .. _v3-2:
 
 3.2 - `master`_

MANIFEST.in

@@ -11,6 +11,7 @@ include pyproject.toml
 
 recursive-include docs *
 recursive-include src/_cffi_src *.py *.c *.h
+recursive-include src/rust Cargo.toml *.rs
 prune docs/_build
 recursive-include tests *.py
 exclude vectors

docs/faq.rst

@@ -118,6 +118,19 @@ series. Since they are no longer receiving security patches from upstream,
 should upgrade to a newer version of OpenSSL (1.0.2 or later). This may require
 you to upgrade to a newer operating system.
 
+Installing ``cryptography`` fails with ``error: Can not find Rust compiler``
+----------------------------------------------------------------------------
+
+Building ``cryptography`` from source requires you have Rust installed and on
+your ``PATH``. You may be able to fix this by upgrading to a newer version of
+``pip`` which will install a pre-compiled ``cryptography`` wheel. If not,
+you'll need to install Rust.
+
+For the current release *only* you can temporarily bypass the requirement to
+have Rust installed by setting the ``CRYPTOGRAPHY_DONT_BUILD_RUST`` environment
+variable. Note that this option will be removed in the next release and not
+having Rust available will be a hard error.
+
 Why are there no wheels for Python 3.6+ on Linux or macOS?
 ----------------------------------------------------------
 

docs/installation.rst

@@ -66,6 +66,8 @@ platforms). ``cryptography`` links against the new 1.1.0 names by default. If
 you need to compile ``cryptography`` against an older version then you **must**
 set ``CRYPTOGRAPHY_WINDOWS_LINK_LEGACY_OPENSSL`` or else installation will fail.
 
+You will also need to have Rust installed and available.
+
 If you need to rebuild ``cryptography`` for any reason be sure to clear the
 local `wheel cache`_.
 
@@ -88,6 +90,8 @@ If you are on Alpine or just want to compile it yourself then
 using ``pypy``), and headers for the OpenSSL and ``libffi`` libraries
 available on your system.
 
+On all Linux distributions you will need to have Rust installed and available.
+
 Alpine
 ~~~~~~
 
@@ -238,6 +242,8 @@ open a terminal window and run:
 This will install a compiler (clang) along with (most of) the required
 development headers.
 
+You will also need to have Rust installed and available.
+
 You'll also need OpenSSL, which you can obtain from `Homebrew`_ or `MacPorts`_.
 Cryptography does **not** support Apple's deprecated OpenSSL distribution.
 

pyproject.toml

@@ -6,6 +6,7 @@ requires = [
     "wheel",
     # Must be kept in sync with the `setup_requirements` in `setup.py`
     "cffi>=1.8,!=1.11.3; platform_python_implementation != 'PyPy'",
+    "setuptools-rust>=0.11.1",
 ]
 build-backend = "setuptools.build_meta"