Integrate Rust into the build process properly

Author: alex

.github/workflows/ci.yml

@@ -15,16 +15,23 @@ jobs:
     strategy:
       matrix:
         PYTHON:
-          - {VERSION: "2.7", TOXENV: "py27", EXTRA_CFLAGS: ""}
           - {VERSION: "3.5", TOXENV: "py35", EXTRA_CFLAGS: ""}
           - {VERSION: "3.8", TOXENV: "py38", EXTRA_CFLAGS: "-DUSE_OSRANDOM_RNG_FOR_TESTING"}
+        RUST:
+          - stable
     name: "Python ${{ matrix.PYTHON.VERSION }} on macOS"
     steps:
       - uses: actions/checkout@master
       - name: Setup python
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.PYTHON.VERSION }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.RUST }}
+          override: true
+          default: true
 
       - run: python -m pip install tox requests coverage
 
@@ -55,14 +62,15 @@ jobs:
     strategy:
       matrix:
         WINDOWS:
-          - {ARCH: 'x86', WINDOWS: 'win32'}
-          - {ARCH: 'x64', WINDOWS: 'win64'}
+          - {ARCH: 'x86', WINDOWS: 'win32', RUST_TRIPLE: 'i686-pc-windows-msvc'}
+          - {ARCH: 'x64', WINDOWS: 'win64', RUST_TRIPLE: 'x86_64-pc-windows-msvc'}
         PYTHON:
-          - {VERSION: "2.7", TOXENV: "py27", MSVC_VERSION: "2010", CL_FLAGS: ""}
           - {VERSION: "3.5", TOXENV: "py35", MSVC_VERSION: "2019", CL_FLAGS: ""}
           - {VERSION: "3.6", TOXENV: "py36", MSVC_VERSION: "2019", CL_FLAGS: ""}
           - {VERSION: "3.7", TOXENV: "py37", MSVC_VERSION: "2019", CL_FLAGS: ""}
           - {VERSION: "3.8", TOXENV: "py38", MSVC_VERSION: "2019", CL_FLAGS: "/D USE_OSRANDOM_RNG_FOR_TESTING"}
+        RUST:
+          - stable
     name: "Python ${{ matrix.PYTHON.VERSION }} on ${{ matrix.WINDOWS.WINDOWS }}"
     steps:
       - uses: actions/checkout@master
@@ -71,14 +79,14 @@ jobs:
         with:
           python-version: ${{ matrix.PYTHON.VERSION }}
           architecture: ${{ matrix.WINDOWS.ARCH }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.RUST }}
+          override: true
+          default: true
+          target: ${{ matrix.WINDOWS.RUST_TRIPLE }}
 
-      - name: Install MSVC for Python 2.7
-        run: |
-            Invoke-WebRequest -Uri https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -OutFile VCForPython27.msi
-            Start-Process msiexec -Wait -ArgumentList @('/i', 'VCForPython27.msi', '/qn', 'ALLUSERS=1')
-            Remove-Item VCForPython27.msi -Force
-        shell: powershell
-        if: matrix.PYTHON.VERSION == '2.7'
       - run: python -m pip install tox requests coverage
       - name: Download OpenSSL
         run: |

.github/workflows/wheel-builder.yml

@@ -11,7 +11,7 @@ jobs:
     container: ${{ matrix.MANYLINUX.CONTAINER }}
     strategy:
       matrix:
-        PYTHON: ["cp27-cp27m", "cp27-cp27mu", "cp35-cp35m"]
+        PYTHON: ["cp35-cp35m"]
         MANYLINUX:
           - NAME: manylinux1_x86_64
             CONTAINER: "pyca/cryptography-manylinux1:x86_64"
@@ -57,10 +57,6 @@ jobs:
     strategy:
       matrix:
         PYTHON:
-          - VERSION: '2.7'
-            ABI_VERSION: '2.7'
-            DOWNLOAD_URL: 'https://www.python.org/ftp/python/2.7.17/python-2.7.17-macosx10.9.pkg'
-            BIN_PATH: '/Library/Frameworks/Python.framework/Versions/2.7/bin/python'
           - VERSION: '3.8'
             ABI_VERSION: '3.5'
             DOWNLOAD_URL: 'https://www.python.org/ftp/python/3.8.2/python-3.8.2-macosx10.9.pkg'
@@ -79,6 +75,12 @@ jobs:
             ${{ matrix.PYTHON.BIN_PATH }} .github/workflows/download_openssl.py macos openssl-macos
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          default: true
 
       - run: ${{ matrix.PYTHON.BIN_PATH }} -m virtualenv venv
       - run: venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'"
@@ -111,10 +113,9 @@ jobs:
     strategy:
       matrix:
         WINDOWS:
-          - {ARCH: 'x86', WINDOWS: 'win32'}
-          - {ARCH: 'x64', WINDOWS: 'win64'}
+          - {ARCH: 'x86', WINDOWS: 'win32', RUST_TRIPLE: 'i686-pc-windows-msvc'}
+          - {ARCH: 'x64', WINDOWS: 'win64', RUST_TRIPLE: 'x86_64-pc-windows-msvc'}
         PYTHON:
-          - {VERSION: "2.7", MSVC_VERSION: "2010"}
           - {VERSION: "3.5", MSVC_VERSION: "2019"}
           - {VERSION: "3.6", MSVC_VERSION: "2019"}
           - {VERSION: "3.7", MSVC_VERSION: "2019"}
@@ -128,13 +129,13 @@ jobs:
         with:
           python-version: ${{ matrix.PYTHON.VERSION }}
           architecture: ${{ matrix.WINDOWS.ARCH }}
-      - name: Install MSVC for Python 2.7
-        run: |
-            Invoke-WebRequest -Uri https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -OutFile VCForPython27.msi
-            Start-Process msiexec -Wait -ArgumentList @('/i', 'VCForPython27.msi', '/qn', 'ALLUSERS=1')
-            Remove-Item VCForPython27.msi -Force
-        shell: powershell
-        if: matrix.PYTHON.VERSION == '2.7'
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          default: true
+          target: ${{ matrix.WINDOWS.RUST_TRIPLE }}
       - run: pip install requests
       - name: Download OpenSSL
         run: |

.gitignore

@@ -12,3 +12,5 @@ htmlcov/
 .eggs/
 *.py[cdo]
 .hypothesis/
+target/
+Cargo.lock

.travis.yml

@@ -19,26 +19,18 @@ branches:
 matrix:
     include:
         - python: 3.8
-          env: TOXENV=pep8,packaging
+          env: TOXENV=pep8,rust,packaging
         # Setting 'python' is just to make travis's UI a bit prettier
         - python: 3.6
           env: TOXENV=py36
           # Travis lists available Pythons (including PyPy) by arch and distro here:
           # https://docs.travis-ci.com/user/languages/python/#python-versions
-        - python: pypy2.7-7.3.1
-          env: TOXENV=pypy-nocoverage
         - python: pypy3.6-7.3.1
           env: TOXENV=pypy3-nocoverage
         - python: 3.8
           env: TOXENV=py38 OPENSSL=1.0.2u
-        - python: 2.7
-          env: TOXENV=py27 OPENSSL=1.1.0l
-        - python: 2.7
-          env: TOXENV=py27-ssh OPENSSL=1.1.0l
         - python: 3.8
           env: TOXENV=py38 OPENSSL=1.1.0l
-        - python: 2.7
-          env: TOXENV=py27 OPENSSL=1.1.1g
         - python: 3.8
           env: TOXENV=py38 OPENSSL=1.1.1g
         - python: 3.8
@@ -54,21 +46,12 @@ matrix:
         - python: 3.8
           env: TOXENV=py38 LIBRESSL=3.2.0
 
-        - python: 2.7
-          services: docker
-          env: TOXENV=py27 DOCKER=pyca/cryptography-runner-centos7
-        - python: 2.7
-          services: docker
-          env: TOXENV=py27 DOCKER=pyca/cryptography-runner-centos8
         - python: 3.6
           services: docker
           env: TOXENV=py36 DOCKER=pyca/cryptography-runner-centos8
         - python: 3.6
           services: docker
           env: TOXENV=py36 OPENSSL_FORCE_FIPS_MODE=1 DOCKER=pyca/cryptography-runner-centos8-fips
-        - python: 2.7
-          services: docker
-          env: TOXENV=py27 DOCKER=pyca/cryptography-runner-stretch
         - python: 3.5
           services: docker
           env: TOXENV=py35 DOCKER=pyca/cryptography-runner-stretch
@@ -87,9 +70,6 @@ matrix:
         - python: 3.8
           services: docker
           env: TOXENV=py38 DOCKER=pyca/cryptography-runner-ubuntu-focal
-        - python: 2.7
-          services: docker
-          env: TOXENV=py27 DOCKER=pyca/cryptography-runner-ubuntu-rolling
         - python: 3.8
           services: docker
           env: TOXENV=py38 DOCKER=pyca/cryptography-runner-ubuntu-rolling
@@ -138,9 +118,11 @@ matrix:
           dist: xenial
 
 install:
+    - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --component rustmft
     - ./.travis/install.sh
 
 script:
+    - source $HOME/.cargo/env
     - ./.travis/run.sh
 
 after_success:

.zuul.playbooks/playbooks/main.yaml

@@ -27,6 +27,10 @@
       become: yes
       when: ansible_distribution == 'CentOS'
 
+    - name: Install rust
+      include_role:
+        name: ensure-rust
+
     - name: Clone wycheproof
       git:
         repo: https://github.com/google/wycheproof

CHANGELOG.rst

@@ -3,6 +3,8 @@ Changelog
 
 .. _v3-1:
 
+TODO: Add Rust to the current release when this is ready to land.
+
 3.1 - `master`_
 ~~~~~~~~~~~~~~~
 

MANIFEST.in

@@ -11,6 +11,7 @@ include pyproject.toml
 
 recursive-include docs *
 recursive-include src/_cffi_src *.py *.c *.h
+recursive-include src/rust Cargo.toml *.rs
 prune docs/_build
 recursive-include tests *.py
 exclude vectors

docs/faq.rst

@@ -118,6 +118,19 @@ series. Since they are no longer receiving security patches from upstream,
 should upgrade to a newer version of OpenSSL (1.0.2 or later). This may require
 you to upgrade to a newer operating system.
 
+Installing ``cryptography`` fails with ``error: Can not find Rust compiler``
+----------------------------------------------------------------------------
+
+Building ``cryptography`` from source requires you have Rust installed and on
+your ``PATH``. You may be able to fix this by upgrading to a newer version of
+``pip`` which will install a pre-compiled ``cryptography`` wheel. If not,
+you'll need to install Rust.
+
+For the current release *only* you can temporarily bypass the requirement to
+have Rust installed by setting the ``CRYPTOGRAPHY_DONT_BUILD_RUST`` environment
+variable. Note that this option will be removed in the next release and not
+having Rust available will be a hard error.
+
 Why are there no wheels for Python 3.6+ on Linux or macOS?
 ----------------------------------------------------------
 

docs/installation.rst

@@ -66,6 +66,8 @@ platforms). ``cryptography`` links against the new 1.1.0 names by default. If
 you need to compile ``cryptography`` against an older version then you **must**
 set ``CRYPTOGRAPHY_WINDOWS_LINK_LEGACY_OPENSSL`` or else installation will fail.
 
+You will also need to have Rust installed and available.
+
 If you need to rebuild ``cryptography`` for any reason be sure to clear the
 local `wheel cache`_.
 
@@ -88,6 +90,8 @@ If you are on Alpine or just want to compile it yourself then
 using ``pypy``), and headers for the OpenSSL and ``libffi`` libraries
 available on your system.
 
+On all Linux distributions you will need to have Rust installed and available.
+
 Alpine
 ~~~~~~
 
@@ -238,6 +242,8 @@ open a terminal window and run:
 This will install a compiler (clang) along with (most of) the required
 development headers.
 
+You will also need to have Rust installed and available.
+
 You'll also need OpenSSL, which you can obtain from `Homebrew`_ or `MacPorts`_.
 Cryptography does **not** support Apple's deprecated OpenSSL distribution.
 

pyproject.toml

@@ -6,6 +6,7 @@ requires = [
     "wheel",
     # Must be kept in sync with the `setup_requirements` in `setup.py`
     "cffi>=1.8,!=1.11.3; platform_python_implementation != 'PyPy'",
+    "setuptools-rust>=0.11.1",
 ]
 build-backend = "setuptools.build_meta"
 

setup.py

@@ -17,6 +17,8 @@
 from setuptools import find_packages, setup
 from setuptools.command.install import install
 
+from setuptools_rust import RustExtension
+
 
 if pkg_resources.parse_version(
     setuptools.__version__
@@ -39,7 +41,7 @@
 
 
 # `setup_requirements` must be kept in sync with `pyproject.toml`
-setup_requirements = ["cffi>=1.8,!=1.11.3"]
+setup_requirements = ["cffi>=1.8,!=1.11.3", "setuptools-rust>=0.11.1"]
 
 if platform.python_implementation() == "PyPy":
     if sys.pypy_version_info < (5, 4):
@@ -152,10 +154,17 @@ def argument_without_setup_requirements(argv, i):
             "src/_cffi_src/build_openssl.py:ffi",
             "src/_cffi_src/build_padding.py:ffi",
         ]
+        if os.environ.get("CRYPTOGRAPHY_DONT_BUILD_RUST"):
+            rust_extensions = []
+        else:
+            rust_extensions = [
+                RustExtension("_rust", "src/rust/Cargo.toml"),
+            ]
 
         return {
             "setup_requires": setup_requirements,
             "cffi_modules": cffi_modules,
+            "rust_extensions": rust_extensions,
         }
 
 

src/rust/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "cryptography-rust"
+version = "0.1.0"
+authors = ["The cryptography developers <[email protected]>"]
+edition = "2018"
+publish = false
+
+[dependencies]
+pyo3 = { version = "0.11", features = ["extension-module"] }
+
+[lib]
+name = "cryptography_rust"
+crate-type = ["cdylib"] 

src/rust/src/lib.rs

@@ -0,0 +1,4 @@
+#[pyo3::prelude::pymodule]
+fn _rust(_py: pyo3::Python<'_>, _m: &pyo3::types::PyModule) -> pyo3::PyResult<()> {
+    Ok(())
+}

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 minversion = 2.4
-envlist = py27,pypy,py35,py36,py37,py38,docs,pep8,packaging
+envlist = py27,pypy,py35,py36,py37,py38,docs,pep8,rust,packaging
 isolated_build = True
 
 [testenv]
@@ -67,6 +67,15 @@ commands =
     flake8 .
     black --check .
 
+[testenv:rust]
+basepython = python3
+changedir = src/rust/
+allowlist_externals =
+    cargo
+commands =
+    cargo fmt --all -- --check
+    cargo clippy
+
 [testenv:packaging]
 deps =
     check-manifest