Integrate Rust into the build process properly

.github/dependabot.yml

@@ -4,3 +4,10 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+  - package-ecosystem: cargo
+    directory: "/src/rust/"
+    schedule:
+      interval: daily
+    allow:
+      # Also update indirect dependencies
+      - dependency-type: all

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         PYTHON:
-          - {VERSION: "3.9", TOXENV: "pep8,packaging,docs", COVERAGE: "false"}
+          - {VERSION: "3.9", TOXENV: "pep8,rust,packaging,docs", COVERAGE: "false"}
           - {VERSION: "pypy3", TOXENV: "pypy3"}
           - {VERSION: "3.9", TOXENV: "py39", OPENSSL: {TYPE: "openssl", VERSION: "1.1.0l"}}
           - {VERSION: "3.9", TOXENV: "py39", OPENSSL: {TYPE: "openssl", VERSION: "1.1.1i"}}
@@ -26,6 +26,8 @@ jobs:
           - {VERSION: "3.9", TOXENV: "py39", OPENSSL: {TYPE: "libressl", VERSION: "3.1.5"}}
           - {VERSION: "3.9", TOXENV: "py39", OPENSSL: {TYPE: "libressl", VERSION: "3.2.3"}}
           - {VERSION: "3.9", TOXENV: "py39", OPENSSL: {TYPE: "libressl", VERSION: "3.3.1"}}
+        RUST:
+          - stable
     name: "${{ matrix.PYTHON.TOXENV }} ${{ matrix.PYTHON.OPENSSL.TYPE }} ${{ matrix.PYTHON.OPENSSL.VERSION }} ${{ matrix.PYTHON.OPENSSL.CONFIG_FLAGS }}"
     timeout-minutes: 30
     steps:
@@ -34,6 +36,12 @@ jobs:
         uses: actions/[email protected]
         with:
           python-version: ${{ matrix.PYTHON.VERSION }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.RUST }}
+          override: true
+          default: true
       - run: git clone --depth=1 https://github.com/google/wycheproof
       - run: python -m pip install tox requests coverage
       - name: Compute config hash and set config vars
@@ -107,17 +115,56 @@ jobs:
       - run: 'tox -- --wycheproof-root="$HOME/wycheproof"'
         env:
           TOXENV: ${{ matrix.IMAGE.TOXENV }}
+          RUSTUP_HOME: /root/.rustup
       - uses: ./.github/actions/upload-coverage
         with:
           name: "${{ matrix.IMAGE.TOXENV }} on ${{ matrix.IMAGE.IMAGE }}"
 
+  linux-rust:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        PYTHON:
+          - {VERSION: "3.9", TOXENV: "py39"}
+        RUST:
+          # Cover MSRV and in-dev versions
+          - 1.45.0
+          - beta
+          - nightly
+    name: "${{ matrix.PYTHON.TOXENV }} with Rust ${{ matrix.RUST }}"
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup python
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ matrix.PYTHON.VERSION }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.RUST }}
+          override: true
+          default: true
+      - run: git clone --depth=1 https://github.com/google/wycheproof
+      - run: python -m pip install tox coverage
+      - name: Tests
+        run: |
+            tox -r --  --color=yes --wycheproof-root=wycheproof
+        env:
+          TOXENV: ${{ matrix.PYTHON.TOXENV }}
+      - uses: ./.github/actions/upload-coverage
+        with:
+          name: "${{ matrix.PYTHON.TOXENV }} with Rust ${{ matrix.RUST }}"
+
   macos:
     runs-on: macos-latest
     strategy:
       matrix:
         PYTHON:
           - {VERSION: "3.6", TOXENV: "py36", EXTRA_CFLAGS: ""}
           - {VERSION: "3.9", TOXENV: "py39", EXTRA_CFLAGS: "-DUSE_OSRANDOM_RNG_FOR_TESTING"}
+        RUST:
+          - stable
     name: "${{ matrix.PYTHON.TOXENV }} on macOS"
     timeout-minutes: 30
     steps:
@@ -126,6 +173,12 @@ jobs:
         uses: actions/[email protected]
         with:
           python-version: ${{ matrix.PYTHON.VERSION }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.RUST }}
+          override: true
+          default: true
 
       - run: python -m pip install tox requests coverage
 
@@ -155,11 +208,13 @@ jobs:
     strategy:
       matrix:
         WINDOWS:
-          - {ARCH: 'x86', WINDOWS: 'win32'}
-          - {ARCH: 'x64', WINDOWS: 'win64'}
+          - {ARCH: 'x86', WINDOWS: 'win32', RUST_TRIPLE: 'i686-pc-windows-msvc'}
+          - {ARCH: 'x64', WINDOWS: 'win64', RUST_TRIPLE: 'x86_64-pc-windows-msvc'}
         PYTHON:
           - {VERSION: "3.6", TOXENV: "py36", MSVC_VERSION: "2019", CL_FLAGS: ""}
           - {VERSION: "3.9", TOXENV: "py39", MSVC_VERSION: "2019", CL_FLAGS: "/D USE_OSRANDOM_RNG_FOR_TESTING"}
+        RUST:
+          - stable
         JOB_NUMBER: [0, 1, 2, 3]
     name: "${{ matrix.PYTHON.TOXENV }} on ${{ matrix.WINDOWS.WINDOWS }} (part ${{ matrix.JOB_NUMBER }})"
     timeout-minutes: 30
@@ -170,6 +225,13 @@ jobs:
         with:
           python-version: ${{ matrix.PYTHON.VERSION }}
           architecture: ${{ matrix.WINDOWS.ARCH }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.RUST }}
+          override: true
+          default: true
+          target: ${{ matrix.WINDOWS.RUST_TRIPLE }}
 
       - run: python -m pip install tox requests coverage
       - name: Download OpenSSL
@@ -203,14 +265,24 @@ jobs:
           - dynamodb-encryption-sdk
           - certbot
           - certbot-josepy
+        RUST:
+          - stable
+        PYTHON:
+          - 3.7
     name: "Downstream tests for ${{ matrix.DOWNSTREAM }}"
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@v2
       - name: Setup python
         uses: actions/[email protected]
         with:
-          python-version: 3.7
+          python-version: ${{ matrix.PYTHON }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.RUST }}
+          override: true
+          default: true
       - run: python -m pip install -U pip wheel
       - run: ./.github/downstream.d/${{ matrix.DOWNSTREAM }}.sh install
       - run: pip uninstall -y enum34
@@ -228,6 +300,12 @@ jobs:
         uses: actions/[email protected]
         with:
           python-version: 3.9
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          default: true
       - run: python -m pip install -U tox
       - run: tox -r --  --color=yes
         env:

.github/workflows/wheel-builder.yml

@@ -13,8 +13,6 @@ jobs:
       matrix:
         PYTHON: ["cp36-cp36m"]
         MANYLINUX:
-          - NAME: manylinux1_x86_64
-            CONTAINER: "cryptography-manylinux1:x86_64"
           - NAME: manylinux2010_x86_64
             CONTAINER: "cryptography-manylinux2010:x86_64"
           - NAME: manylinux2014_x86_64
@@ -23,7 +21,7 @@ jobs:
     steps:
       - run: /opt/python/${{ matrix.PYTHON }}/bin/python -m virtualenv .venv
       - name: Install Python dependencies
-        run: .venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'"
+        run: .venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3' setuptools-rust"
       - run: .venv/bin/pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir tmpwheelhouse
       - run: |
           REGEX="cp3([0-9])*"
@@ -77,9 +75,15 @@ jobs:
             ${{ matrix.PYTHON.BIN_PATH }} .github/workflows/download_openssl.py macos openssl-macos-x86-64
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          default: true
 
       - run: ${{ matrix.PYTHON.BIN_PATH }} -m virtualenv venv
-      - run: venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'"
+      - run: venv/bin/pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3' setuptools-rust"
       - run: venv/bin/pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir wheelhouse
       - name: Build the wheel
         run: |
@@ -104,8 +108,8 @@ jobs:
     strategy:
       matrix:
         WINDOWS:
-          - {ARCH: 'x86', WINDOWS: 'win32'}
-          - {ARCH: 'x64', WINDOWS: 'win64'}
+          - {ARCH: 'x86', WINDOWS: 'win32', RUST_TRIPLE: 'i686-pc-windows-msvc'}
+          - {ARCH: 'x64', WINDOWS: 'win64', RUST_TRIPLE: 'x86_64-pc-windows-msvc'}
         PYTHON:
           - {VERSION: "3.8", MSVC_VERSION: "2019", "ABI_VERSION": "cp36"}
     name: "${{ matrix.PYTHON.VERSION }} ${{ matrix.WINDOWS.WINDOWS }} ${{ matrix.PYTHON.ABI_VERSION }}"
@@ -116,6 +120,14 @@ jobs:
         with:
           python-version: ${{ matrix.PYTHON.VERSION }}
           architecture: ${{ matrix.WINDOWS.ARCH }}
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          default: true
+          target: ${{ matrix.WINDOWS.RUST_TRIPLE }}
+
       - run: pip install requests
       - name: Download OpenSSL
         run: |
@@ -126,7 +138,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         shell: bash
 
-      - run: python -m pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3'"
+      - run: python -m pip install -U pip wheel cffi six ipaddress "enum34; python_version < '3' setuptools-rust"
       - run: pip download cryptography==${{ github.event.inputs.version }} --no-deps --no-binary cryptography && tar zxvf cryptography*.tar.gz && mkdir wheelhouse
         shell: bash
       - run: cd cryptography* && python setup.py bdist_wheel --py-limited-api=${{ matrix.PYTHON.ABI_VERSION }} && mv dist/cryptography*.whl ../wheelhouse

.gitignore

@@ -12,3 +12,4 @@ htmlcov/
 .eggs/
 *.py[cdo]
 .hypothesis/
+target/

.zuul.d/jobs.yaml

@@ -49,10 +49,6 @@
     nodeset: ubuntu-bionic
     vars:
       wheel_builds:
-        - platform: manylinux1_x86_64
-          image: ghcr.io/pyca/cryptography-manylinux1:x86_64
-          pythons:
-            - cp36-cp36m
         - platform: manylinux2010_x86_64
           image: ghcr.io/pyca/cryptography-manylinux2010:x86_64
           pythons:

.zuul.playbooks/playbooks/tox/pre.yaml

@@ -32,3 +32,7 @@
       become: yes
       when: ansible_distribution == 'CentOS'
 
+    - name: Install rust
+      include_role:
+        name: ensure-rust
+

.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/files/build-wheels.sh

@@ -11,7 +11,7 @@ for P in ${PYTHONS}; do
 
     "${PYBIN}"/python -m virtualenv .venv
 
-    .venv/bin/pip install cffi six ipaddress "enum34; python_version < '3'"
+    .venv/bin/pip install cffi six ipaddress "enum34; python_version < '3'" setuptools-rust
 
     REGEX="cp3([0-9])*"
     if [[ "${PYBIN}" =~ $REGEX ]]; then

.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/tasks/main.yaml

@@ -23,6 +23,15 @@
   become: yes
   when: ansible_distribution in ['Debian', 'Ubuntu']
 
+- name: Install rust
+  include_role:
+    name: ensure-rust
+
+- name: Install setuptools-rust
+  pip:
+    name: setuptools-rust
+  become: yes
+
 - name: Create sdist
   command: |
     python3 setup.py sdist

CHANGELOG.rst

@@ -1,14 +1,19 @@
 Changelog
 =========
 
+.. _v3-4:
+
 3.4 - `master`_
 ~~~~~~~~~~~~~~~
 
 .. note:: This version is not yet released and is under active development.
 
 * **BACKWARDS INCOMPATIBLE:** Support for Python 2 has been removed.
-* We now ship ``manylinux2014`` wheels in addition to our ``manylinux1`` and
-  ``manylinux2010`` wheels.
+* We now ship ``manylinux2014`` wheels, and no longer ship ``manylinux1``
+  wheels.
+* ``cryptography`` now incorporates Rust code. Users building ``cryptography``
+  themselves will need to have the Rust toolchain installed, users who use an
+  officially produced wheel will not need to make any changes.
 
 .. _v3-3-1:
 

MANIFEST.in

@@ -10,6 +10,7 @@ include pyproject.toml
 
 recursive-include docs *
 recursive-include src/_cffi_src *.py *.c *.h
+recursive-include src/rust Cargo.toml Cargo.lock *.rs
 prune docs/_build
 recursive-include tests *.py
 exclude vectors

docs/conf.py

@@ -44,6 +44,7 @@
 # extensions  coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     "sphinx.ext.autodoc",
+    "sphinx.ext.autosectionlabel",
     "sphinx.ext.doctest",
     "sphinx.ext.intersphinx",
     "sphinx.ext.viewcode",
@@ -198,3 +199,5 @@
     # Inconsistent small DH params they seem incapable of fixing
     r"https://www.secg.org/sec1-v2.pdf",
 ]
+
+autosectionlabel_prefix_document = True

docs/faq.rst

@@ -117,6 +117,19 @@ upstream, ``cryptography`` is also dropping support for them. To fix this issue
 you should upgrade to a newer version of OpenSSL (1.1.0 or later). This may
 require you to upgrade to a newer operating system.
 
+Installing ``cryptography`` fails with ``error: Can not find Rust compiler``
+----------------------------------------------------------------------------
+
+Building ``cryptography`` from source requires you have :ref:`Rust installed
+and available<installation:Rust>` on your ``PATH``. You may be able to fix this
+by upgrading to a newer version of ``pip`` which will install a pre-compiled
+``cryptography`` wheel. If not, you'll need to install Rust.
+
+For the current release *only* you can temporarily bypass the requirement to
+have Rust installed by setting the ``CRYPTOGRAPHY_DONT_BUILD_RUST`` environment
+variable. Note that this option will be removed in the next release and not
+having Rust available will be a hard error.
+
 Why are there no wheels for my Python3.x version?
 -------------------------------------------------