Skip to content

Commit c63106c

Browse files
ianwianw
authored
Build manylinux wheels with Zuul (#5386)
This adds the Zuul playbooks and role to build manylinux wheels for aarch64 and x86_64 (while aarch64 is the primary goal; it's good for the overall code to keep it flexible). It first builds an sdist from the checkout and then builds the wheels in the appropriate containers. Note this adds the jobs in the gate pipeline, which currently responds to Pull Requests, and the release pipeline, which responds to pushes to refs/tags/.* (see [1]). Note for results of jobs run against tags you will need to find the job directly from https://zuul.opendev.org/t/pyca/builds because there is nowhere to report the results as such (it could be configured to send an email). The wheels are published to the wheelhouse/ directory in the Zuul logs, which is also listed as an artifact on the build results page. [1] https://review.opendev.org/748323
1 parent bda1387 commit c63106c

File tree

7 files changed

+245
-1
lines changed

7 files changed

+245
-1
lines changed

‎.zuul.d/jobs.yaml

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name: pyca-cryptography-base
33
abstract: true
44
description: Run pyca/cryptography unit testing
5-
run: .zuul.playbooks/playbooks/main.yaml
5+
run: .zuul.playbooks/playbooks/tox/main.yaml
66

77
- job:
88
name: pyca-cryptography-ubuntu-focal-py38-arm64
@@ -31,3 +31,38 @@
3131
nodeset: centos-8-arm64
3232
vars:
3333
tox_envlist: py27
34+
35+
- job:
36+
name: pyca-cryptography-build-wheel
37+
abstract: true
38+
run: .zuul.playbooks/playbooks/wheel/main.yaml
39+
40+
- job:
41+
name: pyca-cryptography-build-wheel-arm64
42+
parent: pyca-cryptography-build-wheel
43+
nodeset: ubuntu-bionic-arm64
44+
vars:
45+
wheel_builds:
46+
- platform: manylinux2014_aarch64
47+
image: pyca/cryptography-manylinux2014_aarch64
48+
pythons:
49+
- cp35-cp35m
50+
51+
- job:
52+
name: pyca-cryptography-build-wheel-x86_64
53+
parent: pyca-cryptography-build-wheel
54+
nodeset: ubuntu-bionic
55+
vars:
56+
wheel_builds:
57+
- platform: manylinux1_x86_64
58+
image: pyca/cryptography-manylinux1:x86_64
59+
pythons:
60+
- cp27-cp27m
61+
- cp27-cp27mu
62+
- cp35-cp35m
63+
- platform: manylinux2010_x86_64
64+
image: pyca/cryptography-manylinux2010:x86_64
65+
pythons:
66+
- cp27-cp27m
67+
- cp27-cp27mu
68+
- cp35-cp35m

‎.zuul.d/project.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,13 @@
11
- project:
22
check:
33
jobs:
4+
- pyca-cryptography-build-wheel-arm64
5+
- pyca-cryptography-build-wheel-x86_64
46
- pyca-cryptography-ubuntu-focal-py38-arm64
57
- pyca-cryptography-ubuntu-bionic-py36-arm64
68
- pyca-cryptography-centos-8-py36-arm64
79
- pyca-cryptography-centos-8-py27-arm64
10+
release:
11+
jobs:
12+
- pyca-cryptography-build-wheel-arm64
13+
- pyca-cryptography-build-wheel-x86_64

‎.zuul.playbooks/playbooks/main.yaml renamed to ‎.zuul.playbooks/playbooks/tox/main.yaml

File renamed without changes.

‎.zuul.playbooks/playbooks/wheel/main.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
- hosts: all
2+
tasks:
3+
4+
- name: Build wheel
5+
include_role:
6+
name: build-wheel-manylinux

‎.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/README.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Build manylinux wheels for cryptography

‎.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/files/build-wheels.sh

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/bash -ex
2+
3+
# Compile wheels
4+
cd /io
5+
6+
mkdir -p wheelhouse.final
7+
8+
for P in ${PYTHONS}; do
9+
10+
PYBIN=/opt/python/${P}/bin
11+
12+
"${PYBIN}"/python -m virtualenv .venv
13+
14+
.venv/bin/pip install cffi six ipaddress "enum34; python_version < '3'"
15+
16+
REGEX="cp3([0-9])*"
17+
if [[ "${PYBIN}" =~ $REGEX ]]; then
18+
PY_LIMITED_API="--py-limited-api=cp3${BASH_REMATCH[1]}"
19+
fi
20+
21+
LDFLAGS="-L/opt/pyca/cryptography/openssl/lib" \
22+
CFLAGS="-I/opt/pyca/cryptography/openssl/include -Wl,--exclude-libs,ALL" \
23+
.venv/bin/python setup.py bdist_wheel $PY_LIMITED_API
24+
25+
auditwheel repair --plat ${PLAT} -w wheelhouse/ dist/cryptography*.whl
26+
27+
# Sanity checks
28+
# NOTE(ianw) : no execstack on aarch64, comes from
29+
# prelink, which was never supported. CentOS 8 does
30+
# have it separate, skip for now.
31+
if [[ "${PLAT}" != "manylinux2014_aarch64" ]]; then
32+
for f in wheelhouse/*.whl; do
33+
unzip $f -d execstack.check
34+
35+
results=$(execstack execstack.check/cryptography/hazmat/bindings/*.so)
36+
count=$(echo "$results" | grep -c '^X' || true)
37+
if [ "$count" -ne 0 ]; then
38+
exit 1
39+
fi
40+
rm -rf execstack.check
41+
done
42+
fi
43+
44+
.venv/bin/pip install cryptography --no-index -f wheelhouse/
45+
.venv/bin/python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))"
46+
47+
# Cleanup
48+
mv wheelhouse/* wheelhouse.final
49+
rm -rf .venv dist wheelhouse
50+
51+
done

‎.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/tasks/main.yaml

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
# Wheel builds is a list of dicts, with keys
2+
#
3+
# platform: the manylinux platform name
4+
# image: the docker image to build in
5+
# pythons: list of pythons in the image to build wheels for
6+
- name: Sanity check build list
7+
assert:
8+
that: wheel_builds is defined
9+
10+
- name: Ensure pip installed
11+
include_role:
12+
name: ensure-pip
13+
14+
- name: Run ensure-docker
15+
include_role:
16+
name: ensure-docker
17+
18+
- name: Workaround Linaro aarch64 cloud MTU issues
19+
# NOTE(ianw) : Docker default networking, the Linaro NAT setup and
20+
# *insert random things here* cause PMTU issues, resulting in hung
21+
# connections, particularly to fastly CDN (particularly annoying
22+
# because pypi and pythonhosted live behind that). Can remove after
23+
# upstream changes merge, or we otherwise find a solution in the
24+
# upstream cloud.
25+
# https://review.opendev.org/747062
26+
# https://review.opendev.org/746833
27+
# https://review.opendev.org/747064
28+
when: ansible_architecture == 'aarch64'
29+
block:
30+
- name: Install jq
31+
package:
32+
name: jq
33+
state: present
34+
become: yes
35+
36+
- name: Reset docker MTU
37+
shell: |
38+
jq --arg mtu 1400 '. + {mtu: $mtu|tonumber}' /etc/docker/daemon.json > /etc/docker/daemon.json.new
39+
cat /etc/docker/daemon.json.new
40+
mv /etc/docker/daemon.json.new /etc/docker/daemon.json
41+
service docker restart
42+
become: yes
43+
44+
# We build an sdist of the checkout, and then build wheels from the
45+
# sdist. This ensures that nothing is left out of the sdist.
46+
- name: Install sdist required packages
47+
package:
48+
name:
49+
- build-essential
50+
- libssl-dev
51+
- libffi-dev
52+
- python3-dev
53+
become: yes
54+
when: ansible_distribution in ['Debian', 'Ubuntu']
55+
56+
- name: Create sdist
57+
command: |
58+
python3 setup.py sdist
59+
args:
60+
chdir: '{{ ansible_user_dir }}/{{ zuul.project.src_dir }}'
61+
62+
- name: Find output file
63+
find:
64+
paths: '{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/dist'
65+
file_type: file
66+
patterns: "*.tar.gz"
67+
register: _sdist
68+
69+
- assert:
70+
that:
71+
- _sdist.matched == 1
72+
73+
- name: Create a build area
74+
file:
75+
path: '{{ ansible_user_dir }}/build'
76+
state: directory
77+
78+
- name: Create build area from sdist
79+
unarchive:
80+
src: '{{ _sdist.files[0].path }}'
81+
dest: '{{ ansible_user_dir }}/build'
82+
remote_src: yes
83+
84+
- name: Find cryptography subdir from sdist build dir
85+
set_fact:
86+
_build_dir: "{{ ansible_user_dir }}/build/{{ _sdist.files[0].path | basename | replace('.tar.gz', '') }}"
87+
88+
- name: Show _build_dir
89+
debug:
90+
var: _build_dir
91+
92+
- name: Install build script
93+
copy:
94+
src: build-wheels.sh
95+
dest: '{{ _build_dir }}'
96+
mode: 0755
97+
98+
- name: Pre-pull containers
99+
command: >-
100+
docker pull {{ item.image }}
101+
become: yes
102+
loop: '{{ wheel_builds }}'
103+
104+
- name: Run builds
105+
command: |
106+
docker run --rm \
107+
-e PLAT={{ item.platform }} \
108+
-e PYTHONS="{{ item.pythons | join(' ') }}" \
109+
-v {{ _build_dir }}:/io \
110+
{{ item.image }} \
111+
/io/build-wheels.sh
112+
become: yes
113+
loop: '{{ wheel_builds }}'
114+
115+
- name: Copy sdist to output
116+
synchronize:
117+
src: '{{ _sdist.files[0].path }}'
118+
dest: '{{ zuul.executor.log_root }}'
119+
mode: pull
120+
121+
- name: Return sdist artifact
122+
zuul_return:
123+
data:
124+
zuul:
125+
artifacts:
126+
- name: '{{ _sdist.files[0].path | basename }}'
127+
url: 'sdist/{{ _sdist.files[0].path }}'
128+
metadata:
129+
type: sdist
130+
131+
- name: Copy wheels to output
132+
synchronize:
133+
src: '{{ _build_dir }}/wheelhouse.final/'
134+
dest: '{{ zuul.executor.log_root }}/wheelhouse'
135+
mode: pull
136+
137+
- name: Return wheelhouse artifact
138+
zuul_return:
139+
data:
140+
zuul:
141+
artifacts:
142+
- name: "Wheelhouse"
143+
url: "wheelhouse"
144+
metadata:
145+
type: wheelhouse

0 commit comments

Comments
 (0)