From a3f0459d82e8f934c63b4a3420915bcfa98c92ae Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 4 Aug 2024 09:13:47 +0000 Subject: [PATCH 1/5] CI: split rendering and publication tasks Removes a lot of duplication, and allows getting a copy of the site built for a specific PR. --- .cirrus.yml | 47 ++++++++++++++++++----------------------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index b18f42b..4b82e9a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -67,52 +67,41 @@ task: - debian_and_ubuntu - freebsd -# Generate the site and publish it, for main branch. -publish_task: - only_if: $BRANCH == 'main' +render_task: depends_on: - site gen success container: - image: node:20-bookworm - env: - NETLIFY_AUTH_TOKEN: ENCRYPTED[ec6da05d365917ed543e307555cbd9c618bbd41db7dee828daf7b01825adef48f547db89ead63b99418fe07a9d41b43a] - NETLIFY_SITE_ID: ENCRYPTED[a408add4551cd62fe8a5a912569fda66b5e991dd1097a8eaf0a6479179a07a6ec3e5d9e0c67d16f61c67a6721a728a91] + image: python:3-slim install_script: - - apt-get -y update - - apt-get -y install python3 python3-lxml - # https://github.com/netlify/cli/issues/1870 - - npm install --unsafe-perm=true -g netlify-cli + - pip install lxml script: - python3 -m generate_site $CIRRUS_BUILD_ID - stat _site/index.html - - netlify deploy --auth $NETLIFY_AUTH_TOKEN --site $NETLIFY_SITE_ID --dir=_site --prod + - tar -cf site.tar _site/ + site_artifacts: + path: "site.tar" -# Generates the site but don't publish it, for non-main branches. -fake_publish_task: - only_if: $BRANCH != 'main' +# Publish updates merged to the main branch +publish_task: + skip: $BRANCH != 'main' depends_on: - - site gen success + - render container: image: node:20-bookworm + env: + TOKEN: + ENCRYPTED[ec6da05d365917ed543e307555cbd9c618bbd41db7dee828daf7b01825adef48f547db89ead63b99418fe07a9d41b43a] + SITE_ID: + ENCRYPTED[a408add4551cd62fe8a5a912569fda66b5e991dd1097a8eaf0a6479179a07a6ec3e5d9e0c67d16f61c67a6721a728a91] install_script: - - apt-get -y update - - apt-get -y install python3 python3-lxml # https://github.com/netlify/cli/issues/1870 - npm install --unsafe-perm=true -g netlify-cli script: - - python3 -m generate_site $CIRRUS_BUILD_ID - - stat _site/index.html - -task: - only_if: $BRANCH == 'main' - name: CI success - container: {image: busybox} - depends_on: - - publish + - wget "https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/render/site/site.tar" -O - | tar -vx + - netlify deploy --auth $TOKEN --site $SITE_ID --dir=_site --prod task: - only_if: $BRANCH != 'main' name: CI success container: {image: busybox} depends_on: - - fake_publish + - publish # marked as successful on skip From 7e47946bae72b67aa009d0afa2879eeded49c9d5 Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 4 Aug 2024 09:22:42 +0000 Subject: [PATCH 2/5] CI: refactor `check_versions` task(s) --- .cirrus.yml | 70 +++++++++++++-------------------------- generate_site/__init__.py | 20 ++--------- 2 files changed, 26 insertions(+), 64 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 4b82e9a..d4b3e70 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -6,51 +6,31 @@ lint_task: script: - ruff check -archlinux_and_manjaro_task: - container: - matrix: - - image: archlinux:latest - - image: manjarolinux/base:latest - install_script: pacman -Sy --noconfirm python3 - script: python3 ./check_versions.py - json_artifacts: - path: "source/*.json" - -fedora_task: - container: - matrix: - - image: fedora:latest - script: python3 ./check_versions.py - json_artifacts: - path: "source/*.json" +check_versions_task: + matrix: + - freebsd_instance: {image_family: freebsd-14-0} + - container: {image: fedora:latest} + - container: + matrix: + - image: archlinux:latest + - image: manjarolinux/base:latest + install_script: pacman -Sy --noconfirm python3 -opensuse_task: - container: - matrix: - - image: opensuse/leap:latest - - image: opensuse/tumbleweed:latest - install_script: zypper install --no-confirm python3 - script: python3 ./check_versions.py - json_artifacts: - path: "source/*.json" + - container: + matrix: + - image: debian:oldstable-slim + - image: debian:stable-slim + - image: ubuntu:latest + install_script: + - apt-get -y update + - apt-get -y install python3 -debian_and_ubuntu_task: - container: - matrix: - - image: debian:oldstable-slim - - image: debian:stable-slim - - image: ubuntu:latest # Most recent LTS - see https://hub.docker.com/_/ubuntu - install_script: - - apt-get -y update - - apt-get -y install python3 - script: python3 ./check_versions.py - json_artifacts: - path: "source/*.json" + - container: + matrix: + - image: opensuse/leap:latest + - image: opensuse/tumbleweed:latest + install_script: zypper install --no-confirm python3 -freebsd_task: - freebsd_instance: - matrix: - - image_family: freebsd-14-0 script: python3 ./check_versions.py json_artifacts: path: "source/*.json" @@ -61,11 +41,7 @@ task: image: busybox depends_on: - lint - - archlinux_and_manjaro - - fedora - - opensuse - - debian_and_ubuntu - - freebsd + - check_versions render_task: depends_on: diff --git a/generate_site/__init__.py b/generate_site/__init__.py index 96dbe3c..63685a1 100644 --- a/generate_site/__init__.py +++ b/generate_site/__init__.py @@ -29,12 +29,12 @@ def build_index(directory): ]) -def download_data(build_id, os_name): +def download_data(build_id, task_name = 'check_versions'): """Download relevant Cirrus CI artifacts.""" Path('_data').mkdir(exist_ok=True) uri_domain = 'https://api.cirrus-ci.com' - uri_path = '/v1/artifact/build/{}/{}/json.zip'.format(build_id, os_name) + uri_path = '/v1/artifact/build/{}/{}/json.zip'.format(build_id, task_name) print('Downloading {}{}...'.format(uri_domain, uri_path)) @@ -48,20 +48,6 @@ def download_data(build_id, os_name): zip_file.extract(member, path='_data/') -def download_all(build_id): - """Download Cirrus CI artifacts for all supported operating systems.""" - os_name_list = [ - 'archlinux_and_manjaro', - 'fedora', - 'opensuse', - 'debian_and_ubuntu', - 'freebsd', - ] - - for os_name in os_name_list: - download_data(build_id, os_name) - - def raw_data(): """Get raw data for each .json file in _data/.""" paths = Path('_data/source').glob('*.json') @@ -155,7 +141,7 @@ def main(argv): if build_id.lower() == 'local': shutil.copytree('source/', '_data/source/') else: - download_all(build_id) + download_data(build_id) shutil.copytree('_data/', '_site/data/') From b0a95c6b6579b85829560ed568c9a0f9659e2f67 Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 4 Aug 2024 09:24:11 +0000 Subject: [PATCH 3/5] CI: drop `site gen success` metatask --- .cirrus.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index d4b3e70..b454e24 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -35,17 +35,10 @@ check_versions_task: json_artifacts: path: "source/*.json" -task: - name: site gen success - container: - image: busybox +render_task: depends_on: - lint - check_versions - -render_task: - depends_on: - - site gen success container: image: python:3-slim install_script: From c7505475e532d5d13ffa22523ec37d5cbabdafce Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 4 Aug 2024 13:41:43 +0000 Subject: [PATCH 4/5] check_versions: error-out on unsupported platforms --- check_versions.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/check_versions.py b/check_versions.py index 669f149..3f199fb 100755 --- a/check_versions.py +++ b/check_versions.py @@ -204,7 +204,8 @@ def discover(): if sys.platform.startswith('freebsd'): os_name = 'FreeBSD' os_desc = run('uname -sr') - else: # Assume everything else is Linux. + + elif sys.platform == "linux": # Treat /etc/os-release as a key/value pair of strings, # with optional quotes on the value side. os_release = {k: v[1:-1] if v[0] == '"' else v for (k, v) in [line.split("=") for line in Path("/etc/os-release").read_text().splitlines() if not line.startswith("#")]} @@ -212,6 +213,9 @@ def discover(): os_version = os_release.get('VERSION_ID', '') os_desc = '{} {}'.format(os_name, os_version).strip() + else: + raise NotImplementedError(f"Only FreeBSD and Linux are currenctly supported, not '{sys.platform}'") + return DISTROS[os_name](os_name, os_desc) From 83c2958972ec51ed85be3041cb26a6f51467314f Mon Sep 17 00:00:00 2001 From: nicoo Date: Sun, 4 Aug 2024 09:30:49 +0000 Subject: [PATCH 5/5] CI: use the official `ruff` container image We are not pinning down a specific version, and this is terser and presumably faster --- .cirrus.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index b454e24..c9777c4 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,10 +1,10 @@ lint_task: + environment: + CIRRUS_SHELL: direct container: - image: python:3-slim - install_script: - - pip3 install ruff + image: ghcr.io/astral-sh/ruff script: - - ruff check + - /ruff check check_versions_task: matrix: