From 4353efe612a36ec77e6217b40fd4186df0c64575 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 18 Feb 2025 12:41:55 -0800 Subject: [PATCH] Fixup install.sh output. Also improve docker dev-cmd to support `--inspect` and cache image builds. --- .github/workflows/ci.yml | 18 +--- .github/workflows/release.yml | 3 - install.sh | 16 +-- pyproject.toml | 5 + scripts/docker/uv.py | 187 +++++++++++++++++++++++----------- scripts/docker/uv/Dockerfile | 7 +- uv.lock | 35 +++++++ 7 files changed, 187 insertions(+), 84 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8965897..b92fc80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,9 +17,6 @@ concurrency: group: CI-${{ github.ref }} # Queue on all branches and tags, but only cancel overlapping PR burns. cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }} -env: - # Work around SIGSEGVs and other errors under some qemu targets. - UV_CONCURRENT_BUILDS: 1 jobs: org-check: name: Check GitHub Organization @@ -115,14 +112,14 @@ jobs: path: .mypy_cache # We're using a key suffix / restore-keys prefix trick here to get an updatable cache. # See: https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache - key: ${{ matrix.docker-platform || format('{0}-{1}', matrix.os, runner.arch) }}-a-scie-lift-mypy-v1-${{ github.run_id }} - restore-keys: ${{ matrix.docker-platform || format('{0}-{1}', matrix.os, runner.arch) }}-a-scie-lift-mypy-v1 + key: ${{ matrix.image || format('{0}-{1}', matrix.os, runner.arch) }}-a-scie-lift-mypy-v1-${{ github.run_id }} + restore-keys: ${{ matrix.image || format('{0}-{1}', matrix.os, runner.arch) }}-a-scie-lift-mypy-v1 - name: Check Formatting & Lints if: matrix.image == '' run: | "${UV}" run dev-cmd ci --skip test - name: Check Formatting & Lints - if: matrix.image != '' + if: matrix.image == 'debian' && matrix.arch == 'amd64' run: | "${UV}" run dev-cmd docker -- --image ${{ matrix.image }} --arch ${{ matrix.arch }} \ ci --skip test @@ -143,13 +140,8 @@ jobs: run: | "${UV}" run dev-cmd test -- -vvs - name: Unit Tests - if: matrix.image != '' + if: matrix.image != '' && matrix.arch != 'ppc64le' run: | - if [ "${{ matrix.arch }}" = "ppc64le" ]; then - echo "Skipping tests on ppc64le." - exit 0 - fi - "${UV}" run dev-cmd docker -- --image ${{ matrix.image }} --arch ${{ matrix.arch }} \ test -- -vvs - name: Build & Package @@ -166,7 +158,7 @@ jobs: run: | "${UV}" run dev-cmd doc linkcheck - name: Generate Doc Site - if: matrix.image != '' + if: matrix.image == 'debian' && matrix.arch == 'amd64' run: | "${UV}" run dev-cmd docker -- --image ${{ matrix.image }} --arch ${{ matrix.arch }} \ doc linkcheck diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6aa7504..0bf8e45 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,9 +11,6 @@ on: defaults: run: shell: bash -env: - # Work around SIGSEGVs and other errors under some qemu targets. - UV_CONCURRENT_BUILDS: 1 jobs: org-check: name: Check GitHub Organization diff --git a/install.sh b/install.sh index 63ded43..ad12b81 100755 --- a/install.sh +++ b/install.sh @@ -4,13 +4,13 @@ set -eu -COLOR_RED="\x1b[31m" -COLOR_GREEN="\x1b[32m" -COLOR_YELLOW="\x1b[33m" -COLOR_RESET="\x1b[0m" +COLOR_RED="\e[31m" +COLOR_GREEN="\e[32m" +COLOR_YELLOW="\e[33m" +COLOR_RESET="\e[0m" log() { - echo -e "$@" >&2 + printf "$@\n" >&2 } die() { @@ -107,9 +107,9 @@ fetch() { curl --proto '=https' --tlsv1.2 -SfL --progress-bar -o "${dest}" "${url}" } -ensure_cmd $([ "${OS}" == "macos" ] && echo "shasum" || echo "sha256sum") +ensure_cmd $([ "${OS}" = "macos" ] && echo "shasum" || echo "sha256sum") sha256() { - if [[ "${OS}" == "macos" ]]; then + if [ "${OS}" = "macos" ]; then shasum --algorithm 256 "$@" else sha256sum "$@" @@ -204,7 +204,7 @@ done ARCH="$(determine_arch)" VARIANT="$(determine_variant)" -DIRSEP=$([ "${OS}" == "windows" ] && echo "\\" || echo "/") +DIRSEP=$([ "${OS}" = "windows" ] && echo "\\" || echo "/") EXE_EXT=$([ "${OS}" = "windows" ] && echo ".exe" || echo "") INSTALL_DEST="${INSTALL_PREFIX}${DIRSEP}science${EXE_EXT}" diff --git a/pyproject.toml b/pyproject.toml index 6462015..9a22f7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ include = ["science*"] [dependency-groups] dev = [ "ansicolors", + "coloredlogs", "dev-cmd", "docutils", "mypy", @@ -77,6 +78,10 @@ follow_untyped_imports = true module = "click_log" follow_untyped_imports = true +[[tool.mypy.overrides]] +module = "coloredlogs" +follow_untyped_imports = true + [[tool.mypy.overrides]] module = ["colors.*"] follow_untyped_imports = true diff --git a/scripts/docker/uv.py b/scripts/docker/uv.py index aaa694c..764629c 100755 --- a/scripts/docker/uv.py +++ b/scripts/docker/uv.py @@ -1,7 +1,10 @@ # Copyright 2025 Science project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). - +import hashlib +import json +import logging import os +import shlex import shutil import subprocess import sys @@ -9,13 +12,58 @@ from pathlib import Path from typing import Any +import coloredlogs + from science.platform import CURRENT_OS, Os +logger = logging.getLogger(__name__) + + +def fingerprint_path(path: Path) -> tuple[str, str]: + if path.is_dir(): + tree: dict[str, str] = {} + for r, _, files in os.walk(path): + root = Path(r) + for f in files: + file_path = root / f + tree[str(file_path.relative_to(path))] = hashlib.sha256( + file_path.read_bytes() + ).hexdigest() + + fingerprint = hashlib.sha256(json.dumps(tree, sort_keys=True).encode()).hexdigest() + else: + fingerprint = hashlib.sha256(path.read_bytes()).hexdigest() + + return str(path.resolve().relative_to(Path().resolve())), fingerprint + + +def fingerprint_paths(*paths: Path) -> str: + return hashlib.sha256( + json.dumps(dict(fingerprint_path(path) for path in paths), sort_keys=True).encode() + ).hexdigest() + + +def image_exists(image_name: str) -> bool: + result = subprocess.run( + args=["docker", "image", "ls", "-q", image_name], capture_output=True, text=True + ) + return result.returncode == 0 and bool(result.stdout.strip()) + def main() -> Any: if CURRENT_OS is Os.Windows: return "This script does not work on Windows yet." + coloredlogs.install( + fmt="%(levelname)s %(message)s", + field_styles={ + **coloredlogs.DEFAULT_FIELD_STYLES, + # Default is bold black, we switch to gray; c.f: + # https://coloredlogs.readthedocs.io/en/latest/api.html#available-text-styles-and-colors + "levelname": {"bold": True, "color": 8}, + }, + ) + parser = ArgumentParser() parser.add_argument("--image", default="debian", choices=["alpine", "debian"]) parser.add_argument( @@ -34,6 +82,12 @@ def main() -> Any: "arm/v6", ], ) + parser.add_argument( + "--inspect", + default=False, + action="store_true", + help="Instead of running `uv run dev-cmd` against the extra args, drop into a shell in the image for inspection.", + ) options, args = parser.parse_known_args() platform = f"linux/{options.arch}" @@ -52,69 +106,84 @@ def main() -> Any: parent_dir = Path(__file__).parent arch_tag = options.arch.replace("/", "-") - base_image = f"a-scie/lift/base:{arch_tag}" - # The type-ignores for os.get{uid,gid} cover Windows which we explicitly fail-fast for above. - subprocess.run( - args=[ - "docker", - "buildx", - "build", - "--build-arg", - f"UID={os.getuid()}", # type:ignore[attr-defined] - "--build-arg", - f"GID={os.getgid()}", # type:ignore[attr-defined] - "--platform", - platform, - "--tag", - base_image, - str(parent_dir / options.image), - ], - check=True, - ) + dev_image_context = parent_dir / "uv" + fingerprint = fingerprint_paths(Path("pyproject.toml"), Path("uv.lock"), dev_image_context) + dev_image = f"a-scie/lift/dev:{arch_tag}-{fingerprint}" + if not image_exists(dev_image): + base_image_context = parent_dir / options.image + fingerprint = fingerprint_paths(base_image_context) + base_image = f"a-scie/lift/base:{arch_tag}-{fingerprint}" + if not image_exists(base_image): + # The type-ignores for os.get{uid,gid} cover Windows which we explicitly fail-fast for above. + subprocess.run( + args=[ + "docker", + "buildx", + "build", + "--build-arg", + f"UID={os.getuid()}", # type:ignore[attr-defined] + "--build-arg", + f"GID={os.getgid()}", # type:ignore[attr-defined] + "--platform", + platform, + "--tag", + base_image, + str(base_image_context), + ], + check=True, + ) - dev_image = f"a-scie/lift/dev:{arch_tag}" + ephemeral_build_context = parent_dir / "ephemeral-build-context" + ephemeral_build_context.mkdir(parents=True, exist_ok=True) + shutil.copy(Path("pyproject.toml"), ephemeral_build_context) + shutil.copy(Path("uv.lock"), ephemeral_build_context) - ephemeral_build_context = parent_dir / "ephemeral-build-context" - ephemeral_build_context.mkdir(parents=True, exist_ok=True) - shutil.copy(Path("pyproject.toml"), ephemeral_build_context) - shutil.copy(Path("uv.lock"), ephemeral_build_context) + subprocess.run( + args=[ + "docker", + "buildx", + "build", + "--build-arg", + f"BASE_IMAGE={base_image}", + "--build-context", + f"ephemeral={ephemeral_build_context}", + "--platform", + platform, + "--tag", + dev_image, + str(dev_image_context), + ], + check=True, + ) - subprocess.run( - args=[ - "docker", - "buildx", - "build", - "--build-arg", - f"BASE_IMAGE={base_image}", - "--build-context", - f"ephemeral={ephemeral_build_context}", - "--platform", - platform, - "--tag", - dev_image, - str(parent_dir / "uv"), - ], - check=True, - ) + docker_run_args = [ + "docker", + "run", + "--rm", + "-e", + "FORCE_COLOR", + "-e", + "SCIENCE_AUTH_API_GITHUB_COM_BEARER", + "-v", + f"{Path().absolute()}:/code", + "--platform", + platform, + ] + if options.inspect: + if args: + logger.warning(f"Ignoring extra args in --inspect mode: {shlex.join(args)}") + docker_run_args.append("--interactive") + docker_run_args.append("--tty") + docker_run_args.append("--entrypoint") + docker_run_args.append("sh" if options.image == "alpine" else "bash") + docker_run_args.append(dev_image) + docker_run_args.append("-i") + else: + docker_run_args.append(dev_image) + docker_run_args.extend(args) - subprocess.run( - args=[ - "docker", - "run", - "-e", - "FORCE_COLOR", - "-e", - "SCIENCE_AUTH_API_GITHUB_COM_BEARER", - "-v", - f"{Path().absolute()}:/code", - "--platform", - platform, - dev_image, - *args, - ], - check=True, - ) + subprocess.run(args=docker_run_args, check=True) if __name__ == "__main__": diff --git a/scripts/docker/uv/Dockerfile b/scripts/docker/uv/Dockerfile index 451b910..9b92b45 100644 --- a/scripts/docker/uv/Dockerfile +++ b/scripts/docker/uv/Dockerfile @@ -9,7 +9,12 @@ COPY --from=ephemeral uv.lock . RUN chown -R build:build /code USER build + +# Work around SIGSEGVs and other errors under some qemu targets. +ENV UV_CONCURRENT_BUILDS=1 +# Silence warning about needing to copy since we know /code/.venv lives on a bind mount. ENV UV_LINK_MODE=copy + RUN uv sync --frozen --no-install-project && rm pyproject.toml uv.lock -ENTRYPOINT ["uv", "run", "dev-cmd"] \ No newline at end of file +ENTRYPOINT ["uv", "run", "--frozen", "dev-cmd"] \ No newline at end of file diff --git a/uv.lock b/uv.lock index 35d60cb..c90df36 100644 --- a/uv.lock +++ b/uv.lock @@ -149,6 +149,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, ] +[[package]] +name = "coloredlogs" +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "humanfriendly" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934", size = 46018 }, +] + [[package]] name = "dev-cmd" version = "0.17.1" @@ -229,6 +241,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 }, ] +[[package]] +name = "humanfriendly" +version = "10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyreadline3", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794 }, +] + [[package]] name = "idna" version = "3.10" @@ -432,6 +456,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/3f/01c8b82017c199075f8f788d0d906b9ffbbc5a47dc9918a945e13d5a2bda/pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a", size = 1205513 }, ] +[[package]] +name = "pyreadline3" +version = "3.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/49/4cea918a08f02817aabae639e3d0ac046fef9f9180518a3ad394e22da148/pyreadline3-3.5.4.tar.gz", hash = "sha256:8d57d53039a1c75adba8e50dd3d992b28143480816187ea5efbd5c78e6c885b7", size = 99839 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/dc/491b7661614ab97483abf2056be1deee4dc2490ecbf7bff9ab5cdbac86e1/pyreadline3-3.5.4-py3-none-any.whl", hash = "sha256:eaf8e6cc3c49bcccf145fc6067ba8643d1df34d604a1ec0eccbf7a18e6d3fae6", size = 83178 }, +] + [[package]] name = "pytest" version = "8.3.4" @@ -537,6 +570,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "ansicolors" }, + { name = "coloredlogs" }, { name = "dev-cmd" }, { name = "docutils" }, { name = "mypy" }, @@ -577,6 +611,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "ansicolors" }, + { name = "coloredlogs" }, { name = "dev-cmd" }, { name = "docutils" }, { name = "mypy" },