[deps] record expected hashes with pinned requirements

Makefile

@@ -100,7 +100,9 @@ check-pip-requirements:
 		gear \
 		web_common \
 		batch \
-		ci
+		ci \
+		ci/ci-utils \
+		docker/hailgenetics/hail
 
 .PHONY: install-dev-requirements
 install-dev-requirements:
@@ -140,6 +142,8 @@ generate-pip-lockfiles:
 	./generate-pip-lockfile.sh web_common
 	./generate-pip-lockfile.sh batch
 	./generate-pip-lockfile.sh ci
+	./generate-pip-lockfile.sh ci/ci-utils
+	./generate-pip-lockfile.sh docker/hailgenetics/hail
 
 $(HAILTOP_VERSION):
 	$(MAKE) -C hail python-version-info
@@ -161,7 +165,7 @@ hail-run-image: base-image hail/Dockerfile.hail-run hail/python/pinned-requireme
 	./docker-build.sh . hail/Dockerfile.hail-run $(IMAGE_NAME) --build-arg BASE_IMAGE=$(shell cat base-image)
 	echo $(IMAGE_NAME) > $@
 
-hailgenetics-hail-image: hail-ubuntu-image docker/hailgenetics/hail/Dockerfile $(shell git ls-files hail/src/main hail/python)
+hailgenetics-hail-image: hail-ubuntu-image docker/hailgenetics/hail/Dockerfile docker/hailgenetics/hail/pinned-requirements.txt $(shell git ls-files hail/src/main hail/python)
 	$(MAKE) HAIL_BUILD_MODE=Release -C hail wheel
 	./docker-build.sh . docker/hailgenetics/hail/Dockerfile $(IMAGE_NAME) \
 		--build-arg BASE_IMAGE=$(shell cat hail-ubuntu-image)

build.yaml

@@ -1779,6 +1779,8 @@ steps:
         to: /io/repo/batch
       - from: /repo/ci
         to: /io/repo/ci
+      - from: /repo/docker/hailgenetics/hail
+        to: /io/repo/docker/hailgenetics/hail
     dependsOn:
       - hail_linting_image
       - merge_code

check_pip_requirements.sh

@@ -10,13 +10,25 @@ do
     pinned_no_comments=$(mktemp)
     new_pinned_no_comments=$(mktemp)
 
+    missing=$(awk '/^[A-Za-z0-9]/ { if (req) print req; req = $1; next } /^[[:space:]]*--hash=/ { req = "" } END { if (req) print req }' $pinned)
+    if [ -n "$missing" ]; then
+        echo "$pinned has requirements without hashes:"
+        echo "$missing"
+        echo "Please regenerate the pinned requirements file by running:"
+        echo "> make generate-pip-lockfiles"
+        exit 1
+    fi
+
     uv pip compile \
         --python-version 3.12 \
         --python-platform linux \
+        --generate-hashes \
         --quiet $reqs $pinned --output-file=$new_pinned
-    # Get rid of comments that might differ despite requirements being the same
-    cat $pinned | sed '/#/d' > $pinned_no_comments
-    cat $new_pinned | sed '/#/d' > $new_pinned_no_comments
+    # Compare versions only. Comments can differ despite requirements being the same, and the
+    # hash list for a pinned version grows if PyPI later gains new files for it (pip checks
+    # hashes at install time).
+    sed -e '/#/d' -e '/--hash=/d' -e 's/[[:space:]]*\\$//' $pinned > $pinned_no_comments
+    sed -e '/#/d' -e '/--hash=/d' -e 's/[[:space:]]*\\$//' $new_pinned > $new_pinned_no_comments
     diff $pinned_no_comments $new_pinned_no_comments || {
         echo '>>> up-to-date pinned requirements <<<'
         cat $new_pinned

ci/Dockerfile.ci-utils

@@ -5,11 +5,11 @@ RUN hail-apt-get-install git google-cloud-cli kubectl mysql-client
 
 COPY hail/python/hailtop/pinned-requirements.txt hailtop-requirements.txt
 COPY gear/pinned-requirements.txt gear-requirements.txt
+COPY ci/ci-utils/pinned-requirements.txt ci-utils-requirements.txt
 RUN hail-pip-install \
       -r hailtop-requirements.txt \
       -r gear-requirements.txt \
-      'twine>=6.2.0,<7' \
-      'Jinja2>3,<4'
+      -r ci-utils-requirements.txt
 
 FROM {% if global.dockerhub_prefix %}{{ global.dockerhub_prefix }}/library/{% endif %}golang:1.18 AS skopeo-build
 

ci/ci-utils/requirements.txt

@@ -0,0 +1,4 @@
+-c ../../hail/python/hailtop/pinned-requirements.txt
+-c ../../gear/pinned-requirements.txt
+twine>=6.2.0,<7
+Jinja2>3,<4

dev-docs/pip-dependencies.md

@@ -18,3 +18,21 @@ Note that the full dependency tree for a pip package can
 differ on different operating systems. All services and docker images in CI
 use the fully pinned requirements on a Linux platform, so the pinned requirements
 files are only generated for Linux.
+
+## Hashes
+
+The `pinned-requirements.txt` files include a `--hash` for every published file of each
+pinned version, so pip refuses to install anything whose contents don't match what was
+recorded when the lockfile was generated. Pip turns on hash checking for a whole
+`pip install` call as soon as any requirement *or constraint* has a hash, so:
+
+- Don't mix a pinned file with unpinned packages in a single `pip install`. Give
+  extra packages their own `requirements.txt` (constrained with `-c` by the pinned
+  files they're installed alongside) and add it to `generate-pip-lockfiles` and
+  `check-pip-requirements`; see `ci/ci-utils/` and `docker/hailgenetics/hail/`.
+- When using a pinned file only as a `--constraint` for an unpinned package, pass
+  `--no-require-hashes`.
+
+If an install fails with a hash mismatch, don't just regenerate the lockfile.
+PyPI files are immutable, so a mismatch means the downloaded file is not the one we
+pinned (a tampered download, a bad mirror, or a corrupted cache). Investigate first.

docker/hail-ubuntu/Dockerfile

@@ -48,7 +48,7 @@ RUN hail-apt-get-install g++ gcc
 # Only install google-cloud-profiler if SKIP_GCLOUD_PROFILER is not set to true
 RUN --mount=type=bind,source=gear/pinned-requirements.txt,target=/tmp/gear-requirements.txt,readonly \
     if [ "$SKIP_GCLOUD_PROFILER" != "true" ]; then \
-        hail-pip-install google-cloud-profiler --constraint /tmp/gear-requirements.txt && \
+        hail-pip-install --no-require-hashes google-cloud-profiler --constraint /tmp/gear-requirements.txt && \
         pip wheel --wheel-dir=/tmp/wheels google-cloud-profiler; \
     else \
         touch /tmp/wheels/.no-profiler; \
@@ -61,5 +61,5 @@ COPY --from=wheel-builder /tmp/wheels/ /tmp/wheels/
 # Only install google-cloud-profiler if the wheel exists and SKIP_GCLOUD_PROFILER is not true
 RUN --mount=type=bind,source=gear/pinned-requirements.txt,target=/tmp/gear-requirements.txt,readonly \
     if [ "$SKIP_GCLOUD_PROFILER" != "true" ] && [ -f /tmp/wheels/google_cloud_profiler*.whl ]; then \
-        hail-pip-install /tmp/wheels/google_cloud_profiler*.whl --constraint /tmp/gear-requirements.txt; \
+        hail-pip-install --no-require-hashes /tmp/wheels/google_cloud_profiler*.whl --constraint /tmp/gear-requirements.txt; \
     fi

docker/hailgenetics/hail/Dockerfile

@@ -10,7 +10,8 @@ RUN hail-apt-get-install \
     openjdk-21-jre-headless
 
 COPY hail/python/pinned-requirements.txt requirements.txt
-RUN hail-pip-install -r requirements.txt scikit-learn ipython
+COPY docker/hailgenetics/hail/pinned-requirements.txt extras-requirements.txt
+RUN hail-pip-install -r requirements.txt -r extras-requirements.txt
 
 RUN export SPARK_HOME=$(find_spark_home.py) && \
     curl --fail --silent --show-error --location \