Merge branch 'main' into python3.13

Author: mathbunnyru

.github/workflows/contributed-recipes.yml

@@ -1,5 +1,9 @@
 name: Test the contributed recipes
 
+env:
+  REGISTRY: quay.io
+  OWNER: ${{ github.repository_owner }}
+
 on:
   schedule:
     # Images are rebuilt at 03:00 on Monday UTC
@@ -61,8 +65,31 @@ jobs:
           variant: default
 
       # Not pulling the image, because it might be loaded from previous step or will be downloaded automatically
-      - name: Build recipe 🛠
-        run: docker build --rm --force-rm --tag my-custom-image -f ./${{ matrix.dockerfile }} ./
+      - name: Build recipe with parent image 🛠
+        if: ${{ matrix.parent-image != '' }}
+        run: |
+          docker build \
+            --rm --force-rm \
+            --tag my-custom-image \
+            -f ./${{ matrix.dockerfile }} \
+            --build-arg BASE_IMAGE=${{ env.REGISTRY }}/${{ env.OWNER }}/${{ matrix.parent-image }} \
+            ./
+        env:
+          DOCKER_BUILDKIT: 1
+          # Full logs for CI build
+          BUILDKIT_PROGRESS: plain
+        working-directory: docs/using/recipe_code
+        shell: bash
+
+      # Not pulling the image, because it might be loaded from previous step or will be downloaded automatically
+      - name: Build recipe without parent image 🛠
+        if: ${{ matrix.parent-image == '' }}
+        run: |
+          docker build \
+            --rm --force-rm \
+            --tag my-custom-image \
+            -f ./${{ matrix.dockerfile }} \
+            ./
         env:
           DOCKER_BUILDKIT: 1
           # Full logs for CI build

.github/workflows/docker.yml

@@ -336,6 +336,16 @@ jobs:
     needs: x86_64-pyspark
     if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }}
 
+  contributed-recipes:
+    uses: ./.github/workflows/contributed-recipes.yml
+    with:
+      called-using-workflow-call: true
+    # Contributed recipes only use these images
+    # If recipes using other images will be added, they should be added here as well
+    #
+    # contributed-recipes will give an error if the image is not yet built and uploaded
+    needs: [aarch64-base, x86_64-base, aarch64-minimal, x86_64-minimal]
+
   tag-push:
     uses: ./.github/workflows/docker-tag-push.yml
     with:
@@ -370,6 +380,8 @@ jobs:
             variant: cuda12
     needs:
       [
+        contributed-recipes,
+
         aarch64-foundation,
         aarch64-base,
         aarch64-minimal,
@@ -425,13 +437,3 @@ jobs:
     uses: ./.github/workflows/docker-wiki-update.yml
     needs: tag-push-fast
     if: contains(github.event.pull_request.title, '[FAST_BUILD]')
-
-  contributed-recipes:
-    uses: ./.github/workflows/contributed-recipes.yml
-    with:
-      called-using-workflow-call: true
-    # Contributed recipes only use these images
-    # If recipes using other images will be added, they should be added here as well
-    #
-    # contributed-recipes will give an error if the image is not yet built and uploaded
-    needs: [aarch64-base, x86_64-base, aarch64-minimal, x86_64-minimal]

.pre-commit-config.yaml

@@ -45,6 +45,7 @@ repos:
             "numpy",
             "pytest",
             "requests",
+            "tenacity",
             "urllib3",
             "types-beautifulsoup4",
             "types-python-dateutil",
@@ -98,7 +99,7 @@ repos:
 
   # Lint: YAML
   - repo: https://github.com/adrienverge/yamllint
-    rev: v1.35.1
+    rev: v1.37.0
     hooks:
       - id: yamllint
         args: ["-d {extends: relaxed, rules: {line-length: disable}}", "-s"]
@@ -119,7 +120,7 @@ repos:
 
   # Lint: Python
   - repo: https://github.com/PyCQA/flake8
-    rev: 7.1.2
+    rev: 7.2.0
     hooks:
       - id: flake8
 

docs/using/custom-images.md

@@ -20,6 +20,7 @@ Our repository provides several customization points:
 - `REGISTRY`, `OWNER`, `BASE_IMAGE` (docker arguments) - they allow to specify parent image for all the other images
 - `REGISTRY`, `OWNER` (part of `env` in some GitHub workflows) - these allow to properly tag and refer to images during following steps:
   - [`build-test-upload`](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/docker-build-test-upload.yml)
+  - [`contributed-recipes`](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/contributed-recipes.yml)
   - [`tag-push`](https://github.com/jupyter/docker-stacks/blob/main/.github/workflows/docker-tag-push.yml)
 
 These customization points can't be changed during runtime.

images/docker-stacks-foundation/Dockerfile

@@ -131,6 +131,9 @@ RUN set -x && \
     # Pin major.minor version of python
     # https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#preventing-packages-from-updating-pinning
     mamba list --full-name 'python' | awk 'END{sub("[^.]*$", "*", $2); print $1 " " $2}' >> "${CONDA_DIR}/conda-meta/pinned" && \
+    # Temporarily pin libxml2 to avoid ABI breakage
+    # https://github.com/conda-forge/libxml2-feedstock/issues/145
+    echo 'libxml2<2.14.0' >> /opt/conda/conda-meta/pinned && \
     mamba clean --all -f -y && \
     fix-permissions "${CONDA_DIR}" && \
     fix-permissions "/home/${NB_USER}"

images/tensorflow-notebook/cuda/Dockerfile

@@ -19,7 +19,7 @@ RUN mamba install --yes \
 
 # Install TensorFlow, CUDA and cuDNN with pip
 RUN pip install --no-cache-dir \
-    "tensorflow[and-cuda]<=2.17.1" && \
+    'tensorflow[and-cuda]' && \
     fix-permissions "${CONDA_DIR}" && \
     fix-permissions "/home/${NB_USER}"
 

requirements-dev.txt

@@ -9,3 +9,4 @@ pytest-xdist
 python-dateutil
 requests
 tabulate
+tenacity

tagging/apps/merge_tags.py

@@ -3,10 +3,14 @@
 # Distributed under the terms of the Modified BSD License.
 import logging
 import os
-import time
-from collections.abc import Callable
 
 import plumbum
+from tenacity import (  # type: ignore
+    RetryError,
+    retry,
+    stop_after_attempt,
+    wait_exponential,
+)
 
 from tagging.apps.common_cli_arguments import common_arguments_parser
 from tagging.apps.config import Config
@@ -44,20 +48,11 @@ def read_local_tags_from_files(config: Config) -> tuple[list[str], set[str]]:
     return all_local_tags, merged_local_tags
 
 
-def run_with_retries(func: Callable[[], None]) -> None:
-    ATTEMPTS = 3
-    SLEEP_BACKOFF = 2
-
-    for attempt in range(ATTEMPTS):
-        try:
-            func()
-            break
-        except Exception as e:
-            LOGGER.warning(f"Attempt {attempt + 1} failed: {e}")
-            if attempt + 1 == ATTEMPTS:
-                LOGGER.error(f"Failed after {ATTEMPTS} attempts")
-                raise
-            time.sleep(SLEEP_BACKOFF * (attempt + 1))
+@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4))
+def pull_tag(tag: str) -> None:
+    LOGGER.info(f"Pulling tag: {tag}")
+    docker["pull", tag] & plumbum.FG
+    LOGGER.info(f"Tag {tag} pulled successfully")
 
 
 def pull_missing_tags(merged_tag: str, all_local_tags: list[str]) -> list[str]:
@@ -74,27 +69,35 @@ def pull_missing_tags(merged_tag: str, all_local_tags: list[str]) -> list[str]:
 
         LOGGER.warning(f"Trying to pull: {platform_tag} from registry")
         try:
-            run_with_retries(lambda: docker["pull", platform_tag] & plumbum.FG)
+            pull_tag(platform_tag)
             existing_platform_tags.append(platform_tag)
             LOGGER.info(f"Tag {platform_tag} pulled successfully")
-        except plumbum.ProcessExecutionError:
+        except RetryError:
             LOGGER.warning(f"Pull failed, tag {platform_tag} doesn't exist")
 
     return existing_platform_tags
 
 
-def push_manifest(merged_tag: str, existing_platform_tags: list[str]) -> None:
+@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4))
+def create_manifest(merged_tag: str, existing_platform_tags: list[str]) -> None:
+    # This allows to rerun the script without having to remove the manifest manually
+    try:
+        docker["manifest", "rm", merged_tag] & plumbum.FG
+        LOGGER.warning(f"Manifest {merged_tag} was present locally, removed it")
+    except plumbum.ProcessExecutionError:
+        pass
+
     LOGGER.info(f"Creating manifest for tag: {merged_tag}")
     # Unfortunately, `docker manifest create` requires images to have been already pushed to the registry
     # which is not true for new tags in PRs
-    run_with_retries(
-        lambda: docker["manifest", "create", merged_tag][existing_platform_tags]
-        & plumbum.FG
-    )
+    docker["manifest", "create", merged_tag][existing_platform_tags] & plumbum.FG
     LOGGER.info(f"Successfully created manifest for tag: {merged_tag}")
 
+
+@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4))
+def push_manifest(merged_tag: str) -> None:
     LOGGER.info(f"Pushing manifest for tag: {merged_tag}")
-    run_with_retries(lambda: docker["manifest", "push", merged_tag] & plumbum.FG)
+    docker["manifest", "push", merged_tag] & plumbum.FG
     LOGGER.info(f"Successfully pushed manifest for tag: {merged_tag}")
 
 
@@ -103,16 +106,10 @@ def merge_tags(
 ) -> None:
     LOGGER.info(f"Trying to merge tag: {merged_tag}")
 
-    # This allows to rerun the script without having to remove the manifest manually
-    try:
-        docker["manifest", "rm", merged_tag] & plumbum.FG
-        LOGGER.warning(f"Manifest {merged_tag} was present locally, removed it")
-    except plumbum.ProcessExecutionError:
-        pass
-
     existing_platform_tags = pull_missing_tags(merged_tag, all_local_tags)
     if push_to_registry:
-        push_manifest(merged_tag, existing_platform_tags)
+        create_manifest(merged_tag, existing_platform_tags)
+        push_manifest(merged_tag)
     else:
         LOGGER.info(f"Skipping push for tag: {merged_tag}")
 

tests/by_image/tensorflow-notebook/units/unit_tensorflow.py

@@ -1,5 +1,12 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+import os
+import sys
+
+if "NVIDIA_VISIBLE_DEVICES" in os.environ:
+    print("Not running this test in GPU mode")
+    sys.exit(0)
+
 import tensorflow as tf
 
 print(tf.constant("Hello, TensorFlow"))