Add NVIDIA GPU image for script agent

script-agent/DOCS.md

@@ -404,6 +404,74 @@ running a benchmark does not disturb the assistant — but voice handling
 pauses for its duration. While the model is still loading the page says so and
 the button stays disabled.
 
+## Docker with GPU
+
+`Dockerfile.gpu` builds llama.cpp with CUDA and runs the model on an NVIDIA GPU.
+The host must have the [NVIDIA Container Toolkit][] configured.
+
+From the `script-agent` directory, build and run the image with:
+
+```shell
+export SCRIPT_AGENT_HASS_API=http://192.168.1.100:8123/api
+docker build --file Dockerfile.gpu --tag script-agent:gpu .
+docker run --rm --gpus all \
+  --name script-agent \
+  --env SCRIPT_AGENT_HASS_TOKEN="${SCRIPT_AGENT_HASS_TOKEN}" \
+  --env SCRIPT_AGENT_HASS_API="${SCRIPT_AGENT_HASS_API}" \
+  --publish 10500:10500 \
+  --publish 5000:5000 \
+  --volume script-agent-data:/data \
+  script-agent:gpu
+```
+
+Set `SCRIPT_AGENT_HASS_TOKEN` to a Home Assistant long-lived access token before
+running the command, and replace `192.168.1.100` with the LAN address of your
+Home Assistant server. A `.local` mDNS hostname such as `homeassistant.local`
+may not resolve inside a Docker container, so use an IP address or a hostname
+provided by DNS.
+
+### Docker Compose
+
+A Compose service can instead be configured as follows:
+
+```yaml
+services:
+  script-agent:
+    build:
+      context: .
+      dockerfile: Dockerfile.gpu
+    gpus: all
+    environment:
+      SCRIPT_AGENT_HASS_TOKEN: ${SCRIPT_AGENT_HASS_TOKEN}
+      SCRIPT_AGENT_HASS_API: ${SCRIPT_AGENT_HASS_API}
+    ports:
+      - "10500:10500"
+      - "5000:5000"
+    volumes:
+      - script-agent-data:/data
+
+volumes:
+  script-agent-data:
+```
+
+Configure Home Assistant's Wyoming integration with the Docker host and port
+`10500`.
+
+Every agent CLI option has an uppercase environment variable prefixed with
+`SCRIPT_AGENT_`: `SCRIPT_AGENT_URI`, `SCRIPT_AGENT_HTTP_HOST`,
+`SCRIPT_AGENT_HTTP_PORT`, `SCRIPT_AGENT_HASS_TOKEN`, `SCRIPT_AGENT_HASS_API`,
+`SCRIPT_AGENT_HF_REPO`, `SCRIPT_AGENT_HF_FILENAME`,
+`SCRIPT_AGENT_TOOL_CALL_CACHE_SIZE`, `SCRIPT_AGENT_LLAMA_STATE`,
+`SCRIPT_AGENT_N_CTX`, `SCRIPT_AGENT_N_CTX_OVERHEAD`, `SCRIPT_AGENT_N_THREADS`,
+`SCRIPT_AGENT_N_GPU_LAYERS`, `SCRIPT_AGENT_MAX_TOKENS`,
+`SCRIPT_AGENT_FLASH_ATTENTION`, `SCRIPT_AGENT_BENCHMARK_FIXTURE`,
+`SCRIPT_AGENT_OVERRIDES`, and `SCRIPT_AGENT_DEBUG`. The image defaults
+`SCRIPT_AGENT_N_GPU_LAYERS` to `-1` to offload all model layers. The
+ecosystem-standard `HF_TOKEN` may also be set for authenticated Hugging Face
+downloads. Boolean variables accept `true`/`false`, `yes`/`no`, `on`/`off`, or
+`1`/`0`.
+
+
 ## Benchmarks
 
 Seconds per command with 5 scripts and 35 exposed entities.
@@ -432,5 +500,6 @@ Seconds per command with 5 scripts and 35 exposed entities.
 [official model]: https://huggingface.co/ggml-org/gemma-4-E2B-it-GGUF
 [media player]: https://www.home-assistant.io/integrations/media_player
 [Music Assistant]: https://www.home-assistant.io/integrations/music_assistant/
+[NVIDIA Container Toolkit]: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
 [wyoming]: https://www.home-assistant.io/integrations/wyoming/
 [blueprints]: https://github.com/OHF-Voice/apps/tree/main/script-agent/blueprints

script-agent/Dockerfile.gpu

@@ -0,0 +1,93 @@
+FROM nvidia/cuda:12.8.1-devel-ubuntu24.04 AS llama
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /usr/src
+
+RUN \
+    apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        ninja-build \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3-venv \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN \
+    python3 -m venv .venv \
+    && .venv/bin/pip install --no-cache-dir --upgrade \
+        wheel setuptools
+
+COPY requirements.llama.txt ./
+
+RUN \
+    CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON" \
+    FORCE_CMAKE=1 \
+    .venv/bin/pip install --no-cache-dir \
+        -r ./requirements.llama.txt
+
+# -----------------------------------------------------------------------------
+
+FROM nvidia/cuda:12.8.1-runtime-ubuntu24.04
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+ENV \
+    PYTHONUNBUFFERED=1 \
+    HF_HOME=/data/cache \
+    NVIDIA_VISIBLE_DEVICES=all \
+    NVIDIA_DRIVER_CAPABILITIES=compute,utility \
+    SCRIPT_AGENT_URI=tcp://0.0.0.0:10500 \
+    SCRIPT_AGENT_HTTP_HOST=0.0.0.0 \
+    SCRIPT_AGENT_HTTP_PORT=5000 \
+    SCRIPT_AGENT_HASS_API=http://homeassistant.local:8123/api \
+    SCRIPT_AGENT_HF_REPO=bartowski/google_gemma-4-E2B-it-GGUF \
+    SCRIPT_AGENT_HF_FILENAME=google_gemma-4-E2B-it-Q5_K_M.gguf \
+    SCRIPT_AGENT_TOOL_CALL_CACHE_SIZE=100 \
+    SCRIPT_AGENT_LLAMA_STATE=/data/llama_state.bin \
+    SCRIPT_AGENT_N_CTX=0 \
+    SCRIPT_AGENT_N_CTX_OVERHEAD=128 \
+    SCRIPT_AGENT_N_THREADS=0 \
+    SCRIPT_AGENT_N_GPU_LAYERS=-1 \
+    SCRIPT_AGENT_MAX_TOKENS=128 \
+    SCRIPT_AGENT_FLASH_ATTENTION=true \
+    SCRIPT_AGENT_BENCHMARK_FIXTURE="" \
+    SCRIPT_AGENT_OVERRIDES=/data/overrides.yaml \
+    SCRIPT_AGENT_DEBUG=false
+
+WORKDIR /usr/src
+
+RUN \
+    apt-get update \
+    && apt-get install -y --no-install-recommends \
+        curl \
+        libgomp1 \
+        python3 \
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir -p /data
+
+COPY --from=llama /usr/src/.venv/ ./.venv/
+
+COPY requirements.txt ./
+
+RUN \
+    .venv/bin/pip install --no-cache-dir \
+        -r ./requirements.txt
+
+COPY src/*.py ./
+COPY src/benchmark.yaml ./
+COPY src/templates/ ./templates/
+COPY --chmod=755 docker-entrypoint /usr/local/bin/docker-entrypoint
+
+VOLUME ["/data"]
+EXPOSE 5000 10500
+
+HEALTHCHECK --start-period=10m \
+    CMD curl -f "http://localhost:${SCRIPT_AGENT_HTTP_PORT}/health" || exit 1
+
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]

script-agent/docker-entrypoint

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if (( $# > 0 )) && [[ "$1" != -* ]]; then
+    exec "$@"
+fi
+
+: "${SCRIPT_AGENT_HASS_TOKEN:?SCRIPT_AGENT_HASS_TOKEN must contain a Home Assistant long-lived access token}"
+
+args=(
+    --uri "${SCRIPT_AGENT_URI}"
+    --http-host "${SCRIPT_AGENT_HTTP_HOST}"
+    --http-port "${SCRIPT_AGENT_HTTP_PORT}"
+    --hass-token "${SCRIPT_AGENT_HASS_TOKEN}"
+    --hass-api "${SCRIPT_AGENT_HASS_API}"
+    --hf-repo "${SCRIPT_AGENT_HF_REPO}"
+    --hf-filename "${SCRIPT_AGENT_HF_FILENAME}"
+    --tool-call-cache-size "${SCRIPT_AGENT_TOOL_CALL_CACHE_SIZE}"
+    --llama-state "${SCRIPT_AGENT_LLAMA_STATE}"
+    --n-ctx "${SCRIPT_AGENT_N_CTX}"
+    --n-ctx-overhead "${SCRIPT_AGENT_N_CTX_OVERHEAD}"
+    --n-threads "${SCRIPT_AGENT_N_THREADS}"
+    --n-gpu-layers "${SCRIPT_AGENT_N_GPU_LAYERS}"
+    --max-tokens "${SCRIPT_AGENT_MAX_TOKENS}"
+)
+
+add_boolean_flag() {
+    local value="${1,,}"
+    local enabled_flag="$2"
+    local disabled_flag="${3:-}"
+
+    case "${value}" in
+        1 | true | yes | on)
+            args+=("${enabled_flag}")
+            ;;
+        0 | false | no | off)
+            if [[ -n "${disabled_flag}" ]]; then
+                args+=("${disabled_flag}")
+            fi
+            ;;
+        *)
+            echo "Invalid boolean value '${1}' for ${enabled_flag}" >&2
+            exit 2
+            ;;
+    esac
+}
+
+add_boolean_flag "${SCRIPT_AGENT_FLASH_ATTENTION}" --flash-attention --no-flash-attention
+add_boolean_flag "${SCRIPT_AGENT_DEBUG}" --debug
+
+if [[ -n "${SCRIPT_AGENT_BENCHMARK_FIXTURE}" ]]; then
+    args+=(--benchmark-fixture "${SCRIPT_AGENT_BENCHMARK_FIXTURE}")
+fi
+
+if [[ -n "${SCRIPT_AGENT_OVERRIDES}" ]]; then
+    args+=(--overrides "${SCRIPT_AGENT_OVERRIDES}")
+fi
+
+exec /usr/src/.venv/bin/python3 /usr/src/app.py "${args[@]}" "$@"

script-agent/src/app.py

@@ -84,6 +84,12 @@ async def main() -> None:
         "Throughput is memory-bandwidth bound, so leave headroom for Home "
         "Assistant on the same box rather than using every core.",
     )
+    parser.add_argument(
+        "--n-gpu-layers",
+        type=int,
+        default=0,
+        help="Number of model layers to offload to the GPU (-1 = all)",
+    )
     parser.add_argument(
         "--max-tokens",
         type=int,
@@ -184,6 +190,7 @@ async def main() -> None:
         n_ctx=args.n_ctx if args.n_ctx > 0 else None,
         n_ctx_overhead=args.n_ctx_overhead,
         n_threads=args.n_threads if args.n_threads > 0 else None,
+        n_gpu_layers=args.n_gpu_layers,
         max_tokens=all_overrides.max_tokens or args.max_tokens,
         flash_attn=args.flash_attention,
         debug=args.debug,

script-agent/src/gemma4_recognizer.py

@@ -73,6 +73,7 @@ def __init__(
         n_ctx: Optional[int] = None,
         n_ctx_overhead: int = 128,
         n_threads: Optional[int] = None,
+        n_gpu_layers: int = 0,
         max_tokens: int = DEFAULT_MAX_TOKENS,
         flash_attn: bool = True,
         debug: bool = False,
@@ -90,6 +91,7 @@ def __init__(
         self.n_ctx = n_ctx
         self.n_ctx_overhead = n_ctx_overhead
         self.n_threads = n_threads
+        self.n_gpu_layers = n_gpu_layers
         self.max_tokens = max_tokens
         self.flash_attn = flash_attn
         self.model_path: Optional[Path] = None
@@ -152,6 +154,7 @@ def _create_llm(self, n_ctx: int) -> None:
             chat_template_kwargs={"enable_thinking": self.enable_thinking},
             n_ctx=n_ctx,
             n_threads=self.n_threads,
+            n_gpu_layers=self.n_gpu_layers,
             flash_attn=self.flash_attn,
             verbose=self.debug,
         )
@@ -164,6 +167,7 @@ def _restore_or_build_state(self) -> None:
         runtime_model_id = (
             f"llama-cpp-python/{LLAMA_CPP_VERSION};"
             f"n_ctx={self.llm.n_ctx()};"
+            f"n_gpu_layers={self.n_gpu_layers};"
             f"flash_attn={int(self.flash_attn)};"
             f"model_path={self.model_path};"
             f"{self.repo_id}/{self.filename}"

script-agent/tests/test_gemma4_recognizer.py

@@ -95,6 +95,20 @@ def test_truncated_response_is_not_partially_executed(self):
         self.assertIn("64-token generation limit", text)
 
 
+class ModelCreationTests(unittest.TestCase):
+    @patch("gemma4_recognizer.Llama")
+    @patch("gemma4_recognizer.hf_hub_download", return_value="/model")
+    def test_gpu_layers_are_passed_to_llama(self, _download, llama):
+        recognizer = Gemma4Recognizer(
+            state_path="unused.bin",
+            n_gpu_layers=-1,
+        )
+
+        recognizer._create_llm(256)  # pylint: disable=protected-access
+
+        self.assertEqual(-1, llama.call_args.kwargs["n_gpu_layers"])
+
+
 class PromptTests(unittest.TestCase):
     def test_default_user_prompt_carries_the_current_date(self):
         # Without it the model cannot turn "Saturday" into a date, and answers a
@@ -329,7 +343,7 @@ def test_corrupt_matching_state_is_rebuilt(self):
             recognizer.model_path = Path("/model")
             runtime_model_id = (
                 f"llama-cpp-python/{LLAMA_CPP_VERSION};"
-                "n_ctx=64;flash_attn=1;model_path=/model;"
+                "n_ctx=64;n_gpu_layers=0;flash_attn=1;model_path=/model;"
                 f"{recognizer.repo_id}/{recognizer.filename}"
             )
             state_path.with_suffix(".sha256").write_text(

script-agent/tests/test_release.py

@@ -65,6 +65,40 @@ def test_docker_stages_use_trixie(self):
         )
         self.assertIn("GGML_CPU_ARM_ARCH=armv8.2-a+fp16+dotprod", dockerfile)
 
+    def test_gpu_dockerfile_enables_cuda_and_exposes_cli_environment(self):
+        dockerfile = (self.project_dir / "Dockerfile.gpu").read_text("utf-8")
+        entrypoint = (self.project_dir / "docker-entrypoint").read_text("utf-8")
+        image_sources = dockerfile + entrypoint
+
+        self.assertIn("-DGGML_CUDA=ON", dockerfile)
+        self.assertIn("NVIDIA_VISIBLE_DEVICES=all", dockerfile)
+        for variable in (
+            "SCRIPT_AGENT_URI",
+            "SCRIPT_AGENT_HTTP_HOST",
+            "SCRIPT_AGENT_HTTP_PORT",
+            "SCRIPT_AGENT_HASS_TOKEN",
+            "SCRIPT_AGENT_HASS_API",
+            "SCRIPT_AGENT_HF_REPO",
+            "SCRIPT_AGENT_HF_FILENAME",
+            "SCRIPT_AGENT_TOOL_CALL_CACHE_SIZE",
+            "SCRIPT_AGENT_LLAMA_STATE",
+            "SCRIPT_AGENT_N_CTX",
+            "SCRIPT_AGENT_N_CTX_OVERHEAD",
+            "SCRIPT_AGENT_N_THREADS",
+            "SCRIPT_AGENT_N_GPU_LAYERS",
+            "SCRIPT_AGENT_MAX_TOKENS",
+            "SCRIPT_AGENT_FLASH_ATTENTION",
+            "SCRIPT_AGENT_BENCHMARK_FIXTURE",
+            "SCRIPT_AGENT_OVERRIDES",
+            "SCRIPT_AGENT_DEBUG",
+        ):
+            self.assertTrue(
+                (f"{variable}=" in image_sources)
+                or (f"${{{variable}}}" in image_sources)
+                or (f"${{{variable}:" in image_sources),
+                variable,
+            )
+
     def test_python_environment_matches_app_name(self):
         python_environment = (self.project_dir / ".python-version").read_text("utf-8")
         self.assertEqual("script-agent", python_environment.strip())