diff --git a/script-agent/DOCS.md b/script-agent/DOCS.md index def4a0e..06ea6ac 100644 --- a/script-agent/DOCS.md +++ b/script-agent/DOCS.md @@ -404,6 +404,74 @@ running a benchmark does not disturb the assistant — but voice handling pauses for its duration. While the model is still loading the page says so and the button stays disabled. +## Docker with GPU + +`Dockerfile.gpu` builds llama.cpp with CUDA and runs the model on an NVIDIA GPU. +The host must have the [NVIDIA Container Toolkit][] configured. + +From the `script-agent` directory, build and run the image with: + +```shell +export SCRIPT_AGENT_HASS_API=http://192.168.1.100:8123/api +docker build --file Dockerfile.gpu --tag script-agent:gpu . +docker run --rm --gpus all \ + --name script-agent \ + --env SCRIPT_AGENT_HASS_TOKEN="${SCRIPT_AGENT_HASS_TOKEN}" \ + --env SCRIPT_AGENT_HASS_API="${SCRIPT_AGENT_HASS_API}" \ + --publish 10500:10500 \ + --publish 5000:5000 \ + --volume script-agent-data:/data \ + script-agent:gpu +``` + +Set `SCRIPT_AGENT_HASS_TOKEN` to a Home Assistant long-lived access token before +running the command, and replace `192.168.1.100` with the LAN address of your +Home Assistant server. A `.local` mDNS hostname such as `homeassistant.local` +may not resolve inside a Docker container, so use an IP address or a hostname +provided by DNS. + +### Docker Compose + +A Compose service can instead be configured as follows: + +```yaml +services: + script-agent: + build: + context: . + dockerfile: Dockerfile.gpu + gpus: all + environment: + SCRIPT_AGENT_HASS_TOKEN: ${SCRIPT_AGENT_HASS_TOKEN} + SCRIPT_AGENT_HASS_API: ${SCRIPT_AGENT_HASS_API} + ports: + - "10500:10500" + - "5000:5000" + volumes: + - script-agent-data:/data + +volumes: + script-agent-data: +``` + +Configure Home Assistant's Wyoming integration with the Docker host and port +`10500`. + +Every agent CLI option has an uppercase environment variable prefixed with +`SCRIPT_AGENT_`: `SCRIPT_AGENT_URI`, `SCRIPT_AGENT_HTTP_HOST`, +`SCRIPT_AGENT_HTTP_PORT`, `SCRIPT_AGENT_HASS_TOKEN`, `SCRIPT_AGENT_HASS_API`, +`SCRIPT_AGENT_HF_REPO`, `SCRIPT_AGENT_HF_FILENAME`, +`SCRIPT_AGENT_TOOL_CALL_CACHE_SIZE`, `SCRIPT_AGENT_LLAMA_STATE`, +`SCRIPT_AGENT_N_CTX`, `SCRIPT_AGENT_N_CTX_OVERHEAD`, `SCRIPT_AGENT_N_THREADS`, +`SCRIPT_AGENT_N_GPU_LAYERS`, `SCRIPT_AGENT_MAX_TOKENS`, +`SCRIPT_AGENT_FLASH_ATTENTION`, `SCRIPT_AGENT_BENCHMARK_FIXTURE`, +`SCRIPT_AGENT_OVERRIDES`, and `SCRIPT_AGENT_DEBUG`. The image defaults +`SCRIPT_AGENT_N_GPU_LAYERS` to `-1` to offload all model layers. The +ecosystem-standard `HF_TOKEN` may also be set for authenticated Hugging Face +downloads. Boolean variables accept `true`/`false`, `yes`/`no`, `on`/`off`, or +`1`/`0`. + + ## Benchmarks Seconds per command with 5 scripts and 35 exposed entities. @@ -432,5 +500,6 @@ Seconds per command with 5 scripts and 35 exposed entities. [official model]: https://huggingface.co/ggml-org/gemma-4-E2B-it-GGUF [media player]: https://www.home-assistant.io/integrations/media_player [Music Assistant]: https://www.home-assistant.io/integrations/music_assistant/ +[NVIDIA Container Toolkit]: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html [wyoming]: https://www.home-assistant.io/integrations/wyoming/ [blueprints]: https://github.com/OHF-Voice/apps/tree/main/script-agent/blueprints diff --git a/script-agent/Dockerfile.gpu b/script-agent/Dockerfile.gpu new file mode 100644 index 0000000..751fc02 --- /dev/null +++ b/script-agent/Dockerfile.gpu @@ -0,0 +1,93 @@ +FROM nvidia/cuda:12.8.1-devel-ubuntu24.04 AS llama + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /usr/src + +RUN \ + apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + ninja-build \ + python3 \ + python3-dev \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* + +RUN \ + python3 -m venv .venv \ + && .venv/bin/pip install --no-cache-dir --upgrade \ + wheel setuptools + +COPY requirements.llama.txt ./ + +RUN \ + CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON" \ + FORCE_CMAKE=1 \ + .venv/bin/pip install --no-cache-dir \ + -r ./requirements.llama.txt + +# ----------------------------------------------------------------------------- + +FROM nvidia/cuda:12.8.1-runtime-ubuntu24.04 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ENV \ + PYTHONUNBUFFERED=1 \ + HF_HOME=/data/cache \ + NVIDIA_VISIBLE_DEVICES=all \ + NVIDIA_DRIVER_CAPABILITIES=compute,utility \ + SCRIPT_AGENT_URI=tcp://0.0.0.0:10500 \ + SCRIPT_AGENT_HTTP_HOST=0.0.0.0 \ + SCRIPT_AGENT_HTTP_PORT=5000 \ + SCRIPT_AGENT_HASS_API=http://homeassistant.local:8123/api \ + SCRIPT_AGENT_HF_REPO=bartowski/google_gemma-4-E2B-it-GGUF \ + SCRIPT_AGENT_HF_FILENAME=google_gemma-4-E2B-it-Q5_K_M.gguf \ + SCRIPT_AGENT_TOOL_CALL_CACHE_SIZE=100 \ + SCRIPT_AGENT_LLAMA_STATE=/data/llama_state.bin \ + SCRIPT_AGENT_N_CTX=0 \ + SCRIPT_AGENT_N_CTX_OVERHEAD=128 \ + SCRIPT_AGENT_N_THREADS=0 \ + SCRIPT_AGENT_N_GPU_LAYERS=-1 \ + SCRIPT_AGENT_MAX_TOKENS=128 \ + SCRIPT_AGENT_FLASH_ATTENTION=true \ + SCRIPT_AGENT_BENCHMARK_FIXTURE="" \ + SCRIPT_AGENT_OVERRIDES=/data/overrides.yaml \ + SCRIPT_AGENT_DEBUG=false + +WORKDIR /usr/src + +RUN \ + apt-get update \ + && apt-get install -y --no-install-recommends \ + curl \ + libgomp1 \ + python3 \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /data + +COPY --from=llama /usr/src/.venv/ ./.venv/ + +COPY requirements.txt ./ + +RUN \ + .venv/bin/pip install --no-cache-dir \ + -r ./requirements.txt + +COPY src/*.py ./ +COPY src/benchmark.yaml ./ +COPY src/templates/ ./templates/ +COPY --chmod=755 docker-entrypoint /usr/local/bin/docker-entrypoint + +VOLUME ["/data"] +EXPOSE 5000 10500 + +HEALTHCHECK --start-period=10m \ + CMD curl -f "http://localhost:${SCRIPT_AGENT_HTTP_PORT}/health" || exit 1 + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] diff --git a/script-agent/docker-entrypoint b/script-agent/docker-entrypoint new file mode 100644 index 0000000..565ebc4 --- /dev/null +++ b/script-agent/docker-entrypoint @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (( $# > 0 )) && [[ "$1" != -* ]]; then + exec "$@" +fi + +: "${SCRIPT_AGENT_HASS_TOKEN:?SCRIPT_AGENT_HASS_TOKEN must contain a Home Assistant long-lived access token}" + +args=( + --uri "${SCRIPT_AGENT_URI}" + --http-host "${SCRIPT_AGENT_HTTP_HOST}" + --http-port "${SCRIPT_AGENT_HTTP_PORT}" + --hass-token "${SCRIPT_AGENT_HASS_TOKEN}" + --hass-api "${SCRIPT_AGENT_HASS_API}" + --hf-repo "${SCRIPT_AGENT_HF_REPO}" + --hf-filename "${SCRIPT_AGENT_HF_FILENAME}" + --tool-call-cache-size "${SCRIPT_AGENT_TOOL_CALL_CACHE_SIZE}" + --llama-state "${SCRIPT_AGENT_LLAMA_STATE}" + --n-ctx "${SCRIPT_AGENT_N_CTX}" + --n-ctx-overhead "${SCRIPT_AGENT_N_CTX_OVERHEAD}" + --n-threads "${SCRIPT_AGENT_N_THREADS}" + --n-gpu-layers "${SCRIPT_AGENT_N_GPU_LAYERS}" + --max-tokens "${SCRIPT_AGENT_MAX_TOKENS}" +) + +add_boolean_flag() { + local value="${1,,}" + local enabled_flag="$2" + local disabled_flag="${3:-}" + + case "${value}" in + 1 | true | yes | on) + args+=("${enabled_flag}") + ;; + 0 | false | no | off) + if [[ -n "${disabled_flag}" ]]; then + args+=("${disabled_flag}") + fi + ;; + *) + echo "Invalid boolean value '${1}' for ${enabled_flag}" >&2 + exit 2 + ;; + esac +} + +add_boolean_flag "${SCRIPT_AGENT_FLASH_ATTENTION}" --flash-attention --no-flash-attention +add_boolean_flag "${SCRIPT_AGENT_DEBUG}" --debug + +if [[ -n "${SCRIPT_AGENT_BENCHMARK_FIXTURE}" ]]; then + args+=(--benchmark-fixture "${SCRIPT_AGENT_BENCHMARK_FIXTURE}") +fi + +if [[ -n "${SCRIPT_AGENT_OVERRIDES}" ]]; then + args+=(--overrides "${SCRIPT_AGENT_OVERRIDES}") +fi + +exec /usr/src/.venv/bin/python3 /usr/src/app.py "${args[@]}" "$@" diff --git a/script-agent/src/app.py b/script-agent/src/app.py index 2ff44f4..3dc9570 100644 --- a/script-agent/src/app.py +++ b/script-agent/src/app.py @@ -84,6 +84,12 @@ async def main() -> None: "Throughput is memory-bandwidth bound, so leave headroom for Home " "Assistant on the same box rather than using every core.", ) + parser.add_argument( + "--n-gpu-layers", + type=int, + default=0, + help="Number of model layers to offload to the GPU (-1 = all)", + ) parser.add_argument( "--max-tokens", type=int, @@ -184,6 +190,7 @@ async def main() -> None: n_ctx=args.n_ctx if args.n_ctx > 0 else None, n_ctx_overhead=args.n_ctx_overhead, n_threads=args.n_threads if args.n_threads > 0 else None, + n_gpu_layers=args.n_gpu_layers, max_tokens=all_overrides.max_tokens or args.max_tokens, flash_attn=args.flash_attention, debug=args.debug, diff --git a/script-agent/src/gemma4_recognizer.py b/script-agent/src/gemma4_recognizer.py index 7f4876b..b5d65af 100644 --- a/script-agent/src/gemma4_recognizer.py +++ b/script-agent/src/gemma4_recognizer.py @@ -73,6 +73,7 @@ def __init__( n_ctx: Optional[int] = None, n_ctx_overhead: int = 128, n_threads: Optional[int] = None, + n_gpu_layers: int = 0, max_tokens: int = DEFAULT_MAX_TOKENS, flash_attn: bool = True, debug: bool = False, @@ -90,6 +91,7 @@ def __init__( self.n_ctx = n_ctx self.n_ctx_overhead = n_ctx_overhead self.n_threads = n_threads + self.n_gpu_layers = n_gpu_layers self.max_tokens = max_tokens self.flash_attn = flash_attn self.model_path: Optional[Path] = None @@ -152,6 +154,7 @@ def _create_llm(self, n_ctx: int) -> None: chat_template_kwargs={"enable_thinking": self.enable_thinking}, n_ctx=n_ctx, n_threads=self.n_threads, + n_gpu_layers=self.n_gpu_layers, flash_attn=self.flash_attn, verbose=self.debug, ) @@ -164,6 +167,7 @@ def _restore_or_build_state(self) -> None: runtime_model_id = ( f"llama-cpp-python/{LLAMA_CPP_VERSION};" f"n_ctx={self.llm.n_ctx()};" + f"n_gpu_layers={self.n_gpu_layers};" f"flash_attn={int(self.flash_attn)};" f"model_path={self.model_path};" f"{self.repo_id}/{self.filename}" diff --git a/script-agent/tests/test_gemma4_recognizer.py b/script-agent/tests/test_gemma4_recognizer.py index 9175892..76bfd42 100644 --- a/script-agent/tests/test_gemma4_recognizer.py +++ b/script-agent/tests/test_gemma4_recognizer.py @@ -95,6 +95,20 @@ def test_truncated_response_is_not_partially_executed(self): self.assertIn("64-token generation limit", text) +class ModelCreationTests(unittest.TestCase): + @patch("gemma4_recognizer.Llama") + @patch("gemma4_recognizer.hf_hub_download", return_value="/model") + def test_gpu_layers_are_passed_to_llama(self, _download, llama): + recognizer = Gemma4Recognizer( + state_path="unused.bin", + n_gpu_layers=-1, + ) + + recognizer._create_llm(256) # pylint: disable=protected-access + + self.assertEqual(-1, llama.call_args.kwargs["n_gpu_layers"]) + + class PromptTests(unittest.TestCase): def test_default_user_prompt_carries_the_current_date(self): # Without it the model cannot turn "Saturday" into a date, and answers a @@ -329,7 +343,7 @@ def test_corrupt_matching_state_is_rebuilt(self): recognizer.model_path = Path("/model") runtime_model_id = ( f"llama-cpp-python/{LLAMA_CPP_VERSION};" - "n_ctx=64;flash_attn=1;model_path=/model;" + "n_ctx=64;n_gpu_layers=0;flash_attn=1;model_path=/model;" f"{recognizer.repo_id}/{recognizer.filename}" ) state_path.with_suffix(".sha256").write_text( diff --git a/script-agent/tests/test_release.py b/script-agent/tests/test_release.py index 0f4b214..a4614fc 100644 --- a/script-agent/tests/test_release.py +++ b/script-agent/tests/test_release.py @@ -65,6 +65,40 @@ def test_docker_stages_use_trixie(self): ) self.assertIn("GGML_CPU_ARM_ARCH=armv8.2-a+fp16+dotprod", dockerfile) + def test_gpu_dockerfile_enables_cuda_and_exposes_cli_environment(self): + dockerfile = (self.project_dir / "Dockerfile.gpu").read_text("utf-8") + entrypoint = (self.project_dir / "docker-entrypoint").read_text("utf-8") + image_sources = dockerfile + entrypoint + + self.assertIn("-DGGML_CUDA=ON", dockerfile) + self.assertIn("NVIDIA_VISIBLE_DEVICES=all", dockerfile) + for variable in ( + "SCRIPT_AGENT_URI", + "SCRIPT_AGENT_HTTP_HOST", + "SCRIPT_AGENT_HTTP_PORT", + "SCRIPT_AGENT_HASS_TOKEN", + "SCRIPT_AGENT_HASS_API", + "SCRIPT_AGENT_HF_REPO", + "SCRIPT_AGENT_HF_FILENAME", + "SCRIPT_AGENT_TOOL_CALL_CACHE_SIZE", + "SCRIPT_AGENT_LLAMA_STATE", + "SCRIPT_AGENT_N_CTX", + "SCRIPT_AGENT_N_CTX_OVERHEAD", + "SCRIPT_AGENT_N_THREADS", + "SCRIPT_AGENT_N_GPU_LAYERS", + "SCRIPT_AGENT_MAX_TOKENS", + "SCRIPT_AGENT_FLASH_ATTENTION", + "SCRIPT_AGENT_BENCHMARK_FIXTURE", + "SCRIPT_AGENT_OVERRIDES", + "SCRIPT_AGENT_DEBUG", + ): + self.assertTrue( + (f"{variable}=" in image_sources) + or (f"${{{variable}}}" in image_sources) + or (f"${{{variable}:" in image_sources), + variable, + ) + def test_python_environment_matches_app_name(self): python_environment = (self.project_dir / ".python-version").read_text("utf-8") self.assertEqual("script-agent", python_environment.strip())