run-bench.sh

#!/usr/bin/env bash
# Run llama-bench across the pp/tg sizes referenced in the README.
# Matches the tuned server config (ubatch 2048, q8_0 KV, --fit picks N automatically).

set -euo pipefail

ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
BENCH="$ROOT/llama.cpp/build/bin/llama-bench"

if [[ ! -x "$BENCH" ]]; then
    echo "llama-bench not found at $BENCH" >&2
    exit 1
fi

if [[ -z "${MODEL:-}" ]]; then
    shopt -s nullglob
    candidates=("$ROOT"/models/*.gguf)
    shopt -u nullglob
    if (( ${#candidates[@]} != 1 )); then
        echo "set MODEL=/path/to/file.gguf (found ${#candidates[@]} in models/)" >&2
        exit 1
    fi
    MODEL="${candidates[0]}"
fi

# llama-bench has no --fit; pick N manually. Match whatever the server's
# --fit landed on (check its log for 'offloaded N/41 layers' or the
# 'X overflowing' line). Defaults track the server's desktop-friendly
# preset on a 5600X + 5070 Ti with an active Plasma/Wayland 4K desktop
# (FIT_TARGET=1536 → N~26). N varies with your desktop's VRAM draw —
# re-measure and adjust if your baseline is different.
# For dedicated max-throughput reference numbers on a headless/idle GPU:
# N_CPU_MOE=19 THREADS=$(nproc).
# If bench OOMs on pp2048 (e.g. another process is using VRAM), bump N by
# 2-3 or drop UBATCH.
N_CPU_MOE="${N_CPU_MOE:-26}"
UBATCH="${UBATCH:-2048}"
THREADS="${THREADS:-10}"

exec "$BENCH" \
    -m "$MODEL" \
    -ngl 99 \
    -ncmoe "$N_CPU_MOE" \
    -fa 1 \
    -ctk q8_0 -ctv q8_0 \
    -b "$UBATCH" -ub "$UBATCH" \
    -t "$THREADS" \
    -p 512,2048,4096 \
    -n 128 \
    -r 3