Skip to content

Commit 3caf73f

Browse files
author
Andre
committed
feat: DeepSeek V4 Flash support — custom llama.cpp branch + recipes
Stack changes: - launch.sh: LLAMA_CPP_REPO and LLAMA_CPP_REF env vars to clone+compile from a custom fork/branch at boot. When DSv4 merges to master, just remove these fields from recipes and the builder image works as before. - launch.sh: split-file GGUF discovery via find (handles subdirectories) - vast_up.sh: passes LLAMA_CPP_REPO/REF to container env + onstart cmd - vast_menus.py: shows llama.cpp fork/branch in launch summary, passes recipe llama_cpp_repo/ref fields to vast_up.sh env - editor_menus.py: create wizard asks about custom llama.cpp fork - recipe_editor.py: documents optional llama_cpp_repo/ref fields New recipes (7 DSv4-Flash configurations): - Q2_K on 2×H100 (96GB model, 160GB VRAM) - Q3_K_M on 2×H100 (126GB model, 160GB VRAM) - Q4_K_M on 2×H200 (160GB model, 282GB VRAM, 256K ctx) - Q4_K_M on 4×H100 (160GB model, 320GB VRAM, 2 slots) - MXFP4 on 2×H200 (140GB model, 282GB VRAM) - Q4_K_M on 2×B200 (160GB model, 384GB VRAM, 512K ctx) - Q8_0 on 4×H100 (282GB model, 320GB VRAM, near-lossless) All recipes target fairydreaming/llama.cpp@deepseek-dsa (PR #21149), the DSA implementation favored by llama.cpp maintainers.
1 parent 02c003a commit 3caf73f

6 files changed

Lines changed: 172 additions & 10 deletions

File tree

‎launch.sh‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919
# MODELS_DIR default /workspace/models
2020
# MMPROJ F16 to enable vision
2121
# PORT, HOST default 8000, 127.0.0.1
22+
# LLAMA_CPP_REPO custom llama.cpp fork (default: ggml-org/llama.cpp)
23+
# LLAMA_CPP_REF branch/tag/commit to build from (default: master)
24+
# Use for models needing unmerged PRs, e.g. DeepSeek V4
2225

2326
set -euo pipefail
2427

@@ -43,6 +46,10 @@ HOST="${HOST:-127.0.0.1}"
4346
if [ "${IMAGE_TYPE}" = "builder" ] && [ ! -x /usr/local/bin/llama-server ]; then
4447
log "==> builder image: no pre-compiled llama-server — detecting GPU arch..."
4548

49+
# Custom repo/branch support (for models needing unmerged PRs)
50+
LLAMA_CPP_REPO="${LLAMA_CPP_REPO:-ggml-org/llama.cpp}"
51+
LLAMA_CPP_REF="${LLAMA_CPP_REF:-master}"
52+
4653
# Get compute capability from nvidia-smi, strip the dot: "9.0" → "90"
4754
RAW_CAP="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d ' ')"
4855
if [ -z "${RAW_CAP}" ]; then
@@ -60,10 +67,18 @@ if [ "${IMAGE_TYPE}" = "builder" ] && [ ! -x /usr/local/bin/llama-server ]; then
6067
SRC_DIR="/opt/llama.cpp"
6168
BUILD_DIR="${SRC_DIR}/build"
6269

63-
# Source was cloned into the image at build time — just run cmake
70+
# Source may be cached in image — re-clone if repo/branch differs or missing
6471
if [ ! -d "${SRC_DIR}" ]; then
65-
log " llama.cpp source not found in image — cloning..."
66-
git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "${SRC_DIR}"
72+
log " cloning https://github.com/${LLAMA_CPP_REPO}.git (ref: ${LLAMA_CPP_REF})..."
73+
git clone --depth 1 --branch "${LLAMA_CPP_REF}" \
74+
"https://github.com/${LLAMA_CPP_REPO}.git" "${SRC_DIR}"
75+
elif [ "${LLAMA_CPP_REPO}" != "ggml-org/llama.cpp" ] || [ "${LLAMA_CPP_REF}" != "master" ]; then
76+
# Custom repo/branch requested but image has default source — re-clone
77+
log " custom repo/branch requested — re-cloning..."
78+
log " repo: ${LLAMA_CPP_REPO} ref: ${LLAMA_CPP_REF}"
79+
rm -rf "${SRC_DIR}"
80+
git clone --depth 1 --branch "${LLAMA_CPP_REF}" \
81+
"https://github.com/${LLAMA_CPP_REPO}.git" "${SRC_DIR}"
6782
fi
6883

6984
log " configuring for SM${SM}..."
@@ -134,16 +149,21 @@ else
134149
log "model already present in ${TARGET_DIR}, skipping fetch"
135150
fi
136151

137-
# ── locate weights ─────────────────────────────────────────────────────────────
138-
MODEL_FILE="$(ls -1 "${TARGET_DIR}" | grep -iE "${MODEL_QUANT}.*\.gguf$" | grep -v 'mmproj' | sort | head -n1 || true)"
152+
# ── locate weights ─────────────────────────────────────────────────────────
153+
# Handles both single-file and split GGUFs.
154+
# Split files: llama-server needs the first shard (e.g. -00001-of-00023.gguf)
155+
# Some repos put shards in subdirectories (e.g. Q2_K/model-Q2_K.gguf-00001-of-N)
156+
MODEL_FILE="$(find "${TARGET_DIR}" -maxdepth 2 -name "*.gguf" \
157+
| grep -iE "${MODEL_QUANT}" | grep -v 'mmproj' | sort | head -n1 || true)"
139158
[ -n "${MODEL_FILE}" ] || die "no .gguf matching '${MODEL_QUANT}' in ${TARGET_DIR}"
140-
MODEL_PATH="${TARGET_DIR}/${MODEL_FILE}"
159+
MODEL_PATH="${MODEL_FILE}" # find returns full path
141160

142161
MMPROJ_ARGS=""
143162
if [ -n "${MMPROJ:-}" ]; then
144-
MMPROJ_FILE="$(ls -1 "${TARGET_DIR}" | grep -iE "mmproj-${MMPROJ}.*\.gguf$" | head -n1 || true)"
163+
MMPROJ_FILE="$(find "${TARGET_DIR}" -maxdepth 2 -name "*.gguf" \
164+
| grep -iE "mmproj-${MMPROJ}" | head -n1 || true)"
145165
[ -n "${MMPROJ_FILE}" ] || die "MMPROJ requested but no mmproj file found"
146-
MMPROJ_ARGS="--mmproj ${TARGET_DIR}/${MMPROJ_FILE}"
166+
MMPROJ_ARGS="--mmproj ${MMPROJ_FILE}"
147167
fi
148168

149169
# ── launch ─────────────────────────────────────────────────────────────────────

‎localrouter/menus/editor_menus.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,24 @@ def _create_recipe_wizard(data: dict) -> dict | None:
268268
"Image type:", choices=["prebuilt", "builder"], style=MENU_STYLE
269269
).ask()
270270

271+
# Custom llama.cpp (for models needing unmerged PRs)
272+
custom_llama = questionary.confirm(
273+
"Custom llama.cpp fork/branch? (for unmerged model support, e.g. DSv4)",
274+
default=False, style=MENU_STYLE
275+
).ask()
276+
if custom_llama:
277+
recipe["image_type"] = "builder" # must compile from source
278+
recipe["llama_cpp_repo"] = questionary.text(
279+
"GitHub repo (user/repo):",
280+
default="fairydreaming/llama.cpp",
281+
style=MENU_STYLE,
282+
).ask() or "fairydreaming/llama.cpp"
283+
recipe["llama_cpp_ref"] = questionary.text(
284+
"Branch/tag/commit:",
285+
default="deepseek-dsa",
286+
style=MENU_STYLE,
287+
).ask() or "deepseek-dsa"
288+
271289
# Description (optional for all)
272290
desc = questionary.text("Description (optional):", style=MENU_STYLE).ask()
273291
if desc:

‎localrouter/menus/vast_menus.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,10 @@ def menu_launch(recipes, gpu_tiers, docker_cfg, provider_cfg=None):
290290
t.add_row("Offer ID", offer_id or "auto-select")
291291
t.add_row("Image type", f"[{'green' if image_type == 'prebuilt' else 'yellow'}]{image_type}[/] {cold_start_estimate(image_type)}")
292292
t.add_row("Image", docker_img)
293+
if chosen_recipe.get("llama_cpp_repo") or chosen_recipe.get("llama_cpp_ref"):
294+
repo = chosen_recipe.get("llama_cpp_repo", "ggml-org/llama.cpp")
295+
ref = chosen_recipe.get("llama_cpp_ref", "master")
296+
t.add_row("llama.cpp", f"[yellow]{repo} @ {ref}[/yellow]")
293297
t.add_row("HOST", "[green]127.0.0.1[/green] (tunnel-only)")
294298

295299
# Add cost comparison if provider_cfg is available
@@ -332,6 +336,12 @@ def menu_launch(recipes, gpu_tiers, docker_cfg, provider_cfg=None):
332336
if offer_id:
333337
env["OFFER_ID"] = offer_id
334338

339+
# Custom llama.cpp repo/branch (for models needing unmerged PRs, e.g. DSv4)
340+
if chosen_recipe.get("llama_cpp_repo"):
341+
env["LLAMA_CPP_REPO"] = chosen_recipe["llama_cpp_repo"]
342+
if chosen_recipe.get("llama_cpp_ref"):
343+
env["LLAMA_CPP_REF"] = chosen_recipe["llama_cpp_ref"]
344+
335345
hr("Launching...")
336346
r = subprocess.run(["bash", str(ROOT / "vast_up.sh")], cwd=ROOT, env=env)
337347
if r.returncode == 0:

‎localrouter/recipe_editor.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,11 @@ def list_docker_images(data: dict) -> dict[str, str]:
167167
REQUIRED_RECIPE_FIELDS_LOCAL = {"name", "label", "model_path", "port"}
168168
REQUIRED_RECIPE_FIELDS_TOGETHER = {"name", "label", "model_id"}
169169
REQUIRED_TIER_FIELDS = {"vast_names", "label", "max_price"}
170+
# Optional recipe fields that the editor wizard should know about
171+
OPTIONAL_RECIPE_FIELDS_VAST = {
172+
"parallel", "kv_type", "min_disk_gb", "image_type", "description",
173+
"llama_cpp_repo", "llama_cpp_ref", # custom llama.cpp fork/branch
174+
}
170175

171176

172177
def validate_recipe(recipe: dict, gpu_tiers: dict) -> list[str]:

‎recipes.toml‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
# kv_type q8_0 | q4_0 | bf16 (default q8_0)
2626
# min_disk_gb override tier default disk for heavy recipes
2727
# image_type override tier default image type
28+
# llama_cpp_repo custom llama.cpp fork (user/repo) for unmerged model support
29+
# llama_cpp_ref branch/tag/commit of the custom fork (default: master)
2830
# description free text shown in TUI
2931

3032
# ── docker images ─────────────────────────────────────────────────────────────
@@ -168,6 +170,109 @@ image_type = "builder"
168170
vram_gb = 192
169171
num_gpus = 4
170172

173+
# ── DeepSeek V4 Flash recipes (284B / 13B active, MoE, 1M ctx) ──────────────
174+
# Requires llama.cpp with DSv4 support — built from fairydreaming's DSA branch.
175+
# NOTE: llama_cpp_repo/ref tell launch.sh to clone+compile from this fork.
176+
# When DSv4 support merges to llama.cpp master, remove these two fields.
177+
178+
[[recipes]]
179+
name = "dsv4-flash-q2k-2xh100"
180+
label = "DSv4-Flash 284B Q2_K 128K ctx (2×H100)"
181+
gpu = "h100-sxm-2x"
182+
model_repo = "Preyazz/DeepSeek-V4-Flash-GGUF"
183+
model_quant = "Q2_K"
184+
ctx = 131072
185+
parallel = 1
186+
kv_type = "q8_0"
187+
image_type = "builder"
188+
llama_cpp_repo = "fairydreaming/llama.cpp"
189+
llama_cpp_ref = "deepseek-dsa"
190+
description = "DSv4-Flash Q2_K (96 GB) on 2×H100 (160 GB). Lightest quant, fits with KV headroom."
191+
192+
[[recipes]]
193+
name = "dsv4-flash-q3k-2xh100"
194+
label = "DSv4-Flash 284B Q3_K_M 128K ctx (2×H100)"
195+
gpu = "h100-sxm-2x"
196+
model_repo = "Preyazz/DeepSeek-V4-Flash-GGUF"
197+
model_quant = "Q3_K_M"
198+
ctx = 131072
199+
parallel = 1
200+
kv_type = "q8_0"
201+
image_type = "builder"
202+
llama_cpp_repo = "fairydreaming/llama.cpp"
203+
llama_cpp_ref = "deepseek-dsa"
204+
description = "DSv4-Flash Q3_K_M (126 GB) on 2×H100 (160 GB). Good balance of quality and fit."
205+
206+
[[recipes]]
207+
name = "dsv4-flash-q4k-2xh200"
208+
label = "DSv4-Flash 284B Q4_K_M 256K ctx (2×H200)"
209+
gpu = "h200-sxm-2x"
210+
model_repo = "Preyazz/DeepSeek-V4-Flash-GGUF"
211+
model_quant = "Q4_K_M"
212+
ctx = 262144
213+
parallel = 1
214+
kv_type = "q8_0"
215+
image_type = "builder"
216+
llama_cpp_repo = "fairydreaming/llama.cpp"
217+
llama_cpp_ref = "deepseek-dsa"
218+
description = "DSv4-Flash Q4_K_M (160 GB) on 2×H200 (282 GB). Great quality, generous context."
219+
220+
[[recipes]]
221+
name = "dsv4-flash-q4k-4xh100"
222+
label = "DSv4-Flash 284B Q4_K_M 256K ctx (4×H100)"
223+
gpu = "h100-sxm-4x"
224+
model_repo = "Preyazz/DeepSeek-V4-Flash-GGUF"
225+
model_quant = "Q4_K_M"
226+
ctx = 262144
227+
parallel = 2
228+
kv_type = "q8_0"
229+
image_type = "builder"
230+
llama_cpp_repo = "fairydreaming/llama.cpp"
231+
llama_cpp_ref = "deepseek-dsa"
232+
description = "DSv4-Flash Q4_K_M (160 GB) on 4×H100 (320 GB). 2 parallel slots, room for large ctx."
233+
234+
[[recipes]]
235+
name = "dsv4-flash-mxfp4-2xh200"
236+
label = "DSv4-Flash 284B MXFP4 256K ctx (2×H200)"
237+
gpu = "h200-sxm-2x"
238+
model_repo = "lovedheart/DeepSeek-V4-Flash-GGUF"
239+
model_quant = "MXFP4"
240+
ctx = 262144
241+
parallel = 1
242+
kv_type = "q8_0"
243+
image_type = "builder"
244+
llama_cpp_repo = "fairydreaming/llama.cpp"
245+
llama_cpp_ref = "deepseek-dsa"
246+
description = "DSv4-Flash MXFP4 mixed (140 GB) on 2×H200 (282 GB). MoE experts in FP4, rest in FP8."
247+
248+
[[recipes]]
249+
name = "dsv4-flash-q4k-2xb200"
250+
label = "DSv4-Flash 284B Q4_K_M 512K ctx (2×B200)"
251+
gpu = "b200-sxm-2x"
252+
model_repo = "Preyazz/DeepSeek-V4-Flash-GGUF"
253+
model_quant = "Q4_K_M"
254+
ctx = 524288
255+
parallel = 2
256+
kv_type = "q8_0"
257+
image_type = "builder"
258+
llama_cpp_repo = "fairydreaming/llama.cpp"
259+
llama_cpp_ref = "deepseek-dsa"
260+
description = "DSv4-Flash Q4_K_M (160 GB) on 2×B200 (384 GB). 2 slots, massive ctx headroom."
261+
262+
[[recipes]]
263+
name = "dsv4-flash-q8-4xh100"
264+
label = "DSv4-Flash 284B Q8_0 128K ctx (4×H100)"
265+
gpu = "h100-sxm-4x"
266+
model_repo = "Preyazz/DeepSeek-V4-Flash-Q8_0-GGUF"
267+
model_quant = "Q8_0"
268+
ctx = 131072
269+
parallel = 1
270+
kv_type = "q8_0"
271+
image_type = "builder"
272+
llama_cpp_repo = "fairydreaming/llama.cpp"
273+
llama_cpp_ref = "deepseek-dsa"
274+
description = "DSv4-Flash Q8_0 (282 GB) on 4×H100 (320 GB). Near-lossless quality."
275+
171276
# ── consumer recipes — RTX 5090 32GB ─────────────────────────────────────────
172277

173278
[[recipes]]

‎vast_up.sh‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,10 +215,14 @@ ENV_ARGS=(
215215
)
216216
[ -n "${MMPROJ:-}" ] && ENV_ARGS+=(-e "MMPROJ=${MMPROJ}")
217217
[ -n "${HF_TOKEN_VAL}" ] && ENV_ARGS+=(-e "HF_TOKEN=${HF_TOKEN_VAL}")
218+
[ -n "${LLAMA_CPP_REPO:-}" ] && ENV_ARGS+=(-e "LLAMA_CPP_REPO=${LLAMA_CPP_REPO}")
219+
[ -n "${LLAMA_CPP_REF:-}" ] && ENV_ARGS+=(-e "LLAMA_CPP_REF=${LLAMA_CPP_REF}")
218220

219221
ONSTART_CMD="MODEL_REPO=${MODEL_REPO} MODEL_QUANT=${MODEL_QUANT} CTX=${CTX} KV_TYPE=${KV_TYPE} MODE=${MODE} PARALLEL=${PARALLEL} HOST=127.0.0.1 IMAGE_TYPE=${IMAGE_TYPE}"
220-
[ -n "${MMPROJ:-}" ] && ONSTART_CMD="${ONSTART_CMD} MMPROJ=${MMPROJ}"
221-
[ -n "${HF_TOKEN_VAL}" ] && ONSTART_CMD="${ONSTART_CMD} HF_TOKEN=${HF_TOKEN_VAL}"
222+
[ -n "${MMPROJ:-}" ] && ONSTART_CMD="${ONSTART_CMD} MMPROJ=${MMPROJ}"
223+
[ -n "${HF_TOKEN_VAL}" ] && ONSTART_CMD="${ONSTART_CMD} HF_TOKEN=${HF_TOKEN_VAL}"
224+
[ -n "${LLAMA_CPP_REPO:-}" ] && ONSTART_CMD="${ONSTART_CMD} LLAMA_CPP_REPO=${LLAMA_CPP_REPO}"
225+
[ -n "${LLAMA_CPP_REF:-}" ] && ONSTART_CMD="${ONSTART_CMD} LLAMA_CPP_REF=${LLAMA_CPP_REF}"
222226
ONSTART_CMD="${ONSTART_CMD} bash /app/launch.sh > /var/log/launch.log 2>&1 &"
223227

224228
echo "==> creating instance..."

0 commit comments

Comments
 (0)