Merge pull request #13 from LLMSystems/fix/multi-engine-review-and-selectable-backends

feat(ha): unschedulable visibility (M#5 UI) + capability-based sleep …

Author: milk333445

apps/backend/app/api/models.py

@@ -59,7 +59,14 @@ async def list_models(request: Request, manager: ModelManager = Depends(get_mana
     # (each node backfills its *owned* observed state) — on leader and follower
     # alike, since with per-node actuation (Phase 7) no single registry is complete.
     # SQLite collapsed: the local registry is the truth — identical to before.
-    return [ModelView(**v) for v in await manager.fleet_views(prefer_store=manager.prefer_store_view())]
+    views = await manager.fleet_views(prefer_store=manager.prefer_store_view())
+    # Annotate any model stuck because no live node can run its engine (Medium#5).
+    reasons = await manager.unschedulable_reasons()
+    if reasons:
+        for v in views:
+            if v.get("key") in reasons:
+                v["unschedulable_reason"] = reasons[v["key"]]
+    return [ModelView(**v) for v in views]
 
 
 @router.post("/parse", dependencies=[Depends(require_operator)])

apps/backend/app/api/schemas.py

@@ -31,6 +31,10 @@ class ModelView(BaseModel):
     # HA Phase 3d: which node-agent runs this instance (multi-node deploys). None
     # on a single host / when unknown.
     node_id: Optional[str] = None
+    # Set when this model is desired-running but no live node can run its engine, so
+    # it will stay stopped until a matching node appears — distinguishes "won't ever
+    # start" from "cold-starting" for the dashboard (Medium#5). None = schedulable.
+    unschedulable_reason: Optional[str] = None
 
     @classmethod
     def from_instance(cls, inst: ModelInstance) -> "ModelView":

apps/backend/app/llmops/manager.py

@@ -461,6 +461,34 @@ async def fleet_state_snapshot(self) -> list:
                 out.append(inst)
         return out
 
+    async def unschedulable_reasons(self) -> dict[str, str]:
+        """Desired-running models that no live node can run (their engine is advertised
+        by no live node), each with a human reason. Lets the dashboard tell "won't ever
+        start" from "cold-starting" (Medium#5). Computed from the shared store, so any
+        replica serving the API returns the same answer regardless of who holds the
+        scheduler lease. Empty outside HA/Postgres mode — collapsed single host runs
+        everything. Best-effort: any store issue yields {}."""
+        if not self.prefer_store_view() or not hasattr(self.store, "list_nodes"):
+            return {}
+        from app.llmops.scheduler import node_supports
+        try:
+            desired = await self.store.list_instance_desired()
+            nodes = await self.store.list_nodes()
+        except Exception:
+            logger.debug("unschedulable_reasons: store read failed", exc_info=True)
+            return {}
+        out: dict[str, str] = {}
+        for key, want in desired.items():
+            if want != Desired.RUNNING.value:
+                continue
+            group = key.split("::")[0]
+            engine = getattr(
+                getattr(self.config.LLM_engines.get(group), "settings", None),
+                "engine", "vllm")
+            if not any(node_supports(n, engine) for n in nodes):
+                out[key] = f"no live node runs engine '{engine}'"
+        return out
+
     async def get(self, key: str) -> ModelInstance:
         return self._require(key)
 

apps/backend/tests/unit/test_ha_safety.py

@@ -321,3 +321,28 @@ async def test_edit_rejects_duplicate_lora_within_group(tmp_path):
              "lora_modules": [{"name": "same", "path": "/p1"},
                               {"name": "same", "path": "/p2"}]},
         )
+
+
+# ---- Medium#5: unschedulable visibility (API) ----------------------------
+
+async def test_unschedulable_reason_when_no_node_runs_engine(tmp_path):
+    store = FakeStore()
+    store.desired = {"Qwen3-0.6B::a": Desired.RUNNING.value}  # group engine defaults vllm
+    store.nodes = [{"node_id": "n1", "engines": '["sglang"]'}]  # no vllm node
+    mgr, _ = _manager(tmp_path, store=store)
+    reasons = await mgr.unschedulable_reasons()
+    assert "Qwen3-0.6B::a" in reasons and "vllm" in reasons["Qwen3-0.6B::a"]
+
+
+async def test_not_unschedulable_when_a_node_runs_engine(tmp_path):
+    store = FakeStore()
+    store.desired = {"Qwen3-0.6B::a": Desired.RUNNING.value}
+    store.nodes = [{"node_id": "n1", "engines": '["vllm"]'}]
+    mgr, _ = _manager(tmp_path, store=store)
+    assert await mgr.unschedulable_reasons() == {}
+
+
+async def test_unschedulable_empty_in_collapsed_mode(tmp_path):
+    # No store / SQLite (db_url None) -> nothing computed (single host runs everything).
+    mgr, _ = _manager(tmp_path, store=None)
+    assert await mgr.unschedulable_reasons() == {}

apps/frontend_llmops/src/components/ModelGroupCard.vue

@@ -1,6 +1,6 @@
 <script setup lang="ts">
 import { computed, ref } from 'vue'
-import { Box, ChevronDown, ChevronRight, Gauge, Loader2, Play, Plus, Power, RotateCw, Shuffle, Sparkles, Square } from '@lucide/vue'
+import { AlertTriangle, Box, ChevronDown, ChevronRight, Gauge, Loader2, Play, Plus, Power, RotateCw, Shuffle, Sparkles, Square } from '@lucide/vue'
 import { useI18n } from 'vue-i18n'
 import Card from '@/components/ui/Card.vue'
 import Badge from '@/components/ui/Badge.vue'
@@ -385,6 +385,14 @@ const startLockTitle = computed(() =>
             <RotateCw class="size-3" />{{ model.restart_count }}
           </span>
         </Tooltip>
+        <Tooltip
+          v-if="model.unschedulable_reason"
+          :text="t('modelGroup.unschedulable', { reason: model.unschedulable_reason })"
+        >
+          <span class="flex items-center gap-0.5 text-[10px] text-status-failed" @click.stop>
+            <AlertTriangle class="size-3" />{{ t('modelGroup.unschedulableShort') }}
+          </span>
+        </Tooltip>
 
         <div class="ml-auto flex items-center gap-2.5">
           <Tooltip v-if="metricsFor(model)">

apps/frontend_llmops/src/i18n/locales/en.ts

@@ -1102,6 +1102,8 @@ export default {
     embedding: 'Embedding',
     reranking: 'Reranking',
     crashRestart: 'Crashed and auto-restarted {n} times',
+    unschedulableShort: 'unschedulable',
+    unschedulable: 'Wanted running but {reason} — it stays stopped until a matching backend is available.',
     liveLoad: 'Live load (router /metrics)',
     runningDesc: ' Running — currently generating requests',
     waitingDesc: ' Waiting — queued requests for this instance',

apps/frontend_llmops/src/i18n/locales/zh-TW.ts

@@ -1079,6 +1079,8 @@ export default {
     embedding: '嵌入',
     reranking: '重排序',
     crashRestart: '崩潰後自動重啟 {n} 次',
+    unschedulableShort: '無法排程',
+    unschedulable: '想跑但{reason}——會一直停著,直到有能跑它的 backend 出現。',
     liveLoad: '即時負載（路由器 /metrics）',
     runningDesc: ' 執行中 — 目前正在生成的請求',
     waitingDesc: ' 等待中 — 此實例的排隊請求',

apps/frontend_llmops/src/types/api.ts

@@ -41,6 +41,10 @@ export interface ModelView {
   ready_at: number | null
   updated_at: number | null
   restart_count?: number
+  node_id?: string | null
+  // Set when the model is desired-running but no live node can run its engine
+  // (mixed-engine fleets) — it will stay stopped until a matching backend appears.
+  unschedulable_reason?: string | null
 }
 
 export interface MemoryInfo {

apps/router-server/src/llm_router/metrics_poller.py

@@ -2,6 +2,8 @@
 import logging
 from typing import Any, Dict
 
+from src.llm_router.vllm_metrics_client import engine_sleep_capable
+
 logger = logging.getLogger(__name__)
 
 
@@ -80,7 +82,12 @@ async def poll_metrics_forever(app, interval: float = 1.0):
             # nothing extra.
             sleep_backends: Dict[str, str] = {}
             for model_key, model_cfg in llm_engines.items():
-                if not model_cfg.get("model_config", {}).get("enable_sleep_mode"):
+                mc = model_cfg.get("model_config", {}) or {}
+                # Gate on the engine's *capability*, not just the config toggle: only
+                # engines that actually have /sleep+/is_sleeping are probed, so an
+                # enable_sleep_mode set on a non-sleep engine can't make the router
+                # mark its instances sleeping. (Low#3)
+                if not mc.get("enable_sleep_mode") or not engine_sleep_capable(mc.get("engine", "vllm")):
                     continue
                 for instance in model_cfg.get("instances", []):
                     composite = f"{model_key}\x00{instance['id']}"

apps/router-server/src/llm_router/vllm_metrics_client.py

@@ -88,6 +88,21 @@ def to_dict(self):
 }
 
 
+# Engines that expose vLLM's /sleep + /wake_up (level-1 warm standby that frees VRAM).
+# The router only probes /is_sleeping for these — SGLang and llama.cpp have no such
+# endpoint, so probing them is wasted, and a config that mistakenly set
+# enable_sleep_mode on a non-sleep engine must not make the router treat its instances
+# as sleeping. Kept beside METRIC_NAMES_BY_ENGINE so the router's engine→behaviour
+# knowledge (which metrics an engine speaks, whether it can sleep) lives in one place
+# rather than scattered `engine == "..."` / config-key checks. (Low#3)
+ENGINE_SLEEP_CAPABLE = frozenset({"vllm"})
+
+
+def engine_sleep_capable(engine: str) -> bool:
+    """Whether `engine` supports sleep/wake (and thus /is_sleeping is worth probing)."""
+    return engine in ENGINE_SLEEP_CAPABLE
+
+
 class VLLMMetricsClient:
     # Default (vLLM) names; engine-specific lookups use METRIC_NAMES_BY_ENGINE.
     METRIC_NAMES = METRIC_NAMES_BY_ENGINE["vllm"]