fix(coherence): Debugging to Get Qwen Working In Agentic Coding

.gitignore

@@ -1,9 +1,15 @@
 # Build artifacts
 target/
 **/target/
+# Out-of-tree cargo target dirs used for parallel/experiment builds
+# (target-final, target-fu, target-i84, target-phc, target-rc, ...).
+target-*/
 *.o
 *.ptx
 *.bak
+# Local docker build input: release binary copied in before `docker build`.
+# Regenerated from `target/release/spark`; never tracked.
+docker/gb10/spark-fastbin
 
 # tool-eval-bench run reports — local benchmark output, not source
 /runs/

Cargo.toml

@@ -37,7 +37,12 @@ cudarc = { version = "0.19", features = ["driver", "nvrtc", "cuda-version-from-b
 
 # Serialization (HF config.json)
 serde = { version = "1", features = ["derive"] }
-serde_json = "1"
+# `preserve_order`: keep JSON object keys in insertion/declaration order
+# (IndexMap, not BTreeMap). Required so `serde_json::to_value(ToolDefinition)`
+# and the nested `parameters` JSON-schema Value match transformers'
+# `tojson` (= `json.dumps(sort_keys=False)`) key order in the `<tools>`
+# prompt block. See crates/spark-server/src/tokenizer/jinja_helpers.rs.
+serde_json = { version = "1", features = ["preserve_order"] }
 
 # Error handling
 thiserror = "2"

bench/fp8_dgx2_drift/FP8_COSINE_RESULTS_2026_05_29.txt

@@ -0,0 +1,46 @@
+layer  type       cos    rel_l2    |vllm|   |atlas|
+----------------------------------------------------
+    0   ssm   0.99993   0.01197       0.6       0.6
+    1   ssm   0.99979   0.02077       0.8       0.8
+    2   ssm   0.99902   0.04431       1.0       1.0
+    3  attn   0.99874   0.05058       1.1       1.1  <-- divergence onset (cos<0.999)
+    4   ssm   0.99851   0.05465       1.1       1.1
+    5   ssm   0.99872   0.05059       1.3       1.3
+    6   ssm   0.99914   0.04149       1.3       1.3
+    7  attn   0.99811   0.06280       1.3       1.3
+    8   ssm   0.99804   0.06429       1.3       1.3
+    9   ssm   0.99808   0.06284       1.3       1.4
+   10   ssm   0.99845   0.05645       1.6       1.6
+   11  attn   0.99775   0.06734       1.6       1.6
+   12   ssm   0.99785   0.06570       1.5       1.5
+   13   ssm   0.99785   0.06604       1.5       1.6
+   14   ssm   0.99673   0.08093       1.6       1.6
+   15  attn   0.99188   0.12740       1.7       1.7
+   16   ssm   0.99290   0.11916       1.9       1.9
+   17   ssm   0.99236   0.12357       1.9       1.9
+   18   ssm   0.99410   0.10848       2.2       2.2
+   19  attn   0.99410   0.10851       2.8       2.8
+   20   ssm   0.99430   0.10677       2.8       2.9
+   21   ssm   0.99461   0.10373       2.8       2.8
+   22   ssm   0.99409   0.10868       3.2       3.2
+   23  attn   0.99353   0.11534       3.3       3.2
+   24   ssm   0.99183   0.12967       3.1       3.1
+   25   ssm   0.99296   0.12099       3.5       3.4
+   26   ssm   0.99469   0.10615       3.8       3.7
+   27  attn   0.99464   0.10609       4.2       4.1
+   28   ssm   0.99490   0.10416       4.5       4.4
+   29   ssm   0.99483   0.10377       4.6       4.6
+   30   ssm   0.99341   0.11750       4.9       4.8
+   31  attn   0.99291   0.11950       5.8       5.8
+   32   ssm   0.99209   0.12568       6.3       6.3
+   33   ssm   0.99126   0.13321       7.2       7.1
+   34   ssm   0.99233   0.12482       8.2       8.1
+   35  attn   0.99011   0.14054       9.2       9.2
+   36   ssm   0.98919   0.14737       9.8       9.7
+   37   ssm   0.98809   0.15493      11.3      11.3
+   38   ssm   0.98650   0.16487      14.3      14.2
+   39  attn   0.98766   0.15663      16.9      17.1
+----------------------------------------------------
+worst layer: L38 (ssm) cos=0.98650 rel_l2=0.1649
+final-layer L39 cos=0.98766
+DIVERGENCE ONSET: L3 — inspect this layer's ops (attn/SSM/MoE/norm) next

bench/fp8_dgx2_drift/atlas_tokenize.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""Reproduce Atlas's exact tokenization of the probe JSON.
+
+Uses Atlas's own OpenAI-variant Jinja template (jinja-templates/openai/qwen3_5_moe.jinja)
+to render the same string Atlas sees, then encodes via the model's HF tokenizer.
+
+Writes /tmp/atlas_tokens_dgx2.json in the same format as
+/tmp/atlas_tokens.json so hf_dual_forward.py can consume it.
+"""
+from __future__ import annotations
+
+import json
+import pathlib
+
+import jinja2
+from transformers import AutoTokenizer
+
+TEMPLATE_PATH = pathlib.Path("/workspace/atlas-mtp/jinja-templates/openai/qwen3_5_moe.jinja")
+TOKENIZER_SNAP = "/workspace/.cache/huggingface/Qwen3.6-35B-A3B-FP8-dequanted-BF16"
+PROBE_PATH = pathlib.Path("/workspace/atlas-dumps/numdrift/atlas_turn11_probe.json")
+OUT_PATH = pathlib.Path("/tmp/atlas_tokens_dgx2.json")
+TARGET_TOKEN_COUNT = 9780  # what Atlas reports today
+
+
+def normalize_tool_call_arguments(messages):
+    """Atlas's chat_impl.rs F76: pre-parse tool_call argument strings into dicts."""
+    out = []
+    for m in messages:
+        m2 = dict(m)
+        if m2.get("tool_calls"):
+            new_calls = []
+            for tc in m2["tool_calls"]:
+                tc2 = dict(tc)
+                if tc2.get("function") and isinstance(tc2["function"].get("arguments"), str):
+                    fn2 = dict(tc2["function"])
+                    try:
+                        fn2["arguments"] = json.loads(fn2["arguments"])
+                    except Exception:
+                        pass
+                    tc2["function"] = fn2
+                new_calls.append(tc2)
+            m2["tool_calls"] = new_calls
+        out.append(m2)
+    return out
+
+
+def main() -> None:
+    probe = json.loads(PROBE_PATH.read_text())
+    template_src = TEMPLATE_PATH.read_text()
+
+    env = jinja2.Environment(
+        loader=jinja2.BaseLoader(),
+        trim_blocks=False,
+        lstrip_blocks=False,
+        keep_trailing_newline=True,
+    )
+    tmpl = env.from_string(template_src)
+
+    messages = normalize_tool_call_arguments(probe["messages"])
+    tools = probe.get("tools")
+
+    rendered = tmpl.render(
+        messages=messages,
+        tools=tools,
+        add_generation_prompt=True,
+        enable_thinking=True,
+        reasoning_effort="high",
+        disable_tool_steering=False,
+        add_vision_id=False,
+    )
+    print(f"rendered len chars: {len(rendered)}")
+
+    tok = AutoTokenizer.from_pretrained(TOKENIZER_SNAP)
+    ids = tok(rendered, add_special_tokens=False, return_tensors=None)["input_ids"]
+    print(f"token count: {len(ids)}")
+    print(f"first 10: {ids[:10]}")
+    print(f"last 10:  {ids[-10:]}")
+
+    out = {
+        "prompt_len": len(ids),
+        "all_tokens": ids,
+        "generated_tokens": [],
+    }
+    OUT_PATH.write_text(json.dumps(out))
+    print(f"wrote {OUT_PATH}")
+
+    if len(ids) != TARGET_TOKEN_COUNT:
+        print(
+            f"\nWARN: count {len(ids)} != target {TARGET_TOKEN_COUNT} "
+            f"(Atlas-reported); template/encoder difference may bias HF dump"
+        )
+    else:
+        print(f"\nMATCH: count == {TARGET_TOKEN_COUNT}")
+
+
+if __name__ == "__main__":
+    main()

bench/fp8_dgx2_drift/c1_final_logit_overlap.json

@@ -0,0 +1,17 @@
+{
+  "residual_cos_L39": 0.9895407557487488,
+  "logit_cos": 0.9872766137123108,
+  "argmax_atlas_token": 6820,
+  "argmax_bf16_token": 6820,
+  "top1_agree": true,
+  "topk_jaccard": {
+    "1": 1.0,
+    "5": 1.0,
+    "10": 1.0,
+    "50": 0.8867924528301887,
+    "200": 0.8348623853211009,
+    "1000": 0.8399264029438822
+  },
+  "kl_bf16_vs_atlas": 0.020466946885892377,
+  "kl_atlas_vs_bf16": 0.020438889431768144
+}