Add TWM runtime benchmark gate

Author: zhouning

data_agent/benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Benchmark runners for development-time validation."""

data_agent/benchmarks/twm_runtime_v1/__init__.py

@@ -0,0 +1,5 @@
+"""TWM Runtime Benchmark v1."""
+
+from .runner import run_twm_runtime_benchmark
+
+__all__ = ["run_twm_runtime_benchmark"]

data_agent/benchmarks/twm_runtime_v1/manifest.json

@@ -0,0 +1,18 @@
+{
+  "schema": "territory_world_model.twm_runtime_benchmark_manifest.v1",
+  "suite_id": "twm_runtime_v1",
+  "created_at": "2026-07-03",
+  "state_dataset_dir": "data_agent/test_data/twm_bishan_multi_admin_eval",
+  "synthetic_experiment_foundation": "docs/reports/twm_synthetic_experiment_foundation.csv",
+  "structural_validation_history": "docs/reports/twm_structural_validation_observed_history.csv",
+  "same_case_baselines": [
+    "data_agent/test_data/twm_baseline_metrics/twm_candidate_triage_same_case_outputs.csv",
+    "data_agent/test_data/twm_baseline_metrics/optimization_only_candidate_triage_same_case_outputs.csv"
+  ],
+  "claim_boundary": {
+    "benchmark_scope": "development_runtime_contract",
+    "production_accuracy": "not_supported",
+    "production_decision": "blocked_without_real_observed_history",
+    "flus_superiority": "not_evaluated_by_this_benchmark"
+  }
+}

data_agent/benchmarks/twm_runtime_v1/negative_controls.json

@@ -0,0 +1,30 @@
+{
+  "schema": "territory_world_model.twm_runtime_negative_controls.v1",
+  "controls": [
+    {
+      "control_id": "impossible_action",
+      "description": "Hard-blocked land-use conversion or approval must be blocked.",
+      "required_signal": "block"
+    },
+    {
+      "control_id": "support_material_missing_action",
+      "description": "Low support-material coverage must remain review-only.",
+      "required_signal": "review"
+    },
+    {
+      "control_id": "policy_conflict_action",
+      "description": "Action mask policy conflict must be recalled as blocked.",
+      "required_signal": "block"
+    },
+    {
+      "control_id": "shuffled_action_control",
+      "description": "Performance must drop when action type is shuffled.",
+      "required_signal": "performance_drop"
+    },
+    {
+      "control_id": "shuffled_label_control",
+      "description": "The gate must fail when utility and risk labels are shuffled.",
+      "required_signal": "fail"
+    }
+  ]
+}

data_agent/benchmarks/twm_runtime_v1/runner.py

@@ -0,0 +1,41 @@
+{
+  "dataset_integrity_gate": {
+    "relation_rows_min": 1700,
+    "evidence_rows_min": 200,
+    "synthetic_experiment_rows": 256,
+    "counterfactual_pairs": 128,
+    "train_rows": 128,
+    "validation_rows": 64,
+    "test_rows": 64,
+    "action_type_count": 4,
+    "allowed_rows": 128,
+    "blocked_rows": 128
+  },
+  "renderer_gate": {
+    "object_count_min": 21000,
+    "relation_count_min": 1700,
+    "project_count_min": 90,
+    "rule_evaluation_count_min": 360,
+    "review_task_count_min": 100,
+    "support_material_count_min": 200
+  },
+  "simulator_gate": {
+    "transition_mae_max": 0.05,
+    "constraint_risk_mae_max": 0.05,
+    "planning_utility_mae_max": 0.05,
+    "ranking_correlation_proxy_min": 0.6,
+    "action_mask_accuracy_min": 0.85,
+    "blocked_action_recall_min": 0.85
+  },
+  "planner_gate": {
+    "legal_feasible_topk_precision_min": 0.8,
+    "candidate_rejection_reason_coverage_min": 0.85,
+    "planner_regret_against_human_oracle_max": 0.15
+  },
+  "negative_control_gate": {
+    "impossible_action_block_rate_min": 0.95,
+    "support_missing_review_rate_min": 0.95,
+    "policy_conflict_block_recall_min": 0.9,
+    "shuffled_action_performance_drop_min": 0.15
+  }
+}

data_agent/benchmarks/twm_runtime_v1/thresholds.json

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_twm_runtime_benchmark_blocks_current_facade_without_simulator_trace(tmp_path):
+    from data_agent.benchmarks.twm_runtime_v1.runner import run_twm_runtime_benchmark
+
+    output = tmp_path / "twm_runtime_benchmark_v1.json"
+    markdown_output = tmp_path / "twm_runtime_benchmark_v1.md"
+
+    report = run_twm_runtime_benchmark(
+        output_path=output,
+        markdown_output_path=markdown_output,
+        fail_on_failed=False,
+    )
+
+    assert report["schema"] == "territory_world_model.twm_runtime_benchmark.v1"
+    assert report["suite_id"] == "twm_runtime_v1"
+    assert report["status"] == "fail"
+    assert "simulator_gate" in report["failed_gates"]
+    assert "planner_gate" in report["failed_gates"]
+
+    simulator_gate = report["gates"]["simulator_gate"]
+    assert simulator_gate["status"] == "fail"
+    assert "simulator_trace" in simulator_gate["missing"]
+    assert simulator_gate["checks"]["facade_backend_forbidden"]["status"] == "fail"
+    assert simulator_gate["checks"]["facade_backend_forbidden"]["observed"] == "deterministic_planner_facade"
+
+    planner_gate = report["gates"]["planner_gate"]
+    assert planner_gate["status"] == "fail"
+    assert "planner_consumes_simulator_trace" in planner_gate["missing"]
+
+    claim_boundary = report["claim_boundary"]
+    assert claim_boundary["production_decision"] == "blocked_without_real_observed_history"
+    assert claim_boundary["production_accuracy"] == "not_supported"
+    assert claim_boundary["flus_superiority"] == "not_evaluated_by_this_benchmark"
+    assert claim_boundary["not_for_production_boundary_preserved"] is True
+
+    assert output.exists()
+    assert markdown_output.exists()
+
+
+def test_twm_runtime_benchmark_cli_writes_outputs_and_exits_nonzero_on_failed(tmp_path):
+    output = tmp_path / "cli_report.json"
+    markdown_output = tmp_path / "cli_report.md"
+
+    completed = subprocess.run(
+        [
+            sys.executable,
+            str(REPO_ROOT / "scripts/run_twm_runtime_benchmark.py"),
+            "--suite",
+            "twm_runtime_v1",
+            "--output",
+            str(output),
+            "--markdown-output",
+            str(markdown_output),
+            "--fail-on-failed",
+        ],
+        cwd=REPO_ROOT,
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+
+    assert completed.returncode == 1
+    assert output.exists()
+    assert markdown_output.exists()
+    assert "status=fail" in completed.stdout
+
+
+def test_twm_runtime_benchmark_requires_explicit_leakage_guard_contract(tmp_path):
+    from data_agent.benchmarks.twm_runtime_v1.runner import run_twm_runtime_benchmark
+
+    report = run_twm_runtime_benchmark(
+        output_path=tmp_path / "report.json",
+        markdown_output_path=tmp_path / "report.md",
+    )
+
+    leakage_gate = report["gates"]["leakage_guard_gate"]
+    assert leakage_gate["status"] == "fail"
+    assert "feature_vector_contract" in leakage_gate["missing"]
+    assert leakage_gate["checks"]["target_feature_columns_present_in_source"]["status"] == "review"
+    assert "leakage_guard_gate" in report["failed_gates"]