feat(harness): parse real cost from opencode JSON output (#269)

* fix: capture stderr from Claude Code CLI for error diagnosis

The ClaudeCodeProvider was not passing a stderr callback to
ClaudeAgentOptions, so when the claude CLI exited with code 1,
the actual error message was lost. Logs only showed "Command
failed with exit code 1" with no actionable details.

Now passes a stderr callback that collects output and includes
it in both the error log and the RawResult.error_message field.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: set stderr callback on opts object, not agent_options dict

Avoids test assertion failures caused by unexpected 'stderr' key
in the agent_options dictionary.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* feat(harness): parse real cost from opencode JSON output

Use -f json flag when invoking opencode CLI and parse cost,
prompt_tokens, and completion_tokens from the response. Falls back
to estimate_cli_cost() for older opencode versions that don't
include metrics in their JSON output.

Depends on: opencode-ai/opencode#TBD

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: AbirAbbas

sdk/python/agentfield/harness/providers/opencode.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+import json
 import logging
 import os
 import shutil
@@ -77,6 +78,7 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes
                 f"---\n\nUSER REQUEST:\n{prompt}"
             )
 
+        cmd.extend(["-f", "json"])
         cmd.append(effective_prompt)
 
         env: Dict[str, str] = {}
@@ -119,7 +121,26 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes
             shutil.rmtree(temp_data_dir, ignore_errors=True)
 
         api_ms = int((time.monotonic() - start_api) * 1000)
-        result_text = stdout.strip() if stdout.strip() else None
+
+        parsed_cost: float | None = None
+        parsed_prompt_tokens: int | None = None
+        parsed_completion_tokens: int | None = None
+
+        try:
+            json_output = json.loads(stdout.strip())
+            result_text = json_output.get("response", "").strip() or None
+            raw_cost = json_output.get("cost")
+            if raw_cost is not None and float(raw_cost) > 0:
+                parsed_cost = float(raw_cost)
+            raw_pt = json_output.get("prompt_tokens")
+            if raw_pt is not None:
+                parsed_prompt_tokens = int(raw_pt)
+            raw_ct = json_output.get("completion_tokens")
+            if raw_ct is not None:
+                parsed_completion_tokens = int(raw_ct)
+        except (json.JSONDecodeError, ValueError, TypeError):
+            result_text = stdout.strip() if stdout.strip() else None
+
         clean_stderr = strip_ansi(stderr.strip()) if stderr else ""
 
         logger.info(
@@ -152,19 +173,30 @@ async def _execute_impl(self, prompt: str, options: dict[str, object]) -> RawRes
             is_error = False
             error_message = None
 
-        estimated_cost = estimate_cli_cost(
-            model=str(options.get("model", "")),
-            prompt=effective_prompt,
-            result_text=result_text,
-        )
+        if parsed_cost is not None:
+            final_cost = parsed_cost
+        else:
+            final_cost = estimate_cli_cost(
+                model=str(options.get("model", "")),
+                prompt=effective_prompt,
+                result_text=result_text,
+            )
+
+        usage_data = None
+        if parsed_prompt_tokens is not None or parsed_completion_tokens is not None:
+            usage_data = {
+                "prompt_tokens": parsed_prompt_tokens or 0,
+                "completion_tokens": parsed_completion_tokens or 0,
+            }
 
         return RawResult(
             result=result_text,
             messages=[],
             metrics=Metrics(
                 duration_api_ms=api_ms,
                 num_turns=1 if result_text else 0,
-                total_cost_usd=estimated_cost,
+                total_cost_usd=final_cost,
+                usage=usage_data,
                 session_id="",
             ),
             is_error=is_error,

sdk/python/tests/test_harness_provider_opencode.py

@@ -2,6 +2,7 @@
 
 # pyright: reportMissingImports=false
 
+import json
 from typing import Any
 from unittest.mock import patch
 
@@ -42,6 +43,8 @@ async def fake_run_cli(cmd, *, env=None, cwd=None, timeout=None):
     assert captured["cmd"] == [
         "/usr/local/bin/opencode",
         "run",
+        "-f",
+        "json",
         "hello",
     ]
     assert captured["env"]["A"] == "1"
@@ -125,6 +128,8 @@ async def fake_run_cli(cmd, *, env=None, cwd=None, timeout=None):
         "run",
         "--model",
         "openai/gpt-5",
+        "-f",
+        "json",
         "hello",
     ]
     assert raw.is_error is False
@@ -165,3 +170,141 @@ async def fake_run_cli(cmd, *, env=None, cwd=None, timeout=None):
 
     # No model → estimate_cli_cost gets empty string → returns None
     assert raw.metrics.total_cost_usd is None
+
+
+# ---------------------------------------------------------------------------
+# Functional tests for JSON cost parsing (new opencode -f json output)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.functional
+@pytest.mark.asyncio
+async def test_json_parsing_full_cost_data(monkeypatch: pytest.MonkeyPatch):
+    """JSON output with all cost fields populates metrics correctly."""
+    json_output = json.dumps(
+        {
+            "response": "hello",
+            "cost": 0.04329,
+            "prompt_tokens": 13985,
+            "completion_tokens": 89,
+        }
+    )
+
+    async def fake_run_cli(cmd, *, env=None, cwd=None, timeout=None):
+        _ = (env, cwd, timeout)
+        return json_output, "", 0
+
+    monkeypatch.setattr("agentfield.harness.providers.opencode.run_cli", fake_run_cli)
+
+    provider = OpenCodeProvider(server_url="http://127.0.0.1:9999")
+    raw = await provider.execute("test prompt", {"model": "openai/gpt-4o"})
+
+    assert raw.result == "hello"
+    assert raw.metrics.total_cost_usd == 0.04329
+    assert raw.metrics.usage == {
+        "prompt_tokens": 13985,
+        "completion_tokens": 89,
+    }
+    assert raw.is_error is False
+    assert raw.metrics.num_turns == 1
+
+
+@pytest.mark.functional
+@pytest.mark.asyncio
+async def test_json_without_cost_fields_falls_back_to_estimate(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    """JSON output missing cost fields falls back to estimate_cli_cost."""
+    json_output = json.dumps({"response": "some answer"})
+
+    async def fake_run_cli(cmd, *, env=None, cwd=None, timeout=None):
+        _ = (env, cwd, timeout)
+        return json_output, "", 0
+
+    monkeypatch.setattr("agentfield.harness.providers.opencode.run_cli", fake_run_cli)
+
+    with patch(
+        "agentfield.harness.providers.opencode.estimate_cli_cost",
+        return_value=0.002,
+    ) as mock_estimate:
+        provider = OpenCodeProvider(server_url="http://127.0.0.1:9999")
+        raw = await provider.execute("test", {"model": "openai/gpt-4o"})
+
+    assert raw.result == "some answer"
+    assert raw.metrics.total_cost_usd == 0.002
+    assert raw.metrics.usage is None
+    mock_estimate.assert_called_once()
+
+
+@pytest.mark.functional
+@pytest.mark.asyncio
+async def test_plain_text_fallback_pre_json_opencode(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    """Plain text (non-JSON) stdout falls back to text result + estimate."""
+
+    async def fake_run_cli(cmd, *, env=None, cwd=None, timeout=None):
+        _ = (env, cwd, timeout)
+        return "This is plain text, not JSON.\n", "", 0
+
+    monkeypatch.setattr("agentfield.harness.providers.opencode.run_cli", fake_run_cli)
+
+    with patch(
+        "agentfield.harness.providers.opencode.estimate_cli_cost",
+        return_value=0.001,
+    ) as mock_estimate:
+        provider = OpenCodeProvider(server_url="http://127.0.0.1:9999")
+        raw = await provider.execute("test", {"model": "openai/gpt-4o"})
+
+    assert raw.result == "This is plain text, not JSON."
+    assert raw.metrics.total_cost_usd == 0.001
+    assert raw.metrics.usage is None
+    mock_estimate.assert_called_once()
+
+
+@pytest.mark.functional
+@pytest.mark.asyncio
+async def test_zero_cost_falls_back_to_estimate(monkeypatch: pytest.MonkeyPatch):
+    """Zero cost in JSON is treated as unknown and falls back to estimate."""
+    json_output = json.dumps(
+        {"response": "answer", "cost": 0, "prompt_tokens": 0, "completion_tokens": 0}
+    )
+
+    async def fake_run_cli(cmd, *, env=None, cwd=None, timeout=None):
+        _ = (env, cwd, timeout)
+        return json_output, "", 0
+
+    monkeypatch.setattr("agentfield.harness.providers.opencode.run_cli", fake_run_cli)
+
+    with patch(
+        "agentfield.harness.providers.opencode.estimate_cli_cost",
+        return_value=0.003,
+    ) as mock_estimate:
+        provider = OpenCodeProvider(server_url="http://127.0.0.1:9999")
+        raw = await provider.execute("test", {"model": "openai/gpt-4o"})
+
+    assert raw.result == "answer"
+    # cost <= 0 → parsed_cost stays None → falls back to estimate
+    assert raw.metrics.total_cost_usd == 0.003
+    mock_estimate.assert_called_once()
+
+
+@pytest.mark.functional
+@pytest.mark.asyncio
+async def test_partial_fields_cost_without_tokens(monkeypatch: pytest.MonkeyPatch):
+    """JSON with cost but no token fields: cost is used, usage is None."""
+    json_output = json.dumps({"response": "answer", "cost": 0.05})
+
+    async def fake_run_cli(cmd, *, env=None, cwd=None, timeout=None):
+        _ = (env, cwd, timeout)
+        return json_output, "", 0
+
+    monkeypatch.setattr("agentfield.harness.providers.opencode.run_cli", fake_run_cli)
+
+    provider = OpenCodeProvider(server_url="http://127.0.0.1:9999")
+    raw = await provider.execute("test", {"model": "openai/gpt-4o"})
+
+    assert raw.result == "answer"
+    assert raw.metrics.total_cost_usd == 0.05
+    assert raw.metrics.usage is None
+    assert raw.is_error is False