fix(langchain): preserve anthropic cache metrics

LangChain Anthropic responses report cache reads and cache writes separately from normal input tokens, including TTL-specific cache creation buckets. The previous cached-token fix avoided OpenAI double counting, but it could drop Anthropic cache-write detail from spans and produce totals that were less useful for cost analysis.

Preserve the cache creation metrics users need to understand prompt-cache spend and keep token totals aligned with the prompt-cache semantics, while continuing to avoid double counting OpenAI cached input tokens.

Author: AbhiPrasad

py/src/braintrust/integrations/langchain/callbacks.py

@@ -661,36 +661,40 @@ def _get_metrics_from_response(response: LLMResult):
             input_token_details = usage_metadata.get("input_token_details")
             if input_token_details and isinstance(input_token_details, dict):
                 cache_read = input_token_details.get("cache_read")
-                # langchain-anthropic >= 1.4.0 maps cache_creation_input_tokens to
-                # ephemeral tier fields (ephemeral_5m_input_tokens, ephemeral_1h_input_tokens)
-                # rather than the top-level cache_creation field. Sum both for compat.
                 cache_creation = input_token_details.get("cache_creation")
-                if not cache_creation and (
-                    "ephemeral_5m_input_tokens" in input_token_details
-                    or "ephemeral_1h_input_tokens" in input_token_details
-                ):
-                    cache_creation = input_token_details.get("ephemeral_5m_input_tokens", 0) + input_token_details.get(
-                        "ephemeral_1h_input_tokens", 0
-                    )
+                cache_creation_5m = input_token_details.get("ephemeral_5m_input_tokens")
+                cache_creation_1h = input_token_details.get("ephemeral_1h_input_tokens")
+                has_cache_creation_breakdown = cache_creation_5m is not None or cache_creation_1h is not None
 
                 if cache_read is not None:
                     metrics["prompt_cached_tokens"] = cache_read
-                if cache_creation is not None:
-                    metrics["prompt_cache_creation_tokens"] = cache_creation
-
-                cache_tokens = (cache_read or 0) + (cache_creation or 0)
+                cache_creation_split = (cache_creation_5m or 0) + (cache_creation_1h or 0)
+                if cache_creation is not None or cache_creation_split:
+                    metrics["prompt_cache_creation_tokens"] = max(cache_creation or 0, cache_creation_split)
+                if has_cache_creation_breakdown:
+                    # Anthropic exposes TTL-specific cache creation buckets. Preserve the
+                    # split so downstream cost tooling can price 5m vs 1h writes correctly.
+                    if cache_creation_5m is not None:
+                        metrics["prompt_cache_creation_5m_tokens"] = cache_creation_5m
+                    if cache_creation_1h is not None:
+                        metrics["prompt_cache_creation_1h_tokens"] = cache_creation_1h
+
+                effective_cache_creation = metrics.get("prompt_cache_creation_tokens", 0)
+                cache_tokens = (cache_read or 0) + effective_cache_creation
                 prompt_tokens = metrics.get("prompt_tokens")
                 completion_tokens = metrics.get("completion_tokens")
                 total_tokens = metrics.get("total_tokens")
-                if (
-                    cache_tokens
-                    and prompt_tokens is not None
-                    and completion_tokens is not None
-                    and total_tokens == prompt_tokens + completion_tokens
-                    and _cache_tokens_are_separate_from_input_tokens(input_token_details)
-                ):
-                    metrics["prompt_tokens"] = prompt_tokens + cache_tokens
-                    metrics["total_tokens"] = total_tokens + cache_tokens
+                if prompt_tokens is not None and completion_tokens is not None:
+                    if (
+                        cache_tokens
+                        and total_tokens == prompt_tokens + completion_tokens
+                        and _cache_tokens_are_separate_from_input_tokens(input_token_details)
+                    ):
+                        prompt_tokens += cache_tokens
+                        metrics["prompt_tokens"] = prompt_tokens
+                        if total_tokens is not None:
+                            metrics["total_tokens"] = total_tokens + cache_tokens
+                    metrics["tokens"] = prompt_tokens + completion_tokens
 
     if not metrics or not any(metrics.values()):
         llm_output: dict[str, Any] = response.llm_output or {}

py/src/braintrust/integrations/langchain/test_callbacks.py

@@ -8,12 +8,10 @@
 import pytest
 from braintrust import logger
 from braintrust.integrations.langchain import BraintrustCallbackHandler
-from braintrust.integrations.langchain.callbacks import _get_metrics_from_response
 from braintrust.logger import flush
 from braintrust.test_helpers import init_test_logger
 from langchain_core.callbacks import BaseCallbackHandler
 from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage, ToolMessage
-from langchain_core.outputs import ChatGeneration, LLMResult
 from langchain_core.prompts import ChatPromptTemplate
 from langchain_core.prompts.prompt import PromptTemplate
 from langchain_core.runnables import RunnableMap, RunnableSerializable
@@ -908,34 +906,6 @@ def test_streaming_ttft(logger_memory_logger):
     )
 
 
-def test_openai_cached_tokens_are_not_folded_into_prompt_tokens():
-    response = LLMResult(
-        generations=[
-            [
-                ChatGeneration(
-                    message=AIMessage(
-                        content="Done",
-                        response_metadata={"model_name": "gpt-4o-mini-2024-07-18"},
-                        usage_metadata={
-                            "input_tokens": 1000,
-                            "output_tokens": 200,
-                            "total_tokens": 1200,
-                            "input_token_details": {"cache_read": 500},
-                        },
-                    )
-                )
-            ]
-        ]
-    )
-
-    assert _get_metrics_from_response(response) == {
-        "prompt_tokens": 1000,
-        "completion_tokens": 200,
-        "total_tokens": 1200,
-        "prompt_cached_tokens": 500,
-    }
-
-
 @pytest.mark.vcr
 def test_prompt_caching_tokens(logger_memory_logger):
     from langchain_anthropic import ChatAnthropic
@@ -1114,11 +1084,16 @@ def test_prompt_caching_tokens(logger_memory_logger):
     assert "prompt_tokens" in first_metrics
     assert first_metrics["prompt_tokens"] > 0
 
-    assert "prompt_cache_creation_tokens" in first_metrics
-    assert first_metrics["prompt_cache_creation_tokens"] > 0
+    first_cache_creation_split = first_metrics.get("prompt_cache_creation_5m_tokens", 0) + first_metrics.get(
+        "prompt_cache_creation_1h_tokens", 0
+    )
+    first_cache_creation_tokens = max(first_metrics.get("prompt_cache_creation_tokens", 0), first_cache_creation_split)
+    assert first_cache_creation_tokens > 0
+    if first_cache_creation_split:
+        assert first_metrics["prompt_cache_creation_tokens"] >= first_cache_creation_split
     assert first_metrics["prompt_cached_tokens"] == 0
-    assert first_metrics["prompt_tokens"] >= first_metrics["prompt_cache_creation_tokens"]
-    assert first_metrics["total_tokens"] == first_metrics["prompt_tokens"] + first_metrics["completion_tokens"]
+    assert first_metrics["prompt_tokens"] >= first_cache_creation_tokens
+    assert first_metrics["tokens"] == first_metrics["prompt_tokens"] + first_metrics["completion_tokens"]
 
     second_metrics = None
     for attempt in range(3):
@@ -1147,9 +1122,14 @@ def test_prompt_caching_tokens(logger_memory_logger):
             time.sleep(1)
 
     assert second_metrics is not None
+    second_cache_creation_split = second_metrics.get("prompt_cache_creation_5m_tokens", 0) + second_metrics.get(
+        "prompt_cache_creation_1h_tokens", 0
+    )
+    if second_cache_creation_split:
+        assert second_metrics["prompt_cache_creation_tokens"] >= second_cache_creation_split
     assert second_metrics["prompt_cached_tokens"] > 0
     assert second_metrics["prompt_tokens"] >= second_metrics["prompt_cached_tokens"]
-    assert second_metrics["total_tokens"] == second_metrics["prompt_tokens"] + second_metrics["completion_tokens"]
+    assert second_metrics["tokens"] == second_metrics["prompt_tokens"] + second_metrics["completion_tokens"]
 
 
 @pytest.mark.vcr