feat(anthropic): emit gen_ai.usage.reasoning_tokens from extended thinking

packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/__init__.py

@@ -187,6 +187,16 @@ def is_stream_manager(response):
         )
 
 
+def _get_reasoning_tokens_from_usage(usage):
+    """Extract thinking/reasoning tokens from an Anthropic SDK usage object."""
+    if usage is None:
+        return None
+    output_tokens_details = getattr(usage, "output_tokens_details", None)
+    if output_tokens_details is None:
+        return None
+    return getattr(output_tokens_details, "reasoning_tokens", None)
+
+
 @dont_throw
 async def _aset_token_usage(
     span,
@@ -305,6 +315,11 @@ async def _aset_token_usage(
         cache_creation_tokens,
     )
 
+    reasoning_tokens = _get_reasoning_tokens_from_usage(usage)
+    set_span_attribute(
+        span, SpanAttributes.GEN_AI_USAGE_REASONING_TOKENS, reasoning_tokens
+    )
+
 
 @dont_throw
 def _set_token_usage(
@@ -421,6 +436,11 @@ def _set_token_usage(
         cache_creation_tokens,
     )
 
+    reasoning_tokens = _get_reasoning_tokens_from_usage(usage)
+    set_span_attribute(
+        span, SpanAttributes.GEN_AI_USAGE_REASONING_TOKENS, reasoning_tokens
+    )
+
 
 def _with_chat_telemetry_wrapper(func):
     """Helper for providing tracer for wrapper functions. Includes metric collectors."""

packages/opentelemetry-instrumentation-anthropic/opentelemetry/instrumentation/anthropic/streaming.py

@@ -61,16 +61,27 @@ def _process_response_item(item, complete_response):
         for event in complete_response.get("events", []):
             event["finish_reason"] = item.delta.stop_reason
         if item.usage:
+            item_usage = dict(item.usage)
             if "usage" in complete_response:
-                item_output_tokens = dict(item.usage).get("output_tokens", 0)
+                item_output_tokens = item_usage.get("output_tokens", 0)
                 existing_output_tokens = complete_response["usage"].get(
                     "output_tokens", 0
                 )
                 complete_response["usage"]["output_tokens"] = (
                     item_output_tokens + existing_output_tokens
                 )
+                item_reasoning_details = item_usage.get("output_tokens_details") or {}
+                existing_reasoning_details = (
+                    complete_response["usage"].get("output_tokens_details") or {}
+                )
+                complete_response["usage"]["output_tokens_details"] = {
+                    "reasoning_tokens": (
+                        (item_reasoning_details.get("reasoning_tokens") or 0)
+                        + (existing_reasoning_details.get("reasoning_tokens") or 0)
+                    )
+                }
             else:
-                complete_response["usage"] = dict(item.usage)
+                complete_response["usage"] = item_usage
 
 
 def _set_token_usage(
@@ -105,6 +116,15 @@ def _set_token_usage(
         span, GenAIAttributes.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, cache_creation_tokens
     )
 
+    output_tokens_details = complete_response.get("usage", {}).get("output_tokens_details")
+    reasoning_tokens = None
+    if output_tokens_details:
+        reasoning_tokens = output_tokens_details.get("reasoning_tokens", None)
+
+    set_span_attribute(
+        span, SpanAttributes.GEN_AI_USAGE_REASONING_TOKENS, reasoning_tokens
+    )
+
     set_span_attribute(
         span, GenAIAttributes.GEN_AI_RESPONSE_MODEL, complete_response.get("model")
     )

packages/opentelemetry-instrumentation-anthropic/tests/test_reasoning_usage.py

@@ -0,0 +1,135 @@
+"""Unit tests for reasoning-token usage extraction (gen_ai.usage.reasoning_tokens).
+
+Covers the three paths that consume Anthropic `usage.output_tokens_details`:
+the sync and async non-streaming `_set_token_usage` helpers in `__init__.py`,
+and the streaming path in `streaming.py`.
+"""
+
+from types import SimpleNamespace
+
+import pytest
+
+from opentelemetry.instrumentation.anthropic import _aset_token_usage, _set_token_usage
+from opentelemetry.instrumentation.anthropic.streaming import (
+    _process_response_item,
+    _set_token_usage as _set_streaming_token_usage,
+)
+from opentelemetry.semconv._incubating.attributes import (
+    gen_ai_attributes as GenAIAttributes,
+)
+from opentelemetry.semconv_ai import SpanAttributes
+
+REASONING_TOKENS = SpanAttributes.GEN_AI_USAGE_REASONING_TOKENS
+
+
+def _make_usage(**overrides):
+    usage = {
+        "input_tokens": 10,
+        "output_tokens": 25,
+        "cache_creation_input_tokens": 0,
+        "cache_read_input_tokens": 0,
+        "output_tokens_details": SimpleNamespace(reasoning_tokens=15),
+    }
+    usage.update(overrides)
+    return SimpleNamespace(**usage)
+
+
+def _make_response(usage):
+    return SimpleNamespace(
+        usage=usage,
+        content=[SimpleNamespace(type="text", text="hi")],
+        stop_reason="end_turn",
+        model="claude-3-7-sonnet-20250219",
+    )
+
+
+def _finished_span_attributes(span_exporter):
+    return dict(span_exporter.get_finished_spans()[0].attributes)
+
+
+@pytest.fixture
+def tracer(tracer_provider):
+    return tracer_provider.get_tracer("test-reasoning-usage")
+
+
+def test_set_token_usage_emits_reasoning_tokens(tracer, span_exporter):
+    with tracer.start_as_current_span("test") as span:
+        _set_token_usage(span, None, {}, _make_response(_make_usage()))
+
+    attributes = _finished_span_attributes(span_exporter)
+    assert attributes[REASONING_TOKENS] == 15
+    assert attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 25
+
+
+@pytest.mark.asyncio
+async def test_aset_token_usage_emits_reasoning_tokens(tracer, span_exporter):
+    with tracer.start_as_current_span("test") as span:
+        await _aset_token_usage(span, None, {}, _make_response(_make_usage()))
+
+    attributes = _finished_span_attributes(span_exporter)
+    assert attributes[REASONING_TOKENS] == 15
+
+
+def test_set_token_usage_omits_reasoning_when_details_absent(tracer, span_exporter):
+    with tracer.start_as_current_span("test") as span:
+        _set_token_usage(
+            span, None, {}, _make_response(_make_usage(output_tokens_details=None))
+        )
+
+    attributes = _finished_span_attributes(span_exporter)
+    assert REASONING_TOKENS not in attributes
+
+
+def test_process_response_item_merges_streaming_reasoning_tokens():
+    complete_response = {"events": [], "model": "", "usage": {}, "id": ""}
+
+    _process_response_item(
+        SimpleNamespace(
+            type="message_start",
+            message=SimpleNamespace(
+                model="claude-3-7-sonnet-20250219",
+                id="msg_1",
+                usage={
+                    "input_tokens": 10,
+                    "output_tokens": 0,
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                },
+            ),
+        ),
+        complete_response,
+    )
+    _process_response_item(
+        SimpleNamespace(
+            type="message_delta",
+            delta=SimpleNamespace(stop_reason="end_turn"),
+            usage={
+                "output_tokens": 25,
+                "output_tokens_details": {"reasoning_tokens": 15},
+            },
+        ),
+        complete_response,
+    )
+
+    usage = complete_response["usage"]
+    assert usage["output_tokens"] == 25
+    assert usage["output_tokens_details"]["reasoning_tokens"] == 15
+
+
+def test_streaming_set_token_usage_emits_reasoning_tokens(tracer, span_exporter):
+    complete_response = {
+        "events": [],
+        "model": "claude-3-7-sonnet-20250219",
+        "usage": {
+            "input_tokens": 10,
+            "output_tokens": 25,
+            "output_tokens_details": {"reasoning_tokens": 15},
+        },
+        "id": "msg_1",
+    }
+
+    with tracer.start_as_current_span("test") as span:
+        _set_streaming_token_usage(span, complete_response, 10, 25)
+
+    attributes = _finished_span_attributes(span_exporter)
+    assert attributes[REASONING_TOKENS] == 15