diff --git a/fern/versions/latest/pages/model-server/model-call-capture.mdx b/fern/versions/latest/pages/model-server/model-call-capture.mdx index 3cd079f7c32..16d2325a507 100644 --- a/fern/versions/latest/pages/model-server/model-call-capture.mdx +++ b/fern/versions/latest/pages/model-server/model-call-capture.mdx @@ -245,6 +245,12 @@ includes a gap to show the complete shape. The `gaps` field is omitted when ther reported by the provider; a `null` value means unknown, not zero. For example, Anthropic does not report reasoning-token usage, so `tokens_reasoning` is `null` for that dialect. +Each call's `tokens_in` is the whole prompt size. When the provider reports prompt caching, the call +also carries `cached_tokens` (tokens read from the cache) and `cache_creation_tokens` (tokens written +to the cache); both are part of `tokens_in`. Gym reads cache writes from OpenAI's +`input_tokens_details.cache_write_tokens` (Responses) or `prompt_tokens_details.cache_write_tokens` +(Chat Completions), and from Anthropic's `cache_creation_input_tokens`. + After successful [`ng_trajectory`](/main/observability/rollout-evidence#which-artifact-to-read) projection, attached calls omit request and response payloads; those remain in `CaptureStore`. If projection fails, payloads remain attached and `gaps` includes `trajectory_projection_failed`. LabBench rows with the diff --git a/nemo_gym/trajectory/model_call_record.py b/nemo_gym/trajectory/model_call_record.py index 73664148ce5..234d0af77fd 100644 --- a/nemo_gym/trajectory/model_call_record.py +++ b/nemo_gym/trajectory/model_call_record.py @@ -55,6 +55,11 @@ def extract_token_stats(usage: Any) -> dict[str, Optional[int]]: if tokens_out is None: tokens_out = _token_count(usage.get("completion_tokens")) cache_creation = _token_count(usage.get("cache_creation_input_tokens")) + if cache_creation is None: + # OpenAI reports cache writes as a subset of its prompt count, so they never change tokens_in. + cache_creation = _usage_detail_token( + usage, ("input_tokens_details", "prompt_tokens_details"), ("cache_write_tokens",) + ) tokens_total = _token_count(usage.get("total_tokens")) if tokens_total is None and tokens_in is not None and tokens_out is not None: tokens_total = tokens_in + tokens_out @@ -227,7 +232,7 @@ class ModelCallRecord(BaseModel): reasoning_content: Optional[str] = None # Cache visibility. cached_tokens is the cache-read count; cache_creation_tokens is the - # Anthropic cache-write count (also folded into tokens_in for the true prompt size). + # cache-write count. Both are part of tokens_in. cache_hit: Optional[bool] = None cached_tokens: Optional[int] = None cache_creation_tokens: Optional[int] = None @@ -336,6 +341,8 @@ def _prompt_size(usage: Mapping[str, Any]) -> Optional[int]: - ``input_tokens`` with top-level ``cache_read_input_tokens`` or ``cache_creation_input_tokens`` (Anthropic Messages) excludes it, so the cache counts are added. - ``input_tokens`` alone (Responses) includes it. + + The OpenAI ``*_tokens_details`` cache read and write counts are subsets of the prompt count and are never added. """ prompt_tokens = _token_count(usage.get("prompt_tokens")) if prompt_tokens is not None: diff --git a/tests/unit_tests/trajectory/test_model_call_record.py b/tests/unit_tests/trajectory/test_model_call_record.py index 951a6181f91..36f32066955 100644 --- a/tests/unit_tests/trajectory/test_model_call_record.py +++ b/tests/unit_tests/trajectory/test_model_call_record.py @@ -301,6 +301,85 @@ def test_extract_token_stats_anthropic_fully_cached_zero_base(): 120, None, ), + # OpenAI cache writes are part of input_tokens / prompt_tokens: reported separately, never added again. + "responses nothing cached": ( + { + "input_tokens": 100, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 20, + }, + 100, + 120, + 0, + ), + "responses cache read and write": ( + { + "input_tokens": 12000, + "input_tokens_details": { + "cached_tokens": 4000, + "cache_write_tokens": 7000, + "cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 4000}, + }, + "output_tokens": 20, + "total_tokens": 12020, + }, + 12000, + 12020, + 7000, + ), + "responses malformed cache write": ( + {"input_tokens": 100, "input_tokens_details": {"cache_write_tokens": "60"}, "output_tokens": 20}, + 100, + 120, + None, + ), + # Anthropic's top-level field wins when both are present, an explicit 0 included. + "anthropic zero cache write wins over the openai detail": ( + { + "input_tokens": 100, + "cache_creation_input_tokens": 0, + "input_tokens_details": {"cache_write_tokens": 60}, + "output_tokens": 20, + }, + 100, + 120, + 0, + ), + "anthropic cache write wins over the openai detail": ( + { + "input_tokens": 100, + "cache_creation_input_tokens": 30, + "input_tokens_details": {"cache_write_tokens": 60}, + "output_tokens": 20, + }, + 130, + 150, + 30, + ), + "responses negative cache write": ( + {"input_tokens": 100, "input_tokens_details": {"cache_write_tokens": -5}, "output_tokens": 20}, + 100, + 120, + None, + ), + "responses boolean cache write": ( + {"input_tokens": 100, "input_tokens_details": {"cache_write_tokens": True}, "output_tokens": 20}, + 100, + 120, + None, + ), + "chat completions negative cache write": ( + {"prompt_tokens": 100, "prompt_tokens_details": {"cache_write_tokens": -1}, "completion_tokens": 20}, + 100, + 120, + None, + ), + "chat completions boolean cache write": ( + {"prompt_tokens": 100, "prompt_tokens_details": {"cache_write_tokens": True}, "completion_tokens": 20}, + 100, + 120, + None, + ), "chat completions": ({"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}, 100, 120, None), "chat completions cached subset": ( {"prompt_tokens": 100, "prompt_tokens_details": {"cached_tokens": 80}, "completion_tokens": 20}, @@ -308,6 +387,16 @@ def test_extract_token_stats_anthropic_fully_cached_zero_base(): 120, None, ), + "chat completions cache read and write": ( + { + "prompt_tokens": 100, + "prompt_tokens_details": {"cached_tokens": 30, "cache_write_tokens": 60}, + "completion_tokens": 20, + }, + 100, + 120, + 60, + ), "anthropic messages with cache": ( {"input_tokens": 20, "cache_read_input_tokens": 70, "cache_creation_input_tokens": 10, "output_tokens": 20}, 100,