Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,12 @@ includes a gap to show the complete shape. The `gaps` field is omitted when ther
reported by the provider; a `null` value means unknown, not zero. For example, Anthropic does not
report reasoning-token usage, so `tokens_reasoning` is `null` for that dialect.

Each call's `tokens_in` is the whole prompt size. When the provider reports prompt caching, the call
also carries `cached_tokens` (tokens read from the cache) and `cache_creation_tokens` (tokens written
to the cache); both are part of `tokens_in`. Gym reads cache writes from OpenAI's
`input_tokens_details.cache_write_tokens` (Responses) or `prompt_tokens_details.cache_write_tokens`
(Chat Completions), and from Anthropic's `cache_creation_input_tokens`.

After successful [`ng_trajectory`](/main/observability/rollout-evidence#which-artifact-to-read) projection, attached calls
omit request and response payloads; those remain in `CaptureStore`. If projection fails, payloads
remain attached and `gaps` includes `trajectory_projection_failed`. LabBench rows with the
Expand Down
9 changes: 8 additions & 1 deletion nemo_gym/trajectory/model_call_record.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ def extract_token_stats(usage: Any) -> dict[str, Optional[int]]:
if tokens_out is None:
tokens_out = _token_count(usage.get("completion_tokens"))
cache_creation = _token_count(usage.get("cache_creation_input_tokens"))
if cache_creation is None:
# OpenAI reports cache writes as a subset of its prompt count, so they never change tokens_in.
cache_creation = _usage_detail_token(
usage, ("input_tokens_details", "prompt_tokens_details"), ("cache_write_tokens",)
)
tokens_total = _token_count(usage.get("total_tokens"))
if tokens_total is None and tokens_in is not None and tokens_out is not None:
tokens_total = tokens_in + tokens_out
Expand Down Expand Up @@ -227,7 +232,7 @@ class ModelCallRecord(BaseModel):
reasoning_content: Optional[str] = None

# Cache visibility. cached_tokens is the cache-read count; cache_creation_tokens is the
# Anthropic cache-write count (also folded into tokens_in for the true prompt size).
# cache-write count. Both are part of tokens_in.
cache_hit: Optional[bool] = None
cached_tokens: Optional[int] = None
cache_creation_tokens: Optional[int] = None
Expand Down Expand Up @@ -336,6 +341,8 @@ def _prompt_size(usage: Mapping[str, Any]) -> Optional[int]:
- ``input_tokens`` with top-level ``cache_read_input_tokens`` or ``cache_creation_input_tokens``
(Anthropic Messages) excludes it, so the cache counts are added.
- ``input_tokens`` alone (Responses) includes it.

The OpenAI ``*_tokens_details`` cache read and write counts are subsets of the prompt count and are never added.
"""
prompt_tokens = _token_count(usage.get("prompt_tokens"))
if prompt_tokens is not None:
Expand Down
89 changes: 89 additions & 0 deletions tests/unit_tests/trajectory/test_model_call_record.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,13 +301,102 @@ def test_extract_token_stats_anthropic_fully_cached_zero_base():
120,
None,
),
# OpenAI cache writes are part of input_tokens / prompt_tokens: reported separately, never added again.
"responses nothing cached": (
{
"input_tokens": 100,
"input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0},
"output_tokens": 20,
},
100,
120,
0,
),
"responses cache read and write": (
{
"input_tokens": 12000,
"input_tokens_details": {
"cached_tokens": 4000,
"cache_write_tokens": 7000,
"cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 4000},
},
"output_tokens": 20,
"total_tokens": 12020,
},
12000,
12020,
7000,
),
"responses malformed cache write": (
{"input_tokens": 100, "input_tokens_details": {"cache_write_tokens": "60"}, "output_tokens": 20},
100,
120,
None,
),
# Anthropic's top-level field wins when both are present, an explicit 0 included.
"anthropic zero cache write wins over the openai detail": (
{
"input_tokens": 100,
"cache_creation_input_tokens": 0,
"input_tokens_details": {"cache_write_tokens": 60},
"output_tokens": 20,
},
100,
120,
0,
),
"anthropic cache write wins over the openai detail": (
{
"input_tokens": 100,
"cache_creation_input_tokens": 30,
"input_tokens_details": {"cache_write_tokens": 60},
"output_tokens": 20,
},
130,
150,
30,
),
"responses negative cache write": (
{"input_tokens": 100, "input_tokens_details": {"cache_write_tokens": -5}, "output_tokens": 20},
100,
120,
None,
),
"responses boolean cache write": (
{"input_tokens": 100, "input_tokens_details": {"cache_write_tokens": True}, "output_tokens": 20},
100,
120,
None,
),
"chat completions negative cache write": (
{"prompt_tokens": 100, "prompt_tokens_details": {"cache_write_tokens": -1}, "completion_tokens": 20},
100,
120,
None,
),
"chat completions boolean cache write": (
{"prompt_tokens": 100, "prompt_tokens_details": {"cache_write_tokens": True}, "completion_tokens": 20},
100,
120,
None,
),
"chat completions": ({"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}, 100, 120, None),
"chat completions cached subset": (
{"prompt_tokens": 100, "prompt_tokens_details": {"cached_tokens": 80}, "completion_tokens": 20},
100,
120,
None,
),
"chat completions cache read and write": (
{
"prompt_tokens": 100,
"prompt_tokens_details": {"cached_tokens": 30, "cache_write_tokens": 60},
"completion_tokens": 20,
},
100,
120,
60,
),
"anthropic messages with cache": (
{"input_tokens": 20, "cache_read_input_tokens": 70, "cache_creation_input_tokens": 10, "output_tokens": 20},
100,
Expand Down
Loading