Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 29 additions & 15 deletions nemo_gym/observability/model_calls.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,8 @@ def _usage_detail_token(
def extract_token_stats(usage: Any) -> dict[str, Optional[int]]:
"""Normalize token totals across Responses, Chat Completions, and Anthropic Messages usage.

For native Anthropic ``/v1/messages`` with prompt caching, ``input_tokens`` is only the uncached
remainder, so cache-read + cache-creation tokens are folded into ``tokens_in`` to reflect the true
prompt size (and cache-creation is surfaced separately as ``cache_creation_tokens``). OpenAI /
Responses usage already includes cached tokens in ``input_tokens`` / ``prompt_tokens`` (where
``cached_tokens`` is a subset), so it is left untouched -- no double counting.
``tokens_in`` is the whole prompt size (see ``_prompt_size``); cache creation is also reported
separately as ``cache_creation_tokens``. A count the provider did not report stays ``None``, never 0.

``tokens_in`` is a prompt-*size* metric, not a cost proxy: providers price cache-read (~0.1x) and
cache-creation (~1.25x) differently from base input, so cost-accurate consumers should weight
Expand All @@ -71,20 +68,11 @@ def extract_token_stats(usage: Any) -> dict[str, Optional[int]]:
"tokens_total": None,
"cache_creation_tokens": None,
}
tokens_in = _token_count(usage.get("input_tokens"))
if tokens_in is None:
tokens_in = _token_count(usage.get("prompt_tokens"))
tokens_in = _prompt_size(usage)
tokens_out = _token_count(usage.get("output_tokens"))
if tokens_out is None:
tokens_out = _token_count(usage.get("completion_tokens"))
# Anthropic-native shape: top-level cache_* keys mean input_tokens excludes cached tokens.
cache_read = _token_count(usage.get("cache_read_input_tokens"))
cache_creation = _token_count(usage.get("cache_creation_input_tokens"))
if cache_read is not None or cache_creation is not None:
cache_total = (cache_read or 0) + (cache_creation or 0)
# Zero cache fields alone do not establish a missing prompt count.
if tokens_in is not None or cache_total > 0:
tokens_in = (tokens_in or 0) + cache_total
tokens_total = _token_count(usage.get("total_tokens"))
if tokens_total is None and tokens_in is not None and tokens_out is not None:
tokens_total = tokens_in + tokens_out
Expand Down Expand Up @@ -544,3 +532,29 @@ def merge_model_call_capture_into_record(
capture["gaps"] = [gap.model_dump(mode="json", exclude_none=True) for gap in gaps]
record["ng_model_call_capture"] = capture
return record


def _prompt_size(usage: Mapping[str, Any]) -> Optional[int]:
"""Return the whole prompt size, with cached tokens counted exactly once.

Usage formats disagree on whether the prompt count includes the cache:

- ``prompt_tokens`` (Chat Completions) includes it. It wins whenever present, also when a gateway
adds Anthropic-style ``cache_*`` fields beside it.
- ``input_tokens`` with top-level ``cache_read_input_tokens`` or ``cache_creation_input_tokens``
(Anthropic Messages) excludes it, so the cache counts are added.
- ``input_tokens`` alone (Responses) includes it.
"""
prompt_tokens = _token_count(usage.get("prompt_tokens"))
if prompt_tokens is not None:
return prompt_tokens
input_tokens = _token_count(usage.get("input_tokens"))
cache_read = _token_count(usage.get("cache_read_input_tokens"))
cache_creation = _token_count(usage.get("cache_creation_input_tokens"))
if cache_read is None and cache_creation is None:
return input_tokens
cache_total = (cache_read or 0) + (cache_creation or 0)
# A fully cached Anthropic response may omit input_tokens; zero cache counts alone prove no prompt size.
if input_tokens is None and cache_total == 0:
return None
return (input_tokens or 0) + cache_total
83 changes: 83 additions & 0 deletions tests/unit_tests/test_model_calls.py
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,89 @@ def test_extract_token_stats_anthropic_fully_cached_zero_base():
assert (empty["tokens_in"], empty["tokens_total"], empty["cache_creation_tokens"]) == (None, None, 0)


# One row per usage shape: (usage, tokens_in, tokens_total, cache_creation_tokens).
_USAGE_SHAPES = {
"responses": ({"input_tokens": 100, "output_tokens": 20, "total_tokens": 120}, 100, 120, None),
"responses cached subset": (
{"input_tokens": 100, "input_tokens_details": {"cached_tokens": 80}, "output_tokens": 20},
100,
120,
None,
),
"chat completions": ({"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}, 100, 120, None),
"chat completions cached subset": (
{"prompt_tokens": 100, "prompt_tokens_details": {"cached_tokens": 80}, "completion_tokens": 20},
100,
120,
None,
),
"anthropic messages with cache": (
{"input_tokens": 20, "cache_read_input_tokens": 70, "cache_creation_input_tokens": 10, "output_tokens": 20},
100,
120,
10,
),
"anthropic messages fully cached": ({"cache_read_input_tokens": 100, "output_tokens": 20}, 100, 120, None),
"anthropic messages zero cache, no input count": (
{"cache_read_input_tokens": 0, "output_tokens": 20},
None,
None,
None,
),
# NVIDIA-NeMo/Gym#4721: a gateway adds Anthropic-style cache fields to Chat Completions usage.
"mixed, cache read": (
{"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cache_read_input_tokens": 80},
100,
120,
None,
),
"mixed, cache write": (
{"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cache_creation_input_tokens": 80},
100,
120,
80,
),
"mixed, prompt and input counts": (
{"prompt_tokens": 100, "input_tokens": 20, "cache_read_input_tokens": 80, "completion_tokens": 20},
100,
120,
None,
),
"anthropic messages, nothing cached": (
{"input_tokens": 100, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0, "output_tokens": 20},
100,
120,
0,
),
# A count that is not a non-negative integer is ignored rather than trusted.
"malformed prompt count": (
{"prompt_tokens": "100", "input_tokens": 20, "cache_read_input_tokens": 80, "output_tokens": 20},
100,
120,
None,
),
"no usage": ({}, None, None, None),
"usage is not a mapping": (None, None, None, None),
}


@pytest.mark.parametrize(
("usage", "tokens_in", "tokens_total", "cache_creation"), _USAGE_SHAPES.values(), ids=_USAGE_SHAPES
)
def test_extract_token_stats_counts_the_cache_once_in_every_usage_shape(
usage, tokens_in, tokens_total, cache_creation
):
from nemo_gym.observability.model_calls import extract_token_stats

stats = extract_token_stats(usage)

assert (stats["tokens_in"], stats["tokens_total"], stats["cache_creation_tokens"]) == (
tokens_in,
tokens_total,
cache_creation,
)


def test_extract_token_stats_openai_cached_not_double_counted():
from nemo_gym.observability.model_calls import extract_token_stats

Expand Down
Loading