Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion docs/workflow_development.md
Original file line number Diff line number Diff line change
Expand Up @@ -735,7 +735,14 @@ e2e normalized interactivity (`mean_e2e_norm_intvty`) divides by whole-request
wall clock and is the honest user-visible speed for a long-prefill agentic turn —
the two differ by roughly 3x in practice. Its `p75`/`p90`/`p95` companions are
read off the bottom of the distribution, since for a rate the slow tail is the
low end.
low end. The same per-request rate is also exported under AIPerf's own name,
**E2E Output Token Throughput** (`mean_e2e_output_token_throughput`,
`median_e2e_output_token_throughput`, `p90_e2e_output_token_throughput`,
`p99_e2e_output_token_throughput`), and rendered as the first table in the
report, because it is the number Artificial
Analysis ("speed", average) and OpenRouter ("throughput", P50) publish for a
deployment. `p90`/`p99` follow the same worse-tail convention: `p99_e2e_output_token_throughput`
is the rate only 1% of requests fell below (AIPerf's `p1`).

Several of the fork's tags come in pairs that only mean something together, and
all halves are kept for that reason. `effective_*` throughput averages over the
Expand Down
25 changes: 22 additions & 3 deletions llm_module/drivers/aiperf_agentic_traces.py
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,18 @@ def _int(tag: str, stat: str = "avg") -> int:
"p75_e2e_norm_intvty": e2e_per_user.get("p25"),
"p90_e2e_norm_intvty": e2e_per_user.get("p10"),
"p95_e2e_norm_intvty": e2e_per_user.get("p5"),
# AIPerf's E2E Output Token Throughput (output tokens / whole-request
# wall clock, TTFT included) under its own name, with P50 added. It is
# the number Artificial Analysis ("speed", average) and OpenRouter
# ("throughput", P50) publish, so a release can be read against those
# directly. The tail percentiles follow the worse-tail convention
# OpenRouter now uses (P99 = the rate only 1% of requests fell below),
# so they read AIPerf's low percentiles, like the e2e_norm_intvty ones
# above.
"mean_e2e_output_token_throughput": e2e_per_user.get("avg"),
"median_e2e_output_token_throughput": e2e_per_user.get("p50"),
"p90_e2e_output_token_throughput": e2e_per_user.get("p10"),
"p99_e2e_output_token_throughput": e2e_per_user.get("p1"),
"input_token_throughput": _stat("input_token_throughput"),
"total_token_throughput": _stat("total_token_throughput"),
"request_throughput": _stat("request_throughput"),
Expand Down Expand Up @@ -927,10 +939,17 @@ def _log_run_summary(run: AgenticTracesRun, metrics: Mapping[str, Any]) -> None:
float(metrics.get("p99_tpot_ms", 0) or 0),
)
logger.info(
"[agentic-traces] output tok/s/user streaming/e2e = %.1f/%.1f; "
"total tok/s = %.1f",
"[agentic-traces] E2E output token throughput (tok/s/req) "
"avg/p50/p90/p99 = "
"%.1f/%.1f/%.1f/%.1f",
float(metrics.get("mean_e2e_output_token_throughput", 0) or 0),
float(metrics.get("median_e2e_output_token_throughput", 0) or 0),
float(metrics.get("p90_e2e_output_token_throughput", 0) or 0),
float(metrics.get("p99_e2e_output_token_throughput", 0) or 0),
)
logger.info(
"[agentic-traces] output tok/s/user streaming = %.1f; total tok/s = %.1f",
float(metrics.get("output_token_throughput_per_user", 0) or 0),
float(metrics.get("mean_e2e_norm_intvty", 0) or 0),
float(metrics.get("total_token_throughput", 0) or 0),
)
measured_cache = metrics.get("measured_prefix_cache_hit_pct")
Expand Down
42 changes: 32 additions & 10 deletions report_module/agentic_traces_renderer.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,21 @@
("p90_adj_e2el_ms", "Err-Adj E2EL P90"),
]

# The headline table: AIPerf's E2E Output Token Throughput, i.e. per-request
# output tokens over whole-request wall clock (TTFT included). It is what
# Artificial Analysis ("speed", average) and OpenRouter ("throughput", P50)
# publish. P90/P99 are the slow tail -- the rate 90% / 99% of requests met or
# beat.
E2E_OUTPUT_THROUGHPUT_COLUMNS: List[Tuple[str, str]] = [
("mean_e2e_output_token_throughput", "E2E Output Token Throughput Avg (AA)"),
("median_e2e_output_token_throughput", "E2E Output Token Throughput P50 (OR)"),
("p90_e2e_output_token_throughput", "E2E Output Token Throughput P90"),
("p99_e2e_output_token_throughput", "E2E Output Token Throughput P99"),
]

THROUGHPUT_COLUMNS: List[Tuple[str, str]] = [
("output_token_throughput_per_user", "Out Tok/s/User"),
("median_output_token_throughput_per_user", "Out Tok/s/User P50"),
("mean_e2e_norm_intvty", "E2E Norm Intvty Avg"),
("p75_e2e_norm_intvty", "E2E Norm Intvty P75"),
("p90_e2e_norm_intvty", "E2E Norm Intvty P90"),
("output_token_throughput", "Output Tok/s"),
("total_token_throughput", "Total Tok/s"),
("active_throughput_tok_per_s", "Total Tok/s (Active)"),
Expand Down Expand Up @@ -183,9 +192,10 @@
"output_token_throughput",
"output_token_throughput_per_user",
"median_output_token_throughput_per_user",
"mean_e2e_norm_intvty",
"p75_e2e_norm_intvty",
"p90_e2e_norm_intvty",
"mean_e2e_output_token_throughput",
"median_e2e_output_token_throughput",
"p90_e2e_output_token_throughput",
"p99_e2e_output_token_throughput",
"total_token_throughput",
"active_throughput_tok_per_s",
"effective_prefill_throughput",
Expand Down Expand Up @@ -286,10 +296,13 @@
"**Err-Adj TTFT / E2EL**: percentiles with failed requests folded in, so a "
"run that got fast by erroring out cannot look good. Omitted when a run "
"had no errors.",
"**E2E Tok/s/User**: divides by whole-request wall clock rather than "
"streaming time, so it includes prefill and is the user-visible speed of "
"an agentic turn; expect it to be several times lower than "
"**Out Tok/s/User**.",
"**E2E Output Token Throughput**: AIPerf's per-request output tokens "
"divided by whole-request wall clock, TTFT included -- the user-visible "
'speed of an agentic turn, and the number Artificial Analysis ("speed", '
'**Avg**) and OpenRouter ("throughput", **P50**) publish. **P90 / P99** '
"are the slow tail: the rate 90% / 99% of requests met or beat. Prefill time is part of it, so expect it to be "
"several times lower than **Out Tok/s/User**, and to move less than "
"decode speed does when only decode improves.",
"**Prefill / Decode Tok/s**: the phase split of serving throughput, "
"averaged over the whole run. The **(Active)** variants only count the "
"windows where that phase was actually working, so the ratio between them "
Expand Down Expand Up @@ -602,6 +615,15 @@ def render_agentic_traces(block: Block, metadata: Mapping[str, Any]) -> str:
parts.append("**Benchmarking Tools:**")
parts.extend(_tool_paragraphs(rows))

e2e_throughput = _table(rows, E2E_OUTPUT_THROUGHPUT_COLUMNS)
if e2e_throughput:
parts.append(
"#### Per-run E2E Output Token Throughput (tok/s per request)\n\n"
"Output tokens over whole-request time, TTFT included: the metric "
"Artificial Analysis and OpenRouter publish. P90 / P99 are the "
f"slow tail.\n\n{e2e_throughput}"
)

latency = _table(rows, LATENCY_COLUMNS)
if latency:
parts.append(f"#### Per-run Latency (ms)\n\n{latency}")
Expand Down
6 changes: 6 additions & 0 deletions tests/llm_module/test_agentic_traces.py
Original file line number Diff line number Diff line change
Expand Up @@ -662,6 +662,12 @@ def test_reads_latency_and_throughput_stats(self, tmp_path):
assert metrics["p75_e2e_norm_intvty"] == 3019.83
assert metrics["p90_e2e_norm_intvty"] == 2215.77
assert metrics["p95_e2e_norm_intvty"] == 1813.5
# E2E Output Token Throughput under AIPerf's name, with P50 added and the
# tail on the worse-tail convention: P99 reads AIPerf's p1.
assert metrics["mean_e2e_output_token_throughput"] == 40.41
assert metrics["median_e2e_output_token_throughput"] == 4808.69
assert metrics["p90_e2e_output_token_throughput"] == 2215.77
assert metrics["p99_e2e_output_token_throughput"] == 831.94
assert metrics["measured_benchmark_duration"] == 3605.47

def test_distinguishes_successful_from_completed_counts(self, tmp_path):
Expand Down
42 changes: 42 additions & 0 deletions tests/report_module/test_agentic_traces_renderer.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ def _record(**overrides):
"p99_effective_latency_ms": 137522.2,
"output_token_throughput_per_user": 110.8,
"mean_e2e_norm_intvty": 31.3,
"mean_e2e_output_token_throughput": 31.3,
"median_e2e_output_token_throughput": 76.42,
"p90_e2e_output_token_throughput": 12.05,
"p99_e2e_output_token_throughput": 3.117,
"total_token_throughput": 4856.63,
"output_token_throughput": 41.99,
"request_throughput": 0.042,
Expand Down Expand Up @@ -231,6 +235,44 @@ def test_error_adjusted_percentiles_appear_when_present(self):
assert "8,123.4" in out


_E2E_HEADING = "#### Per-run E2E Output Token Throughput"


class TestE2EOutputThroughputTable:
"""AIPerf's E2E Output Token Throughput is what Artificial Analysis and
OpenRouter publish, so it leads, under AIPerf's own name."""

def test_is_the_first_metric_table(self):
out = _render(_record())
assert f"{_E2E_HEADING} (tok/s per request)" in out
assert out.index(_E2E_HEADING) < out.index("#### Per-run Latency")

def test_reports_avg_p50_p90_p99(self):
table = _render(_record()).split(_E2E_HEADING)[1].split("####")[0]
for header in (
"E2E Output Token Throughput Avg (AA)",
"E2E Output Token Throughput P50 (OR)",
"E2E Output Token Throughput P90",
"E2E Output Token Throughput P99",
):
assert header in table
for cell in ("31.30", "76.42", "12.05", "3.12"):
assert cell in table

def test_is_not_duplicated_in_the_throughput_table(self):
out = _render(_record())
throughput = out.split("#### Per-run Throughput")[1].split("####")[0]
assert "E2E Output Token Throughput" not in throughput
assert "Norm Intvty" not in throughput

def test_is_defined_under_the_aiperf_name(self):
out = _render(_record())
assert "**E2E Output Token Throughput**: AIPerf's per-request" in out

def test_a_source_without_it_gets_no_table(self):
assert _E2E_HEADING not in _render(_swo_record())


class TestConfigTable:
def test_config_is_transposed_to_one_row_per_field(self):
"""As columns, the long dataset and label values set the report width."""
Expand Down
Loading