diff --git a/docs/workflow_development.md b/docs/workflow_development.md index 250bab9be5..c010f8532b 100644 --- a/docs/workflow_development.md +++ b/docs/workflow_development.md @@ -735,7 +735,14 @@ e2e normalized interactivity (`mean_e2e_norm_intvty`) divides by whole-request wall clock and is the honest user-visible speed for a long-prefill agentic turn — the two differ by roughly 3x in practice. Its `p75`/`p90`/`p95` companions are read off the bottom of the distribution, since for a rate the slow tail is the -low end. +low end. The same per-request rate is also exported under AIPerf's own name, +**E2E Output Token Throughput** (`mean_e2e_output_token_throughput`, +`median_e2e_output_token_throughput`, `p90_e2e_output_token_throughput`, +`p99_e2e_output_token_throughput`), and rendered as the first table in the +report, because it is the number Artificial +Analysis ("speed", average) and OpenRouter ("throughput", P50) publish for a +deployment. `p90`/`p99` follow the same worse-tail convention: `p99_e2e_output_token_throughput` +is the rate only 1% of requests fell below (AIPerf's `p1`). Several of the fork's tags come in pairs that only mean something together, and all halves are kept for that reason. `effective_*` throughput averages over the diff --git a/llm_module/drivers/aiperf_agentic_traces.py b/llm_module/drivers/aiperf_agentic_traces.py index 16657d5d67..d36439d2e7 100644 --- a/llm_module/drivers/aiperf_agentic_traces.py +++ b/llm_module/drivers/aiperf_agentic_traces.py @@ -447,6 +447,18 @@ def _int(tag: str, stat: str = "avg") -> int: "p75_e2e_norm_intvty": e2e_per_user.get("p25"), "p90_e2e_norm_intvty": e2e_per_user.get("p10"), "p95_e2e_norm_intvty": e2e_per_user.get("p5"), + # AIPerf's E2E Output Token Throughput (output tokens / whole-request + # wall clock, TTFT included) under its own name, with P50 added. It is + # the number Artificial Analysis ("speed", average) and OpenRouter + # ("throughput", P50) publish, so a release can be read against those + # directly. The tail percentiles follow the worse-tail convention + # OpenRouter now uses (P99 = the rate only 1% of requests fell below), + # so they read AIPerf's low percentiles, like the e2e_norm_intvty ones + # above. + "mean_e2e_output_token_throughput": e2e_per_user.get("avg"), + "median_e2e_output_token_throughput": e2e_per_user.get("p50"), + "p90_e2e_output_token_throughput": e2e_per_user.get("p10"), + "p99_e2e_output_token_throughput": e2e_per_user.get("p1"), "input_token_throughput": _stat("input_token_throughput"), "total_token_throughput": _stat("total_token_throughput"), "request_throughput": _stat("request_throughput"), @@ -927,10 +939,17 @@ def _log_run_summary(run: AgenticTracesRun, metrics: Mapping[str, Any]) -> None: float(metrics.get("p99_tpot_ms", 0) or 0), ) logger.info( - "[agentic-traces] output tok/s/user streaming/e2e = %.1f/%.1f; " - "total tok/s = %.1f", + "[agentic-traces] E2E output token throughput (tok/s/req) " + "avg/p50/p90/p99 = " + "%.1f/%.1f/%.1f/%.1f", + float(metrics.get("mean_e2e_output_token_throughput", 0) or 0), + float(metrics.get("median_e2e_output_token_throughput", 0) or 0), + float(metrics.get("p90_e2e_output_token_throughput", 0) or 0), + float(metrics.get("p99_e2e_output_token_throughput", 0) or 0), + ) + logger.info( + "[agentic-traces] output tok/s/user streaming = %.1f; total tok/s = %.1f", float(metrics.get("output_token_throughput_per_user", 0) or 0), - float(metrics.get("mean_e2e_norm_intvty", 0) or 0), float(metrics.get("total_token_throughput", 0) or 0), ) measured_cache = metrics.get("measured_prefix_cache_hit_pct") diff --git a/report_module/agentic_traces_renderer.py b/report_module/agentic_traces_renderer.py index bff875d414..7fb0e660a8 100644 --- a/report_module/agentic_traces_renderer.py +++ b/report_module/agentic_traces_renderer.py @@ -78,12 +78,21 @@ ("p90_adj_e2el_ms", "Err-Adj E2EL P90"), ] +# The headline table: AIPerf's E2E Output Token Throughput, i.e. per-request +# output tokens over whole-request wall clock (TTFT included). It is what +# Artificial Analysis ("speed", average) and OpenRouter ("throughput", P50) +# publish. P90/P99 are the slow tail -- the rate 90% / 99% of requests met or +# beat. +E2E_OUTPUT_THROUGHPUT_COLUMNS: List[Tuple[str, str]] = [ + ("mean_e2e_output_token_throughput", "E2E Output Token Throughput Avg (AA)"), + ("median_e2e_output_token_throughput", "E2E Output Token Throughput P50 (OR)"), + ("p90_e2e_output_token_throughput", "E2E Output Token Throughput P90"), + ("p99_e2e_output_token_throughput", "E2E Output Token Throughput P99"), +] + THROUGHPUT_COLUMNS: List[Tuple[str, str]] = [ ("output_token_throughput_per_user", "Out Tok/s/User"), ("median_output_token_throughput_per_user", "Out Tok/s/User P50"), - ("mean_e2e_norm_intvty", "E2E Norm Intvty Avg"), - ("p75_e2e_norm_intvty", "E2E Norm Intvty P75"), - ("p90_e2e_norm_intvty", "E2E Norm Intvty P90"), ("output_token_throughput", "Output Tok/s"), ("total_token_throughput", "Total Tok/s"), ("active_throughput_tok_per_s", "Total Tok/s (Active)"), @@ -183,9 +192,10 @@ "output_token_throughput", "output_token_throughput_per_user", "median_output_token_throughput_per_user", - "mean_e2e_norm_intvty", - "p75_e2e_norm_intvty", - "p90_e2e_norm_intvty", + "mean_e2e_output_token_throughput", + "median_e2e_output_token_throughput", + "p90_e2e_output_token_throughput", + "p99_e2e_output_token_throughput", "total_token_throughput", "active_throughput_tok_per_s", "effective_prefill_throughput", @@ -286,10 +296,13 @@ "**Err-Adj TTFT / E2EL**: percentiles with failed requests folded in, so a " "run that got fast by erroring out cannot look good. Omitted when a run " "had no errors.", - "**E2E Tok/s/User**: divides by whole-request wall clock rather than " - "streaming time, so it includes prefill and is the user-visible speed of " - "an agentic turn; expect it to be several times lower than " - "**Out Tok/s/User**.", + "**E2E Output Token Throughput**: AIPerf's per-request output tokens " + "divided by whole-request wall clock, TTFT included -- the user-visible " + 'speed of an agentic turn, and the number Artificial Analysis ("speed", ' + '**Avg**) and OpenRouter ("throughput", **P50**) publish. **P90 / P99** ' + "are the slow tail: the rate 90% / 99% of requests met or beat. Prefill time is part of it, so expect it to be " + "several times lower than **Out Tok/s/User**, and to move less than " + "decode speed does when only decode improves.", "**Prefill / Decode Tok/s**: the phase split of serving throughput, " "averaged over the whole run. The **(Active)** variants only count the " "windows where that phase was actually working, so the ratio between them " @@ -602,6 +615,15 @@ def render_agentic_traces(block: Block, metadata: Mapping[str, Any]) -> str: parts.append("**Benchmarking Tools:**") parts.extend(_tool_paragraphs(rows)) + e2e_throughput = _table(rows, E2E_OUTPUT_THROUGHPUT_COLUMNS) + if e2e_throughput: + parts.append( + "#### Per-run E2E Output Token Throughput (tok/s per request)\n\n" + "Output tokens over whole-request time, TTFT included: the metric " + "Artificial Analysis and OpenRouter publish. P90 / P99 are the " + f"slow tail.\n\n{e2e_throughput}" + ) + latency = _table(rows, LATENCY_COLUMNS) if latency: parts.append(f"#### Per-run Latency (ms)\n\n{latency}") diff --git a/tests/llm_module/test_agentic_traces.py b/tests/llm_module/test_agentic_traces.py index 9c23f11a2e..f6cbdf09c5 100644 --- a/tests/llm_module/test_agentic_traces.py +++ b/tests/llm_module/test_agentic_traces.py @@ -662,6 +662,12 @@ def test_reads_latency_and_throughput_stats(self, tmp_path): assert metrics["p75_e2e_norm_intvty"] == 3019.83 assert metrics["p90_e2e_norm_intvty"] == 2215.77 assert metrics["p95_e2e_norm_intvty"] == 1813.5 + # E2E Output Token Throughput under AIPerf's name, with P50 added and the + # tail on the worse-tail convention: P99 reads AIPerf's p1. + assert metrics["mean_e2e_output_token_throughput"] == 40.41 + assert metrics["median_e2e_output_token_throughput"] == 4808.69 + assert metrics["p90_e2e_output_token_throughput"] == 2215.77 + assert metrics["p99_e2e_output_token_throughput"] == 831.94 assert metrics["measured_benchmark_duration"] == 3605.47 def test_distinguishes_successful_from_completed_counts(self, tmp_path): diff --git a/tests/report_module/test_agentic_traces_renderer.py b/tests/report_module/test_agentic_traces_renderer.py index ce2d33e328..0fcc5b427b 100644 --- a/tests/report_module/test_agentic_traces_renderer.py +++ b/tests/report_module/test_agentic_traces_renderer.py @@ -58,6 +58,10 @@ def _record(**overrides): "p99_effective_latency_ms": 137522.2, "output_token_throughput_per_user": 110.8, "mean_e2e_norm_intvty": 31.3, + "mean_e2e_output_token_throughput": 31.3, + "median_e2e_output_token_throughput": 76.42, + "p90_e2e_output_token_throughput": 12.05, + "p99_e2e_output_token_throughput": 3.117, "total_token_throughput": 4856.63, "output_token_throughput": 41.99, "request_throughput": 0.042, @@ -231,6 +235,44 @@ def test_error_adjusted_percentiles_appear_when_present(self): assert "8,123.4" in out +_E2E_HEADING = "#### Per-run E2E Output Token Throughput" + + +class TestE2EOutputThroughputTable: + """AIPerf's E2E Output Token Throughput is what Artificial Analysis and + OpenRouter publish, so it leads, under AIPerf's own name.""" + + def test_is_the_first_metric_table(self): + out = _render(_record()) + assert f"{_E2E_HEADING} (tok/s per request)" in out + assert out.index(_E2E_HEADING) < out.index("#### Per-run Latency") + + def test_reports_avg_p50_p90_p99(self): + table = _render(_record()).split(_E2E_HEADING)[1].split("####")[0] + for header in ( + "E2E Output Token Throughput Avg (AA)", + "E2E Output Token Throughput P50 (OR)", + "E2E Output Token Throughput P90", + "E2E Output Token Throughput P99", + ): + assert header in table + for cell in ("31.30", "76.42", "12.05", "3.12"): + assert cell in table + + def test_is_not_duplicated_in_the_throughput_table(self): + out = _render(_record()) + throughput = out.split("#### Per-run Throughput")[1].split("####")[0] + assert "E2E Output Token Throughput" not in throughput + assert "Norm Intvty" not in throughput + + def test_is_defined_under_the_aiperf_name(self): + out = _render(_record()) + assert "**E2E Output Token Throughput**: AIPerf's per-request" in out + + def test_a_source_without_it_gets_no_table(self): + assert _E2E_HEADING not in _render(_swo_record()) + + class TestConfigTable: def test_config_is_transposed_to_one_row_per_field(self): """As columns, the long dataset and label values set the report width."""