From 96d6b0ab86665d9048c2147b9bce35f0ed2ec77e Mon Sep 17 00:00:00 2001 From: Divy Wadhwani Date: Fri, 13 Nov 2026 14:47:11 -0500 Subject: [PATCH 1/2] AgentX: report per-request E2E output throughput as headline "speed" Export avg/P50/P90/P99 of AIPerf's e2e_output_token_throughput as mean/median/p90/p99_speed_tps and render them as the first table of the agentic-traces report. This is the metric Artificial Analysis (avg) and OpenRouter (P50) publish. P90/P99 use the worse-tail convention, so they read AIPerf's p10/p1. The duplicate "E2E Norm Intvty" columns are dropped from the throughput table; the e2e_norm_intvty fields stay in the payload and agenticSweep export, so requirements grading is unchanged. Closes #5570 Co-Authored-By: Claude Opus 5.5 --- docs/workflow_development.md | 7 +++- llm_module/drivers/aiperf_agentic_traces.py | 23 +++++++++-- report_module/agentic_traces_renderer.py | 41 ++++++++++++++----- tests/llm_module/test_agentic_traces.py | 6 +++ .../test_agentic_traces_renderer.py | 33 +++++++++++++++ 5 files changed, 96 insertions(+), 14 deletions(-) diff --git a/docs/workflow_development.md b/docs/workflow_development.md index 250bab9be5..516dd6109e 100644 --- a/docs/workflow_development.md +++ b/docs/workflow_development.md @@ -735,7 +735,12 @@ e2e normalized interactivity (`mean_e2e_norm_intvty`) divides by whole-request wall clock and is the honest user-visible speed for a long-prefill agentic turn — the two differ by roughly 3x in practice. Its `p75`/`p90`/`p95` companions are read off the bottom of the distribution, since for a rate the slow tail is the -low end. +low end. The same per-request rate is also exported as **speed** +(`mean_speed_tps`, `median_speed_tps`, `p90_speed_tps`, `p99_speed_tps`) and +rendered as the first table in the report, because it is the number Artificial +Analysis ("speed", average) and OpenRouter ("throughput", P50) publish for a +deployment. `p90`/`p99` follow the same worse-tail convention: `p99_speed_tps` +is the rate only 1% of requests fell below (AIPerf's `p1`). Several of the fork's tags come in pairs that only mean something together, and all halves are kept for that reason. `effective_*` throughput averages over the diff --git a/llm_module/drivers/aiperf_agentic_traces.py b/llm_module/drivers/aiperf_agentic_traces.py index 16657d5d67..87b27bf017 100644 --- a/llm_module/drivers/aiperf_agentic_traces.py +++ b/llm_module/drivers/aiperf_agentic_traces.py @@ -447,6 +447,17 @@ def _int(tag: str, stat: str = "avg") -> int: "p75_e2e_norm_intvty": e2e_per_user.get("p25"), "p90_e2e_norm_intvty": e2e_per_user.get("p10"), "p95_e2e_norm_intvty": e2e_per_user.get("p5"), + # "Speed": the same per-request rate (output tokens / whole-request + # wall clock, TTFT included), named for what Artificial Analysis + # ("speed", average) and OpenRouter ("throughput", P50) publish, so a + # release can be read against those numbers directly. The tail + # percentiles follow the worse-tail convention OpenRouter now uses + # (P99 = the rate only 1% of requests fell below), so they read + # AIPerf's low percentiles, like the e2e_norm_intvty ones above. + "mean_speed_tps": e2e_per_user.get("avg"), + "median_speed_tps": e2e_per_user.get("p50"), + "p90_speed_tps": e2e_per_user.get("p10"), + "p99_speed_tps": e2e_per_user.get("p1"), "input_token_throughput": _stat("input_token_throughput"), "total_token_throughput": _stat("total_token_throughput"), "request_throughput": _stat("request_throughput"), @@ -927,10 +938,16 @@ def _log_run_summary(run: AgenticTracesRun, metrics: Mapping[str, Any]) -> None: float(metrics.get("p99_tpot_ms", 0) or 0), ) logger.info( - "[agentic-traces] output tok/s/user streaming/e2e = %.1f/%.1f; " - "total tok/s = %.1f", + "[agentic-traces] speed (e2e tok/s/req) avg/p50/p90/p99 = " + "%.1f/%.1f/%.1f/%.1f", + float(metrics.get("mean_speed_tps", 0) or 0), + float(metrics.get("median_speed_tps", 0) or 0), + float(metrics.get("p90_speed_tps", 0) or 0), + float(metrics.get("p99_speed_tps", 0) or 0), + ) + logger.info( + "[agentic-traces] output tok/s/user streaming = %.1f; total tok/s = %.1f", float(metrics.get("output_token_throughput_per_user", 0) or 0), - float(metrics.get("mean_e2e_norm_intvty", 0) or 0), float(metrics.get("total_token_throughput", 0) or 0), ) measured_cache = metrics.get("measured_prefix_cache_hit_pct") diff --git a/report_module/agentic_traces_renderer.py b/report_module/agentic_traces_renderer.py index bff875d414..73405a63a0 100644 --- a/report_module/agentic_traces_renderer.py +++ b/report_module/agentic_traces_renderer.py @@ -78,12 +78,20 @@ ("p90_adj_e2el_ms", "Err-Adj E2EL P90"), ] +# The headline table: per-request output tokens over whole-request wall clock +# (TTFT included), which is what Artificial Analysis ("speed", average) and +# OpenRouter ("throughput", P50) publish. P90/P99 are the slow tail -- the rate +# 90% / 99% of requests met or beat. +SPEED_COLUMNS: List[Tuple[str, str]] = [ + ("mean_speed_tps", "Speed Avg (AA)"), + ("median_speed_tps", "Speed P50 (OR)"), + ("p90_speed_tps", "Speed P90"), + ("p99_speed_tps", "Speed P99"), +] + THROUGHPUT_COLUMNS: List[Tuple[str, str]] = [ ("output_token_throughput_per_user", "Out Tok/s/User"), ("median_output_token_throughput_per_user", "Out Tok/s/User P50"), - ("mean_e2e_norm_intvty", "E2E Norm Intvty Avg"), - ("p75_e2e_norm_intvty", "E2E Norm Intvty P75"), - ("p90_e2e_norm_intvty", "E2E Norm Intvty P90"), ("output_token_throughput", "Output Tok/s"), ("total_token_throughput", "Total Tok/s"), ("active_throughput_tok_per_s", "Total Tok/s (Active)"), @@ -183,9 +191,10 @@ "output_token_throughput", "output_token_throughput_per_user", "median_output_token_throughput_per_user", - "mean_e2e_norm_intvty", - "p75_e2e_norm_intvty", - "p90_e2e_norm_intvty", + "mean_speed_tps", + "median_speed_tps", + "p90_speed_tps", + "p99_speed_tps", "total_token_throughput", "active_throughput_tok_per_s", "effective_prefill_throughput", @@ -286,10 +295,13 @@ "**Err-Adj TTFT / E2EL**: percentiles with failed requests folded in, so a " "run that got fast by erroring out cannot look good. Omitted when a run " "had no errors.", - "**E2E Tok/s/User**: divides by whole-request wall clock rather than " - "streaming time, so it includes prefill and is the user-visible speed of " - "an agentic turn; expect it to be several times lower than " - "**Out Tok/s/User**.", + "**Speed**: output tokens divided by whole-request wall clock, TTFT " + "included, per request -- the user-visible speed of an agentic turn and " + "the number Artificial Analysis (**Avg**) and OpenRouter (**P50**) " + "publish. **P90 / P99** are the slow tail: the speed 90% / 99% of " + "requests met or beat. Prefill time is part of it, so expect it to be " + "several times lower than **Out Tok/s/User**, and to move less than " + "decode speed does when only decode improves.", "**Prefill / Decode Tok/s**: the phase split of serving throughput, " "averaged over the whole run. The **(Active)** variants only count the " "windows where that phase was actually working, so the ratio between them " @@ -602,6 +614,15 @@ def render_agentic_traces(block: Block, metadata: Mapping[str, Any]) -> str: parts.append("**Benchmarking Tools:**") parts.extend(_tool_paragraphs(rows)) + speed = _table(rows, SPEED_COLUMNS) + if speed: + parts.append( + "#### Per-run Speed (tok/s per request)\n\n" + "Output tokens over whole-request time, TTFT included: the metric " + "Artificial Analysis and OpenRouter publish. P90 / P99 are the " + f"slow tail.\n\n{speed}" + ) + latency = _table(rows, LATENCY_COLUMNS) if latency: parts.append(f"#### Per-run Latency (ms)\n\n{latency}") diff --git a/tests/llm_module/test_agentic_traces.py b/tests/llm_module/test_agentic_traces.py index 9c23f11a2e..779b552453 100644 --- a/tests/llm_module/test_agentic_traces.py +++ b/tests/llm_module/test_agentic_traces.py @@ -662,6 +662,12 @@ def test_reads_latency_and_throughput_stats(self, tmp_path): assert metrics["p75_e2e_norm_intvty"] == 3019.83 assert metrics["p90_e2e_norm_intvty"] == 2215.77 assert metrics["p95_e2e_norm_intvty"] == 1813.5 + # Speed is the same rate under the AA/OR name, with P50 added and the + # tail on the worse-tail convention: P99 reads AIPerf's p1. + assert metrics["mean_speed_tps"] == 40.41 + assert metrics["median_speed_tps"] == 4808.69 + assert metrics["p90_speed_tps"] == 2215.77 + assert metrics["p99_speed_tps"] == 831.94 assert metrics["measured_benchmark_duration"] == 3605.47 def test_distinguishes_successful_from_completed_counts(self, tmp_path): diff --git a/tests/report_module/test_agentic_traces_renderer.py b/tests/report_module/test_agentic_traces_renderer.py index ce2d33e328..616f48b6b4 100644 --- a/tests/report_module/test_agentic_traces_renderer.py +++ b/tests/report_module/test_agentic_traces_renderer.py @@ -58,6 +58,10 @@ def _record(**overrides): "p99_effective_latency_ms": 137522.2, "output_token_throughput_per_user": 110.8, "mean_e2e_norm_intvty": 31.3, + "mean_speed_tps": 31.3, + "median_speed_tps": 76.42, + "p90_speed_tps": 12.05, + "p99_speed_tps": 3.117, "total_token_throughput": 4856.63, "output_token_throughput": 41.99, "request_throughput": 0.042, @@ -231,6 +235,35 @@ def test_error_adjusted_percentiles_appear_when_present(self): assert "8,123.4" in out +class TestSpeedTable: + """Speed is what Artificial Analysis and OpenRouter publish, so it leads.""" + + def test_speed_is_the_first_metric_table(self): + out = _render(_record()) + assert "#### Per-run Speed (tok/s per request)" in out + assert out.index("#### Per-run Speed") < out.index("#### Per-run Latency") + + def test_reports_avg_p50_p90_p99(self): + speed = _render(_record()).split("#### Per-run Speed")[1].split("####")[0] + for header in ("Speed Avg (AA)", "Speed P50 (OR)", "Speed P90", "Speed P99"): + assert header in speed + for cell in ("31.30", "76.42", "12.05", "3.12"): + assert cell in speed + + def test_speed_is_not_duplicated_in_the_throughput_table(self): + out = _render(_record()) + throughput = out.split("#### Per-run Throughput")[1].split("####")[0] + assert "Speed" not in throughput + assert "Norm Intvty" not in throughput + + def test_speed_is_defined(self): + out = _render(_record()) + assert "**Speed**: output tokens divided by whole-request wall clock" in out + + def test_a_source_without_speed_gets_no_speed_table(self): + assert "#### Per-run Speed" not in _render(_swo_record()) + + class TestConfigTable: def test_config_is_transposed_to_one_row_per_field(self): """As columns, the long dataset and label values set the report width.""" From f063e538a4f0e38c447e749ee770259a410f636a Mon Sep 17 00:00:00 2001 From: Divy Wadhwani Date: Fri, 13 Nov 2026 15:05:36 -0500 Subject: [PATCH 2/2] AgentX: use AIPerf's "E2E Output Token Throughput" name Rename the speed fields to mean/median/p90/p99_e2e_output_token_throughput and label the report table, columns, definition and log line with AIPerf's official term. "speed" remains only where the text maps the metric to what Artificial Analysis publishes. Co-Authored-By: Claude Opus 5.5 --- docs/workflow_development.md | 10 ++-- llm_module/drivers/aiperf_agentic_traces.py | 34 +++++++------- report_module/agentic_traces_renderer.py | 45 +++++++++--------- tests/llm_module/test_agentic_traces.py | 10 ++-- .../test_agentic_traces_renderer.py | 47 +++++++++++-------- 5 files changed, 80 insertions(+), 66 deletions(-) diff --git a/docs/workflow_development.md b/docs/workflow_development.md index 516dd6109e..c010f8532b 100644 --- a/docs/workflow_development.md +++ b/docs/workflow_development.md @@ -735,11 +735,13 @@ e2e normalized interactivity (`mean_e2e_norm_intvty`) divides by whole-request wall clock and is the honest user-visible speed for a long-prefill agentic turn — the two differ by roughly 3x in practice. Its `p75`/`p90`/`p95` companions are read off the bottom of the distribution, since for a rate the slow tail is the -low end. The same per-request rate is also exported as **speed** -(`mean_speed_tps`, `median_speed_tps`, `p90_speed_tps`, `p99_speed_tps`) and -rendered as the first table in the report, because it is the number Artificial +low end. The same per-request rate is also exported under AIPerf's own name, +**E2E Output Token Throughput** (`mean_e2e_output_token_throughput`, +`median_e2e_output_token_throughput`, `p90_e2e_output_token_throughput`, +`p99_e2e_output_token_throughput`), and rendered as the first table in the +report, because it is the number Artificial Analysis ("speed", average) and OpenRouter ("throughput", P50) publish for a -deployment. `p90`/`p99` follow the same worse-tail convention: `p99_speed_tps` +deployment. `p90`/`p99` follow the same worse-tail convention: `p99_e2e_output_token_throughput` is the rate only 1% of requests fell below (AIPerf's `p1`). Several of the fork's tags come in pairs that only mean something together, and diff --git a/llm_module/drivers/aiperf_agentic_traces.py b/llm_module/drivers/aiperf_agentic_traces.py index 87b27bf017..d36439d2e7 100644 --- a/llm_module/drivers/aiperf_agentic_traces.py +++ b/llm_module/drivers/aiperf_agentic_traces.py @@ -447,17 +447,18 @@ def _int(tag: str, stat: str = "avg") -> int: "p75_e2e_norm_intvty": e2e_per_user.get("p25"), "p90_e2e_norm_intvty": e2e_per_user.get("p10"), "p95_e2e_norm_intvty": e2e_per_user.get("p5"), - # "Speed": the same per-request rate (output tokens / whole-request - # wall clock, TTFT included), named for what Artificial Analysis - # ("speed", average) and OpenRouter ("throughput", P50) publish, so a - # release can be read against those numbers directly. The tail - # percentiles follow the worse-tail convention OpenRouter now uses - # (P99 = the rate only 1% of requests fell below), so they read - # AIPerf's low percentiles, like the e2e_norm_intvty ones above. - "mean_speed_tps": e2e_per_user.get("avg"), - "median_speed_tps": e2e_per_user.get("p50"), - "p90_speed_tps": e2e_per_user.get("p10"), - "p99_speed_tps": e2e_per_user.get("p1"), + # AIPerf's E2E Output Token Throughput (output tokens / whole-request + # wall clock, TTFT included) under its own name, with P50 added. It is + # the number Artificial Analysis ("speed", average) and OpenRouter + # ("throughput", P50) publish, so a release can be read against those + # directly. The tail percentiles follow the worse-tail convention + # OpenRouter now uses (P99 = the rate only 1% of requests fell below), + # so they read AIPerf's low percentiles, like the e2e_norm_intvty ones + # above. + "mean_e2e_output_token_throughput": e2e_per_user.get("avg"), + "median_e2e_output_token_throughput": e2e_per_user.get("p50"), + "p90_e2e_output_token_throughput": e2e_per_user.get("p10"), + "p99_e2e_output_token_throughput": e2e_per_user.get("p1"), "input_token_throughput": _stat("input_token_throughput"), "total_token_throughput": _stat("total_token_throughput"), "request_throughput": _stat("request_throughput"), @@ -938,12 +939,13 @@ def _log_run_summary(run: AgenticTracesRun, metrics: Mapping[str, Any]) -> None: float(metrics.get("p99_tpot_ms", 0) or 0), ) logger.info( - "[agentic-traces] speed (e2e tok/s/req) avg/p50/p90/p99 = " + "[agentic-traces] E2E output token throughput (tok/s/req) " + "avg/p50/p90/p99 = " "%.1f/%.1f/%.1f/%.1f", - float(metrics.get("mean_speed_tps", 0) or 0), - float(metrics.get("median_speed_tps", 0) or 0), - float(metrics.get("p90_speed_tps", 0) or 0), - float(metrics.get("p99_speed_tps", 0) or 0), + float(metrics.get("mean_e2e_output_token_throughput", 0) or 0), + float(metrics.get("median_e2e_output_token_throughput", 0) or 0), + float(metrics.get("p90_e2e_output_token_throughput", 0) or 0), + float(metrics.get("p99_e2e_output_token_throughput", 0) or 0), ) logger.info( "[agentic-traces] output tok/s/user streaming = %.1f; total tok/s = %.1f", diff --git a/report_module/agentic_traces_renderer.py b/report_module/agentic_traces_renderer.py index 73405a63a0..7fb0e660a8 100644 --- a/report_module/agentic_traces_renderer.py +++ b/report_module/agentic_traces_renderer.py @@ -78,15 +78,16 @@ ("p90_adj_e2el_ms", "Err-Adj E2EL P90"), ] -# The headline table: per-request output tokens over whole-request wall clock -# (TTFT included), which is what Artificial Analysis ("speed", average) and -# OpenRouter ("throughput", P50) publish. P90/P99 are the slow tail -- the rate -# 90% / 99% of requests met or beat. -SPEED_COLUMNS: List[Tuple[str, str]] = [ - ("mean_speed_tps", "Speed Avg (AA)"), - ("median_speed_tps", "Speed P50 (OR)"), - ("p90_speed_tps", "Speed P90"), - ("p99_speed_tps", "Speed P99"), +# The headline table: AIPerf's E2E Output Token Throughput, i.e. per-request +# output tokens over whole-request wall clock (TTFT included). It is what +# Artificial Analysis ("speed", average) and OpenRouter ("throughput", P50) +# publish. P90/P99 are the slow tail -- the rate 90% / 99% of requests met or +# beat. +E2E_OUTPUT_THROUGHPUT_COLUMNS: List[Tuple[str, str]] = [ + ("mean_e2e_output_token_throughput", "E2E Output Token Throughput Avg (AA)"), + ("median_e2e_output_token_throughput", "E2E Output Token Throughput P50 (OR)"), + ("p90_e2e_output_token_throughput", "E2E Output Token Throughput P90"), + ("p99_e2e_output_token_throughput", "E2E Output Token Throughput P99"), ] THROUGHPUT_COLUMNS: List[Tuple[str, str]] = [ @@ -191,10 +192,10 @@ "output_token_throughput", "output_token_throughput_per_user", "median_output_token_throughput_per_user", - "mean_speed_tps", - "median_speed_tps", - "p90_speed_tps", - "p99_speed_tps", + "mean_e2e_output_token_throughput", + "median_e2e_output_token_throughput", + "p90_e2e_output_token_throughput", + "p99_e2e_output_token_throughput", "total_token_throughput", "active_throughput_tok_per_s", "effective_prefill_throughput", @@ -295,11 +296,11 @@ "**Err-Adj TTFT / E2EL**: percentiles with failed requests folded in, so a " "run that got fast by erroring out cannot look good. Omitted when a run " "had no errors.", - "**Speed**: output tokens divided by whole-request wall clock, TTFT " - "included, per request -- the user-visible speed of an agentic turn and " - "the number Artificial Analysis (**Avg**) and OpenRouter (**P50**) " - "publish. **P90 / P99** are the slow tail: the speed 90% / 99% of " - "requests met or beat. Prefill time is part of it, so expect it to be " + "**E2E Output Token Throughput**: AIPerf's per-request output tokens " + "divided by whole-request wall clock, TTFT included -- the user-visible " + 'speed of an agentic turn, and the number Artificial Analysis ("speed", ' + '**Avg**) and OpenRouter ("throughput", **P50**) publish. **P90 / P99** ' + "are the slow tail: the rate 90% / 99% of requests met or beat. Prefill time is part of it, so expect it to be " "several times lower than **Out Tok/s/User**, and to move less than " "decode speed does when only decode improves.", "**Prefill / Decode Tok/s**: the phase split of serving throughput, " @@ -614,13 +615,13 @@ def render_agentic_traces(block: Block, metadata: Mapping[str, Any]) -> str: parts.append("**Benchmarking Tools:**") parts.extend(_tool_paragraphs(rows)) - speed = _table(rows, SPEED_COLUMNS) - if speed: + e2e_throughput = _table(rows, E2E_OUTPUT_THROUGHPUT_COLUMNS) + if e2e_throughput: parts.append( - "#### Per-run Speed (tok/s per request)\n\n" + "#### Per-run E2E Output Token Throughput (tok/s per request)\n\n" "Output tokens over whole-request time, TTFT included: the metric " "Artificial Analysis and OpenRouter publish. P90 / P99 are the " - f"slow tail.\n\n{speed}" + f"slow tail.\n\n{e2e_throughput}" ) latency = _table(rows, LATENCY_COLUMNS) diff --git a/tests/llm_module/test_agentic_traces.py b/tests/llm_module/test_agentic_traces.py index 779b552453..f6cbdf09c5 100644 --- a/tests/llm_module/test_agentic_traces.py +++ b/tests/llm_module/test_agentic_traces.py @@ -662,12 +662,12 @@ def test_reads_latency_and_throughput_stats(self, tmp_path): assert metrics["p75_e2e_norm_intvty"] == 3019.83 assert metrics["p90_e2e_norm_intvty"] == 2215.77 assert metrics["p95_e2e_norm_intvty"] == 1813.5 - # Speed is the same rate under the AA/OR name, with P50 added and the + # E2E Output Token Throughput under AIPerf's name, with P50 added and the # tail on the worse-tail convention: P99 reads AIPerf's p1. - assert metrics["mean_speed_tps"] == 40.41 - assert metrics["median_speed_tps"] == 4808.69 - assert metrics["p90_speed_tps"] == 2215.77 - assert metrics["p99_speed_tps"] == 831.94 + assert metrics["mean_e2e_output_token_throughput"] == 40.41 + assert metrics["median_e2e_output_token_throughput"] == 4808.69 + assert metrics["p90_e2e_output_token_throughput"] == 2215.77 + assert metrics["p99_e2e_output_token_throughput"] == 831.94 assert metrics["measured_benchmark_duration"] == 3605.47 def test_distinguishes_successful_from_completed_counts(self, tmp_path): diff --git a/tests/report_module/test_agentic_traces_renderer.py b/tests/report_module/test_agentic_traces_renderer.py index 616f48b6b4..0fcc5b427b 100644 --- a/tests/report_module/test_agentic_traces_renderer.py +++ b/tests/report_module/test_agentic_traces_renderer.py @@ -58,10 +58,10 @@ def _record(**overrides): "p99_effective_latency_ms": 137522.2, "output_token_throughput_per_user": 110.8, "mean_e2e_norm_intvty": 31.3, - "mean_speed_tps": 31.3, - "median_speed_tps": 76.42, - "p90_speed_tps": 12.05, - "p99_speed_tps": 3.117, + "mean_e2e_output_token_throughput": 31.3, + "median_e2e_output_token_throughput": 76.42, + "p90_e2e_output_token_throughput": 12.05, + "p99_e2e_output_token_throughput": 3.117, "total_token_throughput": 4856.63, "output_token_throughput": 41.99, "request_throughput": 0.042, @@ -235,33 +235,42 @@ def test_error_adjusted_percentiles_appear_when_present(self): assert "8,123.4" in out -class TestSpeedTable: - """Speed is what Artificial Analysis and OpenRouter publish, so it leads.""" +_E2E_HEADING = "#### Per-run E2E Output Token Throughput" - def test_speed_is_the_first_metric_table(self): + +class TestE2EOutputThroughputTable: + """AIPerf's E2E Output Token Throughput is what Artificial Analysis and + OpenRouter publish, so it leads, under AIPerf's own name.""" + + def test_is_the_first_metric_table(self): out = _render(_record()) - assert "#### Per-run Speed (tok/s per request)" in out - assert out.index("#### Per-run Speed") < out.index("#### Per-run Latency") + assert f"{_E2E_HEADING} (tok/s per request)" in out + assert out.index(_E2E_HEADING) < out.index("#### Per-run Latency") def test_reports_avg_p50_p90_p99(self): - speed = _render(_record()).split("#### Per-run Speed")[1].split("####")[0] - for header in ("Speed Avg (AA)", "Speed P50 (OR)", "Speed P90", "Speed P99"): - assert header in speed + table = _render(_record()).split(_E2E_HEADING)[1].split("####")[0] + for header in ( + "E2E Output Token Throughput Avg (AA)", + "E2E Output Token Throughput P50 (OR)", + "E2E Output Token Throughput P90", + "E2E Output Token Throughput P99", + ): + assert header in table for cell in ("31.30", "76.42", "12.05", "3.12"): - assert cell in speed + assert cell in table - def test_speed_is_not_duplicated_in_the_throughput_table(self): + def test_is_not_duplicated_in_the_throughput_table(self): out = _render(_record()) throughput = out.split("#### Per-run Throughput")[1].split("####")[0] - assert "Speed" not in throughput + assert "E2E Output Token Throughput" not in throughput assert "Norm Intvty" not in throughput - def test_speed_is_defined(self): + def test_is_defined_under_the_aiperf_name(self): out = _render(_record()) - assert "**Speed**: output tokens divided by whole-request wall clock" in out + assert "**E2E Output Token Throughput**: AIPerf's per-request" in out - def test_a_source_without_speed_gets_no_speed_table(self): - assert "#### Per-run Speed" not in _render(_swo_record()) + def test_a_source_without_it_gets_no_table(self): + assert _E2E_HEADING not in _render(_swo_record()) class TestConfigTable: