diff --git a/.github/codeowner-signoff-verify-prompt.md b/.github/codeowner-signoff-verify-prompt.md index 0d77ac41e7..84f01fb7f9 100644 --- a/.github/codeowner-signoff-verify-prompt.md +++ b/.github/codeowner-signoff-verify-prompt.md @@ -927,9 +927,12 @@ Entries are permanent, so this fails here; PR-body claims stay under Check 15. ### (g) Removed points were off the frontier A removed sweep point is allowed only when it was not on the combined Pareto frontier, -or a measured head point dominates it in both `tput_per_gpu` and `mean_intvty`. Per -config key, `check_18g_points` lists `undominated_removed` (FAIL, unless head-point -measurements the facts missed dominate them) and `unmeasured_removed`. For unmeasured +or the measured head curve covers it: a head point dominates it in both `tput_per_gpu` +and `mean_intvty`, or it lies on or under the straight line between the two adjacent +non-dominated head points whose `mean_intvty` brackets its own (no extrapolation past +the head's range). `dominated_by` names the point or segment. Per config key, +`check_18g_points` lists `undominated_removed` (FAIL, unless head-point measurements +the facts missed cover them) and `unmeasured_removed`. For unmeasured points, take measurements from the most recent sweep run that measured them (`gh run download`) and run `signoff_facts dominance`. Points with the same shape and concurrency but a new recipe fingerprint are re-tuned, not removed; they appear under `retuned_deltas`. diff --git a/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py b/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py index 8a3a3682ed..e8fc0572ee 100644 --- a/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py +++ b/inferencex-e2e/infx/tests/workflows/test_signoff_facts.py @@ -168,6 +168,21 @@ def test_dominance_requires_removed_points_to_be_dominated(): assert report["deltas"] == [{"label": "keep", "tput_per_gpu_pct": 10.0, "mean_intvty_pct": 0.0}] +@pytest.mark.parametrize( + ("removed", "covered_by"), + [ + (point("c56", 13058, 114.2), "between c64 and c48"), + (point("above", 14500, 114.2), None), + (point("past", 5000, 140.0), None), + ], +) +def test_dominance_covers_points_under_the_head_frontier_segment(removed, covered_by): + # "slow" is dominated by c48; as a segment end it would drop the line under c56. + after = [point("c48", 11792, 133.7), point("c64", 15322, 103.5), point("slow", 9000, 120.0)] + report = signoff_facts.dominance([removed], after) + assert report["removed"] == [{"label": removed["label"], "dominated_by": covered_by}] + + def test_dominance_rejects_missing_metrics(): with pytest.raises(ValueError, match="positive"): signoff_facts.dominance([point("a", 0, 50)], []) diff --git a/inferencex-e2e/infx/workflows/signoff_facts.py b/inferencex-e2e/infx/workflows/signoff_facts.py index a5391b84c5..c5d32549bf 100644 --- a/inferencex-e2e/infx/workflows/signoff_facts.py +++ b/inferencex-e2e/infx/workflows/signoff_facts.py @@ -8,6 +8,7 @@ import argparse import io +import itertools import json import math import re @@ -287,8 +288,29 @@ def _dominates(a: dict, b: dict) -> bool: return a[x] >= b[x] and a[y] >= b[y] and (a[x] > b[x] or a[y] > b[y]) +def _cover(point: dict, others: list[dict]) -> str | None: + """What covers a removed point: a dominating point, or the frontier segment over it. + + The segment is the straight line between the two adjacent non-dominated points that + bracket the point's interactivity. Nothing covers a point outside their range. + """ + cover = next((other for other in others if _dominates(other, point)), None) + if cover: + return cover["label"] + x, y = "mean_intvty", "tput_per_gpu" + frontier = sorted( + (other for other in others if not any(_dominates(o, other) for o in others)), + key=lambda other: other[x], + ) + for low, high in itertools.pairwise(frontier): + if low[x] < point[x] < high[x]: + line = low[y] + (high[y] - low[y]) * (point[x] - low[x]) / (high[x] - low[x]) + return f"between {low['label']} and {high['label']}" if line >= point[y] else None + return None + + def dominance(before: list[dict], after: list[dict]) -> dict[str, Any]: - """Removed points must be dominated by an after point; shared points report deltas.""" + """Removed points must lie under the after frontier; shared points report deltas.""" for point in (*before, *after): if not isinstance(point.get("label"), str) or not all( isinstance(point.get(metric), int | float) @@ -304,8 +326,7 @@ def dominance(before: list[dict], after: list[dict]) -> dict[str, Any]: for point in before: if point["label"] in after_by_label: continue - cover = next((other for other in after if _dominates(other, point)), None) - removed.append({"label": point["label"], "dominated_by": cover and cover["label"]}) + removed.append({"label": point["label"], "dominated_by": _cover(point, after)}) deltas = [ { "label": point["label"], @@ -424,7 +445,7 @@ def points_diff(base: dict, head: dict, ingested: list[Any], new_run: list[Any]) Points match measurements by recipe fingerprint and concurrency. Base points use the ingested curve; head points prefer the PR's validated sweep, then the ingested curve. - A removed point passes only when a measured head point dominates it. + A removed point passes only when the measured head frontier covers it. """ from infx.matrix.changed import compare @@ -441,18 +462,16 @@ def measured(point: dict, table: dict[tuple, dict]) -> dict: measured(point, after) for point in head.get(change["config-key"], ({}, []))[0].values() ] added = [measured(point, after) for point in change["added"]] + covers = [ + other["measured"] | {"label": other["label"]} + for other in head_points + if other["measured"] + ] removed = [] for point in change["removed"]: item = measured(point, before) if item["measured"]: - item["dominated_by"] = next( - ( - other["label"] - for other in head_points - if other["measured"] and _dominates(other["measured"], item["measured"]) - ), - None, - ) + item["dominated_by"] = _cover(item["measured"], covers) removed.append(item) added_labels = {item["label"] for item in added} replaced = {item["label"]: item for item in added if item["measured"]}