Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions packages/app/cypress/e2e/measured-power-overlay.cy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ describe('Measured power on unofficial-run overlay', () => {
cy.visit('/inference?unofficialrun=26312107787', {
onBeforeLoad(win) {
win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now()));
win.localStorage.setItem('inferencex-feature-gate', '1');
},
});
cy.get('[data-testid="inference-chart-display"]', { timeout: 30_000 }).should('exist');
Expand All @@ -19,7 +18,7 @@ describe('Measured power on unofficial-run overlay', () => {
cy.get('[data-testid="yaxis-metric-selector"]').click();
cy.get('[data-slot="select-content"]').should('exist');

// Verify the gated "Measured Energy" group + both options. The select list is a
// Verify the "Measured Energy" group + both options. The select list is a
// scroll container (max-h-72 overflow-y-auto), and this group sits below the fold,
// so scroll each target into view before asserting visibility.
cy.contains('[data-slot="select-content"]', 'Measured Energy')
Expand Down
81 changes: 52 additions & 29 deletions packages/app/src/components/inference/InferenceContext.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,7 @@ export function InferenceProvider({

const {
graphs,
selectionPoints,
loading: chartDataLoading,
error: chartDataError,
hardwareConfig,
Expand Down Expand Up @@ -775,6 +776,10 @@ export function InferenceProvider({
),
[graphs, effectivePrecisions],
);
const hwSelectablePoints = useMemo(
() => selectionPoints.filter((point) => effectivePrecisions.includes(point.precision)),
[selectionPoints, effectivePrecisions],
);
const extractHwKey = useCallback((point: InferenceData) => point.hwKey as string, []);

const comparisonExclusion = useMemo(
Expand Down Expand Up @@ -891,11 +896,22 @@ export function InferenceProvider({
[resolveHwSelection, setActiveHwTypes, setActiveHwTypesDispatch],
);

const hwTypesWithData = useChartDataFilter(
hwFilteredPoints,
// `selectableHwTypes` is the universe the legend selection lives in: every
// config in scope for the current model / sequence / precision, whether or
// not it carries the selected y-metric. `hwTypesWithData` stays metric-aware
// and drives what the legend renders and what the chart draws. Keeping the
// two apart is what makes a y-axis switch non-destructive — the selection is
// never intersected with metric coverage, so nothing has to be restored when
// the user switches back.
const selectableHwTypes = useChartDataFilter(
hwSelectablePoints,
setActiveHwTypesWithFilter,
extractHwKey,
);
const hwTypesWithData = useMemo(
() => new Set(hwFilteredPoints.map(extractHwKey)),
[hwFilteredPoints, extractHwKey],
);

const bestHwTypes = useMemo(() => {
const wantedType = selectedXAxisMode === 'interactivity' ? 'interactivity' : 'e2e';
Expand All @@ -920,37 +936,42 @@ export function InferenceProvider({
const setBestPerSkuAndApply = useCallback(
(enabled: boolean) => {
setBestPerSku(enabled);
const target = enabled ? bestHwTypes : hwTypesWithData;
const target = enabled ? bestHwTypes : selectableHwTypes;
setActiveHwTypes(resolveHwSelection(target).result);
setActivePresetId(null);
presetHwFilterRef.current = null;
},
[bestHwTypes, hwTypesWithData, resolveHwSelection, setActiveHwTypes],
[bestHwTypes, selectableHwTypes, resolveHwSelection, setActiveHwTypes],
);

// Direct fallback: apply pendingHwFilter when hwTypesWithData is already populated
// Direct fallback: apply pendingHwFilter when selectableHwTypes is already populated
// but useChartDataFilter didn't fire (e.g. re-selecting the same preset).
useEffect(() => {
if (!pendingHwFilter || hwTypesWithData.size === 0) return;
if (!pendingHwFilter || selectableHwTypes.size === 0) return;
const filtered = new Set(
[...hwTypesWithData].filter((k) => matchesPresetHwFilter(k, pendingHwFilter, selectedModel)),
[...selectableHwTypes].filter((k) =>
matchesPresetHwFilter(k, pendingHwFilter, selectedModel),
),
);
if (filtered.size > 0) {
setActiveHwTypes(resolveHwSelection(filtered).result);
setPendingHwFilter(null);
}
}, [pendingHwFilter, hwTypesWithData, selectedModel, resolveHwSelection, setActiveHwTypes]);
}, [pendingHwFilter, selectableHwTypes, selectedModel, resolveHwSelection, setActiveHwTypes]);

const toggleHwType = useCallback(
(hw: string) => {
const next = toggleComparisonSelection(activeHwTypes, hw, hwTypesWithData);
// Toggle against the selection universe, not the metric-filtered legend:
// computeToggle's "everything is on, so solo this one" branch compares
// set sizes, and activeHwTypes is sized against selectableHwTypes.
const next = toggleComparisonSelection(activeHwTypes, hw, selectableHwTypes);
if (!next) return;
setActiveHwTypes(next);
setBestPerSku(false);
setActivePresetId(null);
presetHwFilterRef.current = null;
},
[activeHwTypes, hwTypesWithData, setActiveHwTypes, toggleComparisonSelection],
[activeHwTypes, selectableHwTypes, setActiveHwTypes, toggleComparisonSelection],
);

const removeHwType = useCallback(
Expand Down Expand Up @@ -980,20 +1001,20 @@ export function InferenceProvider({
const selectAllHwTypes = useCallback(() => {
setBestPerSku(false);
if (exclusion) {
const { result, droppedGroups } = resolveHwSelection(hwTypesWithData, activeHwTypes);
const { result, droppedGroups } = resolveHwSelection(selectableHwTypes, activeHwTypes);
setActiveHwTypes(result);
if (droppedGroups.length > 0) {
setEngineConflict({
kind: 'resolved',
...exclusionResolutionFamilies(hwTypesWithData, result, exclusion),
...exclusionResolutionFamilies(selectableHwTypes, result, exclusion),
});
}
return;
}
selectAllHwRaw(hwTypesWithData);
selectAllHwRaw(selectableHwTypes);
}, [
selectAllHwRaw,
hwTypesWithData,
selectableHwTypes,
activeHwTypes,
exclusion,
resolveHwSelection,
Expand All @@ -1011,7 +1032,7 @@ export function InferenceProvider({
// Skip the reset when a preset hw filter is pending — the fallback effect below handles it.
// When a preset is still active (presetHwFilterRef), re-apply the filter instead of resetting
// to all GPUs — this handles deferred effectivePrecisions changes from late availability data.
// Track the last applied key with a ref and include hwTypesWithData in the deps so the
// Track the last applied key with a ref and include selectableHwTypes in the deps so the
// reset commits as soon as data for the new model arrives — without this, switching models
// bails on the empty-data tick and never re-fires, leaving the legend at the prior intersection.
const precisionsKey = effectivePrecisions.join(',');
Expand All @@ -1021,26 +1042,26 @@ export function InferenceProvider({
const lastHwResetKeyRef = useRef('');

// Restore legend-active selection from URL on first availability of
// hwTypesWithData. Sets lastHwResetKeyRef so the reset effect below treats
// selectableHwTypes. Sets lastHwResetKeyRef so the reset effect below treats
// the current key as already-applied and bails. Empty intersections fall back
// to all available configs before the active exclusion policy is applied.
useEffect(() => {
if (!pendingActiveHwTypes) return;
if (pendingHwFilterRef.current) return;
if (hwTypesWithData.size === 0) return;
if (selectableHwTypes.size === 0) return;
// Match exact hwKeys (URL-restored) AND bare GPU prefixes (used by
// /compare/[a]-vs-[b] pages, which know the GPU key but not which framework
// configs exist for it).
const prefixes = [...pendingActiveHwTypes].filter((k) => !k.includes('_'));
let restored = new Set(
[...hwTypesWithData].filter(
[...selectableHwTypes].filter(
(k) =>
pendingActiveHwTypes.has(k) || prefixes.some((p) => k.startsWith(`${p}_`) || k === p),
),
);
// Empty intersection (e.g. URL referenced GPUs no longer in availability,
// or every referenced key disappeared) falls back to all available configs.
if (restored.size === 0) restored = hwTypesWithData;
if (restored.size === 0) restored = selectableHwTypes;
if (exclusion) {
const proposed = restored;
const resolved = resolveHwSelection(restored, new Set());
Expand All @@ -1057,7 +1078,7 @@ export function InferenceProvider({
setPendingActiveHwTypes(null);
}, [
pendingActiveHwTypes,
hwTypesWithData,
selectableHwTypes,
exclusion,
selectedModel,
effectiveSequence,
Expand All @@ -1070,13 +1091,13 @@ export function InferenceProvider({
useEffect(() => {
if (pendingHwFilterRef.current) return;
if (pendingActiveHwTypes) return;
if (hwTypesWithData.size === 0) return;
if (selectableHwTypes.size === 0) return;
if (lastHwResetKeyRef.current === hwResetKey) return;
lastHwResetKeyRef.current = hwResetKey;
const presetFilter = presetHwFilterRef.current;
if (presetFilter) {
const filtered = new Set(
[...hwTypesWithData].filter((k) => matchesPresetHwFilter(k, presetFilter, selectedModel)),
[...selectableHwTypes].filter((k) => matchesPresetHwFilter(k, presetFilter, selectedModel)),
);
if (filtered.size > 0) {
// Presets explicitly choose configs. Resolve any engine conflict
Expand All @@ -1090,7 +1111,7 @@ export function InferenceProvider({
// Scenarios that restrict standard-token engines (8K/1K, AgentX) keep one
// sticky group so their charts remain useful; variant-only rules retain
// the existing clear-all behavior.
const automaticSelection = bestPerSku ? bestHwTypes : hwTypesWithData;
const automaticSelection = bestPerSku ? bestHwTypes : selectableHwTypes;
const { result, droppedGroups } = resolveHwSelection(automaticSelection);
setActiveHwTypes(result);
if (droppedGroups.length > 0) {
Expand All @@ -1101,13 +1122,13 @@ export function InferenceProvider({
}
return;
}
setActiveHwTypes(bestPerSku ? bestHwTypes : hwTypesWithData);
setActiveHwTypes(bestPerSku ? bestHwTypes : selectableHwTypes);
Comment thread
edwingao28 marked this conversation as resolved.
Resolved
}, [
selectedModel,
effectiveSequence,
precisionsKey,
hwResetKey,
hwTypesWithData,
selectableHwTypes,
bestHwTypes,
bestPerSku,
exclusion,
Expand Down Expand Up @@ -1215,7 +1236,9 @@ export function InferenceProvider({
// ── URL sync ──────────────────────────────────────────────────────────────

// Serialize the legend-active set, omitting (empty string → URL default) when
// it equals the full set of items with data. Keeps share URLs short.
// it equals the full selectable set. Comparing against the selectable set
// rather than the metric-filtered one keeps a Measured Energy axis from
// baking that axis's coverage into the share URL. Keeps share URLs short.
const iActiveStr = useMemo(() => {
if (activeHwTypes.size === 0) return '';
if (bestPerSku && activeHwTypes.size === bestHwTypes.size) {
Expand All @@ -1228,18 +1251,18 @@ export function InferenceProvider({
}
if (same) return '';
}
if (activeHwTypes.size === hwTypesWithData.size) {
if (activeHwTypes.size === selectableHwTypes.size) {
let same = true;
for (const k of activeHwTypes) {
if (!hwTypesWithData.has(k)) {
if (!selectableHwTypes.has(k)) {
same = false;
break;
}
}
if (same) return '';
}
return [...activeHwTypes].toSorted().join(',');
}, [activeHwTypes, hwTypesWithData, bestHwTypes, bestPerSku]);
}, [activeHwTypes, selectableHwTypes, bestHwTypes, bestPerSku]);

const serializedLabelState = serializeLabelState(labelScenarioKind, {
showPointLabels,
Expand Down
58 changes: 58 additions & 0 deletions packages/app/src/components/inference/hooks/useChartData.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,12 @@ import { describe, it, expect } from 'vitest';

import chartDefinitions from '@/components/inference/inference-chart-config.json';

import type { InferenceData } from '@/components/inference/types';
import { EMPTY_QUICK_FILTERS } from '@/components/inference/utils/quickFilters';

import {
applyAgenticPercentileToXLabel,
applyScopeFilters,
buildComparisonDates,
dedupeRowsToLatestPerConfig,
filterOverviewHistoryRows,
Expand Down Expand Up @@ -281,3 +285,57 @@ describe('derived higher-is-better x-axis rooflines', () => {
expect(derivedModeRoofline(undefined, true)).toBeUndefined();
});
});

const scopePoint = (hwKey: string, extra: Record<string, unknown> = {}): InferenceData =>
({
x: 10,
y: 100,
hwKey,
model: 'DeepSeek-V4-Pro',
date: '2026-08-01',
tp: 8,
conc: 16,
precision: 'fp8',
framework: hwKey.split('_')[1],
disagg: false,
spec_method: 'none',
...extra,
}) as unknown as InferenceData;

describe('applyScopeFilters', () => {
// The legend's selection universe is built from this filter chain. It must
// depend on the GPU picker / quick filters / compare scope and on nothing
// else — in particular NOT on which y-axis metric is drawn. See the
// `selectionPoints` comment in useChartData: reconcileActiveSet intersects
// the user's legend selection with whatever set it is handed and never
// re-widens, so a universe that shrinks when a Measured Energy axis is
// picked deletes the telemetry-less configs for good.
const withTelemetry = scopePoint('b200_sglang', { measuredAvgPower: { y: 900, roof: false } });
const withoutTelemetry = scopePoint('h200_vllm');
const points = [withTelemetry, withoutTelemetry];

it('keeps configs that carry no measured-power telemetry', () => {
const scoped = applyScopeFilters(points, [], EMPTY_QUICK_FILTERS, null);
expect(scoped.map((p) => p.hwKey)).toEqual(['b200_sglang', 'h200_vllm']);
});

it('still applies the GPU picker', () => {
const scoped = applyScopeFilters(points, ['b200_sglang'], EMPTY_QUICK_FILTERS, null);
expect(scoped.map((p) => p.hwKey)).toEqual(['b200_sglang']);
});

it('still applies the framework quick filter', () => {
const scoped = applyScopeFilters(
points,
[],
{ ...EMPTY_QUICK_FILTERS, frameworks: ['vllm'] },
null,
);
expect(scoped.map((p) => p.hwKey)).toEqual(['h200_vllm']);
});

it('still applies the two-GPU compare scope', () => {
const scoped = applyScopeFilters(points, [], EMPTY_QUICK_FILTERS, ['h200', 'mi355x']);
expect(scoped.map((p) => p.hwKey)).toEqual(['h200_vllm']);
});
});
Loading