Skip to content

Commit d141e71

Browse files
committed
fix: preserve diagnostic sidecars and legacy result processing
只忽略已定义的诊断 sidecar,保留损坏与缺失点门禁;旧处理器逐点处理并累计失败,新版导入错误不回退。
1 parent bc8684e commit d141e71

3 files changed

Lines changed: 136 additions & 2 deletions

File tree

‎.github/workflows/benchmark-multinode-tmpl.yml‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -480,7 +480,47 @@ jobs:
480480
export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)"
481481
echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA"
482482
fi
483-
python3 utils/process_result.py --all
483+
# Older measured refs predate the batch CLI. Import failures in a
484+
# present processor are errors, not a reason to downgrade validation.
485+
has_batch=$(python3 - <<'PY'
486+
import importlib
487+
from pathlib import Path
488+
if Path('infx/results/fixed_sequence.py').is_file():
489+
module = importlib.import_module('infx.results.fixed_sequence')
490+
print(int(hasattr(module, 'process_multinode_results')))
491+
else:
492+
print(0)
493+
PY
494+
)
495+
if [ "$has_batch" = 1 ]; then
496+
python3 utils/process_result.py --all
497+
else
498+
batch_rc=0
499+
observed=()
500+
for result_file in "${RESULT_FILENAME}"_*.json; do
501+
[ -f "$result_file" ] || continue
502+
case "$result_file" in
503+
*.pytorch.json|"${RESULT_FILENAME}_gpu_metrics_context.json"|"${RESULT_FILENAME}_gpu_metrics_identity.json") continue ;;
504+
esac
505+
if [[ "$result_file" =~ _(c|conc|concurrency_)([0-9]+)(_req_rate_[^_]+)?_gpus_([0-9]+)(_ctx_([0-9]+)_gen_([0-9]+))?\.json$ ]]; then
506+
observed+=("${BASH_REMATCH[2]}")
507+
gpus=${BASH_REMATCH[4]}
508+
prefill_gpus=${BASH_REMATCH[6]:-$gpus}
509+
decode_gpus=${BASH_REMATCH[7]:-0}
510+
RESULT_FILENAME=${result_file%.json} IS_MULTINODE=true PREFILL_GPUS="$prefill_gpus" DECODE_GPUS="$decode_gpus" \
511+
python3 utils/process_result.py || batch_rc=1
512+
else
513+
echo "Unrecognized result filename: $result_file" >&2
514+
batch_rc=1
515+
fi
516+
done
517+
read -r -a expected <<< "$CONC_LIST"
518+
if [[ "$(printf '%s\n' "${observed[@]}" | sort -n)" != "$(printf '%s\n' "${expected[@]}" | sort -n)" ]]; then
519+
echo 'Legacy result batch has missing, duplicate or unexpected concurrencies' >&2
520+
batch_rc=1
521+
fi
522+
exit "$batch_rc"
523+
fi
484524
485525
- name: Upload result
486526
if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }}

‎infx/results/fixed_sequence.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -318,9 +318,16 @@ def process_multinode_results(env: Mapping[str, str]) -> int:
318318
if not expected or min(expected) <= 0:
319319
raise ValueError('CONC_LIST must contain positive concurrencies')
320320
points: list[dict[str, Any]] = []
321+
ignored_sidecars: list[str] = []
321322
observed: set[int] = set()
322323
status = 0
323324
for path in sorted(Path('.').glob(f"{env['RESULT_FILENAME']}_*.json")):
325+
if path.name.endswith('.pytorch.json') or path.name in {
326+
f"{env['RESULT_FILENAME']}_gpu_metrics_context.json",
327+
f"{env['RESULT_FILENAME']}_gpu_metrics_identity.json",
328+
}:
329+
ignored_sidecars.append(path.name)
330+
continue
324331
point: dict[str, Any] = {'source': path.name}
325332
try:
326333
match = re.search(
@@ -364,7 +371,7 @@ def process_multinode_results(env: Mapping[str, str]) -> int:
364371
status = max(status, int(bool(missing or unexpected)))
365372
summary = {'expected_concurrencies': sorted(expected),
366373
'missing_concurrencies': missing, 'unexpected_concurrencies': unexpected,
367-
'points': points, 'exit_code': status}
374+
'points': points, 'ignored_sidecars': ignored_sidecars, 'exit_code': status}
368375
Path(f"result_processing_{env['RESULT_FILENAME']}.json").write_text(json.dumps(summary, indent=2))
369376
if missing or unexpected:
370377
print(f'[process_result] incomplete sweep: missing={missing}, unexpected={unexpected}', file=sys.stderr)

‎utils/test_process_result.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1623,3 +1623,90 @@ def test_public_power_audit_bounds_text_and_device_identifiers():
16231623
assert audit['exporter_image_sha256'] == 'a' * 64
16241624
assert len(audit['observed_gpu_ids']) == 1024
16251625
assert audit['observed_gpu_ids'][:2] == ['gpu0', 'gpu1']
1626+
1627+
1628+
@pytest.mark.parametrize('sidecar', ['run_recipe_conc4_gpus_4_ctx_2_gen_2.pytorch.json',
1629+
'run_gpu_metrics_context.json', 'run_gpu_metrics_identity.json'])
1630+
@pytest.mark.parametrize('point_state', ['valid', 'missing', 'malformed'])
1631+
def test_multinode_batch_retains_sidecars_without_counting_them_as_points(
1632+
tmp_path, multinode_env_vars, sample_benchmark_result, sidecar, point_state,
1633+
):
1634+
(tmp_path / sidecar).write_text('{"diagnostic": true}')
1635+
if point_state != 'missing':
1636+
(tmp_path / 'run_recipe_conc4_gpus_4_ctx_2_gen_2.json').write_text(
1637+
json.dumps({**sample_benchmark_result, 'max_concurrency': 4})
1638+
if point_state == 'valid' else '{broken')
1639+
env = {**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run', 'CONC_LIST': '4',
1640+
'PYTHONPATH': str(REPO_ROOT)}
1641+
result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path, env=env,
1642+
capture_output=True, text=True)
1643+
assert result.returncode == int(point_state != 'valid'), result.stderr
1644+
receipt = json.loads((tmp_path / 'result_processing_run.json').read_text())
1645+
assert receipt['ignored_sidecars'] == [sidecar]
1646+
assert receipt['missing_concurrencies'] == ([] if point_state == 'valid' else [4])
1647+
assert (tmp_path / sidecar).read_text() == '{"diagnostic": true}'
1648+
1649+
1650+
def test_multinode_batch_rejects_unknown_point_filename(
1651+
tmp_path, multinode_env_vars, sample_benchmark_result,
1652+
):
1653+
for name in ['run_conc4_gpus_4_ctx_2_gen_2.json', 'run_conc16_gpus_bad.json']:
1654+
(tmp_path / name).write_text(json.dumps({**sample_benchmark_result, 'max_concurrency': 4}))
1655+
result = subprocess.run([*MODULE_COMMAND, '--all'], cwd=tmp_path,
1656+
env={**os.environ, **multinode_env_vars, 'RESULT_FILENAME': 'run',
1657+
'CONC_LIST': '4 16', 'PYTHONPATH': str(REPO_ROOT)},
1658+
capture_output=True, text=True)
1659+
assert result.returncode == 1
1660+
receipt = json.loads((tmp_path / 'result_processing_run.json').read_text())
1661+
assert receipt['missing_concurrencies'] == [16]
1662+
assert any('filename lacks' in point.get('error', '') for point in receipt['points'])
1663+
1664+
1665+
@pytest.mark.parametrize('first_rc', [0, 7])
1666+
@pytest.mark.parametrize('expected_concs', ['4 16', '4 8 16', '4'])
1667+
def test_multinode_workflow_processes_every_point_with_legacy_processor(tmp_path, first_rc, expected_concs):
1668+
(tmp_path / 'utils').mkdir()
1669+
(tmp_path / 'utils/process_result.py').write_text('''import json, os, sys
1670+
from pathlib import Path
1671+
stem = os.environ['RESULT_FILENAME']
1672+
data = json.loads(Path(stem + '.json').read_text())
1673+
Path('agg_' + stem + '.json').write_text(json.dumps(data))
1674+
sys.exit(data['exit_code'])
1675+
''')
1676+
for conc, rc in [(4, first_rc), (16, 0)]:
1677+
(tmp_path / f'run_conc{conc}_gpus_4_ctx_2_gen_2.json').write_text(
1678+
json.dumps({'exit_code': rc}))
1679+
workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml').read_text())
1680+
step = next(step for job in workflow['jobs'].values() for step in job.get('steps', [])
1681+
if step.get('name') == 'Process result')
1682+
result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step['run']], cwd=tmp_path,
1683+
env={**os.environ, 'RESULT_FILENAME': 'run', 'POWER_PRODUCER_SHA': '',
1684+
'CONC_LIST': expected_concs, 'PYTHONPATH': '',
1685+
'PATH': f"{Path(sys.executable).parent}:{os.environ['PATH']}"},
1686+
capture_output=True, text=True, timeout=10)
1687+
assert result.returncode == int(bool(first_rc) or expected_concs != '4 16'), result.stderr
1688+
for conc in [4, 16]:
1689+
assert (tmp_path / f'agg_run_conc{conc}_gpus_4_ctx_2_gen_2.json').is_file()
1690+
1691+
1692+
def test_multinode_workflow_does_not_downgrade_processor_import_errors(tmp_path):
1693+
(tmp_path / 'infx/results').mkdir(parents=True)
1694+
(tmp_path / 'infx/__init__.py').touch()
1695+
(tmp_path / 'infx/results/__init__.py').touch()
1696+
(tmp_path / 'infx/results/fixed_sequence.py').write_text(
1697+
'raise RuntimeError("broken processor import")\n')
1698+
(tmp_path / 'utils').mkdir()
1699+
(tmp_path / 'utils/process_result.py').write_text(
1700+
'from pathlib import Path\nPath("legacy-called").touch()\n')
1701+
(tmp_path / 'run_conc4_gpus_4.json').write_text('{}')
1702+
workflow = yaml.safe_load((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml').read_text())
1703+
step = next(step for job in workflow['jobs'].values() for step in job.get('steps', [])
1704+
if step.get('name') == 'Process result')
1705+
result = subprocess.run(['bash', '-euo', 'pipefail', '-c', step['run']], cwd=tmp_path,
1706+
env={**os.environ, 'RESULT_FILENAME': 'run', 'POWER_PRODUCER_SHA': '',
1707+
'CONC_LIST': '4', 'PYTHONPATH': '',
1708+
'PATH': f"{Path(sys.executable).parent}:{os.environ['PATH']}"},
1709+
capture_output=True, text=True, timeout=10)
1710+
assert result.returncode != 0
1711+
assert 'broken processor import' in result.stderr
1712+
assert not (tmp_path / 'legacy-called').exists()

0 commit comments

Comments
 (0)