@@ -1623,3 +1623,90 @@ def test_public_power_audit_bounds_text_and_device_identifiers():
16231623 assert audit ['exporter_image_sha256' ] == 'a' * 64
16241624 assert len (audit ['observed_gpu_ids' ]) == 1024
16251625 assert audit ['observed_gpu_ids' ][:2 ] == ['gpu0' , 'gpu1' ]
1626+
1627+
1628+ @pytest .mark .parametrize ('sidecar' , ['run_recipe_conc4_gpus_4_ctx_2_gen_2.pytorch.json' ,
1629+ 'run_gpu_metrics_context.json' , 'run_gpu_metrics_identity.json' ])
1630+ @pytest .mark .parametrize ('point_state' , ['valid' , 'missing' , 'malformed' ])
1631+ def test_multinode_batch_retains_sidecars_without_counting_them_as_points (
1632+ tmp_path , multinode_env_vars , sample_benchmark_result , sidecar , point_state ,
1633+ ):
1634+ (tmp_path / sidecar ).write_text ('{"diagnostic": true}' )
1635+ if point_state != 'missing' :
1636+ (tmp_path / 'run_recipe_conc4_gpus_4_ctx_2_gen_2.json' ).write_text (
1637+ json .dumps ({** sample_benchmark_result , 'max_concurrency' : 4 })
1638+ if point_state == 'valid' else '{broken' )
1639+ env = {** os .environ , ** multinode_env_vars , 'RESULT_FILENAME' : 'run' , 'CONC_LIST' : '4' ,
1640+ 'PYTHONPATH' : str (REPO_ROOT )}
1641+ result = subprocess .run ([* MODULE_COMMAND , '--all' ], cwd = tmp_path , env = env ,
1642+ capture_output = True , text = True )
1643+ assert result .returncode == int (point_state != 'valid' ), result .stderr
1644+ receipt = json .loads ((tmp_path / 'result_processing_run.json' ).read_text ())
1645+ assert receipt ['ignored_sidecars' ] == [sidecar ]
1646+ assert receipt ['missing_concurrencies' ] == ([] if point_state == 'valid' else [4 ])
1647+ assert (tmp_path / sidecar ).read_text () == '{"diagnostic": true}'
1648+
1649+
1650+ def test_multinode_batch_rejects_unknown_point_filename (
1651+ tmp_path , multinode_env_vars , sample_benchmark_result ,
1652+ ):
1653+ for name in ['run_conc4_gpus_4_ctx_2_gen_2.json' , 'run_conc16_gpus_bad.json' ]:
1654+ (tmp_path / name ).write_text (json .dumps ({** sample_benchmark_result , 'max_concurrency' : 4 }))
1655+ result = subprocess .run ([* MODULE_COMMAND , '--all' ], cwd = tmp_path ,
1656+ env = {** os .environ , ** multinode_env_vars , 'RESULT_FILENAME' : 'run' ,
1657+ 'CONC_LIST' : '4 16' , 'PYTHONPATH' : str (REPO_ROOT )},
1658+ capture_output = True , text = True )
1659+ assert result .returncode == 1
1660+ receipt = json .loads ((tmp_path / 'result_processing_run.json' ).read_text ())
1661+ assert receipt ['missing_concurrencies' ] == [16 ]
1662+ assert any ('filename lacks' in point .get ('error' , '' ) for point in receipt ['points' ])
1663+
1664+
1665+ @pytest .mark .parametrize ('first_rc' , [0 , 7 ])
1666+ @pytest .mark .parametrize ('expected_concs' , ['4 16' , '4 8 16' , '4' ])
1667+ def test_multinode_workflow_processes_every_point_with_legacy_processor (tmp_path , first_rc , expected_concs ):
1668+ (tmp_path / 'utils' ).mkdir ()
1669+ (tmp_path / 'utils/process_result.py' ).write_text ('''import json, os, sys
1670+ from pathlib import Path
1671+ stem = os.environ['RESULT_FILENAME']
1672+ data = json.loads(Path(stem + '.json').read_text())
1673+ Path('agg_' + stem + '.json').write_text(json.dumps(data))
1674+ sys.exit(data['exit_code'])
1675+ ''' )
1676+ for conc , rc in [(4 , first_rc ), (16 , 0 )]:
1677+ (tmp_path / f'run_conc{ conc } _gpus_4_ctx_2_gen_2.json' ).write_text (
1678+ json .dumps ({'exit_code' : rc }))
1679+ workflow = yaml .safe_load ((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml' ).read_text ())
1680+ step = next (step for job in workflow ['jobs' ].values () for step in job .get ('steps' , [])
1681+ if step .get ('name' ) == 'Process result' )
1682+ result = subprocess .run (['bash' , '-euo' , 'pipefail' , '-c' , step ['run' ]], cwd = tmp_path ,
1683+ env = {** os .environ , 'RESULT_FILENAME' : 'run' , 'POWER_PRODUCER_SHA' : '' ,
1684+ 'CONC_LIST' : expected_concs , 'PYTHONPATH' : '' ,
1685+ 'PATH' : f"{ Path (sys .executable ).parent } :{ os .environ ['PATH' ]} " },
1686+ capture_output = True , text = True , timeout = 10 )
1687+ assert result .returncode == int (bool (first_rc ) or expected_concs != '4 16' ), result .stderr
1688+ for conc in [4 , 16 ]:
1689+ assert (tmp_path / f'agg_run_conc{ conc } _gpus_4_ctx_2_gen_2.json' ).is_file ()
1690+
1691+
1692+ def test_multinode_workflow_does_not_downgrade_processor_import_errors (tmp_path ):
1693+ (tmp_path / 'infx/results' ).mkdir (parents = True )
1694+ (tmp_path / 'infx/__init__.py' ).touch ()
1695+ (tmp_path / 'infx/results/__init__.py' ).touch ()
1696+ (tmp_path / 'infx/results/fixed_sequence.py' ).write_text (
1697+ 'raise RuntimeError("broken processor import")\n ' )
1698+ (tmp_path / 'utils' ).mkdir ()
1699+ (tmp_path / 'utils/process_result.py' ).write_text (
1700+ 'from pathlib import Path\n Path("legacy-called").touch()\n ' )
1701+ (tmp_path / 'run_conc4_gpus_4.json' ).write_text ('{}' )
1702+ workflow = yaml .safe_load ((REPO_ROOT / '.github/workflows/benchmark-multinode-tmpl.yml' ).read_text ())
1703+ step = next (step for job in workflow ['jobs' ].values () for step in job .get ('steps' , [])
1704+ if step .get ('name' ) == 'Process result' )
1705+ result = subprocess .run (['bash' , '-euo' , 'pipefail' , '-c' , step ['run' ]], cwd = tmp_path ,
1706+ env = {** os .environ , 'RESULT_FILENAME' : 'run' , 'POWER_PRODUCER_SHA' : '' ,
1707+ 'CONC_LIST' : '4' , 'PYTHONPATH' : '' ,
1708+ 'PATH' : f"{ Path (sys .executable ).parent } :{ os .environ ['PATH' ]} " },
1709+ capture_output = True , text = True , timeout = 10 )
1710+ assert result .returncode != 0
1711+ assert 'broken processor import' in result .stderr
1712+ assert not (tmp_path / 'legacy-called' ).exists ()
0 commit comments