Skip to content

Commit f131a0d

Browse files
pgmpablo157321pgmpablo157321
authored
Update seeds for inference v6.0 (#2437)
1 parent 6f86c96 commit f131a0d

7 files changed

Lines changed: 34 additions & 37 deletions

File tree

‎.github/workflows/test-resnet50.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,8 @@ jobs:
3636
- name: Install dependencies
3737
run: |
3838
python3 -m pip install mlcflow
39-
mlc pull repo mlcommons@mlperf-automations --branch=subcheckmod_change
39+
mlc pull repo mlcommons@mlperf-automations --branch=dev
4040
#python3 -m pip install mlc-scripts
4141
- name: Test Resnet50 and end to end submission generation
4242
run: |
43-
mlcr run,mlperf,inference,generate-run-cmds,_submission,_short,_r6.0 --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.submission_checker_refactor --adr.inference-src.version=custom --adr.inference-src-loadgen.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}
44-
#mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --adr.inference-src-loadgen.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}
43+
mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src.version=custom --adr.inference-src-loadgen.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}

‎.github/workflows/test-retinanet.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,8 @@ jobs:
3434
- name: Install dependencies
3535
run: |
3636
python3 -m pip install mlcflow
37-
mlc pull repo mlcommons@mlperf-automations --branch=subcheckmod_change
37+
mlc pull repo mlcommons@mlperf-automations --branch=dev
3838
#python3 -m pip install mlc-scripts
3939
- name: Test Retinanet and end to end submission generation
4040
run: |
41-
mlcr run,mlperf,inference,generate-run-cmds,_submission,_short,_r6.0 --quiet --submitter="MLCommons" --hw_name=default --model=retinanet --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=10 --adr.compiler.tags=gcc --adr.accuracy-check-src.tags=_branch.submission_checker_refactor --adr.inference-src.version=custom --adr.inference-src.tags=_branch.submission_checker_refactor --adr.inference-src-loadgen.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom
42-
#mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=retinanet --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=10 --adr.compiler.tags=gcc --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.${{ github.event.pull_request.head.ref }} --adr.inference-src-loadgen.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom
41+
mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --hw_name=default --model=retinanet --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=10 --adr.compiler.tags=gcc --adr.inference-src.version=custom --adr.inference-src.tags=_repo.${{ github.event.pull_request.head.repo.html_url }},_branch.${{ github.event.pull_request.head.ref }} --adr.inference-src-loadgen.tags=_branch.${{ github.event.pull_request.head.ref }},_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom

‎.github/workflows/test-rgat.yml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,8 @@ jobs:
3333
- name: Install dependencies
3434
run: |
3535
python3 -m pip install mlcflow
36-
mlc pull repo mlcommons@mlperf-automations --branch=subcheckmod_change
36+
mlc pull repo mlcommons@mlperf-automations --branch=dev
3737
#python3 -m pip install mlc-scripts
3838
- name: Test R-GAT and end to end submission generation
3939
run: |
40-
mlcr run,mlperf,inference,generate-run-cmds,_submission,_short,_r6.0 --quiet --submitter="MLCommons" --category=datacenter --hw_name=default --model=rgat --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.submission_checker_refactor --adr.inference-src-loadgen.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}
41-
#mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --category=datacenter --hw_name=default --model=rgat --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}
40+
mlcr run,mlperf,inference,generate-run-cmds,_submission,_short --quiet --submitter="MLCommons" --category=datacenter --hw_name=default --model=rgat --implementation=reference --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --adr.inference-src.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.tags=_branch.$PR_HEAD_REF,_repo.${{ github.event.pull_request.head.repo.html_url }} --adr.inference-src-loadgen.version=custom --adr.loadgen.version=custom ${{ matrix.loadgen-flag }}

‎loadgen/mlperf.conf‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ qwen3-vl-235b-a22b.*.performance_sample_count_override = 48289
3232
3d-unet.*.performance_sample_count_override = 0
3333

3434
# Set seeds.
35-
*.*.qsl_rng_seed = 1780908523862526354
36-
*.*.sample_index_rng_seed = 14771362308971278857
37-
*.*.schedule_rng_seed = 18209322760996052031
35+
*.*.qsl_rng_seed = 2465351861681999779
36+
*.*.sample_index_rng_seed = 14276810075590677512
37+
*.*.schedule_rng_seed = 3936089224930324775
3838

3939
# Set seeds for TEST_05 (not needed from v5.0 onwards)
4040
*.*.test05_qsl_rng_seed = 7975553102935885558

‎text_to_image/tools/sample_ids.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ def get_args():
2020
default=10,
2121
help="Dataset download location")
2222
parser.add_argument(
23-
"--seed", "-s", type=int, default=927722784, help="Dataset download location"
23+
"--seed", "-s", type=int, default=1999337312, help="Dataset download location"
2424
)
2525
args = parser.parse_args()
2626
return args

‎text_to_image/tools/sample_ids.txt‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
2747
2-
2235
3-
2165
4-
1515
5-
1538
6-
1367
7-
2419
8-
4629
9-
3657
10-
4532
1+
1311
2+
2476
3+
3644
4+
2188
5+
4114
6+
52
7+
388
8+
1195
9+
3427
10+
2289

‎tools/submission/submission_checker.py‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -305,9 +305,9 @@
305305
},
306306
"seeds": {
307307
# TODO: Update random seeds
308-
"qsl_rng_seed": 1780908523862526354,
309-
"sample_index_rng_seed": 14771362308971278857,
310-
"schedule_rng_seed": 18209322760996052031,
308+
"qsl_rng_seed": 2465351861681999779,
309+
"sample_index_rng_seed": 14276810075590677512,
310+
"schedule_rng_seed": 3936089224930324775,
311311
},
312312
"ignore_errors": [],
313313
"latency-constraint": {
@@ -688,18 +688,18 @@
688688
]
689689
REQUIRED_ACC_BENCHMARK = {
690690
"stable-diffusion-xl": {
691-
"v5.0": {
691+
"v6.0": {
692692
"images": [
693-
"4459",
694-
"4015",
695-
"2705",
696-
"1682",
697-
"4048",
698-
"4683",
699-
"3757",
700-
"1578",
701-
"3319",
702-
"95",
693+
"1311",
694+
"2476",
695+
"3644",
696+
"2188",
697+
"4114",
698+
"52",
699+
"388",
700+
"1195",
701+
"3427",
702+
"2289",
703703
]
704704
},
705705
"v5.1": {

0 commit comments

Comments
 (0)