Skip to content

Commit 8e36925

Browse files
Submission checker version 4.0 (#1560)
* Submission checker version 4.0 * Fix Llama2-70b name * Add 4.0 random seeds
1 parent 007ec3c commit 8e36925

File tree

10 files changed

+307
-34
lines changed

10 files changed

+307
-34
lines changed

language/llama2-70b/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
# Reference Implementation for Llama-v2-70B
1+
# Reference Implementation for llama2-70b
22

3-
**Basic implementation for Llama-v2-70B. Few noteworthy items:**
3+
**Basic implementation for llama2-70b. Few noteworthy items:**
44

55
+ Processing of Validation dataset is not finalized yet. Decision on input token lengths is pending
66
+ Streamer for communicating with loadgen has quite some overhead. This is only meant to provide functional implementation

language/llama2-70b/main.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ def main():
4747
settings = lg.TestSettings()
4848
settings.scenario = scenario_map[args.scenario.lower()]
4949
# Need to update the conf
50-
settings.FromConfig(args.mlperf_conf, "llama-v2-70b", args.scenario)
51-
settings.FromConfig(args.user_conf, "llama-v2-70b", args.scenario)
50+
settings.FromConfig(args.mlperf_conf, "llama2-70b", args.scenario)
51+
settings.FromConfig(args.user_conf, "llama2-70b", args.scenario)
5252

5353
if args.accuracy:
5454
settings.mode = lg.TestMode.AccuracyOnly

language/llama2-70b/mlperf.conf

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,9 @@ rnnt.Server.target_latency = 1000
5656
gptj.Server.target_latency = 20000
5757

5858
# Falcon Server scenario requires two latency constraints
59-
llama-v2-70b.Server.target_latency = 2000
60-
llama-v2-70b.Server.ttft_latency = 2000
61-
llama-v2-70b.Server.tpot_latency = 200
59+
llama2-70b.Server.target_latency = 2000
60+
llama2-70b.Server.ttft_latency = 2000
61+
llama2-70b.Server.tpot_latency = 200
6262

6363
*.Offline.target_latency_percentile = 90
6464
*.Offline.min_duration = 600000

loadgen/CMakeLists.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ cmake_minimum_required(VERSION 3.1)
33
project(mlperf_loadgen)
44

55
# The mlperf_loadgen version.
6-
set(mlperf_loadgen_VERSION_MAJOR 3)
7-
set(mlperf_loadgen_VERSION_MINOR 1)
6+
set(mlperf_loadgen_VERSION_MAJOR 4)
7+
set(mlperf_loadgen_VERSION_MINOR 0)
88
message("mlperf_loadgen v${mlperf_loadgen_VERSION_MAJOR}.${mlperf_loadgen_VERSION_MINOR}")
99

1010
# Set build options. NB: CXX_STANDARD is supported since CMake 3.1.

loadgen/setup.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,13 @@
7676

7777
mlperf_loadgen_module = Pybind11Extension(
7878
"mlperf_loadgen",
79-
define_macros=[("MAJOR_VERSION", "3"), ("MINOR_VERSION", "1")],
79+
define_macros=[("MAJOR_VERSION", "4"), ("MINOR_VERSION", "0")],
8080
include_dirs=[".", get_include()],
8181
sources=mlperf_loadgen_sources,
8282
depends=mlperf_loadgen_headers)
8383

8484
setup(name="mlperf_loadgen",
85-
version="3.1",
85+
version="4.0",
8686
description="MLPerf Inference LoadGen python bindings",
8787
url="https://mlcommons.org/",
8888
cmdclass={"build_ext": build_ext},

loadgen/version_generator.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ def generate_loadgen_version_definitions(cc_filename, loadgen_root):
9494
ofile.write("// DO NOT EDIT: Autogenerated by version_generator.py.\n\n")
9595
ofile.write("#include <string>\n\n")
9696
ofile.write("namespace mlperf {\n\n")
97-
ofile.write(func_def("Version", "\"3.1\""))
97+
ofile.write(func_def("Version", "\"4.0\""))
9898

9999
date_time_now_local = datetime.datetime.now().isoformat()
100100
date_time_now_utc = datetime.datetime.utcnow().isoformat()

mlperf.conf

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,13 @@ stable-diffusion-xl.*.performance_sample_count_override = 5000
1717
3d-unet.*.performance_sample_count_override = 0
1818

1919
# Set seeds. The seeds will be distributed two weeks before the submission.
20-
*.*.qsl_rng_seed = 148687905518835231
21-
*.*.sample_index_rng_seed = 520418551913322573
22-
*.*.schedule_rng_seed = 811580660758947900
20+
*.*.qsl_rng_seed = 13281865557512327830
21+
*.*.sample_index_rng_seed = 198141574272810017
22+
*.*.schedule_rng_seed = 7575108116881280410
2323
# Set seeds for TEST_05. The seeds will be distributed two weeks before the submission.
24-
*.*.test05_qsl_rng_seed = 793197339507417767
25-
*.*.test05_sample_index_rng_seed = 255610748586851044
26-
*.*.test05_schedule_rng_seed = 352213341366340113
24+
*.*.test05_qsl_rng_seed = 2376919268182438552
25+
*.*.test05_sample_index_rng_seed = 11176391829184272374
26+
*.*.test05_schedule_rng_seed = 3911940905271271337
2727

2828

2929
*.SingleStream.target_latency_percentile = 90

text_to_image/tools/sample_ids.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,16 @@ def get_args():
1515
parser.add_argument(
1616
"--n", type=int, default=10, help="Dataset download location"
1717
)
18+
parser.add_argument(
19+
"--seed", "-s", type=int, default=926019364, help="Dataset download location"
20+
)
1821
args = parser.parse_args()
1922
return args
2023

2124

2225
if __name__ == "__main__":
2326
args = get_args()
24-
np.random.seed(42)
27+
np.random.seed(args.seed)
2528
df_annotations = pd.read_csv(f"{args.tsv_path}", sep="\t")
2629
sample_ids = list(np.random.choice(df_annotations.shape[0], args.n))
2730
with open(args.output_path, "w+") as f:

text_to_image/tools/sample_ids.txt

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
860
2-
3772
3-
3092
4-
466
5-
4426
6-
3444
7-
3171
8-
2919
9-
130
10-
1685
1+
4459
2+
4015
3+
2705
4+
1682
5+
4048
6+
4683
7+
3757
8+
1578
9+
3319
10+
95

0 commit comments

Comments
 (0)