Skip to content

Commit 85dea30

Browse files
pgmpablo157321pgmpablo157321
committed
Add v6.0 constants to new submission checker
1 parent f651383 commit 85dea30

File tree

1 file changed

+361
-0
lines changed

1 file changed

+361
-0
lines changed

‎tools/submission/submission_checker/constants.py‎

Lines changed: 361 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,365 @@
11
MODEL_CONFIG = {
2+
"v6.0": {
3+
"models": [
4+
"resnet",
5+
"retinanet",
6+
"bert-99",
7+
"bert-99.9",
8+
"3d-unet-99",
9+
"3d-unet-99.9",
10+
"llama3.1-8b",
11+
"llama3.1-8b-edge",
12+
"llama2-70b-99",
13+
"llama2-70b-99.9",
14+
"stable-diffusion-xl",
15+
"mixtral-8x7b",
16+
"llama3.1-405b",
17+
"rgat",
18+
"pointpainting",
19+
"deepseek-r1",
20+
"whisper",
21+
"gpt-oss-120b",
22+
"wan-2.2-t2v-a14b",
23+
"qwen3-vl-235b-a22b",
24+
"dlrm-v3",
25+
],
26+
"required-scenarios-datacenter": {
27+
"retinanet": ["Server", "Offline"],
28+
"dlrm-v3": ["Server", "Offline"],
29+
"3d-unet-99": ["Offline"],
30+
"3d-unet-99.9": ["Offline"],
31+
"llama3.1-8b": ["Offline"],
32+
"llama2-70b-99": ["Offline"],
33+
"llama2-70b-99.9": ["Offline"],
34+
"stable-diffusion-xl": ["Server", "Offline"],
35+
"mixtral-8x7b": ["Server", "Offline"],
36+
"llama3.1-405b": ["Offline"],
37+
"rgat": ["Offline"],
38+
"whisper": ["Offline"],
39+
"deepseek-r1": ["Offline"],
40+
"gpt-oss-120b": ["Offline"],
41+
"qwen3-vl-235b-a22b": ["Server", "Offline"],
42+
"dlrm-v3": ["Server", "Offline"],
43+
},
44+
"optional-scenarios-datacenter": {
45+
"llama2-70b-99": ["Interactive", "Server"],
46+
"llama2-70b-99.9": ["Interactive", "Server"],
47+
"llama3.1-405b": ["Interactive", "Server"],
48+
"llama3.1-8b": ["Interactive", "Server"],
49+
"deepseek-r1": ["Interactive", "Server"],
50+
"gpt-oss-120b": ["Interactive", "Server"],
51+
},
52+
"required-scenarios-edge": {
53+
"resnet": ["SingleStream", "MultiStream", "Offline"],
54+
"retinanet": ["SingleStream", "MultiStream", "Offline"],
55+
"bert-99": ["SingleStream", "Offline"],
56+
"bert-99.9": ["SingleStream", "Offline"],
57+
"3d-unet-99": ["SingleStream", "Offline"],
58+
"3d-unet-99.9": ["SingleStream", "Offline"],
59+
"llama3.1-8b-edge": ["SingleStream", "Offline"],
60+
"stable-diffusion-xl": ["SingleStream", "Offline"],
61+
"pointpainting": ["SingleStream"],
62+
"whisper": ["Offline"],
63+
},
64+
"optional-scenarios-edge": {},
65+
"required-scenarios-datacenter-edge": {
66+
"resnet": ["SingleStream", "MultiStream", "Offline", "Server"],
67+
"retinanet": ["SingleStream", "Offline", "MultiStream", "Server"],
68+
"bert-99": ["SingleStream", "Offline"],
69+
"bert-99.9": ["SingleStream", "Offline"],
70+
"3d-unet-99": ["SingleStream", "Offline"],
71+
"3d-unet-99.9": ["SingleStream", "Offline"],
72+
"llama3.1-8b": ["Offline"],
73+
"llama3.1-8b-edge": ["SingleStream", "Offline"],
74+
"llama2-70b-99": ["Offline"],
75+
"llama2-70b-99.9": ["Offline"],
76+
"stable-diffusion-xl": ["SingleStream", "Offline", "Server"],
77+
"mixtral-8x7b": ["Server", "Offline"],
78+
"llama3.1-405b": ["Offline"],
79+
"rgat": ["Offline"],
80+
"pointpainting": ["SingleStream"],
81+
"deepseek-r1": ["Offline"],
82+
"whisper": ["Offline"],
83+
"gpt-oss-120b": ["Offline"],
84+
"qwen3-vl-235b-a22b": ["Offline"],
85+
"dlrm-v3": ["Offline", "Server"],
86+
},
87+
"optional-scenarios-datacenter-edge": {
88+
"llama2-70b-99": ["Interactive", "Server"],
89+
"llama2-70b-99.9": ["Interactive", "Server"],
90+
"llama3.1-405b": ["Interactive", "Server"],
91+
"llama3.1-8b": ["Interactive", "Server"],
92+
"deepseek-r1": ["Interactive", "Server"],
93+
"gpt-oss-120b": ["Interactive", "Server"],
94+
"qwen3-vl-235b-a22b": ["Interactive", "Server"],
95+
},
96+
"accuracy-target": {
97+
"resnet": ("acc", 76.46 * 0.99),
98+
"retinanet": ("mAP", 37.55 * 0.99),
99+
"bert-99": ("F1", 90.874 * 0.99),
100+
"bert-99.9": ("F1", 90.874 * 0.999),
101+
"dlrm-v2-99": ("AUC", 80.31 * 0.99),
102+
"dlrm-v2-99.9": ("AUC", 80.31 * 0.999),
103+
"3d-unet-99": ("DICE", 0.86170 * 0.99),
104+
"3d-unet-99.9": ("DICE", 0.86170 * 0.999),
105+
106+
"llama3.1-8b": (
107+
"ROUGE1",
108+
38.7792 * 0.99,
109+
"ROUGE2",
110+
15.9075 * 0.99,
111+
"ROUGEL",
112+
24.4957 * 0.99,
113+
"ROUGELSUM",
114+
35.793 * 0.99,
115+
"GEN_LEN",
116+
8167644 * 0.9,
117+
),
118+
"llama3.1-8b-edge": (
119+
"ROUGE1",
120+
39.06 * 0.99,
121+
"ROUGE2",
122+
16.1147 * 0.99,
123+
"ROUGEL",
124+
24.6375 * 0.99,
125+
"ROUGELSUM",
126+
36.124 * 0.99,
127+
"GEN_LEN",
128+
3051113 * 0.9,
129+
),
130+
"llama2-70b-99": (
131+
"ROUGE1",
132+
44.4312 * 0.99,
133+
"ROUGE2",
134+
22.0352 * 0.99,
135+
"ROUGEL",
136+
28.6162 * 0.99,
137+
"TOKENS_PER_SAMPLE",
138+
294.45 * 0.9,
139+
),
140+
"llama2-70b-99.9": (
141+
"ROUGE1",
142+
44.4312 * 0.999,
143+
"ROUGE2",
144+
22.0352 * 0.999,
145+
"ROUGEL",
146+
28.6162 * 0.999,
147+
"TOKENS_PER_SAMPLE",
148+
294.45 * 0.9,
149+
),
150+
"stable-diffusion-xl": (
151+
"CLIP_SCORE",
152+
31.68631873,
153+
"FID_SCORE",
154+
23.01085758,
155+
),
156+
"mixtral-8x7b": (
157+
"ROUGE1",
158+
45.5989 * 0.99,
159+
"ROUGE2",
160+
23.3526 * 0.99,
161+
"ROUGEL",
162+
30.4608 * 0.99,
163+
"TOKENS_PER_SAMPLE",
164+
144.84 * 0.9,
165+
"gsm8k_accuracy",
166+
73.66 * 0.99,
167+
"mbxp_accuracy",
168+
60.16 * 0.99,
169+
),
170+
"llama3.1-405b": (
171+
"ROUGEL",
172+
21.6666 * 0.99,
173+
"exact_match",
174+
90.1335 * 0.99,
175+
"TOKENS_PER_SAMPLE",
176+
684.68 * 0.9,
177+
),
178+
"rgat": ("acc", 0.7286 * 0.99),
179+
"pointpainting": ("mAP", 0.5425 * 0.999),
180+
"deepseek-r1": ("exact_match", 0.99 * 81.3582, "TOKENS_PER_SAMPLE", 0.9 * 3886.2274),
181+
"whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99),
182+
# TODO: Placeholder for now
183+
"gpt-oss-120b": ("exact_match", 83.13 * 0.99),
184+
# TODO: Placeholder for now
185+
"qwen3-vl-235b-a22b": ("F1", 0.7903 * 0.99),
186+
"dlrm-v3": ("AUC", 78.663 * 0.99), # TODO: Placeholder for now
187+
},
188+
"accuracy-upper-limit": {
189+
"stable-diffusion-xl": (
190+
"CLIP_SCORE",
191+
31.81331801,
192+
"FID_SCORE",
193+
23.95007626,
194+
),
195+
"llama2-70b-99": ("TOKENS_PER_SAMPLE", 294.45 * 1.1),
196+
"llama2-70b-99.9": ("TOKENS_PER_SAMPLE", 294.45 * 1.1),
197+
"mixtral-8x7b": ("TOKENS_PER_SAMPLE", 145.9 * 1.1),
198+
"llama3.1-405b": ("TOKENS_PER_SAMPLE", 684.68 * 1.1),
199+
"llama3.1-8b": ("GEN_LEN", 8167644 * 1.1),
200+
"llama3.1-8b-edge": ("GEN_LEN", 3051113 * 1.1),
201+
"deepseek-r1": ("TOKENS_PER_SAMPLE", 1.1 * 3886.2274),
202+
# TODO: Placeholder for now
203+
"gpt-oss-120b": ("TOKENS_PER_SAMPLE", 1.1 * 9999),
204+
},
205+
"accuracy-delta-perc": {
206+
"stable-diffusion-xl": {"CLIP_SCORE": 1, "FID_SCORE": 2}
207+
},
208+
"performance-sample-count": {
209+
"resnet": 1024,
210+
"retinanet": 64,
211+
"bert-99": 10833,
212+
"bert-99.9": 10833,
213+
"dlrm-v2-99": 204800,
214+
"dlrm-v2-99.9": 204800,
215+
"3d-unet-99": 43,
216+
"3d-unet-99.9": 43,
217+
"llama3.1-8b": 13368,
218+
"llama3.1-8b-edge": 5000,
219+
"llama2-70b-99": 24576,
220+
"llama2-70b-99.9": 24576,
221+
"stable-diffusion-xl": 5000,
222+
"mixtral-8x7b": 15000,
223+
"llama3.1-405b": 8313,
224+
"rgat": 788379,
225+
"pointpainting": 1024,
226+
"deepseek-r1": 4388,
227+
"whisper": 1633,
228+
# TODO: Need to add accuracy sample count checkers as well (4395)
229+
"gpt-oss-120b": 6396,
230+
"qwen3-vl-235b-a22b": 48289,
231+
"dlrm-v3": 34996,
232+
},
233+
"dataset-size": {
234+
"resnet": 50000,
235+
"retinanet": 24781,
236+
"bert-99": 10833,
237+
"bert-99.9": 10833,
238+
"dlrm-v2-99": 330067,
239+
"dlrm-v2-99.9": 330067,
240+
"3d-unet-99": 43,
241+
"3d-unet-99.9": 43,
242+
"llama3.1-8b": 13368,
243+
"llama3.1-8b-edge": 5000,
244+
"llama2-70b-99": 24576,
245+
"llama2-70b-99.9": 24576,
246+
"stable-diffusion-xl": 5000,
247+
"mixtral-8x7b": 15000,
248+
"llama3.1-405b": 8313,
249+
"rgat": 788379,
250+
"pointpainting": 39987,
251+
"deepseek-r1": 4388,
252+
"whisper": 1633,
253+
# TODO: Need to add accuracy sample count checkers as well (4395)
254+
"gpt-oss-120b": 6396,
255+
"qwen3-vl-235b-a22b": 48289,
256+
"dlrm-v3": 34996,
257+
},
258+
# model_mapping.json is expected in the root directory of the
259+
# submission folder for open submissions and so the below dictionary is
260+
# not really needed
261+
"model_mapping": {
262+
# map model names to the official mlperf model class
263+
"ssd-resnet34": "retinanet",
264+
"mobilenet": "resnet",
265+
"resnet50": "resnet",
266+
"llama3_1-405b": "llama3.1-405b",
267+
"llama3_1-8b": "llama3.1-8b",
268+
"llama3_1-8b-edge": "llama3.1-8b-edge",
269+
},
270+
"seeds": {
271+
# TODO: Update random seeds
272+
"qsl_rng_seed": 1780908523862526354,
273+
"sample_index_rng_seed": 14771362308971278857,
274+
"schedule_rng_seed": 18209322760996052031,
275+
},
276+
"ignore_errors": [],
277+
"latency-constraint": {
278+
"resnet": {"Server": 15000000},
279+
"retinanet": {"Server": 100000000},
280+
"dlrm-v2-99": {"Server": 60000000},
281+
"dlrm-v2-99.9": {"Server": 60000000},
282+
"llama3.1-8b": {"Server": 20000000000},
283+
"stable-diffusion-xl": {"Server": 20000000000},
284+
"llama2-70b-99": {"Server": 20000000000},
285+
"llama2-70b-99.9": {"Server": 20000000000},
286+
"mixtral-8x7b": {"Server": 20000000000},
287+
"llama3.1-405b": {"Server": 60000000000},
288+
"deepseek-r1": {"Server": 60000000000},
289+
"gpt-oss-120b": {"Server": 60000000000},
290+
"qwen3-vl-235b-a22b": {"Server": 60000000000},
291+
"dlrm-v3": {"Server": 60000000000},
292+
},
293+
"min-queries": {
294+
"resnet": {
295+
"SingleStream": 1024,
296+
"MultiStream": 270336,
297+
"Server": 270336,
298+
"Offline": 1,
299+
},
300+
"retinanet": {
301+
"SingleStream": 1024,
302+
"MultiStream": 270336,
303+
"Server": 270336,
304+
"Offline": 1,
305+
},
306+
"bert-99": {"SingleStream": 1024, "Offline": 1},
307+
"bert-99.9": {"SingleStream": 1024, "Offline": 1},
308+
"dlrm-v2-99": {"Server": 270336, "Offline": 1},
309+
"dlrm-v2-99.9": {"Server": 270336, "Offline": 1},
310+
"3d-unet-99": {"SingleStream": 1024, "Offline": 1},
311+
"3d-unet-99.9": {"SingleStream": 1024, "Offline": 1},
312+
"llama3.1-8b": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
313+
"llama3.1-8b-edge": {"SingleStream": 1024, "Offline": 1},
314+
"llama2-70b-99": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
315+
"llama2-70b-99.9": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
316+
"stable-diffusion-xl": {
317+
"SingleStream": 1024,
318+
"Server": 270336,
319+
"Offline": 1,
320+
},
321+
"mixtral-8x7b": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
322+
"llama3.1-405b": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
323+
"rgat": {"SingleStream": 1024, "Offline": 1},
324+
"pointpainting": {"SingleStream": 1024},
325+
"deepseek-r1": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
326+
"whisper": {"SingleStream": 1024, "Offline": 1},
327+
"gpt-oss-120b": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
328+
"qwen3-vl-235b-a22b": {"SingleStream": 1024, "Server": 270336, "Offline": 1},
329+
"dlrm-v3": {"Server": 270336, "Offline": 1},
330+
},
331+
"models_TEST01": [
332+
"resnet",
333+
"retinanet",
334+
"bert-99",
335+
"bert-99.9",
336+
"dlrm-v2-99",
337+
"dlrm-v2-99.9",
338+
"3d-unet-99",
339+
"3d-unet-99.9",
340+
"stable-diffusion-xl",
341+
"rgat",
342+
"pointpainting",
343+
"whisper",
344+
],
345+
"models_TEST04": [
346+
"resnet",
347+
"stable-diffusion-xl",
348+
"pointpainting",
349+
],
350+
"models_TEST06": [
351+
"llama2-70b-99",
352+
"llama2-70b-99.9",
353+
"llama2-70b-interactive-99",
354+
"llama2-70b-interactive-99.9",
355+
"llama3.1-405b",
356+
"llama3.1-8b",
357+
"llama3.1-8b-interactive",
358+
"llama3.1-405b-interactive",
359+
"mixtral-8x7b",
360+
"deepseek-r1",
361+
]
362+
},
2363
"v5.0": {
3364
"models": [
4365
"resnet",

0 commit comments

Comments
 (0)