Add v5.1 submission checker

.github/workflows/auto-update-dev.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master  # Trigger workflow on commits to 'master' branch
+
 
 jobs:
   update-dev:

language/llama3.1-8b/ref_eval.py

@@ -17,7 +17,8 @@ def rouge(label, pred):
 
 
 def niah_em(label, pred):
-    label_uuids = re.findall(r'[\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12}', label)
+    label_uuids = re.findall(
+        r'[\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12}', label)
     pred_uuids = re.findall(r'[\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12}', pred)
 
     # https://github.com/hsiehjackson/RULER/blob/main/scripts/eval/synthetic/constants.py#L28
@@ -43,7 +44,8 @@ def qa_em(label, pred):
         return {'exact_match': 100.0}
 
     normalized_answer = re.sub(r'\s+', '', answer_substring).lower()
-    label_entries = [re.sub(r'\s+', '', entry).lower() for entry in label.split('|')]
+    label_entries = [re.sub(r'\s+', '', entry).lower()
+                     for entry in label.split('|')]
 
     match_found = any(entry in normalized_answer for entry in label_entries)
     return {'exact_match': 100.0 if match_found else 0.0}
@@ -63,7 +65,12 @@ def process_row(row):
 
 def run_evaluation(df):
     with Pool(cpu_count()) as pool:
-        accuracies = list(tqdm(pool.imap(process_row, df.to_dict('records')), total=len(df)))
+        accuracies = list(
+            tqdm(
+                pool.imap(
+                    process_row,
+                    df.to_dict('records')),
+                total=len(df)))
 
     df['accuracy'] = accuracies
     return df
@@ -74,10 +81,10 @@ def run_evaluation(df):
     df = pd.read_pickle(fname)
 
     df = run_evaluation(df)
-    #df.to_pickle(str(fname).replace(".pkl", "_eval.pkl"))
+    # df.to_pickle(str(fname).replace(".pkl", "_eval.pkl"))
     print(f"WROTE: {str(fname).replace('.pkl', '_eval.pkl')}")
 
     accuracy = df.accuracy.apply(pd.Series)
     print(df.dataset.value_counts())
     print(accuracy.describe())
-    print(df.describe())
+    print(df.describe())

loadgen/mlperf.conf

@@ -27,9 +27,9 @@ whisper.*.performance_sample_count_override = 1633
 3d-unet.*.performance_sample_count_override = 0
 
 # Set seeds.
-*.*.qsl_rng_seed = 6023615788873153749
-*.*.sample_index_rng_seed = 15036839855038426416
-*.*.schedule_rng_seed = 9933818062894767841
+*.*.qsl_rng_seed = 1780908523862526354
+*.*.sample_index_rng_seed = 14771362308971278857
+*.*.schedule_rng_seed = 18209322760996052031
 
 # Set seeds for TEST_05 (not needed from v5.0 onwards)
 *.*.test05_qsl_rng_seed = 7975553102935885558

text_to_image/tools/sample_ids.py

@@ -20,7 +20,7 @@ def get_args():
         default=10,
         help="Dataset download location")
     parser.add_argument(
-        "--seed", "-s", type=int, default=265673710, help="Dataset download location"
+        "--seed", "-s", type=int, default=927722784, help="Dataset download location"
     )
     args = parser.parse_args()
     return args

text_to_image/tools/sample_ids.txt

@@ -1,10 +1,10 @@
-716
-2387
-3694
-273
-1221
-2942
-2047
-1691
-1510
-4399
+2747
+2235
+2165
+1515
+1538
+1367
+2419
+4629
+3657
+4532

tools/submission/generate_final_report.py

@@ -135,26 +135,24 @@ def main():
         [
             "resnet",
             "retinanet",
-            "3d-unet-99",
-            "3d-unet-99.9",
-            "rnnt",
             "bert-99",
             "bert-99.9",
             "dlrm-v2-99",
             "dlrm-v2-99.9",
-            "gptj-99",
-            "gptj-99.9",
-            "stable-diffusion-xl",
+            "3d-unet-99",
+            "3d-unet-99.9",
+            "llama3.1-8b",
             "llama2-70b-99",
             "llama2-70b-99.9",
-            "llama2-70b-interactive-99",
-            "llama2-70b-interactive-99.9",
-            "llama3.1-405b",
+            "stable-diffusion-xl",
             "mixtral-8x7b",
-            "pointpainting",
+            "llama3.1-405b",
             "rgat",
+            "pointpainting",
+            "deepseek-r1",
+            "whisper"
         ],
-        ["SingleStream", "MultiStream", "Server", "Offline"],
+        ["SingleStream", "MultiStream", "Server", "Offline", "Interactive"],
         [
             "Latency (ms)",
             "Samples/s",
@@ -165,14 +163,13 @@ def main():
         ],
     ]
 
-    if args.version == "4.1":
+    if args.version == "5.0":
         filter_scenarios = {
             "datacenter": {
                 "resnet": ["Server", "Offline"],
                 "retinanet": ["Server", "Offline"],
-                "rnnt": ["Server", "Offline"],
-                "bert-99": ["Server", "Offline"],
-                "bert-99.9": ["Server", "Offline"],
+                "bert-99": [],
+                "bert-99.9": [],
                 "dlrm-v2-99": ["Server", "Offline"],
                 "dlrm-v2-99.9": ["Server", "Offline"],
                 "3d-unet-99": ["Offline"],
@@ -182,21 +179,32 @@ def main():
                 "stable-diffusion-xl": ["Server", "Offline"],
                 "llama2-70b-99": ["Server", "Offline"],
                 "llama2-70b-99.9": ["Server", "Offline"],
+                "llama2-70b-interactive-99": ["Server", "Offline"],
+                "llama2-70b-interactive-99.9": ["Server", "Offline"],
                 "mixtral-8x7b": ["Server", "Offline"],
+                "rgat": ["Offline"],
+                "llama3.1-405b": ["Offline", "Server"],
+                "pointpainting": []
             },
             "edge": {
                 "resnet": ["SingleStream", "MultiStream", "Offline"],
                 "retinanet": ["SingleStream", "MultiStream", "Offline"],
-                "rnnt": ["SingleStream", "Offline"],
                 "bert-99": ["SingleStream", "Offline"],
-                "bert-99.9": [],
+                "bert-99.9": ["SingleStream", "Offline"],
                 "dlrm-v2-99": [],
                 "dlrm-v2-99.9": [],
                 "3d-unet-99": ["SingleStream", "Offline"],
                 "3d-unet-99.9": ["SingleStream", "Offline"],
+                "llama2-70b-99": [],
+                "llama2-70b-99.9": [],
+                "llama2-70b-interactive-99": [],
+                "llama2-70b-interactive-99.9": [],
+                "llama3.1-405b": [],
                 "gptj-99": ["SingleStream", "Offline"],
                 "gptj-99.9": ["SingleStream", "Offline"],
+                "rgat": [],
                 "stable-diffusion-xl": ["SingleStream", "Offline"],
+                "pointpainting": ["SingleStream"],
             },
         }
     else:
@@ -210,17 +218,16 @@ def main():
                 "dlrm-v2-99.9": ["Server", "Offline"],
                 "3d-unet-99": ["Offline"],
                 "3d-unet-99.9": ["Offline"],
-                "gptj-99": ["Server", "Offline"],
-                "gptj-99.9": ["Server", "Offline"],
                 "stable-diffusion-xl": ["Server", "Offline"],
-                "llama2-70b-99": ["Server", "Offline"],
-                "llama2-70b-99.9": ["Server", "Offline"],
-                "llama2-70b-interactive-99": ["Server", "Offline"],
-                "llama2-70b-interactive-99.9": ["Server", "Offline"],
+                "llama2-70b-99": ["Server", "Offline", "Interactive"],
+                "llama2-70b-99.9": ["Server", "Offline", "Interactive"],
                 "mixtral-8x7b": ["Server", "Offline"],
                 "rgat": ["Offline"],
                 "llama3.1-405b": ["Offline", "Server"],
-                "pointpainting": []
+                "pointpainting": [],
+                "llama3.1-8b": ["Server", "Offline"],
+                "deepseek-r1": ["Server", "Offline"],
+                "whisper": ["Offline"],
             },
             "edge": {
                 "resnet": ["SingleStream", "MultiStream", "Offline"],
@@ -233,14 +240,12 @@ def main():
                 "3d-unet-99.9": ["SingleStream", "Offline"],
                 "llama2-70b-99": [],
                 "llama2-70b-99.9": [],
-                "llama2-70b-interactive-99": [],
-                "llama2-70b-interactive-99.9": [],
                 "llama3.1-405b": [],
-                "gptj-99": ["SingleStream", "Offline"],
-                "gptj-99.9": ["SingleStream", "Offline"],
                 "rgat": [],
                 "stable-diffusion-xl": ["SingleStream", "Offline"],
                 "pointpainting": ["SingleStream"],
+                "llama3.1-8b": ["SingleStream", "Offline"],
+                "whisper": ["Offline"],
             },
         }