Report improvement - support output of IDs to a json file (#2059)

* Updated final report generation script to output the list of ids being used
* Update generate_final_report.py

Author: arjunsuresh

main.py

@@ -502,7 +502,8 @@ def get_common_info(spaces, implementation, model):
         info += f"\n{pre_space}!!! tip\n\n"
         info += f"{pre_space}    - Number of threads could be adjusted using `--threads=#`, where `#` is the desired number of threads. This option works only if the implementation in use supports threading.\n\n"
         info += f"{pre_space}    - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n"
-        info += f"{pre_space}    - `_r4.1-dev` could also be given instead of `_r5.0-dev` if you want to run the benchmark with the MLPerf version being 4.1.\n\n"
+        if implementation.lower() == "reference":
+            info += f"{pre_space}    - `_r4.1-dev` could also be given instead of `_r5.0-dev` if you want to run the benchmark with the MLPerf version being 4.1.\n\n"
         if model == "rgat":
             info += f"{pre_space}    - Add `--env.CM_DATASET_IGBH_PATH=<Path to IGBH dataset>` if you have already downloaded the dataset. The path will be automatically mounted when using docker run.\n\n"
             info += f"{pre_space}    - Add `--env.CM_ML_MODEL_RGAT_CHECKPOINT_PATH=<Path to R-GAT model checkpoint>` if you have already downloaded the model. The path will be automatically mounted when using docker run.\n\n"

tools/submission/generate_final_report.py

@@ -202,7 +202,8 @@ def main():
             "datacenter": {
                 "resnet": ["Server", "Offline"],
                 "retinanet": ["Server", "Offline"],
-                "rnnt": ["Server", "Offline"],
+                "bert-99": [],
+                "bert-99.9": [],
                 "dlrm-v2-99": ["Server", "Offline"],
                 "dlrm-v2-99.9": ["Server", "Offline"],
                 "3d-unet-99": ["Offline"],
@@ -216,18 +217,26 @@ def main():
                 "llama2-70b-interactive-99.9": ["Server", "Offline"],
                 "mixtral-8x7b": ["Server", "Offline"],
                 "rgat": ["Offline"],
-                "llama3.1-405b": ["Offline", "Server"]
+                "llama3.1-405b": ["Offline", "Server"],
+                "pointpainting": []
             },
             "edge": {
                 "resnet": ["SingleStream", "MultiStream", "Offline"],
                 "retinanet": ["SingleStream", "MultiStream", "Offline"],
-                "rnnt": ["SingleStream", "Offline"],
                 "bert-99": ["SingleStream", "Offline"],
                 "bert-99.9": ["SingleStream", "Offline"],
+                "dlrm-v2-99": [],
+                "dlrm-v2-99.9": [],
                 "3d-unet-99": ["SingleStream", "Offline"],
                 "3d-unet-99.9": ["SingleStream", "Offline"],
+                "llama2-70b-99": [],
+                "llama2-70b-99.9": [],
+                "llama2-70b-interactive-99": [],
+                "llama2-70b-interactive-99.9": [],
+                "llama3.1-405b": [],
                 "gptj-99": ["SingleStream", "Offline"],
                 "gptj-99.9": ["SingleStream", "Offline"],
+                "rgat": [],
                 "stable-diffusion-xl": ["SingleStream", "Offline"],
                 "pointpainting": ["SingleStream"],
             },
@@ -285,16 +294,34 @@ def MakeUniqueID(x):
             "Unique ID (e.g. for Audit)"],
         inplace=True,
     )
-    id_dict = {
-        key: 1 + value
-        for (value, key) in enumerate(pd.unique(df["Unique ID (e.g. for Audit)"]))
+    if os.path.exists("ids.json"):
+        with open("ids.json", "r") as f:
+            id_dict = json.load(f)
+    else:
+        id_dict = {}
+    cur_keys = id_dict.keys()
+    cur_ids = id_dict.values()
+    new_keys = [
+        a for a in pd.unique(
+            df["Unique ID (e.g. for Audit)"]) if a not in cur_keys]
+    if cur_ids:
+        max_cur_id = max(cur_ids)
+    else:
+        max_cur_id = 0
+    id_dict_new = {
+        key: 1 + value + max_cur_id
+        for (value, key) in enumerate(new_keys)
     }
+    id_dict.update(id_dict_new)
+
     df["ID"] = df.apply(
         lambda x: "{}-{:04}".format(
             args.version, id_dict[x["Unique ID (e.g. for Audit)"]]
         ),
         axis=1,
     )
+    with open("ids.json", "w") as f:
+        f.write(json.dumps(id_dict, indent=4))
 
     for category in ["closed", "open", "network"]:
         for suite in ["datacenter", "edge"]: