Check equal issue for open division + check accuracy run covers all the dataset

Author: pgmpablo157321

tools/submission/submission_checker.py

@@ -188,6 +188,22 @@
             "stable-diffusion-xl": 5000,
             "mixtral-8x7b": 15000,
         },
+        "dataset-size": {
+            "resnet": 50000,
+            "retinanet": 24781,
+            "bert-99": 10833,
+            "bert-99.9": 10833,
+            "dlrm-v2-99": 204800,
+            "dlrm-v2-99.9": 204800,
+            "3d-unet-99": 43,
+            "3d-unet-99.9": 43,
+            "gptj-99": 13368,
+            "gptj-99.9": 13368,
+            "llama2-70b-99": 24576,
+            "llama2-70b-99.9": 24576,
+            "stable-diffusion-xl": 5000,
+            "mixtral-8x7b": 15000,
+        },
         # TODO: Update this list.
         "model_mapping": {
             # map model names to the official mlperf model class
@@ -466,6 +482,27 @@
             "rgat": 788379,
             "pointpainting": 1024,
         },
+        "dataset-size" : {
+            "resnet": 50000,
+            "retinanet": 24781,
+            "bert-99": 10833,
+            "bert-99.9": 10833,
+            "dlrm-v2-99": 204800,
+            "dlrm-v2-99.9": 204800,
+            "3d-unet-99": 43,
+            "3d-unet-99.9": 43,
+            "gptj-99": 13368,
+            "gptj-99.9": 13368,
+            "llama2-70b-99": 24576,
+            "llama2-70b-99.9": 24576,
+            "llama2-70b-interactive-99": 24576,
+            "llama2-70b-interactive-99.9": 24576,
+            "stable-diffusion-xl": 5000,
+            "mixtral-8x7b": 15000,
+            "llama3.1-405b": 8313,
+            "rgat": 788379,
+            "pointpainting": 39987,
+        },
         # model_mapping.json is expected in the root directory of the
         # submission folder for open submissions and so the below dictionary is
         # not really needed
@@ -888,6 +925,7 @@ def __init__(
         self.accuracy_delta_perc = self.base["accuracy-delta-perc"]
         self.accuracy_upper_limit = self.base.get("accuracy-upper-limit", {})
         self.performance_sample_count = self.base["performance-sample-count"]
+        self.dataset_size = self.base["dataset-size"]
         self.latency_constraint = self.base.get("latency-constraint", {})
         self.min_queries = self.base.get("min-queries", {})
         self.required = None
@@ -981,6 +1019,12 @@ def get_min_query_count(self, model, scenario):
         if model not in self.min_queries:
             raise ValueError("model not known: " + model)
         return self.min_queries[model].get(scenario)
+    
+    def get_dataset_size(self, model):
+        model = self.get_mlperf_model(model)
+        if model not in self.dataset_size:
+            raise ValueError("model not known: " + model)
+        return self.dataset_size[model]
 
     def get_delta_perc(self, model, metric):
         if model in self.accuracy_delta_perc:
@@ -1000,10 +1044,9 @@ def has_new_logging_format(self):
     def uses_early_stopping(self, scenario):
         return scenario in ["Server", "SingleStream", "MultiStream"]
 
-    def requires_equal_issue(self, model, division):
+    def requires_equal_issue(self, model):
         return (
-            division in ["closed", "network"]
-            and model
+            model
             in [
                 "3d-unet-99",
                 "3d-unet-99.9",
@@ -1306,6 +1349,15 @@ def check_accuracy_dir(config, model, path, verbose):
             "%s has loadgen errors, number of errors: %s", path, mlperf_log.num_errors()
         )
 
+    # check the whole dataset was used in the accuracy run
+    mlperf_log = MLPerfLog(fname)
+    qsl_total_count = mlperf_log["qsl_reported_total_count"]
+    expected_qsl_total_count = config.get_dataset_size(model)
+    if qsl_total_count != expected_qsl_total_count:
+        log.error(
+            "%s accurcy run does not cover all dataset, accuracy samples: %s, dataset size: %s", path, qsl_total_count, expected_qsl_total_count
+        )
+
     return is_valid, result_acc
 
 
@@ -1423,7 +1475,7 @@ def check_performance_dir(
     equal_issue_used_check = (
         mlperf_log["effective_sample_concatenate_permutation"] == True
     )
-    if not config.requires_equal_issue(model, division):
+    if not config.requires_equal_issue(model):
         equal_issue_used_check = True
     if not equal_issue_used_check:
         log.error(