v5.1 and lower compatibility: skip dataset test

Author: pgmpablo157321

tools/submission/submission_checker/checks/accuracy_check.py

@@ -127,6 +127,11 @@ def loadgen_errors_check(self):
         return True
 
     def dataset_check(self):
+        if self.config.skip_dataset_size_check:
+            self.log.info(
+                "%s Skipping dataset size check", self.path
+            )
+            return True
         qsl_total_count = self.mlperf_log["qsl_reported_total_count"]
         expected_qsl_total_count = self.config.get_dataset_size(self.model)
         if qsl_total_count != expected_qsl_total_count:

tools/submission/submission_checker/configuration/configuration.py

@@ -17,7 +17,8 @@ def __init__(
         skip_extra_files_in_root_check=False,
         skip_extra_accuracy_files_check=False,
         skip_all_systems_have_results_check=False,
-        skip_calibration_check=False
+        skip_calibration_check=False,
+        skip_dataset_size_check=False
     ):
         self.base = MODEL_CONFIG.get(version)
         self.extra_model_benchmark_map = extra_model_benchmark_map
@@ -34,6 +35,7 @@ def __init__(
         self.skip_extra_accuracy_files_check = skip_extra_accuracy_files_check
         self.skip_all_systems_have_results_check = skip_all_systems_have_results_check
         self.skip_calibration_check = skip_calibration_check
+        self.skip_dataset_size_check = skip_dataset_size_check
         self.load_config(version)
 
 

tools/submission/submission_checker/main.py

@@ -101,6 +101,11 @@ def get_args():
         action="store_true",
         help="skips the check that the calibration documentation should exist",
     )
+    parser.add_argument(
+        "--skip-dataset-size-check",
+        action="store_true",
+        help="skips dataset size check, only for backwards compatibility",
+    )
     args = parser.parse_args()
     return args
 
@@ -121,6 +126,7 @@ def main():
         skip_extra_accuracy_files_check=args.skip_extra_accuracy_files_check,
         skip_all_systems_have_results_check=args.skip_all_systems_have_results_check,
         skip_calibration_check=args.skip_calibration_check,
+        skip_dataset_size_check=args.skip_dataset_size_check
     )
 
     if args.scenarios_to_skip: