Skip to content

[Post Mortem] Check equal issue for open division + check accuracy run covers all the dataset #2170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pgmpablo157321 merged 2 commits into from
May 20, 2025
Merged

[Post Mortem] Check equal issue for open division + check accuracy run covers all the dataset #2170

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1c873d0
Check equal issue for open division + check accuracy run covers all t…
pgmpablo157321 Apr 22, 2025
13200cf
Merge branch 'master' into open_checker_fix
mrmhodak May 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 56 additions & 4 deletions tools/submission/submission_checker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,22 @@
"stable-diffusion-xl": 5000,
"mixtral-8x7b": 15000,
},
"dataset-size": {
"resnet": 50000,
"retinanet": 24781,
"bert-99": 10833,
"bert-99.9": 10833,
"dlrm-v2-99": 204800,
"dlrm-v2-99.9": 204800,
"3d-unet-99": 43,
"3d-unet-99.9": 43,
"gptj-99": 13368,
"gptj-99.9": 13368,
"llama2-70b-99": 24576,
"llama2-70b-99.9": 24576,
"stable-diffusion-xl": 5000,
"mixtral-8x7b": 15000,
},
# TODO: Update this list.
"model_mapping": {
# map model names to the official mlperf model class
Expand Down Expand Up @@ -466,6 +482,27 @@
"rgat": 788379,
"pointpainting": 1024,
},
"dataset-size" : {
"resnet": 50000,
"retinanet": 24781,
"bert-99": 10833,
"bert-99.9": 10833,
"dlrm-v2-99": 204800,
"dlrm-v2-99.9": 204800,
"3d-unet-99": 43,
"3d-unet-99.9": 43,
"gptj-99": 13368,
"gptj-99.9": 13368,
"llama2-70b-99": 24576,
"llama2-70b-99.9": 24576,
"llama2-70b-interactive-99": 24576,
"llama2-70b-interactive-99.9": 24576,
"stable-diffusion-xl": 5000,
"mixtral-8x7b": 15000,
"llama3.1-405b": 8313,
"rgat": 788379,
"pointpainting": 39987,
},
# model_mapping.json is expected in the root directory of the
# submission folder for open submissions and so the below dictionary is
# not really needed
Expand Down Expand Up @@ -890,6 +927,7 @@ def __init__(
self.accuracy_delta_perc = self.base["accuracy-delta-perc"]
self.accuracy_upper_limit = self.base.get("accuracy-upper-limit", {})
self.performance_sample_count = self.base["performance-sample-count"]
self.dataset_size = self.base["dataset-size"]
self.latency_constraint = self.base.get("latency-constraint", {})
self.min_queries = self.base.get("min-queries", {})
self.required = None
Expand Down Expand Up @@ -985,6 +1023,12 @@ def get_min_query_count(self, model, scenario):
if model not in self.min_queries:
raise ValueError("model not known: " + model)
return self.min_queries[model].get(scenario)

def get_dataset_size(self, model):
model = self.get_mlperf_model(model)
if model not in self.dataset_size:
raise ValueError("model not known: " + model)
return self.dataset_size[model]

def get_delta_perc(self, model, metric):
if model in self.accuracy_delta_perc:
Expand All @@ -1004,10 +1048,9 @@ def has_new_logging_format(self):
def uses_early_stopping(self, scenario):
return scenario in ["Server", "SingleStream", "MultiStream"]

def requires_equal_issue(self, model, division):
def requires_equal_issue(self, model):
return (
division in ["closed", "network"]
and model
model
in [
"3d-unet-99",
"3d-unet-99.9",
Expand Down Expand Up @@ -1320,6 +1363,15 @@ def check_accuracy_dir(config, model, path, verbose):
"%s has loadgen errors, number of errors: %s", path, mlperf_log.num_errors()
)

# check the whole dataset was used in the accuracy run
mlperf_log = MLPerfLog(fname)
qsl_total_count = mlperf_log["qsl_reported_total_count"]
expected_qsl_total_count = config.get_dataset_size(model)
if qsl_total_count != expected_qsl_total_count:
log.error(
"%s accurcy run does not cover all dataset, accuracy samples: %s, dataset size: %s", path, qsl_total_count, expected_qsl_total_count
)

return is_valid, result_acc


Expand Down Expand Up @@ -1437,7 +1489,7 @@ def check_performance_dir(
equal_issue_used_check = (
mlperf_log["effective_sample_concatenate_permutation"] == True
)
if not config.requires_equal_issue(model, division):
if not config.requires_equal_issue(model):
equal_issue_used_check = True
if not equal_issue_used_check:
log.error(
Expand Down
Loading