Skip to content

Commit 72b20e1

Browse files
pgmpablo157321pgmpablo157321
committed
Format loader file
1 parent 87dba7b commit 72b20e1

File tree

1 file changed

+180
-57
lines changed

1 file changed

+180
-57
lines changed

‎tools/submission/submission_checker/loader.py‎

Lines changed: 180 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@
1414

1515

1616
class SubmissionLogs:
17-
def __init__(self, performance_log = None, accuracy_log = None, accuracy_result = None, accuracy_json = None, system_json = None, measurements_json = None, loader_data = {}) -> None:
17+
def __init__(self, performance_log=None, accuracy_log=None, accuracy_result=None,
18+
accuracy_json=None, system_json=None, measurements_json=None, loader_data={}) -> None:
1819
self.performance_log = performance_log
1920
self.accuracy_log = accuracy_log
2021
self.accuracy_result = accuracy_result
@@ -29,42 +30,81 @@ def __init__(self, root, version) -> None:
2930
self.root = root
3031
self.version = version
3132
self.logger = logging.getLogger("LoadgenParser")
32-
self.perf_log_path = os.path.join(self.root, PERFORMANCE_LOG_PATH.get(version, PERFORMANCE_LOG_PATH["default"]))
33-
self.perf_summary_path = os.path.join(self.root, PERFORMANCE_SUMMARY_PATH.get(version, PERFORMANCE_SUMMARY_PATH["default"]))
34-
self.acc_log_path = os.path.join(self.root, ACCURACY_LOG_PATH.get(version, ACCURACY_LOG_PATH["default"]))
35-
self.acc_result_path = os.path.join(self.root, ACCURACY_RESULT_PATH.get(version, ACCURACY_RESULT_PATH["default"]))
36-
self.acc_json_path = os.path.join(self.root, ACCURACY_JSON_PATH.get(version, ACCURACY_JSON_PATH["default"]))
37-
self.system_log_path = os.path.join(self.root, SYSTEM_PATH.get(version, SYSTEM_PATH["default"]))
38-
self.measurements_path = os.path.join(self.root, MEASUREMENTS_PATH.get(version, MEASUREMENTS_PATH["default"]))
39-
self.compliance_path = os.path.join(self.root, COMPLIANCE_PATH.get(version, COMPLIANCE_PATH["default"]))
40-
self.test01_perf_path = os.path.join(self.root, TEST01_PERF_PATH.get(version, TEST01_PERF_PATH["default"]))
41-
self.test01_acc_path = os.path.join(self.root, TEST01_ACC_PATH.get(version, TEST01_ACC_PATH["default"]))
42-
self.test04_perf_path = os.path.join(self.root, TEST04_PERF_PATH.get(version, TEST04_PERF_PATH["default"]))
43-
self.test04_acc_path = os.path.join(self.root, TEST04_ACC_PATH.get(version, TEST04_ACC_PATH["default"]))
44-
self.test06_acc_path = os.path.join(self.root, TEST06_ACC_PATH.get(version, TEST06_ACC_PATH["default"]))
45-
self.power_dir_path = os.path.join(self.root, POWER_DIR_PATH.get(version, POWER_DIR_PATH["default"]))
46-
self.src_path = os.path.join(self.root, SRC_PATH.get(version, SRC_PATH["default"]))
33+
self.perf_log_path = os.path.join(
34+
self.root, PERFORMANCE_LOG_PATH.get(
35+
version, PERFORMANCE_LOG_PATH["default"]))
36+
self.perf_summary_path = os.path.join(
37+
self.root, PERFORMANCE_SUMMARY_PATH.get(
38+
version, PERFORMANCE_SUMMARY_PATH["default"]))
39+
self.acc_log_path = os.path.join(
40+
self.root, ACCURACY_LOG_PATH.get(
41+
version, ACCURACY_LOG_PATH["default"]))
42+
self.acc_result_path = os.path.join(
43+
self.root, ACCURACY_RESULT_PATH.get(
44+
version, ACCURACY_RESULT_PATH["default"]))
45+
self.acc_json_path = os.path.join(
46+
self.root, ACCURACY_JSON_PATH.get(
47+
version, ACCURACY_JSON_PATH["default"]))
48+
self.system_log_path = os.path.join(
49+
self.root, SYSTEM_PATH.get(
50+
version, SYSTEM_PATH["default"]))
51+
self.measurements_path = os.path.join(
52+
self.root, MEASUREMENTS_PATH.get(
53+
version, MEASUREMENTS_PATH["default"]))
54+
self.compliance_path = os.path.join(
55+
self.root, COMPLIANCE_PATH.get(
56+
version, COMPLIANCE_PATH["default"]))
57+
self.test01_perf_path = os.path.join(
58+
self.root, TEST01_PERF_PATH.get(
59+
version, TEST01_PERF_PATH["default"]))
60+
self.test01_acc_path = os.path.join(
61+
self.root, TEST01_ACC_PATH.get(
62+
version, TEST01_ACC_PATH["default"]))
63+
self.test04_perf_path = os.path.join(
64+
self.root, TEST04_PERF_PATH.get(
65+
version, TEST04_PERF_PATH["default"]))
66+
self.test04_acc_path = os.path.join(
67+
self.root, TEST04_ACC_PATH.get(
68+
version, TEST04_ACC_PATH["default"]))
69+
self.test06_acc_path = os.path.join(
70+
self.root, TEST06_ACC_PATH.get(
71+
version, TEST06_ACC_PATH["default"]))
72+
self.power_dir_path = os.path.join(
73+
self.root, POWER_DIR_PATH.get(
74+
version, POWER_DIR_PATH["default"]))
75+
self.src_path = os.path.join(
76+
self.root, SRC_PATH.get(
77+
version, SRC_PATH["default"]))
4778

48-
49-
def get_measurement_path(self, path, division, submitter, system, benchmark, scenario):
79+
def get_measurement_path(self, path, division,
80+
submitter, system, benchmark, scenario):
5081
measurements_file = None
5182
if "{file}" in path:
52-
files = list_files(str(os.path.dirname(path)).format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario))
83+
files = list_files(
84+
str(
85+
os.path.dirname(path)).format(
86+
division=division,
87+
submitter=submitter,
88+
system=system,
89+
benchmark=benchmark,
90+
scenario=scenario))
5391
for i in files:
5492
if i.startswith(system) and i.endswith(
5593
"_" + scenario + ".json"):
5694
measurements_file = i
57-
#end = len("_" + scenario + ".json")
95+
# end = len("_" + scenario + ".json")
5896
break
5997
elif i.startswith(system) and i.endswith(".json"):
6098
measurements_file = i
61-
#end = len(".json")
99+
# end = len(".json")
62100
break
63-
return path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario, file = measurements_file)
64-
return path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
65-
101+
return path.format(division=division, submitter=submitter, system=system,
102+
benchmark=benchmark, scenario=scenario, file=measurements_file)
103+
return path.format(division=division, submitter=submitter,
104+
system=system, benchmark=benchmark, scenario=scenario)
66105

67-
def load_single_log(self, path, log_type: Literal["Performance", "Accuracy", "AccuracyResult", "AccuracyJSON", "Test", "System", "Measurements"]):
106+
def load_single_log(self, path, log_type: Literal["Performance", "Accuracy",
107+
"AccuracyResult", "AccuracyJSON", "Test", "System", "Measurements"]):
68108
log = None
69109
if os.path.exists(path):
70110
self.logger.info("Loading %s log from %s", log_type, path)
@@ -79,58 +119,141 @@ def load_single_log(self, path, log_type: Literal["Performance", "Accuracy", "Ac
79119
elif log_type in ["AccuracyJSON"]:
80120
log = path
81121
else:
82-
self.logger.info("Could not load %s log from %s, log type not recognized", log_type, path)
122+
self.logger.info(
123+
"Could not load %s log from %s, log type not recognized",
124+
log_type,
125+
path)
83126
else:
84-
self.logger.info("Could not load %s log from %s, path does not exist", log_type, path)
127+
self.logger.info(
128+
"Could not load %s log from %s, path does not exist",
129+
log_type,
130+
path)
85131
return log
86132

87-
88133
def load(self) -> Generator[SubmissionLogs, None, None]:
89134
for division in list_dir(self.root):
90135
if division not in VALID_DIVISIONS:
91136
continue
92137
division_path = os.path.join(self.root, division)
93138
for submitter in list_dir(division_path):
94-
results_path = os.path.join(division_path, submitter, "results")
139+
results_path = os.path.join(
140+
division_path, submitter, "results")
95141
model_mapping = {}
96-
if division == "open" and os.path.exists(os.path.join(division_path, submitter, "model_mapping.json")):
97-
model_mapping = self.load_single_log(os.path.join(division_path, submitter, "model_mapping.json"), "System")
142+
if division == "open" and os.path.exists(os.path.join(
143+
division_path, submitter, "model_mapping.json")):
144+
model_mapping = self.load_single_log(os.path.join(
145+
division_path, submitter, "model_mapping.json"), "System")
98146
for system in list_dir(results_path):
99147
system_path = os.path.join(results_path, system)
100-
system_json_path = self.system_log_path.format(division = division, submitter = submitter, system = system)
101-
system_json = self.load_single_log(system_json_path, "System")
148+
system_json_path = self.system_log_path.format(
149+
division=division, submitter=submitter, system=system)
150+
system_json = self.load_single_log(
151+
system_json_path, "System")
102152
for benchmark in list_dir(system_path):
103153
benchmark_path = os.path.join(system_path, benchmark)
104154
for scenario in list_dir(benchmark_path):
105-
scenario_path = os.path.join(benchmark_path, benchmark)
155+
scenario_path = os.path.join(
156+
benchmark_path, benchmark)
106157
# Format Paths for a specific submission
107-
perf_path = self.perf_log_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
108-
acc_path = self.acc_log_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
109-
acc_result_path = self.acc_result_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
110-
acc_json_path = self.acc_json_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
111-
power_dir_path = self.power_dir_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
112-
measurements_path = self.get_measurement_path(self.measurements_path, division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
113-
compliance_path = self.compliance_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
114-
test01_perf_path = self.test01_perf_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
115-
test01_acc_path = self.test01_acc_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
116-
test04_perf_path = self.test04_perf_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
117-
test04_acc_path = self.test04_acc_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
118-
test06_acc_path = self.test06_acc_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
119-
src_path = self.src_path.format(division = division, submitter = submitter)
158+
perf_path = self.perf_log_path.format(
159+
division=division,
160+
submitter=submitter,
161+
system=system,
162+
benchmark=benchmark,
163+
scenario=scenario)
164+
acc_path = self.acc_log_path.format(
165+
division=division,
166+
submitter=submitter,
167+
system=system,
168+
benchmark=benchmark,
169+
scenario=scenario)
170+
acc_result_path = self.acc_result_path.format(
171+
division=division,
172+
submitter=submitter,
173+
system=system,
174+
benchmark=benchmark,
175+
scenario=scenario)
176+
acc_json_path = self.acc_json_path.format(
177+
division=division,
178+
submitter=submitter,
179+
system=system,
180+
benchmark=benchmark,
181+
scenario=scenario)
182+
power_dir_path = self.power_dir_path.format(
183+
division=division,
184+
submitter=submitter,
185+
system=system,
186+
benchmark=benchmark,
187+
scenario=scenario)
188+
measurements_path = self.get_measurement_path(
189+
self.measurements_path,
190+
division=division,
191+
submitter=submitter,
192+
system=system,
193+
benchmark=benchmark,
194+
scenario=scenario)
195+
compliance_path = self.compliance_path.format(
196+
division=division,
197+
submitter=submitter,
198+
system=system,
199+
benchmark=benchmark,
200+
scenario=scenario)
201+
test01_perf_path = self.test01_perf_path.format(
202+
division=division,
203+
submitter=submitter,
204+
system=system,
205+
benchmark=benchmark,
206+
scenario=scenario)
207+
test01_acc_path = self.test01_acc_path.format(
208+
division=division,
209+
submitter=submitter,
210+
system=system,
211+
benchmark=benchmark,
212+
scenario=scenario)
213+
test04_perf_path = self.test04_perf_path.format(
214+
division=division,
215+
submitter=submitter,
216+
system=system,
217+
benchmark=benchmark,
218+
scenario=scenario)
219+
test04_acc_path = self.test04_acc_path.format(
220+
division=division,
221+
submitter=submitter,
222+
system=system,
223+
benchmark=benchmark,
224+
scenario=scenario)
225+
test06_acc_path = self.test06_acc_path.format(
226+
division=division,
227+
submitter=submitter,
228+
system=system,
229+
benchmark=benchmark,
230+
scenario=scenario)
231+
src_path = self.src_path.format(
232+
division=division, submitter=submitter)
120233

121234
# Load logs
122-
perf_log = self.load_single_log(perf_path, "Performance")
123-
acc_log = self.load_single_log(acc_path, "Accuracy")
124-
acc_result = self.load_single_log(acc_result_path, "AccuracyResult")
125-
acc_json = self.load_single_log(acc_json_path, "AccuracyJSON")
126-
measurements_json = self.load_single_log(measurements_path, "Measurements")
235+
perf_log = self.load_single_log(
236+
perf_path, "Performance")
237+
acc_log = self.load_single_log(
238+
acc_path, "Accuracy")
239+
acc_result = self.load_single_log(
240+
acc_result_path, "AccuracyResult")
241+
acc_json = self.load_single_log(
242+
acc_json_path, "AccuracyJSON")
243+
measurements_json = self.load_single_log(
244+
measurements_path, "Measurements")
127245

128246
# Load test logs
129-
test01_perf_log = self.load_single_log(test01_perf_path, "Performance")
130-
test01_acc_result = self.load_single_log(test01_acc_path, "AccuracyResult")
131-
test04_perf_log = self.load_single_log(test04_perf_path, "Performance")
132-
test04_acc_result = self.load_single_log(test04_acc_path, "AccuracyResult")
133-
test06_acc_result = self.load_single_log(test06_acc_path, "AccuracyResult")
247+
test01_perf_log = self.load_single_log(
248+
test01_perf_path, "Performance")
249+
test01_acc_result = self.load_single_log(
250+
test01_acc_path, "AccuracyResult")
251+
test04_perf_log = self.load_single_log(
252+
test04_perf_path, "Performance")
253+
test04_acc_result = self.load_single_log(
254+
test04_acc_path, "AccuracyResult")
255+
test06_acc_result = self.load_single_log(
256+
test06_acc_path, "AccuracyResult")
134257

135258
loader_data = {
136259
# Submission info

0 commit comments

Comments
 (0)