Skip to content

Commit 9d45a18

Browse files
jeffng-orjeffng-or
committed
Adding gen_at_report plus unit tests
Signed-off-by: Jeff Ng <[email protected]>
1 parent 804812e commit 9d45a18

File tree

17 files changed

+1346
-2
lines changed

17 files changed

+1346
-2
lines changed

‎tools/AutoTuner/src/autotuner/utils/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# To get relative imports to work
2+
import os, sys; sys.path.append(os.path.dirname(os.path.realpath(__file__)))

‎tools/AutoTuner/src/autotuner/utils/at_dir_extractor.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#!/usr/bin/env python3
2+
#############################################################################
3+
##
4+
## Copyright (c) 2024, Precision Innovations Inc.
5+
## All rights reserved.
6+
##
7+
## BSD 3-Clause License
8+
##
9+
## Redistribution and use in source and binary forms, with or without
10+
## modification, are permitted provided that the following conditions are met:
11+
##
12+
## * Redistributions of source code must retain the above copyright notice, this
13+
## list of conditions and the following disclaimer.
14+
##
15+
## * Redistributions in binary form must reproduce the above copyright notice,
16+
## this list of conditions and the following disclaimer in the documentation
17+
## and/or other materials provided with the distribution.
18+
##
19+
## * Neither the name of the copyright holder nor the names of its
20+
## contributors may be used to endorse or promote products derived from
21+
## this software without specific prior written permission.
22+
##
23+
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24+
## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25+
## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26+
## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
27+
## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28+
## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29+
## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30+
## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31+
## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32+
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33+
## POSSIBILITY OF SUCH DAMAGE.
34+
###############################################################################
35+
36+
import os
37+
import sys
38+
import time
39+
import argparse
40+
import json
41+
from at_extractor_base import ATExtractorBase
42+
43+
class ATDirExtractor(ATExtractorBase):
44+
def __init__(self, obj, completion_cbk, result_cbk):
45+
"""Initializer"""
46+
47+
super().__init__(obj, completion_cbk, result_cbk)
48+
49+
def extract_dir(self, dir_name):
50+
"""Extracts the data from the log directory"""
51+
52+
for trial_name in os.listdir(dir_name):
53+
if trial_name.startswith("variant-AutoTunerBase-") and trial_name.endswith("-ray"):
54+
self._extract_dir(trial_name,
55+
os.path.join(dir_name, trial_name))
56+
57+
def _extract_dir(self, trial_name, dir_name):
58+
"""Extracts the data from the result.json file"""
59+
60+
results_file = os.path.join(dir_name, "result.json")
61+
if os.path.exists(results_file):
62+
with open(results_file, "r") as fh:
63+
data = json.load(fh)
64+
metrics = data.copy()
65+
metrics["metric"] = round(metrics["metric"], 1)
66+
self._result_cbk(self._obj, trial_name, metrics)
67+
run_time = time.strftime("%H:%M:%S",
68+
time.gmtime(data["time_this_iter_s"]))
69+
self._completion_cbk(self._obj, trial_name, data["timestamp"],
70+
run_time)
71+
72+
@staticmethod
73+
def main():
74+
"""Standalone test driver"""
75+
parser = argparse.ArgumentParser(prog="at_dir_extractor.py")
76+
parser.add_argument(
77+
"-d", "--dir_name", help="ORFS log directory for AutoTuner run",
78+
required=True
79+
)
80+
args = parser.parse_args()
81+
rep = ATDirExtractor(None, ATExtractorBase.sample_completion_callback,
82+
ATExtractorBase.sample_results_callback)
83+
rep.extract_dir(args.dir_name)
84+
85+
86+
if __name__ == "__main__":
87+
ATDirExtractor.main()
88+
sys.exit(0)
89+

‎tools/AutoTuner/src/autotuner/utils/at_extractor_base.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/usr/bin/env python3
2+
#############################################################################
3+
##
4+
## Copyright (c) 2024, Precision Innovations Inc.
5+
## All rights reserved.
6+
##
7+
## BSD 3-Clause License
8+
##
9+
## Redistribution and use in source and binary forms, with or without
10+
## modification, are permitted provided that the following conditions are met:
11+
##
12+
## * Redistributions of source code must retain the above copyright notice, this
13+
## list of conditions and the following disclaimer.
14+
##
15+
## * Redistributions in binary form must reproduce the above copyright notice,
16+
## this list of conditions and the following disclaimer in the documentation
17+
## and/or other materials provided with the distribution.
18+
##
19+
## * Neither the name of the copyright holder nor the names of its
20+
## contributors may be used to endorse or promote products derived from
21+
## this software without specific prior written permission.
22+
##
23+
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24+
## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25+
## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26+
## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
27+
## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28+
## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29+
## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30+
## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31+
## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32+
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33+
## POSSIBILITY OF SUCH DAMAGE.
34+
###############################################################################
35+
36+
from datetime import datetime
37+
38+
class ATExtractorBase:
39+
"""Base class for AutoTuner extractors"""
40+
41+
def __init__(self, obj, completion_cbk, result_cbk):
42+
"""Registers context object, completion callback and results callback"""
43+
44+
self._completion_cbk = completion_cbk
45+
self._result_cbk = result_cbk
46+
self._obj = obj
47+
48+
@staticmethod
49+
def get_completion_time(timestamp):
50+
"""Returns a string for the completion time timestamp"""
51+
datetime_obj = datetime.fromtimestamp(timestamp)
52+
datetime_str = datetime_obj.strftime("%Y-%m-%d %H:%M:%S")
53+
return datetime_str
54+
55+
@staticmethod
56+
def sample_completion_callback(obj, trial_name, completion_timestamp, run_time):
57+
""" Sample completion callback """
58+
completion_time = ATExtractorBase.get_completion_time(completion_timestamp)
59+
print(f"Trial {trial_name} finished at {completion_time} with run time {run_time}")
60+
61+
@staticmethod
62+
def sample_results_callback(obj, trial_name, metrics):
63+
""" Sample results callback """
64+
print(f"Trial {trial_name} metrics {metrics}")

‎tools/AutoTuner/src/autotuner/utils/at_log_extractor.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
#!/usr/bin/env python3
2+
#############################################################################
3+
##
4+
## Copyright (c) 2024, Precision Innovations Inc.
5+
## All rights reserved.
6+
##
7+
## BSD 3-Clause License
8+
##
9+
## Redistribution and use in source and binary forms, with or without
10+
## modification, are permitted provided that the following conditions are met:
11+
##
12+
## * Redistributions of source code must retain the above copyright notice, this
13+
## list of conditions and the following disclaimer.
14+
##
15+
## * Redistributions in binary form must reproduce the above copyright notice,
16+
## this list of conditions and the following disclaimer in the documentation
17+
## and/or other materials provided with the distribution.
18+
##
19+
## * Neither the name of the copyright holder nor the names of its
20+
## contributors may be used to endorse or promote products derived from
21+
## this software without specific prior written permission.
22+
##
23+
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24+
## AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25+
## IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26+
## ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
27+
## LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28+
## CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29+
## SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30+
## INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31+
## CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32+
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33+
## POSSIBILITY OF SUCH DAMAGE.
34+
###############################################################################
35+
36+
import re
37+
import sys
38+
import argparse
39+
from datetime import datetime
40+
from at_extractor_base import ATExtractorBase
41+
42+
class ATLogExtractor(ATExtractorBase):
43+
"""Class to extract AutoTuner results from the AutoTuner output"""
44+
45+
def __init__(self, obj, completion_cbk, result_cbk):
46+
"""Initializes RE's to parse log file"""
47+
48+
super().__init__(obj, completion_cbk, result_cbk)
49+
self._trial_completed_re = re.compile(
50+
"Trial\s+(\S+)\s+completed\s+after\s+\d+\s+iterations\s+at\s+(\S+)\s+(\S+)\.\s+Total\s+running\s+time\:\s+((\d+)h\s+)?((\d+)min\s+)?(\d+)s"
51+
)
52+
self._trial_result_table_start_re = re.compile("Trial\s+(\S+)\s+result")
53+
self._trial_result_table_metric_re = re.compile("\s+(\w+)\s+(-?\d+(\.\d+)?)\s+")
54+
55+
def extract_file(self, file_name):
56+
"""Opens and reads the log file to populate the trial dictionary"""
57+
58+
with open(file_name, "r") as fh:
59+
self._extract(fh)
60+
61+
### Private API
62+
63+
def _get_timestamp(self, date_str, time_str):
64+
"""Converts the date and time strings to a timestamp"""
65+
66+
date_time_str = f"{date_str} {time_str}"
67+
date_time_obj = datetime.strptime(date_time_str, "%Y-%m-%d %H:%M:%S")
68+
return date_time_obj.timestamp()
69+
70+
def _read_result_table(self, fh, trial_name):
71+
"""Reads the metrics from the results table and returns it in a dict"""
72+
73+
metrics = {}
74+
line = fh.readline()
75+
while line:
76+
if line == "\n":
77+
return metrics
78+
result = self._trial_result_table_metric_re.search(line)
79+
if result:
80+
metric_name = result.group(1)
81+
metric_value = result.group(2)
82+
metrics[metric_name] = metric_value
83+
line = fh.readline()
84+
85+
def _add_completed_trial(self, result):
86+
"""Extracts results from RE and adds data to trial dictionary"""
87+
88+
trial_name = result.group(1)
89+
completion_date = result.group(2)
90+
completion_time = result.group(3)
91+
run_time = self._get_runtime(result)
92+
completion_timestamp = self._get_timestamp(completion_date, completion_time)
93+
self._completion_cbk(self._obj, trial_name, completion_timestamp, run_time)
94+
95+
def _add_result(self, fh, trial_name):
96+
"""Extracts results from table and calls result callback"""
97+
98+
metrics = self._read_result_table(fh, trial_name)
99+
self._result_cbk(self._obj, trial_name, metrics)
100+
101+
def _extract(self, fh):
102+
"""Parses the stream and extracts the data"""
103+
104+
for line in fh:
105+
result = self._trial_completed_re.match(line)
106+
if result:
107+
self._add_completed_trial(result)
108+
else:
109+
result = self._trial_result_table_start_re.search(line)
110+
if result:
111+
trial_name = result.group(1)
112+
self._add_result(fh, trial_name)
113+
114+
def _get_runtime(self, result):
115+
"""
116+
Returns the runtime string in h:mm:ss format, based on the RE match
117+
input
118+
"""
119+
120+
runtime_hr = runtime_min = runtime_sec = 0
121+
# 4 is the optional wrapper around hours
122+
# 5 is the hours
123+
if result.group(4):
124+
runtime_hr = int(result.group(5))
125+
# 6 is the optional wrapper around mins
126+
# 7 is the minutes
127+
if result.group(6):
128+
runtime_min = int(result.group(7))
129+
runtime_sec = int(result.group(8))
130+
run_time = f"{runtime_hr}:{runtime_min:02}:{runtime_sec:02}"
131+
return run_time
132+
133+
@staticmethod
134+
def main(): # pragma: nocover
135+
"""Standalone test driver"""
136+
137+
parser = argparse.ArgumentParser(prog="at_log_extractor.py")
138+
parser.add_argument(
139+
"-i", "--input_file", help="log file from AutoTuner", required=True
140+
)
141+
args = parser.parse_args()
142+
rep = ATLogExtractor(None, ATExtractorBase.sample_completion_callback,
143+
ATExtractorBase.sample_results_callback)
144+
rep.extract_file(args.input_file)
145+
146+
147+
if __name__ == "__main__": # pragma: nocover
148+
ATLogExtractor.main()
149+
sys.exit(0)
150+

0 commit comments

Comments
 (0)