Skip to content

Commit 6193059

Browse files
ianaylianayl
committed
Appease check formatting
1 parent 38c9bed commit 6193059

File tree

3 files changed

+142
-123
lines changed

3 files changed

+142
-123
lines changed

‎devops/scripts/benchmarking/aggregate.py‎

Lines changed: 81 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,18 @@
66

77
import common
88

9+
910
# Simple median calculation
1011
class SimpleMedian:
1112

12-
def __init__(self):
13-
self.elements = []
13+
def __init__(self):
14+
self.elements = []
1415

15-
def add(self, n: float):
16-
self.elements.append(n)
16+
def add(self, n: float):
17+
self.elements.append(n)
1718

18-
def get_median(self) -> float:
19-
return statistics.median(elements)
19+
def get_median(self) -> float:
20+
return statistics.median(elements)
2021

2122

2223
# Calculate medians incrementally using a heap: Useful for when dealing with
@@ -26,88 +27,100 @@ def get_median(self) -> float:
2627
# with precommit in mind, but if this only runs nightly, it would actually be
2728
# faster to do a normal median calculation.
2829
class StreamingMedian:
29-
30+
3031
def __init__(self):
31-
# Gist: we keep a minheap and a maxheap, and store the median as the top
32-
# of the minheap. When a new element comes it gets put into the heap
33-
# based on if the element is bigger than the current median. Then, the
34-
# heaps are heapified and the median is repopulated by heapify.
32+
# Gist: we keep a minheap and a maxheap, and store the median as the top
33+
# of the minheap. When a new element comes it gets put into the heap
34+
# based on if the element is bigger than the current median. Then, the
35+
# heaps are heapified and the median is repopulated by heapify.
3536
self.minheap_larger = []
3637
self.maxheap_smaller = []
37-
# Note: numbers on maxheap should be negative, as heapq
38-
# is minheap by default
38+
39+
# Note: numbers on maxheap should be negative, as heapq
40+
# is minheap by default
3941

4042
def add(self, n: float):
4143
if len(self.maxheap_smaller) == 0 or -self.maxheap_smaller[0] >= n:
4244
heapq.heappush(self.maxheap_smaller, -n)
4345
else:
4446
heapq.heappush(self.minheap_larger, n)
4547

46-
# Ensure minheap has more elements than maxheap
48+
# Ensure minheap has more elements than maxheap
4749
if len(self.maxheap_smaller) > len(self.minheap_larger) + 1:
48-
heapq.heappush(self.minheap_larger,
49-
-heapq.heappop(self.maxheap_smaller))
50+
heapq.heappush(self.minheap_larger, -heapq.heappop(self.maxheap_smaller))
5051
elif len(self.maxheap_smaller) < len(self.minheap_larger):
51-
heapq.heappush(self.maxheap_smaller,
52-
-heapq.heappop(self.minheap_larger))
52+
heapq.heappush(self.maxheap_smaller, -heapq.heappop(self.minheap_larger))
5353

5454
def get_median(self) -> float:
5555
if len(self.maxheap_smaller) == len(self.minheap_larger):
56-
# Equal number of elements smaller and larger than "median":
57-
# thus, there are two median values. The median would then become
58-
# the average of both median values.
56+
# Equal number of elements smaller and larger than "median":
57+
# thus, there are two median values. The median would then become
58+
# the average of both median values.
5959
return (-self.maxheap_smaller[0] + self.minheap_larger[0]) / 2.0
6060
else:
61-
# Otherwise, median is always in minheap, as minheap is always
62-
# bigger
61+
# Otherwise, median is always in minheap, as minheap is always
62+
# bigger
6363
return -self.maxheap_smaller[0]
6464

6565

6666
def aggregate_median(runner: str, benchmark: str, cutoff: str):
6767

68-
# Get all .csv benchmark samples for the requested runner + benchmark
69-
def csv_samples() -> list[str]:
70-
# TODO check that the path below is valid directory
71-
cache_dir = Path(f"{common.PERF_RES_PATH}/{runner}/{benchmark}")
72-
# TODO check for time range; What time range do I want?
73-
return filter(lambda f: f.is_file() and
74-
common.valid_timestamp(str(f)[-19:-4]) and str(f)[-19:-4] > cutoff,
75-
cache_dir.glob(f"{benchmark}-*_*.csv"))
76-
77-
# Calculate median of every desired metric:
78-
aggregate_s = dict()
79-
for sample_path in csv_samples():
80-
with open(sample_path, 'r') as sample_file:
81-
for s in csv.DictReader(sample_file):
82-
test_case = s["TestCase"]
83-
# Construct entry in aggregate_s for test case if it does not
84-
# exist already:
85-
if test_case not in aggregate_s:
86-
aggregate_s[test_case] = \
87-
{ metric: SimpleMedian() for metric in common.metrics_variance }
88-
89-
for metric in common.metrics_variance:
90-
aggregate_s[test_case][metric].add(common.sanitize(s[metric]))
91-
92-
# Write calculated median (aggregate_s) as a new .csv file:
93-
with open(f"{common.PERF_RES_PATH}/{runner}/{benchmark}/{benchmark}-median.csv", 'w') as output_csv:
94-
writer = csv.DictWriter(output_csv,
95-
fieldnames=["TestCase", *common.metrics_variance.keys()])
96-
writer.writeheader()
97-
for test_case in aggregate_s:
98-
writer.writerow({ "TestCase": test_case } |
99-
{ metric: aggregate_s[test_case][metric].get_median()
100-
for metric in common.metrics_variance })
101-
102-
68+
# Get all .csv benchmark samples for the requested runner + benchmark
69+
def csv_samples() -> list[str]:
70+
# TODO check that the path below is valid directory
71+
cache_dir = Path(f"{common.PERF_RES_PATH}/{runner}/{benchmark}")
72+
# TODO check for time range; What time range do I want?
73+
return filter(
74+
lambda f: f.is_file()
75+
and common.valid_timestamp(str(f)[-19:-4])
76+
and str(f)[-19:-4] > cutoff,
77+
cache_dir.glob(f"{benchmark}-*_*.csv"),
78+
)
79+
80+
# Calculate median of every desired metric:
81+
aggregate_s = dict()
82+
for sample_path in csv_samples():
83+
with open(sample_path, "r") as sample_file:
84+
for s in csv.DictReader(sample_file):
85+
test_case = s["TestCase"]
86+
# Construct entry in aggregate_s for test case if it does not
87+
# exist already:
88+
if test_case not in aggregate_s:
89+
aggregate_s[test_case] = {
90+
metric: SimpleMedian() for metric in common.metrics_variance
91+
}
92+
93+
for metric in common.metrics_variance:
94+
aggregate_s[test_case][metric].add(common.sanitize(s[metric]))
95+
96+
# Write calculated median (aggregate_s) as a new .csv file:
97+
with open(
98+
f"{common.PERF_RES_PATH}/{runner}/{benchmark}/{benchmark}-median.csv", "w"
99+
) as output_csv:
100+
writer = csv.DictWriter(
101+
output_csv, fieldnames=["TestCase", *common.metrics_variance.keys()]
102+
)
103+
writer.writeheader()
104+
for test_case in aggregate_s:
105+
writer.writerow(
106+
{"TestCase": test_case}
107+
| {
108+
metric: aggregate_s[test_case][metric].get_median()
109+
for metric in common.metrics_variance
110+
}
111+
)
112+
113+
103114
if __name__ == "__main__":
104-
if len(sys.argv) < 4:
105-
print(f"Usage: {sys.argv[0]} <runner name> <test case name> <cutoff date YYYYMMDD_HHMMSS>")
106-
exit(1)
107-
if not common.valid_timestamp(sys.argv[3]):
108-
print(sys.argv)
109-
print(f"Bad cutoff timestamp, please use YYYYMMDD_HHMMSS.")
110-
exit(1)
111-
common.load_configs()
112-
# <runner>, <test case>, <cutoff>
113-
aggregate_median(sys.argv[1], sys.argv[2], sys.argv[3])
115+
if len(sys.argv) < 4:
116+
print(
117+
f"Usage: {sys.argv[0]} <runner name> <test case name> <cutoff date YYYYMMDD_HHMMSS>"
118+
)
119+
exit(1)
120+
if not common.valid_timestamp(sys.argv[3]):
121+
print(sys.argv)
122+
print(f"Bad cutoff timestamp, please use YYYYMMDD_HHMMSS.")
123+
exit(1)
124+
common.load_configs()
125+
# <runner>, <test case>, <cutoff>
126+
aggregate_median(sys.argv[1], sys.argv[2], sys.argv[3])

‎devops/scripts/benchmarking/common.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@
88

99

1010
def sanitize(stat: str) -> float:
11-
# Get rid of %
12-
if stat[-1] == '%':
13-
stat = stat[:-1]
14-
return float(stat)
11+
# Get rid of %
12+
if stat[-1] == '%':
13+
stat = stat[:-1]
14+
return float(stat)
1515

1616

1717
def load_configs():
@@ -26,11 +26,11 @@ def load_configs():
2626

2727
global PERF_RES_PATH, metrics_variance, metrics_recorded
2828
global BENCHMARK_ERROR_LOG, BENCHMARK_SLOW_LOG
29-
perf_res_re = re.compile(r'^PERF_RES_PATH=(.*)$', re.M)
29+
perf_res_re = re.compile(r'^PERF_RES_PATH=(.*)$', re.M)
3030
m_variance_re = re.compile(r'^METRICS_VARIANCE=(.*)$', re.M)
3131
m_recorded_re = re.compile(r'^METRICS_RECORDED=(.*)$', re.M)
32-
b_slow_re = re.compile(r'^BENCHMARK_SLOW_LOG=(.*)$', re.M)
33-
b_error_re = re.compile(r'^BENCHMARK_ERROR_LOG=(.*)$', re.M)
32+
b_slow_re = re.compile(r'^BENCHMARK_SLOW_LOG=(.*)$', re.M)
33+
b_error_re = re.compile(r'^BENCHMARK_ERROR_LOG=(.*)$', re.M)
3434

3535
with open(benchmarking_ci_conf_path, 'r') as configs_file:
3636
configs_str = configs_file.read()

‎devops/scripts/benchmarking/compare.py‎

Lines changed: 54 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -3,57 +3,63 @@
33
import sys
44
from pathlib import Path
55

6-
import common
6+
import common
7+
78

89
# TODO compare_to(metric) instead?
910
def compare_to_median(runner: str, test_name: str, test_csv_path: str):
10-
median_path = f"{common.PERF_RES_PATH}/{runner}/{test_name}/{test_name}-median.csv"
11-
12-
if not os.path.isfile(test_csv_path):
13-
print("Invalid test file provided: " + test_csv_path)
14-
exit(-1)
15-
if not os.path.isfile(median_path):
16-
print(f"Median file for test {test_name} not found at {median_path}.\n" +
17-
"Please build the median using the aggregate workflow.")
18-
exit(-1)
19-
20-
median = dict()
21-
with open(median_path, 'r') as median_csv:
22-
for stat in csv.DictReader(median_csv):
23-
median[stat["TestCase"]] = \
24-
{ metric: float(stat[metric]) for metric in common.metrics_variance }
25-
26-
# TODO read status codes from a config file instead?
27-
status = 0
28-
failure_counts = { metric: 0 for metric in common.metrics_variance }
29-
with open(test_csv_path, 'r') as sample_csv:
30-
for sample in csv.DictReader(sample_csv):
31-
# Ignore test cases we haven't profiled before
32-
if sample["TestCase"] not in median:
33-
continue
34-
test_median = median[sample["TestCase"]]
35-
for metric, threshold in common.metrics_variance.items():
36-
max_tolerated = test_median[metric] * (1 + threshold)
37-
if common.sanitize(sample[metric]) > max_tolerated:
38-
print("vvv FAILED vvv")
39-
print(sample['TestCase'])
40-
print(f"{metric}: {common.sanitize(sample[metric])} -- Historic avg. {test_median[metric]} (max tolerance {threshold*100}%: {max_tolerated})")
41-
print("^^^^^^^^^^^^^^")
42-
with open(common.BENCHMARK_SLOW_LOG, 'a') as slow_log:
43-
slow_log.write(
44-
f"-- {test_name}::{sample['TestCase']}\n"
45-
f" {metric}: {common.sanitize(sample[metric])} -- Historic avg. {test_median[metric]} (max tol. {threshold*100}%: {max_tolerated})\n"
46-
)
47-
status = 1
48-
failure_counts[metric] += 1
49-
if status != 0:
50-
print(f"Failure counts: {failure_counts}")
51-
return status
11+
median_path = f"{common.PERF_RES_PATH}/{runner}/{test_name}/{test_name}-median.csv"
12+
13+
if not os.path.isfile(test_csv_path):
14+
print("Invalid test file provided: " + test_csv_path)
15+
exit(-1)
16+
if not os.path.isfile(median_path):
17+
print(
18+
f"Median file for test {test_name} not found at {median_path}.\n"
19+
+ "Please build the median using the aggregate workflow."
20+
)
21+
exit(-1)
22+
23+
median = dict()
24+
with open(median_path, "r") as median_csv:
25+
for stat in csv.DictReader(median_csv):
26+
median[stat["TestCase"]] = {
27+
metric: float(stat[metric]) for metric in common.metrics_variance
28+
}
29+
30+
# TODO read status codes from a config file instead?
31+
status = 0
32+
failure_counts = {metric: 0 for metric in common.metrics_variance}
33+
with open(test_csv_path, "r") as sample_csv:
34+
for sample in csv.DictReader(sample_csv):
35+
# Ignore test cases we haven't profiled before
36+
if sample["TestCase"] not in median:
37+
continue
38+
test_median = median[sample["TestCase"]]
39+
for metric, threshold in common.metrics_variance.items():
40+
max_tolerated = test_median[metric] * (1 + threshold)
41+
if common.sanitize(sample[metric]) > max_tolerated:
42+
print("vvv FAILED vvv")
43+
print(sample["TestCase"])
44+
print(
45+
f"{metric}: {common.sanitize(sample[metric])} -- Historic avg. {test_median[metric]} (max tolerance {threshold*100}%: {max_tolerated})"
46+
)
47+
print("^^^^^^^^^^^^^^")
48+
with open(common.BENCHMARK_SLOW_LOG, "a") as slow_log:
49+
slow_log.write(
50+
f"-- {test_name}::{sample['TestCase']}\n"
51+
f" {metric}: {common.sanitize(sample[metric])} -- Historic avg. {test_median[metric]} (max tol. {threshold*100}%: {max_tolerated})\n"
52+
)
53+
status = 1
54+
failure_counts[metric] += 1
55+
if status != 0:
56+
print(f"Failure counts: {failure_counts}")
57+
return status
5258

5359

5460
if __name__ == "__main__":
55-
if len(sys.argv) < 4:
56-
print(f"Usage: {sys.argv[0]} <runner name> <test name> <test csv path>")
57-
exit(-1)
58-
common.load_configs()
59-
exit(compare_to_median(sys.argv[1], sys.argv[2], sys.argv[3]))
61+
if len(sys.argv) < 4:
62+
print(f"Usage: {sys.argv[0]} <runner name> <test name> <test csv path>")
63+
exit(-1)
64+
common.load_configs()
65+
exit(compare_to_median(sys.argv[1], sys.argv[2], sys.argv[3]))

0 commit comments

Comments
 (0)