Skip to content

Commit 730b4c1

Browse files
cosminbascacosminbasca
committed
[GR-9830] unittest statistics
PullRequest: graalpython-open/44
2 parents fce43ce + 7f2fcfa commit 730b4c1

File tree

2 files changed

+175
-2
lines changed

2 files changed

+175
-2
lines changed

‎scripts/process_unittests.py

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
# Copyright (c) 2018, Oracle and/or its affiliates.
2+
#
3+
# The Universal Permissive License (UPL), Version 1.0
4+
#
5+
# Subject to the condition set forth below, permission is hereby granted to any
6+
# person obtaining a copy of this software, associated documentation and/or data
7+
# (collectively the "Software"), free of charge and under any and all copyright
8+
# rights in the Software, and any and all patent rights owned or freely
9+
# licensable by each licensor hereunder covering either (i) the unmodified
10+
# Software as contributed to or provided by such licensor, or (ii) the Larger
11+
# Works (as defined below), to deal in both
12+
#
13+
# (a) the Software, and
14+
# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
15+
# one is included with the Software (each a "Larger Work" to which the
16+
# Software is contributed by such licensors),
17+
#
18+
# without restriction, including without limitation the rights to copy, create
19+
# derivative works of, display, perform, and distribute the Software and make,
20+
# use, sell, offer for sale, import, export, have made, and have sold the
21+
# Software and the Larger Work(s), and to sublicense the foregoing rights on
22+
# either these or other terms.
23+
#
24+
# This license is subject to the following condition:
25+
#
26+
# The above copyright notice and either this complete permission notice or at a
27+
# minimum a reference to the UPL must be included in all copies or substantial
28+
# portions of the Software.
29+
#
30+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
31+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
32+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
33+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
34+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
35+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36+
# SOFTWARE.
37+
38+
import os
39+
import sys
40+
import re
41+
import csv
42+
from json import dumps
43+
from collections import defaultdict
44+
from pprint import pprint, pformat
45+
46+
PTRN_ERROR = re.compile(r'^(?P<error>[A-Z][a-z][a-zA-Z]+):(?P<message>.*)$')
47+
PTRN_UNITTEST = re.compile(r'^#### running: graalpython/lib-python/3/test/(?P<unittest>.*)$')
48+
PTRN_NUM_TESTS = re.compile(r'^Ran (?P<num_tests>\d+) test.*$')
49+
PTRN_NUM_ERRORS = re.compile(r'^FAILED \((failures=(?P<failures>\d+))?(, )?((errors=(?P<errors>\d+)))?(, )?((skipped=(?P<skipped>\d+)))?\)$')
50+
51+
52+
def process_output(output):
53+
unittests = []
54+
error_messages = defaultdict(set)
55+
56+
class StatEntry(object):
57+
def __init__(self):
58+
self.num_tests = -1
59+
self.num_errors = -1
60+
self.num_fails = -1
61+
self.num_skipped = -1
62+
63+
def all_ok(self):
64+
self.num_errors = 0
65+
self.num_fails = 0
66+
self.num_skipped = 0
67+
68+
@property
69+
def num_passes(self):
70+
if self.num_tests > 0:
71+
return self.num_tests - (self.num_fails + self.num_errors + self.num_skipped)
72+
return -1
73+
74+
stats = defaultdict(StatEntry)
75+
76+
with open(output, 'r') as OUT:
77+
for line in OUT:
78+
match = re.match(PTRN_UNITTEST, line)
79+
if match:
80+
unittests.append(match.group('unittest'))
81+
continue
82+
83+
# extract python reported python error messages
84+
match = re.match(PTRN_ERROR, line)
85+
if match:
86+
error_messages[unittests[-1]].add((match.group('error'), match.group('message')))
87+
continue
88+
89+
# stats
90+
if line.strip() == 'OK':
91+
stats[unittests[-1]].all_ok()
92+
continue
93+
94+
match = re.match(PTRN_NUM_TESTS, line)
95+
if match:
96+
stats[unittests[-1]].num_tests = int(match.group('num_tests'))
97+
continue
98+
99+
match = re.match(PTRN_NUM_ERRORS, line)
100+
if match:
101+
fails = match.group('failures')
102+
errs = match.group('errors')
103+
skipped = match.group('skipped')
104+
if not fails and not errs and not skipped:
105+
continue
106+
107+
stats[unittests[-1]].num_fails = int(fails) if fails else 0
108+
stats[unittests[-1]].num_errors = int(errs) if errs else 0
109+
stats[unittests[-1]].num_skipped = int(skipped) if skipped else 0
110+
111+
with open('unittests.csv', 'w') as CSV:
112+
fieldnames = ['unittest', 'num_tests', 'num_fails', 'num_errors', 'num_skipped', 'num_passes', 'python_errors']
113+
writer = csv.DictWriter(CSV, fieldnames=fieldnames)
114+
writer.writeheader()
115+
116+
totals = {
117+
'num_tests': 0,
118+
'num_fails': 0,
119+
'num_errors': 0,
120+
'num_skipped': 0,
121+
'num_passes': 0,
122+
}
123+
total_not_run_at_all = 0
124+
total_pass_all = 0
125+
126+
for unittest in unittests:
127+
unittest_stats = stats[unittest]
128+
unittest_errmsg = error_messages[unittest]
129+
writer.writerow({
130+
'unittest': unittest,
131+
'num_tests': unittest_stats.num_tests,
132+
'num_fails': unittest_stats.num_fails,
133+
'num_errors': unittest_stats.num_errors,
134+
'num_skipped': unittest_stats.num_skipped,
135+
'num_passes': unittest_stats.num_passes,
136+
'python_errors': dumps(list(unittest_errmsg))
137+
})
138+
139+
# update totals that ran in some way
140+
if unittest_stats.num_tests > 0:
141+
totals['num_tests'] += unittest_stats.num_tests
142+
totals['num_fails'] += unittest_stats.num_fails
143+
totals['num_errors'] += unittest_stats.num_errors
144+
totals['num_skipped'] += unittest_stats.num_skipped
145+
totals['num_passes'] += unittest_stats.num_passes
146+
if unittest_stats.num_tests == unittest_stats.num_passes:
147+
total_pass_all += 1
148+
else:
149+
total_not_run_at_all += 1
150+
151+
_all_runs = len(unittests)-total_not_run_at_all
152+
_all_total = len(unittests)
153+
_percent_all_runs = float(_all_runs) / float(_all_total) * 100.0
154+
_percent_all_full_passes = float(total_pass_all) / float(_all_total) * 100.0
155+
156+
_test_runs = totals['num_passes']
157+
_test_total = totals['num_tests']
158+
_percent_test_runs = float(_test_runs) / float(_test_total) * 100.0
159+
160+
writer.writerow({
161+
'unittest': 'TOTAL',
162+
'num_tests': totals['num_tests'],
163+
'num_fails': totals['num_fails'],
164+
'num_errors': totals['num_errors'],
165+
'num_skipped': totals['num_skipped'],
166+
'num_passes': totals['num_passes'],
167+
'python_errors': 'Could run {0}/{1} unittests ({2:.2f}%). Unittests which pass completely: {3:.2f}%. Of the ones which ran, could run: {4}/{5} tests ({6:.2f}%)'.format(
168+
_all_runs, _all_total, _percent_all_runs, _percent_all_full_passes,
169+
_test_runs, _test_total, _percent_test_runs)
170+
})
171+
172+
if __name__ == '__main__':
173+
process_output(sys.argv[1])

‎scripts/py_unittests.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ cd ..
4242
UNIT_TESTS_PATH="graalpython/lib-python/3/test/test_*.py"
4343
for TEST in ${UNIT_TESTS_PATH}
4444
do
45-
echo "----------------------------------------------------------------------"
46-
echo "running: ${TEST}"
45+
echo "##############################################################"
46+
echo "#### running: ${TEST}"
4747
mx python3 --python.CatchAllExceptions=true ${TEST}
4848
done
4949

0 commit comments

Comments
 (0)