Skip to content

Commit fcbce8e

Browse files
msimacekmsimacek
committed
[GR-22564] Improve unittest retagger, part 1
PullRequest: graalpython/923
2 parents 7ae61e2 + 19294f1 commit fcbce8e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+318
-3
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ Python3.g4.stamp
5656
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/*.sha256
5757
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/PointerEquality_Primitive.c
5858
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/buffer.c
59+
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/dealloc.c
5960
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/format_specifier_w_star.c
6061
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/memoryview_member_flags.c
6162
/graalpython/com.oracle.graal.python.test/src/tests/cpyext/parseargs_H.c

‎graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,17 +101,39 @@ def fun(self):
101101
del test_f
102102

103103

104+
# This function has a unittest in test_tagger
105+
def parse_unittest_output(output):
106+
# The whole reason for this function's complexity is that we want to consume arbitrary
107+
# warnings after the '...' part without accidentally consuming the next test result
108+
import re
109+
re_test_result = re.compile(r"""\b(test\S+) \(([^\s]+)\)(?:\n.*?)?? \.\.\. """, re.MULTILINE | re.DOTALL)
110+
re_test_status = re.compile(r"""\b(ok|skipped (?:'[^']*'|"[^"]*")|FAIL|ERROR)$""", re.MULTILINE | re.DOTALL)
111+
pos = 0
112+
current_result = None
113+
while True:
114+
result_match = re_test_result.search(output, pos)
115+
status_match = re_test_status.search(output, pos)
116+
if current_result and status_match and (not result_match or status_match.start() < result_match.start()):
117+
yield current_result.group(1), current_result.group(2), status_match.group(1)
118+
current_result = None
119+
pos = status_match.end()
120+
elif result_match:
121+
current_result = result_match
122+
pos = result_match.end()
123+
else:
124+
return
125+
126+
104127
if __name__ == "__main__":
105128
# find working tests
106129
import re
107130

108131
executable = sys.executable.split(" ") # HACK: our sys.executable on Java is a cmdline
109-
re_test_result = re.compile(r"""(test\S+) \(([^\s]+)\)(?:\n.*?)? \.\.\. \b(ok|skipped(?: ["'][^\n]+)?|ERROR|FAIL)$""", re.MULTILINE | re.DOTALL)
110132
kwargs = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True, "check": False}
111133

112134
glob_pattern = os.path.join(os.path.dirname(test.__file__), "test_*.py")
113135
retag = False
114-
maxrepeats = 2
136+
maxrepeats = 4
115137
for arg in sys.argv[1:]:
116138
if arg == "--retag":
117139
retag = True
@@ -189,7 +211,7 @@ def get_pass_name(funcname, classname):
189211

190212
# n.b.: we add a '*' in the front, so that unittests doesn't add
191213
# its own asterisks, because now this is already a pattern
192-
for funcname, classname, result in re_test_result.findall(stderr):
214+
for funcname, classname, result in parse_unittest_output(stderr):
193215
# We consider skipped tests as passing in order to avoid a situation where a Linux run
194216
# untags a Darwin-only test and vice versa
195217
if result == 'ok' or result.startswith('skipped'):
@@ -209,13 +231,19 @@ def get_pass_name(funcname, classname):
209231
f.write("\n")
210232
if not passing_only_patterns:
211233
os.unlink(tagfile)
234+
print("No successful tests remaining")
235+
break
212236

213237
if p.returncode == 0:
238+
print(f"Suite succeeded with {len(passing_only_patterns)} tests")
214239
break
240+
else:
241+
print(f"Suite failed, retrying with {len(passing_only_patterns)} tests")
215242

216243
else:
217244
# we tried the last time and failed, so our tags don't work for
218245
# some reason
246+
print("The suite failed even in the last attempt, untagging completely")
219247
try:
220248
os.unlink(tagfile)
221249
except Exception:

‎graalpython/com.oracle.graal.python.test/src/tests/test_tagger.py

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
2+
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
3+
#
4+
# The Universal Permissive License (UPL), Version 1.0
5+
#
6+
# Subject to the condition set forth below, permission is hereby granted to any
7+
# person obtaining a copy of this software, associated documentation and/or
8+
# data (collectively the "Software"), free of charge and under any and all
9+
# copyright rights in the Software, and any and all patent rights owned or
10+
# freely licensable by each licensor hereunder covering either (i) the
11+
# unmodified Software as contributed to or provided by such licensor, or (ii)
12+
# the Larger Works (as defined below), to deal in both
13+
#
14+
# (a) the Software, and
15+
#
16+
# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
17+
# one is included with the Software each a "Larger Work" to which the Software
18+
# is contributed by such licensors),
19+
#
20+
# without restriction, including without limitation the rights to copy, create
21+
# derivative works of, display, perform, and distribute the Software and make,
22+
# use, sell, offer for sale, import, export, have made, and have sold the
23+
# Software and the Larger Work(s), and to sublicense the foregoing rights on
24+
# either these or other terms.
25+
#
26+
# This license is subject to the following condition:
27+
#
28+
# The above copyright notice and either this complete permission notice or at a
29+
# minimum a reference to the UPL must be included in all copies or substantial
30+
# portions of the Software.
31+
#
32+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
33+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
34+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
35+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
36+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
37+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38+
# SOFTWARE.
39+
40+
# test for the output parsing in test_tagged_unittests
41+
from . import test_tagged_unittests
42+
43+
data = """
44+
test_1 (module) ... ok
45+
test2 (module) ... FAIL
46+
random output
47+
test_2 (module) ... skipped 'requires Windows'
48+
test_3 (module) ... skipped 'some
49+
multiline reason'
50+
some unrelated output
51+
test_x (module)
52+
Some docstring that
53+
spans
54+
multiple
55+
lines ... ERROR
56+
test_y (module)
57+
Some docstring ... Some warning
58+
ok
59+
unrelated stuff
60+
test_z (module) ... ok
61+
testA (module) ... test_B (module) ... ok
62+
test_T (module) ... the test printed ok somewhere
63+
skipped 'foo'
64+
test_foo (module) ... test_bar (module) ... null
65+
"""
66+
67+
expected = [
68+
('test_1', 'module', 'ok'),
69+
('test2', 'module', 'FAIL'),
70+
('test_2', 'module', "skipped 'requires Windows'"),
71+
('test_3', 'module', "skipped 'some\nmultiline reason'"),
72+
('test_x', 'module', 'ERROR'),
73+
('test_y', 'module', 'ok'),
74+
('test_z', 'module', 'ok'),
75+
('test_B', 'module', 'ok'),
76+
('test_T', 'module', "skipped 'foo'"),
77+
]
78+
79+
80+
def test_re():
81+
parsed = list(test_tagged_unittests.parse_unittest_output(data))
82+
assert parsed == expected, parsed

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_ast.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
*AST_Tests.test_AST_garbage_collection
22
*AST_Tests.test_field_attr_writable
3+
*AST_Tests.test_issue31592
34
*AST_Tests.test_realtype

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_atexit.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
*SubinterpreterTest.test_callback_on_subinterpreter_teardown
2+
*SubinterpreterTest.test_callbacks_leak
3+
*SubinterpreterTest.test_callbacks_leak_refcycle

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_builtin.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
*BuiltinTest.test_all
22
*BuiltinTest.test_any
3+
*BuiltinTest.test_bin

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_call.txt

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,40 @@
2424
*CFunctionCalls.test_varargs2
2525
*CFunctionCalls.test_varargs2_ext
2626
*CFunctionCalls.test_varargs2_kw
27+
*CFunctionCallsErrorMessages.test_oldargs0_1
28+
*CFunctionCallsErrorMessages.test_oldargs0_1_kw
29+
*CFunctionCallsErrorMessages.test_oldargs0_2
30+
*CFunctionCallsErrorMessages.test_oldargs0_2_kw
31+
*CFunctionCallsErrorMessages.test_oldargs1_0
32+
*CFunctionCallsErrorMessages.test_oldargs1_0_kw
33+
*CFunctionCallsErrorMessages.test_oldargs1_1_kw
34+
*CFunctionCallsErrorMessages.test_oldargs1_2
35+
*CFunctionCallsErrorMessages.test_oldargs1_2_kw
36+
*CFunctionCallsErrorMessages.test_varargs0
37+
*CFunctionCallsErrorMessages.test_varargs10_kw
38+
*CFunctionCallsErrorMessages.test_varargs11_kw
39+
*CFunctionCallsErrorMessages.test_varargs12_kw
40+
*CFunctionCallsErrorMessages.test_varargs13_kw
41+
*CFunctionCallsErrorMessages.test_varargs14_kw
42+
*CFunctionCallsErrorMessages.test_varargs15_kw
43+
*CFunctionCallsErrorMessages.test_varargs16_kw
44+
*CFunctionCallsErrorMessages.test_varargs17_kw
45+
*CFunctionCallsErrorMessages.test_varargs1_kw
46+
*CFunctionCallsErrorMessages.test_varargs1max
47+
*CFunctionCallsErrorMessages.test_varargs1min
48+
*CFunctionCallsErrorMessages.test_varargs2
49+
*CFunctionCallsErrorMessages.test_varargs2_kw
50+
*CFunctionCallsErrorMessages.test_varargs2max
51+
*CFunctionCallsErrorMessages.test_varargs2min
52+
*CFunctionCallsErrorMessages.test_varargs3
53+
*CFunctionCallsErrorMessages.test_varargs3_kw
54+
*CFunctionCallsErrorMessages.test_varargs4_kw
55+
*CFunctionCallsErrorMessages.test_varargs5_kw
56+
*CFunctionCallsErrorMessages.test_varargs6_kw
57+
*CFunctionCallsErrorMessages.test_varargs7_kw
58+
*CFunctionCallsErrorMessages.test_varargs8_kw
59+
*CFunctionCallsErrorMessages.test_varargs9_kw
60+
*FastCallTests.test_fastcall
2761
*FastCallTests.test_fastcall_clearing_dict
62+
*FastCallTests.test_vectorcall
63+
*FastCallTests.test_vectorcall_dict

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_class.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,6 @@
22
*ClassTests.testGetSetAndDel
33
*ClassTests.testInit
44
*ClassTests.testMisc
5+
*ClassTests.testSFBug532646
6+
*ClassTests.testSetattrWrapperNameIntern
7+
*ClassTests.testUnaryOps

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_code.txt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
*CodeConstsTest.test_interned_string
2+
*CodeConstsTest.test_interned_string_default
3+
*CodeConstsTest.test_interned_string_in_frozenset
4+
*CodeConstsTest.test_interned_string_in_tuple
5+
*CodeConstsTest.test_interned_string_with_null
6+
*CodeTest.test_closure_injection
7+
*CodeTest.test_constructor
8+
*CodeTest.test_newempty
9+
*CodeTest.test_replace

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_coroutines.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
*AsyncBadSyntaxTest.test_badsyntax_2
22
*AsyncBadSyntaxTest.test_badsyntax_3
33
*AsyncBadSyntaxTest.test_badsyntax_4
4+
*CAPITest.test_tp_await_1
5+
*CAPITest.test_tp_await_2
6+
*CAPITest.test_tp_await_3
47
*CoroutineTest.test_comp_7
58
*CoroutineTest.test_for_tuple
69
*CoroutineTest.test_gen_1

0 commit comments

Comments
 (0)