Skip to content

Commit 62e4d2f

Browse files
mi-acV8-internal LUCI CQ
mi-ac
authored and
V8-internal LUCI CQ
committed
Make transpiler results format more fine-grained.
This enhances the results format after test transpilation. Before, we had only one level of: {num_tests: int, failures: [{path: string, output: string}]} Now we'll key the two lowest directory levels in Test262, e.g. for a typical path like: language/literals/boolean/S7.8.2_A1_T1.js, the key would be language/literals. All results under this directory will be listed as a dict value, with numbers and failures as previously, further directories accordingly: { language/literals: {num_tests: ..., failures: ...}, language/identifiers: ... ... } We will now transpile all Test262 tests in one run and won't need to exclude any subdirectories, like staging, as we can now report separate numbers anyways. This also updates the merge script to the new format and adds additional unit tests for some helper functions. Bug: 442444727 Change-Id: Idf23c650c646bc970d81fc8a318d4a8c76797a4d Reviewed-on: https://chrome-internal-review.googlesource.com/c/v8/fuzzilli/+/8841396 Commit-Queue: Michael Achenbach <[email protected]> Reviewed-by: Liviu Rau <[email protected]>
1 parent 60b6d27 commit 62e4d2f

File tree

4 files changed

+242
-66
lines changed

4 files changed

+242
-66
lines changed

‎Tools/transpile_tests/merge_json_results.py‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,22 @@
2020
import json
2121
import sys
2222

23+
from collections import defaultdict, Counter
24+
2325

2426
def merge_test_results(inputs):
25-
num_tests = 0
26-
failures = []
27+
num_tests = Counter()
28+
failures = defaultdict(list)
29+
2730
for result in inputs:
28-
num_tests += result["num_tests"]
29-
failures += result["failures"]
31+
for key, value in result.items():
32+
num_tests[key] += value["num_tests"]
33+
failures[key] += value["failures"]
3034

31-
return {
32-
'num_tests': num_tests,
33-
'failures': failures,
34-
}
35+
return dict(
36+
(key, {'num_tests': num_tests[key], 'failures': failures[key]})
37+
for key in sorted(num_tests.keys())
38+
)
3539

3640

3741
def parse_args(args):
@@ -57,8 +61,7 @@ def main(args):
5761
with open(options.json_output, 'w') as f:
5862
json.dump(result, f, sort_keys=True, indent=2)
5963

60-
print(f'Merged results for {result["num_tests"]} tests '
61-
f'and {len(result["failures"])} failures.')
64+
print(f'Successfully merged results.')
6265

6366

6467
if __name__ == '__main__':

‎Tools/transpile_tests/test_merge_json_results.py‎

Lines changed: 45 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -28,19 +28,37 @@ class TestMergeResults(fake_filesystem_unittest.TestCase):
2828
def test_full_run(self, fs):
2929
with open('/in1.json', 'w') as f:
3030
json.dump({
31-
'num_tests': 2,
32-
'failures': [
33-
{'path': 'path/to/failure1', 'output': 'foo'},
34-
]
31+
'a/a': {
32+
'num_tests': 2,
33+
'failures': [],
34+
},
35+
'a/b': {
36+
'num_tests': 2,
37+
'failures': [
38+
{'path': 'a/b/c/failure1', 'output': 'foo'},
39+
],
40+
},
3541
}, f)
3642

3743
with open('/in2.json', 'w') as f:
3844
json.dump({
39-
'num_tests': 3,
40-
'failures': [
41-
{'path': 'path/to/failure2', 'output': 'bar 42\nbar 43'},
42-
{'path': 'path/to/failure3', 'output': 'baz'},
43-
]
45+
'a': {
46+
'num_tests': 1,
47+
'failures': [
48+
{'path': 'a/failure4', 'output': 'foo'},
49+
],
50+
},
51+
'a/a': {
52+
'num_tests': 1,
53+
'failures': [],
54+
},
55+
'a/b': {
56+
'num_tests': 3,
57+
'failures': [
58+
{'path': 'a/b/c/failure2', 'output': 'bar 42\nbar 43'},
59+
{'path': 'a/b/d/failure3', 'output': 'baz'},
60+
]
61+
},
4462
}, f)
4563

4664
f = io.StringIO()
@@ -53,21 +71,32 @@ def test_full_run(self, fs):
5371

5472
# Verify the output.
5573
self.assertEqual(
56-
'Merged results for 5 tests and 3 failures.',
74+
'Successfully merged results.',
5775
f.getvalue().strip())
5876

5977
# Verify the results written to the json output file.
6078
with open('/output.json') as f:
6179
actual_results = json.load(f)
6280

6381
expected_results = {
64-
'num_tests': 5,
65-
'failures': [
66-
{'path': 'path/to/failure1', 'output': 'foo'},
67-
{'path': 'path/to/failure2', 'output': 'bar 42\nbar 43'},
68-
{'path': 'path/to/failure3', 'output': 'baz'},
69-
],
82+
'a': {
83+
'failures': [{'output': 'foo', 'path': 'a/failure4'}],
84+
'num_tests': 1,
85+
},
86+
'a/a': {
87+
'failures': [],
88+
'num_tests': 3,
89+
},
90+
'a/b': {
91+
'failures': [
92+
{'output': 'foo', 'path': 'a/b/c/failure1'},
93+
{'output': 'bar 42\nbar 43', 'path': 'a/b/c/failure2'},
94+
{'output': 'baz', 'path': 'a/b/d/failure3'},
95+
],
96+
'num_tests': 5,
97+
},
7098
}
99+
71100
self.assertEqual(expected_results, actual_results)
72101

73102

‎Tools/transpile_tests/test_transpile_tests.py‎

Lines changed: 105 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -97,13 +97,14 @@ def test_full_run(self, fs):
9797
actual_results = json.load(f)
9898

9999
expected_results = {
100-
'num_tests': 4,
101-
'failures': [
102-
{
103-
'output': 'Failed!',
104-
'path': 'test/test262/data/test/folder2/Test4_fail.js',
105-
},
106-
],
100+
'folder1/subfolder1': {
101+
'failures': [],
102+
'num_tests': 2,
103+
},
104+
'folder2': {
105+
'failures': [{'output': 'Failed!', 'path': 'folder2/Test4_fail.js'}],
106+
'num_tests': 2,
107+
},
107108
}
108109
self.assertEqual(expected_results, actual_results)
109110

@@ -145,15 +146,105 @@ def test_shard_run(self, fs):
145146
actual_results = json.load(f)
146147

147148
expected_results = {
148-
'num_tests': 2,
149-
'failures': [
150-
{
151-
'output': 'Failed!',
152-
'path': 'test/test262/data/test/folder2/Test4_fail.js',
153-
},
154-
],
149+
'folder1/subfolder1': {
150+
'failures': [],
151+
'num_tests': 1,
152+
},
153+
'folder2': {
154+
'failures': [{'output': 'Failed!', 'path': 'folder2/Test4_fail.js'}],
155+
'num_tests': 1,
156+
},
155157
}
156158
self.assertEqual(expected_results, actual_results)
157159

160+
161+
Options = namedtuple('Options', 'verbose')
162+
163+
class UnitTests(unittest.TestCase):
164+
def test_level_key_0(self):
165+
counter = transpile_tests.TestCounter(0, Options(False))
166+
self.assertEqual('', counter.level_key(Path('test.js')))
167+
self.assertEqual('', counter.level_key(Path('level1/test.js')))
168+
169+
def test_level_key_2(self):
170+
counter = transpile_tests.TestCounter(2, Options(False))
171+
self.assertEqual(
172+
'', counter.level_key(Path('test.js')))
173+
self.assertEqual(
174+
'level1', counter.level_key(Path('level1/test.js')))
175+
self.assertEqual(
176+
'level1/level2',
177+
counter.level_key(Path('level1/level2/test.js')))
178+
self.assertEqual(
179+
'level1/level2',
180+
counter.level_key(Path('level1/level2/level3/test.js')))
181+
182+
def test_simple_counts(self):
183+
counter = transpile_tests.TestCounter(1, Options(False))
184+
self.assertEqual({}, counter.results())
185+
186+
counter.count(0, Path('pass1.js'), 'good')
187+
self.assertEqual(
188+
{
189+
'': {'failures': [], 'num_tests': 1},
190+
},
191+
counter.results())
192+
193+
counter.count(0, Path('a/b/pass1.js'), 'good')
194+
self.assertEqual(
195+
{
196+
'': {'failures': [], 'num_tests': 1},
197+
'a': {'failures': [], 'num_tests': 1},
198+
},
199+
counter.results())
200+
201+
counter.count(1, Path('a/fail1.js'), 'bad')
202+
self.assertEqual(
203+
{
204+
'': {'failures': [], 'num_tests': 1},
205+
'a': {
206+
'failures': [{'output': 'bad', 'path': 'a/fail1.js'}],
207+
'num_tests': 2,
208+
},
209+
},
210+
counter.results())
211+
212+
def test_complex_count(self):
213+
counter = transpile_tests.TestCounter(2, Options(False))
214+
counter.count(0, Path('A1/A2/3/pass1.js'), 'good')
215+
counter.count(0, Path('A1/B2/3/pass1.js'), 'good')
216+
counter.count(1, Path('A1/B2/3/fail1.js'), 'bad')
217+
counter.count(0, Path('A1/A2/3/pass2.js'), 'good')
218+
counter.count(0, Path('A1/A2/3/pass3.js'), 'good')
219+
counter.count(1, Path('fail4.js'), 'bad')
220+
counter.count(0, Path('B1/A2/3/pass1.js'), 'good')
221+
counter.count(1, Path('B1/A2/fail2.js'), 'bad')
222+
counter.count(1, Path('B1/fail3.js'), 'bad')
223+
counter.count(1, Path('fail5.js'), 'bad')
224+
counter.count(1, Path('fail6.js'), 'bad')
225+
counter.count(0, Path('A1/B2/3/pass4.js'), 'good')
226+
227+
self.assertEqual(
228+
{
229+
'': {'num_tests': 3, 'failures':[
230+
{'output': 'bad', 'path': 'fail4.js'},
231+
{'output': 'bad', 'path': 'fail5.js'},
232+
{'output': 'bad', 'path': 'fail6.js'},
233+
]},
234+
'B1': {'num_tests': 1, 'failures':[
235+
{'output': 'bad', 'path': 'B1/fail3.js'},
236+
]},
237+
'A1/A2': {'num_tests': 3, 'failures':[]},
238+
'A1/B2': {'num_tests': 3, 'failures':[
239+
{'output': 'bad', 'path': 'A1/B2/3/fail1.js'},
240+
]},
241+
'B1/A2': {'num_tests': 2, 'failures':[
242+
{'output': 'bad', 'path': 'B1/A2/fail2.js'},
243+
]},
244+
},
245+
counter.results()
246+
)
247+
248+
158249
if __name__ == '__main__':
159250
unittest.main()

0 commit comments

Comments
 (0)