Skip to content

Commit 6ed6c4c

Browse files
boomanaiden154boomanaiden154
committed
[CI] Setup generate_report to describe ninja failures
This patch makes it so that generate_report will add information about failed build actions to the summary report. This makes it significantly easier to find compilation failures, especially given we run ninja with -k 0. This patch only does the integration into generate_report (along with testing). Actual utilization in the script is split into a separate patch to try and keep things clean. Pull Request: llvm#152621
1 parent 8777e10 commit 6ed6c4c

File tree

2 files changed

+167
-22
lines changed

2 files changed

+167
-22
lines changed

‎.ci/generate_test_report_lib.py

Lines changed: 62 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,25 @@ def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, s
6767
return failures
6868

6969

70+
def _format_ninja_failures(ninja_failures: list[tuple[str, str]]) -> list[str]:
71+
"""Formats ninja failures into summary views for the report."""
72+
output = []
73+
for build_failure in ninja_failures:
74+
failed_action, failure_message = build_failure
75+
output.extend(
76+
[
77+
"<details>",
78+
f"<summary>{failed_action}</summary>",
79+
"",
80+
"```",
81+
failure_message,
82+
"```",
83+
"</details>",
84+
]
85+
)
86+
return output
87+
88+
7089
# Set size_limit to limit the byte size of the report. The default is 1MB as this
7190
# is the most that can be put into an annotation. If the generated report exceeds
7291
# this limit and failures are listed, it will be generated again without failures
@@ -77,6 +96,7 @@ def generate_report(
7796
title,
7897
return_code,
7998
junit_objects,
99+
ninja_logs: list[list[str]],
80100
size_limit=1024 * 1024,
81101
list_failures=True,
82102
):
@@ -114,15 +134,34 @@ def generate_report(
114134
]
115135
)
116136
else:
117-
report.extend(
118-
[
119-
"The build failed before running any tests.",
120-
"",
121-
SEE_BUILD_FILE_STR,
122-
"",
123-
UNRELATED_FAILURES_STR,
124-
]
125-
)
137+
ninja_failures = find_failure_in_ninja_logs(ninja_logs)
138+
if not ninja_failures:
139+
report.extend(
140+
[
141+
"The build failed before running any tests. Detailed "
142+
"information about the build failure could not be "
143+
"automatically obtained.",
144+
"",
145+
SEE_BUILD_FILE_STR,
146+
"",
147+
UNRELATED_FAILURES_STR,
148+
]
149+
)
150+
else:
151+
report.extend(
152+
[
153+
"The build failed before running any tests. Click on a "
154+
"failure below to see the details.",
155+
"",
156+
]
157+
)
158+
report.extend(_format_ninja_failures(ninja_failures))
159+
report.extend(
160+
[
161+
"",
162+
UNRELATED_FAILURES_STR,
163+
]
164+
)
126165
return "\n".join(report)
127166

128167
tests_passed = tests_run - tests_skipped - tests_failed
@@ -167,14 +206,20 @@ def plural(num_tests):
167206
elif return_code != 0:
168207
# No tests failed but the build was in a failed state. Bring this to the user's
169208
# attention.
170-
report.extend(
171-
[
172-
"",
173-
"All tests passed but another part of the build **failed**.",
174-
"",
175-
SEE_BUILD_FILE_STR,
176-
]
177-
)
209+
ninja_failures = find_failure_in_ninja_logs(ninja_logs)
210+
if not ninja_failures:
211+
report.extend(
212+
[
213+
"",
214+
"All tests passed but another part of the build **failed**. "
215+
"Information about the build failure could not be automatically "
216+
"obtained.",
217+
"",
218+
SEE_BUILD_FILE_STR,
219+
]
220+
)
221+
else:
222+
report.extend(_format_ninja_failures(ninja_failures))
178223

179224
if failures or return_code != 0:
180225
report.extend(["", UNRELATED_FAILURES_STR])

‎.ci/generate_test_report_lib_test.py

Lines changed: 105 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ def test_ninja_log_multiple_failures(self):
126126

127127
def test_title_only(self):
128128
self.assertEqual(
129-
generate_test_report_lib.generate_report("Foo", 0, []),
129+
generate_test_report_lib.generate_report("Foo", 0, [], []),
130130
dedent(
131131
"""\
132132
# Foo
@@ -137,19 +137,58 @@ def test_title_only(self):
137137

138138
def test_title_only_failure(self):
139139
self.assertEqual(
140-
generate_test_report_lib.generate_report("Foo", 1, []),
140+
generate_test_report_lib.generate_report("Foo", 1, [], []),
141141
dedent(
142142
"""\
143143
# Foo
144144
145-
The build failed before running any tests.
145+
The build failed before running any tests. Detailed information about the build failure could not be automatically obtained.
146146
147147
Download the build's log file to see the details.
148148
149149
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
150150
),
151151
)
152152

153+
def test_title_only_failure_ninja_log(self):
154+
self.assertEqual(
155+
generate_test_report_lib.generate_report(
156+
"Foo",
157+
1,
158+
[],
159+
[
160+
[
161+
"[1/5] test/1.stamp",
162+
"[2/5] test/2.stamp",
163+
"[3/5] test/3.stamp",
164+
"[4/5] test/4.stamp",
165+
"FAILED: test/4.stamp",
166+
"touch test/4.stamp",
167+
"Wow! Risk!",
168+
"[5/5] test/5.stamp",
169+
]
170+
],
171+
),
172+
dedent(
173+
"""\
174+
# Foo
175+
176+
The build failed before running any tests. Click on a failure below to see the details.
177+
178+
<details>
179+
<summary>test/4.stamp</summary>
180+
181+
```
182+
FAILED: test/4.stamp
183+
touch test/4.stamp
184+
Wow! Risk!
185+
```
186+
</details>
187+
188+
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
189+
),
190+
)
191+
153192
def test_no_tests_in_testsuite(self):
154193
self.assertEqual(
155194
generate_test_report_lib.generate_report(
@@ -167,12 +206,13 @@ def test_no_tests_in_testsuite(self):
167206
)
168207
)
169208
],
209+
[],
170210
),
171211
dedent(
172212
"""\
173213
# Foo
174214
175-
The build failed before running any tests.
215+
The build failed before running any tests. Detailed information about the build failure could not be automatically obtained.
176216
177217
Download the build's log file to see the details.
178218
@@ -198,6 +238,7 @@ def test_no_failures(self):
198238
)
199239
)
200240
],
241+
[],
201242
),
202243
(
203244
dedent(
@@ -227,6 +268,7 @@ def test_no_failures_build_failed(self):
227268
)
228269
)
229270
],
271+
[],
230272
),
231273
(
232274
dedent(
@@ -235,7 +277,7 @@ def test_no_failures_build_failed(self):
235277
236278
* 1 test passed
237279
238-
All tests passed but another part of the build **failed**.
280+
All tests passed but another part of the build **failed**. Information about the build failure could not be automatically obtained.
239281
240282
Download the build's log file to see the details.
241283
@@ -244,6 +286,58 @@ def test_no_failures_build_failed(self):
244286
),
245287
)
246288

289+
def test_no_failures_build_failed_ninja_log(self):
290+
self.assertEqual(
291+
generate_test_report_lib.generate_report(
292+
"Foo",
293+
1,
294+
[
295+
junit_from_xml(
296+
dedent(
297+
"""\
298+
<?xml version="1.0" encoding="UTF-8"?>
299+
<testsuites time="0.00">
300+
<testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
301+
<testcase classname="Bar/test_1" name="test_1" time="0.00"/>
302+
</testsuite>
303+
</testsuites>"""
304+
)
305+
)
306+
],
307+
[
308+
[
309+
"[1/5] test/1.stamp",
310+
"[2/5] test/2.stamp",
311+
"[3/5] test/3.stamp",
312+
"[4/5] test/4.stamp",
313+
"FAILED: test/4.stamp",
314+
"touch test/4.stamp",
315+
"Wow! Close To You!",
316+
"[5/5] test/5.stamp",
317+
]
318+
],
319+
),
320+
(
321+
dedent(
322+
"""\
323+
# Foo
324+
325+
* 1 test passed
326+
<details>
327+
<summary>test/4.stamp</summary>
328+
329+
```
330+
FAILED: test/4.stamp
331+
touch test/4.stamp
332+
Wow! Close To You!
333+
```
334+
</details>
335+
336+
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
337+
)
338+
),
339+
)
340+
247341
def test_report_single_file_single_testsuite(self):
248342
self.assertEqual(
249343
generate_test_report_lib.generate_report(
@@ -271,6 +365,7 @@ def test_report_single_file_single_testsuite(self):
271365
)
272366
)
273367
],
368+
[],
274369
),
275370
(
276371
dedent(
@@ -366,6 +461,7 @@ def test_report_single_file_multiple_testsuites(self):
366461
)
367462
)
368463
],
464+
[],
369465
),
370466
self.MULTI_SUITE_OUTPUT,
371467
)
@@ -407,6 +503,7 @@ def test_report_multiple_files_multiple_testsuites(self):
407503
)
408504
),
409505
],
506+
[],
410507
),
411508
self.MULTI_SUITE_OUTPUT,
412509
)
@@ -431,6 +528,7 @@ def test_report_dont_list_failures(self):
431528
)
432529
)
433530
],
531+
[],
434532
list_failures=False,
435533
),
436534
(
@@ -467,6 +565,7 @@ def test_report_dont_list_failures_link_to_log(self):
467565
)
468566
)
469567
],
568+
[],
470569
list_failures=False,
471570
),
472571
(
@@ -506,6 +605,7 @@ def test_report_size_limit(self):
506605
)
507606
)
508607
],
608+
[],
509609
size_limit=512,
510610
),
511611
(

0 commit comments

Comments
 (0)