Skip to content

Commit 3094627

Browse files
boomanaiden154boomanaiden154
committed
[CI] Setup generate_report to describe ninja failures
This patch makes it so that generate_report will add information about failed build actions to the summary report. This makes it significantly easier to find compilation failures, especially given we run ninja with -k 0. This patch only does the integration into generate_report (along with testing). Actual utilization in the script is split into a separate patch to try and keep things clean. Pull Request: llvm#152621
1 parent 66bdf0d commit 3094627

File tree

2 files changed

+172
-22
lines changed

2 files changed

+172
-22
lines changed

‎.ci/generate_test_report_lib.py

Lines changed: 67 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ def generate_report(
7575
title,
7676
return_code,
7777
junit_objects,
78+
ninja_logs: list[list[str]],
7879
size_limit=1024 * 1024,
7980
list_failures=True,
8081
):
@@ -112,15 +113,46 @@ def generate_report(
112113
]
113114
)
114115
else:
115-
report.extend(
116-
[
117-
"The build failed before running any tests.",
118-
"",
119-
SEE_BUILD_FILE_STR,
120-
"",
121-
UNRELATED_FAILURES_STR,
122-
]
123-
)
116+
ninja_failures = find_failure_in_ninja_logs(ninja_logs)
117+
if not ninja_failures:
118+
report.extend(
119+
[
120+
"The build failed before running any tests. Detailed "
121+
"information about the build failure could not be "
122+
"automatically obtained.",
123+
"",
124+
SEE_BUILD_FILE_STR,
125+
"",
126+
UNRELATED_FAILURES_STR,
127+
]
128+
)
129+
else:
130+
report.extend(
131+
[
132+
"The build failed before running any tests. Click on the "
133+
"failure below to see the details.",
134+
"",
135+
]
136+
)
137+
for build_failure in ninja_failures:
138+
failed_action, failure_message = build_failure
139+
report.extend(
140+
[
141+
"<details>",
142+
f"<summary>{failed_action}</summary>",
143+
"",
144+
"```",
145+
failure_message,
146+
"```",
147+
"</details>",
148+
]
149+
)
150+
report.extend(
151+
[
152+
"",
153+
UNRELATED_FAILURES_STR,
154+
]
155+
)
124156
return "\n".join(report)
125157

126158
tests_passed = tests_run - tests_skipped - tests_failed
@@ -165,14 +197,32 @@ def plural(num_tests):
165197
elif return_code != 0:
166198
# No tests failed but the build was in a failed state. Bring this to the user's
167199
# attention.
168-
report.extend(
169-
[
170-
"",
171-
"All tests passed but another part of the build **failed**.",
172-
"",
173-
SEE_BUILD_FILE_STR,
174-
]
175-
)
200+
ninja_failures = find_failure_in_ninja_logs(ninja_logs)
201+
if not ninja_failures:
202+
report.extend(
203+
[
204+
"",
205+
"All tests passed but another part of the build **failed**. "
206+
"Detailed information about the build failure could not be "
207+
"automatically obtained.",
208+
"",
209+
SEE_BUILD_FILE_STR,
210+
]
211+
)
212+
else:
213+
for build_failure in ninja_failures:
214+
failed_action, failure_message = build_failure
215+
report.extend(
216+
[
217+
"<details>",
218+
f"<summary>{failed_action}</summary>",
219+
"",
220+
"```",
221+
failure_message,
222+
"```",
223+
"</details>",
224+
]
225+
)
176226

177227
if failures or return_code != 0:
178228
report.extend(["", UNRELATED_FAILURES_STR])

‎.ci/generate_test_report_lib_test.py

Lines changed: 105 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ def test_ninja_log_multiple_failures(self):
128128

129129
def test_title_only(self):
130130
self.assertEqual(
131-
generate_test_report_lib.generate_report("Foo", 0, []),
131+
generate_test_report_lib.generate_report("Foo", 0, [], []),
132132
dedent(
133133
"""\
134134
# Foo
@@ -139,19 +139,58 @@ def test_title_only(self):
139139

140140
def test_title_only_failure(self):
141141
self.assertEqual(
142-
generate_test_report_lib.generate_report("Foo", 1, []),
142+
generate_test_report_lib.generate_report("Foo", 1, [], []),
143143
dedent(
144144
"""\
145145
# Foo
146146
147-
The build failed before running any tests.
147+
The build failed before running any tests. Detailed information about the build failure could not be automatically obtained.
148148
149149
Download the build's log file to see the details.
150150
151151
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
152152
),
153153
)
154154

155+
def test_title_only_failure_ninja_log(self):
156+
self.assertEqual(
157+
generate_test_report_lib.generate_report(
158+
"Foo",
159+
1,
160+
[],
161+
[
162+
[
163+
"[1/5] test/1.stamp",
164+
"[2/5] test/2.stamp",
165+
"[3/5] test/3.stamp",
166+
"[4/5] test/4.stamp",
167+
"FAILED: test/4.stamp",
168+
"touch test/4.stamp",
169+
"Wow! Risk!",
170+
"[5/5] test/5.stamp",
171+
]
172+
],
173+
),
174+
dedent(
175+
"""\
176+
# Foo
177+
178+
The build failed before running any tests. Click on the failure below to see the details.
179+
180+
<details>
181+
<summary>test/4.stamp</summary>
182+
183+
```
184+
FAILED: test/4.stamp
185+
touch test/4.stamp
186+
Wow! Risk!
187+
```
188+
</details>
189+
190+
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
191+
),
192+
)
193+
155194
def test_no_tests_in_testsuite(self):
156195
self.assertEqual(
157196
generate_test_report_lib.generate_report(
@@ -169,12 +208,13 @@ def test_no_tests_in_testsuite(self):
169208
)
170209
)
171210
],
211+
[],
172212
),
173213
dedent(
174214
"""\
175215
# Foo
176216
177-
The build failed before running any tests.
217+
The build failed before running any tests. Detailed information about the build failure could not be automatically obtained.
178218
179219
Download the build's log file to see the details.
180220
@@ -200,6 +240,7 @@ def test_no_failures(self):
200240
)
201241
)
202242
],
243+
[],
203244
),
204245
(
205246
dedent(
@@ -229,6 +270,7 @@ def test_no_failures_build_failed(self):
229270
)
230271
)
231272
],
273+
[],
232274
),
233275
(
234276
dedent(
@@ -237,7 +279,7 @@ def test_no_failures_build_failed(self):
237279
238280
* 1 test passed
239281
240-
All tests passed but another part of the build **failed**.
282+
All tests passed but another part of the build **failed**. Detailed information about the build failure could not be automatically obtained.
241283
242284
Download the build's log file to see the details.
243285
@@ -246,6 +288,58 @@ def test_no_failures_build_failed(self):
246288
),
247289
)
248290

291+
def test_no_failures_build_failed_ninja_log(self):
292+
self.assertEqual(
293+
generate_test_report_lib.generate_report(
294+
"Foo",
295+
1,
296+
[
297+
junit_from_xml(
298+
dedent(
299+
"""\
300+
<?xml version="1.0" encoding="UTF-8"?>
301+
<testsuites time="0.00">
302+
<testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
303+
<testcase classname="Bar/test_1" name="test_1" time="0.00"/>
304+
</testsuite>
305+
</testsuites>"""
306+
)
307+
)
308+
],
309+
[
310+
[
311+
"[1/5] test/1.stamp",
312+
"[2/5] test/2.stamp",
313+
"[3/5] test/3.stamp",
314+
"[4/5] test/4.stamp",
315+
"FAILED: test/4.stamp",
316+
"touch test/4.stamp",
317+
"Wow! Close To You!",
318+
"[5/5] test/5.stamp",
319+
]
320+
],
321+
),
322+
(
323+
dedent(
324+
"""\
325+
# Foo
326+
327+
* 1 test passed
328+
<details>
329+
<summary>test/4.stamp</summary>
330+
331+
```
332+
FAILED: test/4.stamp
333+
touch test/4.stamp
334+
Wow! Close To You!
335+
```
336+
</details>
337+
338+
If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
339+
)
340+
),
341+
)
342+
249343
def test_report_single_file_single_testsuite(self):
250344
self.assertEqual(
251345
generate_test_report_lib.generate_report(
@@ -273,6 +367,7 @@ def test_report_single_file_single_testsuite(self):
273367
)
274368
)
275369
],
370+
[],
276371
),
277372
(
278373
dedent(
@@ -368,6 +463,7 @@ def test_report_single_file_multiple_testsuites(self):
368463
)
369464
)
370465
],
466+
[],
371467
),
372468
self.MULTI_SUITE_OUTPUT,
373469
)
@@ -409,6 +505,7 @@ def test_report_multiple_files_multiple_testsuites(self):
409505
)
410506
),
411507
],
508+
[],
412509
),
413510
self.MULTI_SUITE_OUTPUT,
414511
)
@@ -433,6 +530,7 @@ def test_report_dont_list_failures(self):
433530
)
434531
)
435532
],
533+
[],
436534
list_failures=False,
437535
),
438536
(
@@ -469,6 +567,7 @@ def test_report_dont_list_failures_link_to_log(self):
469567
)
470568
)
471569
],
570+
[],
472571
list_failures=False,
473572
),
474573
(
@@ -508,6 +607,7 @@ def test_report_size_limit(self):
508607
)
509608
)
510609
],
610+
[],
511611
size_limit=512,
512612
),
513613
(

0 commit comments

Comments
 (0)