diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index df95db6a1d6b0..75e3ca0e7d32d 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -73,6 +73,25 @@ def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, s
return failures
+def _format_ninja_failures(ninja_failures: list[tuple[str, str]]) -> list[str]:
+ """Formats ninja failures into summary views for the report."""
+ output = []
+ for build_failure in ninja_failures:
+ failed_action, failure_message = build_failure
+ output.extend(
+ [
+ "",
+ f"{failed_action}
",
+ "",
+ "```",
+ failure_message,
+ "```",
+ " ",
+ ]
+ )
+ return output
+
+
# Set size_limit to limit the byte size of the report. The default is 1MB as this
# is the most that can be put into an annotation. If the generated report exceeds
# this limit and failures are listed, it will be generated again without failures
@@ -83,6 +102,7 @@ def generate_report(
title,
return_code,
junit_objects,
+ ninja_logs: list[list[str]],
size_limit=1024 * 1024,
list_failures=True,
):
@@ -120,15 +140,34 @@ def generate_report(
]
)
else:
- report.extend(
- [
- "The build failed before running any tests.",
- "",
- SEE_BUILD_FILE_STR,
- "",
- UNRELATED_FAILURES_STR,
- ]
- )
+ ninja_failures = find_failure_in_ninja_logs(ninja_logs)
+ if not ninja_failures:
+ report.extend(
+ [
+ "The build failed before running any tests. Detailed "
+ "information about the build failure could not be "
+ "automatically obtained.",
+ "",
+ SEE_BUILD_FILE_STR,
+ "",
+ UNRELATED_FAILURES_STR,
+ ]
+ )
+ else:
+ report.extend(
+ [
+ "The build failed before running any tests. Click on a "
+ "failure below to see the details.",
+ "",
+ ]
+ )
+ report.extend(_format_ninja_failures(ninja_failures))
+ report.extend(
+ [
+ "",
+ UNRELATED_FAILURES_STR,
+ ]
+ )
return "\n".join(report)
tests_passed = tests_run - tests_skipped - tests_failed
@@ -173,14 +212,28 @@ def plural(num_tests):
elif return_code != 0:
# No tests failed but the build was in a failed state. Bring this to the user's
# attention.
- report.extend(
- [
- "",
- "All tests passed but another part of the build **failed**.",
- "",
- SEE_BUILD_FILE_STR,
- ]
- )
+ ninja_failures = find_failure_in_ninja_logs(ninja_logs)
+ if not ninja_failures:
+ report.extend(
+ [
+ "",
+ "All tests passed but another part of the build **failed**. "
+ "Information about the build failure could not be automatically "
+ "obtained.",
+ "",
+ SEE_BUILD_FILE_STR,
+ ]
+ )
+ else:
+ report.extend(
+ [
+ "",
+ "All tests passed but another part of the build **failed**. Click on "
+ "a failure below to see the details.",
+ "",
+ ]
+ )
+ report.extend(_format_ninja_failures(ninja_failures))
if failures or return_code != 0:
report.extend(["", UNRELATED_FAILURES_STR])
@@ -200,7 +253,5 @@ def plural(num_tests):
def generate_report_from_files(title, return_code, junit_files):
return generate_report(
- title,
- return_code,
- [JUnitXml.fromfile(p) for p in junit_files],
+ title, return_code, [JUnitXml.fromfile(p) for p in junit_files], []
)
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index 41f3eae591e19..389d781042e23 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -126,7 +126,7 @@ def test_ninja_log_multiple_failures(self):
def test_title_only(self):
self.assertEqual(
- generate_test_report_lib.generate_report("Foo", 0, []),
+ generate_test_report_lib.generate_report("Foo", 0, [], []),
dedent(
"""\
# Foo
@@ -137,12 +137,12 @@ def test_title_only(self):
def test_title_only_failure(self):
self.assertEqual(
- generate_test_report_lib.generate_report("Foo", 1, []),
+ generate_test_report_lib.generate_report("Foo", 1, [], []),
dedent(
"""\
# Foo
- The build failed before running any tests.
+ The build failed before running any tests. Detailed information about the build failure could not be automatically obtained.
Download the build's log file to see the details.
@@ -150,6 +150,45 @@ def test_title_only_failure(self):
),
)
+ def test_title_only_failure_ninja_log(self):
+ self.assertEqual(
+ generate_test_report_lib.generate_report(
+ "Foo",
+ 1,
+ [],
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: test/4.stamp",
+ "touch test/4.stamp",
+ "Wow! Risk!",
+ "[5/5] test/5.stamp",
+ ]
+ ],
+ ),
+ dedent(
+ """\
+ # Foo
+
+ The build failed before running any tests. Click on a failure below to see the details.
+
+
+ test/4.stamp
+
+ ```
+ FAILED: test/4.stamp
+ touch test/4.stamp
+ Wow! Risk!
+ ```
+
+
+ If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+ ),
+ )
+
def test_no_tests_in_testsuite(self):
self.assertEqual(
generate_test_report_lib.generate_report(
@@ -167,12 +206,13 @@ def test_no_tests_in_testsuite(self):
)
)
],
+ [],
),
dedent(
"""\
# Foo
- The build failed before running any tests.
+ The build failed before running any tests. Detailed information about the build failure could not be automatically obtained.
Download the build's log file to see the details.
@@ -198,6 +238,7 @@ def test_no_failures(self):
)
)
],
+ [],
),
(
dedent(
@@ -227,6 +268,7 @@ def test_no_failures_build_failed(self):
)
)
],
+ [],
),
(
dedent(
@@ -235,7 +277,7 @@ def test_no_failures_build_failed(self):
* 1 test passed
- All tests passed but another part of the build **failed**.
+ All tests passed but another part of the build **failed**. Information about the build failure could not be automatically obtained.
Download the build's log file to see the details.
@@ -244,6 +286,155 @@ def test_no_failures_build_failed(self):
),
)
+ def test_no_failures_build_failed_ninja_log(self):
+ self.assertEqual(
+ generate_test_report_lib.generate_report(
+ "Foo",
+ 1,
+ [
+ junit_from_xml(
+ dedent(
+ """\
+
+
+
+
+
+ """
+ )
+ )
+ ],
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: test/4.stamp",
+ "touch test/4.stamp",
+ "Wow! Close To You!",
+ "[5/5] test/5.stamp",
+ ]
+ ],
+ ),
+ (
+ dedent(
+ """\
+ # Foo
+
+ * 1 test passed
+
+ All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
+
+ test/4.stamp
+
+ ```
+ FAILED: test/4.stamp
+ touch test/4.stamp
+ Wow! Close To You!
+ ```
+
+
+ If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+ )
+ ),
+ )
+
+ def test_no_failures_multiple_build_failed_ninja_log(self):
+ test = generate_test_report_lib.generate_report(
+ "Foo",
+ 1,
+ [
+ junit_from_xml(
+ dedent(
+ """\
+
+
+
+
+
+ """
+ )
+ )
+ ],
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "FAILED: touch test/2.stamp",
+ "Wow! Be Kind!",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: touch test/4.stamp",
+ "Wow! I Dare You!",
+ "[5/5] test/5.stamp",
+ ]
+ ],
+ )
+ print(test)
+ self.assertEqual(
+ generate_test_report_lib.generate_report(
+ "Foo",
+ 1,
+ [
+ junit_from_xml(
+ dedent(
+ """\
+
+
+
+
+
+ """
+ )
+ )
+ ],
+ [
+ [
+ "[1/5] test/1.stamp",
+ "[2/5] test/2.stamp",
+ "FAILED: touch test/2.stamp",
+ "Wow! Be Kind!",
+ "[3/5] test/3.stamp",
+ "[4/5] test/4.stamp",
+ "FAILED: touch test/4.stamp",
+ "Wow! I Dare You!",
+ "[5/5] test/5.stamp",
+ ]
+ ],
+ ),
+ (
+ dedent(
+ """\
+ # Foo
+
+ * 1 test passed
+
+ All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
+
+ test/2.stamp
+
+ ```
+ FAILED: touch test/2.stamp
+ Wow! Be Kind!
+ ```
+
+
+ test/4.stamp
+
+ ```
+ FAILED: touch test/4.stamp
+ Wow! I Dare You!
+ ```
+
+
+ If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+ )
+ ),
+ )
+
def test_report_single_file_single_testsuite(self):
self.assertEqual(
generate_test_report_lib.generate_report(
@@ -271,6 +462,7 @@ def test_report_single_file_single_testsuite(self):
)
)
],
+ [],
),
(
dedent(
@@ -366,6 +558,7 @@ def test_report_single_file_multiple_testsuites(self):
)
)
],
+ [],
),
self.MULTI_SUITE_OUTPUT,
)
@@ -407,6 +600,7 @@ def test_report_multiple_files_multiple_testsuites(self):
)
),
],
+ [],
),
self.MULTI_SUITE_OUTPUT,
)
@@ -431,6 +625,7 @@ def test_report_dont_list_failures(self):
)
)
],
+ [],
list_failures=False,
),
(
@@ -467,6 +662,7 @@ def test_report_dont_list_failures_link_to_log(self):
)
)
],
+ [],
list_failures=False,
),
(
@@ -506,6 +702,7 @@ def test_report_size_limit(self):
)
)
],
+ [],
size_limit=512,
),
(