diff --git a/.ci/generate_test_report_github.py b/.ci/generate_test_report_github.py index 4b7f3a2367d50..7242264723cbf 100644 --- a/.ci/generate_test_report_github.py +++ b/.ci/generate_test_report_github.py @@ -16,11 +16,13 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("return_code", help="The build's return code.", type=int) - parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*") + parser.add_argument( + "build_test_logs", help="Paths to JUnit report files and ninja logs.", nargs="*" + ) args = parser.parse_args() report = generate_test_report_lib.generate_report_from_files( - PLATFORM_TITLES[platform.system()], args.return_code, args.junit_files + PLATFORM_TITLES[platform.system()], args.return_code, args.build_test_logs ) print(report) diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py index 75e3ca0e7d32d..d868c08ab69ef 100644 --- a/.ci/generate_test_report_lib.py +++ b/.ci/generate_test_report_lib.py @@ -251,7 +251,19 @@ def plural(num_tests): return report -def generate_report_from_files(title, return_code, junit_files): +def generate_report_from_files(title, return_code, build_log_files): + junit_files = [ + junit_file for junit_file in build_log_files if junit_file.endswith(".xml") + ] + ninja_log_files = [ + ninja_log for ninja_log in build_log_files if ninja_log.endswith(".log") + ] + ninja_logs = [] + for ninja_log_file in ninja_log_files: + with open(ninja_log_file, "r") as ninja_log_file_handle: + ninja_logs.append( + [log_line.strip() for log_line in ninja_log_file_handle.readlines()] + ) return generate_report( - title, return_code, [JUnitXml.fromfile(p) for p in junit_files], [] + title, return_code, [JUnitXml.fromfile(p) for p in junit_files], ninja_logs ) diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py index 389d781042e23..466a8234776dc 100644 --- a/.ci/generate_test_report_lib_test.py +++ b/.ci/generate_test_report_lib_test.py @@ -8,6 +8,8 @@ import unittest from io import StringIO from textwrap import dedent +import tempfile +import os from junitparser import JUnitXml @@ -718,3 +720,59 @@ def test_report_size_limit(self): ) ), ) + + def test_generate_report_end_to_end(self): + with tempfile.TemporaryDirectory() as temp_dir: + junit_xml_file = os.path.join(temp_dir, "junit.xml") + with open(junit_xml_file, "w") as junit_xml_handle: + junit_xml_handle.write( + dedent( + """\ + + + + + + """ + ) + ) + ninja_log_file = os.path.join(temp_dir, "ninja.log") + with open(ninja_log_file, "w") as ninja_log_handle: + ninja_log_handle.write( + dedent( + """\ + [1/5] test/1.stamp + [2/5] test/2.stamp + [3/5] test/3.stamp + [4/5] test/4.stamp + FAILED: test/4.stamp + touch test/4.stamp + Wow! That's so True! + [5/5] test/5.stamp""" + ) + ) + self.assertEqual( + generate_test_report_lib.generate_report_from_files( + "Foo", 1, [junit_xml_file, ninja_log_file] + ), + dedent( + """\ + # Foo + + * 1 test passed + + All tests passed but another part of the build **failed**. Click on a failure below to see the details. + +
+ test/4.stamp + + ``` + FAILED: test/4.stamp + touch test/4.stamp + Wow! That's so True! + ``` +
+ + If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label.""" + ), + ) diff --git a/.ci/utils.sh b/.ci/utils.sh index 6656ffe06666a..30bf2d9df5ca1 100644 --- a/.ci/utils.sh +++ b/.ci/utils.sh @@ -33,7 +33,8 @@ function at-exit { if [[ "$GITHUB_STEP_SUMMARY" != "" ]]; then python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py \ - $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY + $retcode "${BUILD_DIR}"/test-results.*.xml "${BUILD_DIR}"/ninja*.log \ + >> $GITHUB_STEP_SUMMARY fi } trap at-exit EXIT