diff --git a/.ci/fifo_read.py b/.ci/fifo_read.py new file mode 100755 index 0000000000000..5ed31bf70d72c --- /dev/null +++ b/.ci/fifo_read.py @@ -0,0 +1,14 @@ +import os +import sys +import tempfile + +if __name__ == "__main__": + while True: + fifo_path = sys.argv[1] + with open(sys.argv[1], 'rb') as fifo: + filename, ext = os.path.splitext(os.path.basename(fifo_path)) + fd, _ = tempfile.mkstemp( + suffix=ext, prefix=f"{filename}.", dir=os.path.dirname(fifo_path)) + with os.fdopen(fd, 'wb') as out: + out.write(fifo.read()) + diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py new file mode 100644 index 0000000000000..f2ae116ace99a --- /dev/null +++ b/.ci/generate_test_report.py @@ -0,0 +1,328 @@ +# Script to parse many JUnit XML result files and send a report to the buildkite +# agent as an annotation. +# +# To run the unittests: +# python3 -m unittest discover -p generate_test_report.py + +import argparse +import unittest +from io import StringIO +from junitparser import JUnitXml, Failure +from textwrap import dedent +from subprocess import check_call + + +def junit_from_xml(xml): + return JUnitXml.fromfile(StringIO(xml)) + + +class TestReports(unittest.TestCase): + def test_title_only(self): + self.assertEqual(_generate_report("Foo", []), ("", None)) + + def test_no_tests_in_testsuite(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + """ + ) + ) + ], + ), + ("", None), + ) + + def test_no_failures(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + + """ + ) + ) + ], + ), + ( + dedent( + """\ + # Foo + + * 1 test passed""" + ), + "success", + ), + ) + + def test_report_single_file_single_testsuite(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + + + + + + + """ + ) + ) + ], + ), + ( + dedent( + """\ + # Foo + + * 1 test passed + * 1 test skipped + * 2 tests failed + + ## Failed tests + (click to see output) + + ### Bar +
+ Bar/test_3/test_3 + + ``` + Output goes here + ``` +
+
+ Bar/test_4/test_4 + + ``` + Other output goes here + ``` +
""" + ), + "error", + ), + ) + + MULTI_SUITE_OUTPUT = ( + dedent( + """\ + # ABC and DEF + + * 1 test passed + * 1 test skipped + * 2 tests failed + + ## Failed tests + (click to see output) + + ### ABC +
+ ABC/test_2/test_2 + + ``` + ABC/test_2 output goes here + ``` +
+ + ### DEF +
+ DEF/test_2/test_2 + + ``` + DEF/test_2 output goes here + ``` +
""" + ), + "error", + ) + + def test_report_single_file_multiple_testsuites(self): + self.assertEqual( + _generate_report( + "ABC and DEF", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + + + + + + + + + """ + ) + ) + ], + ), + self.MULTI_SUITE_OUTPUT, + ) + + def test_report_multiple_files_multiple_testsuites(self): + self.assertEqual( + _generate_report( + "ABC and DEF", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + """ + ) + ), + junit_from_xml( + dedent( + """\ + + + + + + + + + + + """ + ) + ), + ], + ), + self.MULTI_SUITE_OUTPUT, + ) + + +def _generate_report(title, junit_objects): + style = None + + if not junit_objects: + return ("", style) + + failures = {} + tests_run = 0 + tests_skipped = 0 + tests_failed = 0 + + for results in junit_objects: + for testsuite in results: + tests_run += testsuite.tests + tests_skipped += testsuite.skipped + tests_failed += testsuite.failures + + for test in testsuite: + if ( + not test.is_passed + and test.result + and isinstance(test.result[0], Failure) + ): + if failures.get(testsuite.name) is None: + failures[testsuite.name] = [] + failures[testsuite.name].append( + (test.classname + "/" + test.name, test.result[0].text) + ) + + if not tests_run: + return ("", style) + + style = "error" if tests_failed else "success" + report = [f"# {title}", ""] + + tests_passed = tests_run - tests_skipped - tests_failed + + def plural(num_tests): + return "test" if num_tests == 1 else "tests" + + if tests_passed: + report.append(f"* {tests_passed} {plural(tests_passed)} passed") + if tests_skipped: + report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped") + if tests_failed: + report.append(f"* {tests_failed} {plural(tests_failed)} failed") + + if failures: + report.extend(["", "## Failed tests", "(click to see output)"]) + for testsuite_name, failures in failures.items(): + report.extend(["", f"### {testsuite_name}"]) + for name, output in failures: + report.extend( + [ + "
", + f"{name}", + "", + "```", + output, + "```", + "
", + ] + ) + + return "\n".join(report), style + + +def generate_report(title, junit_files): + return _generate_report(title, [JUnitXml.fromfile(p) for p in junit_files]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "title", help="Title of the test report, without Markdown formatting." + ) + parser.add_argument("context", help="Annotation context to write to.") + parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*") + args = parser.parse_args() + + report, style = generate_report(args.title, args.junit_files) + check_call( + [ + "buildkite-agent", + "annotate", + "--context", + args.context, + "--style", + style, + report, + ] + ) diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh index b78dc59432b65..d1910f04081c0 100755 --- a/.ci/monolithic-linux.sh +++ b/.ci/monolithic-linux.sh @@ -20,6 +20,7 @@ MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}" BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}" INSTALL_DIR="${BUILD_DIR}/install" rm -rf "${BUILD_DIR}" +mkdir -p ${BUILD_DIR} ccache --zero-stats @@ -28,11 +29,22 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then ccache --clear fi -function show-stats { +# This will capture each time lit results out results and put each in its own file. +test_result_path="${BUILD_DIR}"/test-results.xml +mkfifo $test_result_path +python3 "${MONOREPO_ROOT}"/.ci/fifo_read.py $test_result_path & +fifo_pid=$! + +function at-exit { + kill $fifo_pid + rm $test_result_path + python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64 Test Results" \ + "linux-x64-test-results" "${BUILD_DIR}"/test-results*.xml + mkdir -p artifacts ccache --print-stats > artifacts/ccache_stats.txt } -trap show-stats EXIT +trap at-exit EXIT projects="${1}" targets="${2}" @@ -40,6 +52,7 @@ targets="${2}" echo "--- cmake" pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt +pip install -q junitparser==3.2.0 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_ENABLE_PROJECTS="${projects}" \ -G Ninja \ diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh index 91e719c52d436..bd5330d579644 100755 --- a/.ci/monolithic-windows.sh +++ b/.ci/monolithic-windows.sh @@ -20,23 +20,36 @@ MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}" BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}" rm -rf "${BUILD_DIR}" +mkdir -p ${BUILD_DIR} if [[ -n "${CLEAR_CACHE:-}" ]]; then echo "clearing sccache" rm -rf "$SCCACHE_DIR" fi +# This will capture each time lit results out results and put each in its own file. +test_result_path="${BUILD_DIR}"/test-results.xml +mkfifo $test_result_path +python "${MONOREPO_ROOT}"/.ci/fifo_read.py $test_result_path & +fifo_pid=$! + sccache --zero-stats -function show-stats { +function at-exit { + kill $fifo_pid + rm $test_result_path + python "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":windows: Windows x64 Test Results" \ + "windows-x64-test-results" "${BUILD_DIR}"/test-results*.xml + mkdir -p artifacts sccache --show-stats >> artifacts/sccache_stats.txt } -trap show-stats EXIT +trap at-exit EXIT projects="${1}" targets="${2}" echo "--- cmake" +pip install junitparser==3.2.0 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt # The CMAKE_*_LINKER_FLAGS to disable the manifest come from research diff --git a/clang/README.md b/clang/README.md index b98182d8a3f68..94b1e1a7a0743 100644 --- a/clang/README.md +++ b/clang/README.md @@ -23,3 +23,5 @@ If you're interested in more (including how to build Clang) it is best to read t * If you find a bug in Clang, please file it in the LLVM bug tracker: https://github.com/llvm/llvm-project/issues + +test change \ No newline at end of file diff --git a/clang/test/Driver/cl-pch.c b/clang/test/Driver/cl-pch.c index 36d83a11242dc..1a9527458ebf4 100644 --- a/clang/test/Driver/cl-pch.c +++ b/clang/test/Driver/cl-pch.c @@ -25,7 +25,7 @@ // CHECK-YCTP-SAME: -o // CHECK-YCTP-SAME: pchfile.pch // CHECK-YCTP-SAME: -x -// CHECK-YCTP-SAME: c++-header +// CHECK-YCTP-SAME: c++-header -- this will fail tests on windows! // Except if a later /TC changes it back. // RUN: %clang_cl -Werror /Yc%S/Inputs/pchfile.h /FI%S/Inputs/pchfile.h /c /Fo%t/pchfile.obj /Fp%t/pchfile.pch -v -- %s 2>&1 \ diff --git a/llvm/README.txt b/llvm/README.txt index b9b71a3b6daff..ba60b8ffdd072 100644 --- a/llvm/README.txt +++ b/llvm/README.txt @@ -15,3 +15,5 @@ documentation setup. If you are writing a package for LLVM, see docs/Packaging.rst for our suggestions. + +test change \ No newline at end of file