diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge index 7676ff716c418..190dd1e5ba5af 100755 --- a/.ci/generate-buildkite-pipeline-premerge +++ b/.ci/generate-buildkite-pipeline-premerge @@ -272,7 +272,7 @@ if [[ "${linux_projects}" != "" ]]; then artifact_paths: - 'artifacts/**/*' - '*_result.json' - - 'build/test-results.xml' + - 'build/test-results.*.xml' agents: ${LINUX_AGENTS} retry: automatic: @@ -295,7 +295,7 @@ if [[ "${windows_projects}" != "" ]]; then artifact_paths: - 'artifacts/**/*' - '*_result.json' - - 'build/test-results.xml' + - 'build/test-results.*.xml' agents: ${WINDOWS_AGENTS} retry: automatic: diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py new file mode 100644 index 0000000000000..f2ae116ace99a --- /dev/null +++ b/.ci/generate_test_report.py @@ -0,0 +1,328 @@ +# Script to parse many JUnit XML result files and send a report to the buildkite +# agent as an annotation. +# +# To run the unittests: +# python3 -m unittest discover -p generate_test_report.py + +import argparse +import unittest +from io import StringIO +from junitparser import JUnitXml, Failure +from textwrap import dedent +from subprocess import check_call + + +def junit_from_xml(xml): + return JUnitXml.fromfile(StringIO(xml)) + + +class TestReports(unittest.TestCase): + def test_title_only(self): + self.assertEqual(_generate_report("Foo", []), ("", None)) + + def test_no_tests_in_testsuite(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + """ + ) + ) + ], + ), + ("", None), + ) + + def test_no_failures(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + + """ + ) + ) + ], + ), + ( + dedent( + """\ + # Foo + + * 1 test passed""" + ), + "success", + ), + ) + + def test_report_single_file_single_testsuite(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + + + + + + + """ + ) + ) + ], + ), + ( + dedent( + """\ + # Foo + + * 1 test passed + * 1 test skipped + * 2 tests failed + + ## Failed tests + (click to see output) + + ### Bar +
+ Bar/test_3/test_3 + + ``` + Output goes here + ``` +
+
+ Bar/test_4/test_4 + + ``` + Other output goes here + ``` +
""" + ), + "error", + ), + ) + + MULTI_SUITE_OUTPUT = ( + dedent( + """\ + # ABC and DEF + + * 1 test passed + * 1 test skipped + * 2 tests failed + + ## Failed tests + (click to see output) + + ### ABC +
+ ABC/test_2/test_2 + + ``` + ABC/test_2 output goes here + ``` +
+ + ### DEF +
+ DEF/test_2/test_2 + + ``` + DEF/test_2 output goes here + ``` +
""" + ), + "error", + ) + + def test_report_single_file_multiple_testsuites(self): + self.assertEqual( + _generate_report( + "ABC and DEF", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + + + + + + + + + """ + ) + ) + ], + ), + self.MULTI_SUITE_OUTPUT, + ) + + def test_report_multiple_files_multiple_testsuites(self): + self.assertEqual( + _generate_report( + "ABC and DEF", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + """ + ) + ), + junit_from_xml( + dedent( + """\ + + + + + + + + + + + """ + ) + ), + ], + ), + self.MULTI_SUITE_OUTPUT, + ) + + +def _generate_report(title, junit_objects): + style = None + + if not junit_objects: + return ("", style) + + failures = {} + tests_run = 0 + tests_skipped = 0 + tests_failed = 0 + + for results in junit_objects: + for testsuite in results: + tests_run += testsuite.tests + tests_skipped += testsuite.skipped + tests_failed += testsuite.failures + + for test in testsuite: + if ( + not test.is_passed + and test.result + and isinstance(test.result[0], Failure) + ): + if failures.get(testsuite.name) is None: + failures[testsuite.name] = [] + failures[testsuite.name].append( + (test.classname + "/" + test.name, test.result[0].text) + ) + + if not tests_run: + return ("", style) + + style = "error" if tests_failed else "success" + report = [f"# {title}", ""] + + tests_passed = tests_run - tests_skipped - tests_failed + + def plural(num_tests): + return "test" if num_tests == 1 else "tests" + + if tests_passed: + report.append(f"* {tests_passed} {plural(tests_passed)} passed") + if tests_skipped: + report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped") + if tests_failed: + report.append(f"* {tests_failed} {plural(tests_failed)} failed") + + if failures: + report.extend(["", "## Failed tests", "(click to see output)"]) + for testsuite_name, failures in failures.items(): + report.extend(["", f"### {testsuite_name}"]) + for name, output in failures: + report.extend( + [ + "
", + f"{name}", + "", + "```", + output, + "```", + "
", + ] + ) + + return "\n".join(report), style + + +def generate_report(title, junit_files): + return _generate_report(title, [JUnitXml.fromfile(p) for p in junit_files]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "title", help="Title of the test report, without Markdown formatting." + ) + parser.add_argument("context", help="Annotation context to write to.") + parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*") + args = parser.parse_args() + + report, style = generate_report(args.title, args.junit_files) + check_call( + [ + "buildkite-agent", + "annotate", + "--context", + args.context, + "--style", + style, + report, + ] + ) diff --git a/.ci/lit-wrapper.py b/.ci/lit-wrapper.py new file mode 100644 index 0000000000000..0eaea3ce550c0 --- /dev/null +++ b/.ci/lit-wrapper.py @@ -0,0 +1,29 @@ +#! /usr/bin/env/python3 + +import sys +import os +import subprocess +import tempfile + +result_path = None +for idx, option in enumerate(sys.argv): + if option == "--xunit-xml-output": + result_path = sys.argv[idx + 1] + break + +dirname, _ = os.path.split(os.path.abspath(__file__)) +res = subprocess.run( + [sys.executable, os.path.join(dirname, "llvm-lit-actual.py"), *sys.argv[1:]], + check=False, +) + +if result_path is not None: + with open(result_path, "rb") as results_file: + filename, ext = os.path.splitext(os.path.basename(result_path)) + fd, _ = tempfile.mkstemp( + suffix=ext, prefix=f"{filename}.", dir=os.path.dirname(result_path) + ) + with os.fdopen(fd, "wb") as out: + out.write(results_file.read()) + +sys.exit(res.returncode) diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh index b78dc59432b65..a371a15af7697 100755 --- a/.ci/monolithic-linux.sh +++ b/.ci/monolithic-linux.sh @@ -28,18 +28,23 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then ccache --clear fi -function show-stats { +function at-exit { + python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64 Test Results" \ + "linux-x64-test-results" "${BUILD_DIR}"/test-results.*.xml + mkdir -p artifacts ccache --print-stats > artifacts/ccache_stats.txt } -trap show-stats EXIT +trap at-exit EXIT projects="${1}" targets="${2}" +llvm_lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" echo "--- cmake" pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt +pip install -q junitparser==3.2.0 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_ENABLE_PROJECTS="${projects}" \ -G Ninja \ @@ -47,13 +52,18 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_ENABLE_ASSERTIONS=ON \ -D LLVM_BUILD_EXAMPLES=ON \ -D COMPILER_RT_BUILD_LIBFUZZER=OFF \ - -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \ + -D LLVM_LIT_ARGS="$llvm_lit_args" \ -D LLVM_ENABLE_LLD=ON \ -D CMAKE_CXX_FLAGS=-gmlt \ -D LLVM_CCACHE_BUILD=ON \ -D MLIR_ENABLE_BINDINGS_PYTHON=ON \ -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" +# Configure installs llvm-lit into bin. Replace this with the wrapper script. +# TODO: make a function for this +mv "${BUILD_DIR}"/bin/llvm-lit "${BUILD_DIR}"/bin/llvm-lit-actual.py +cp "${MONOREPO_ROOT}"/.ci/lit-wrapper.py "${BUILD_DIR}"/bin/llvm-lit + echo "--- ninja" # Targets are not escaped as they are passed as separate arguments. ninja -C "${BUILD_DIR}" -k 0 ${targets} @@ -87,7 +97,11 @@ if [[ "${runtimes}" != "" ]]; then -D CMAKE_BUILD_TYPE=RelWithDebInfo \ -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ -D LIBCXX_TEST_PARAMS="std=c++03" \ - -D LIBCXXABI_TEST_PARAMS="std=c++03" + -D LIBCXXABI_TEST_PARAMS="std=c++03" \ + -D LLVM_LIT_ARGS="$llvm_lit_args" + + mv "${RUNTIMES_BUILD_DIR}"/bin/llvm-lit "${RUNTIMES_BUILD_DIR}"/bin/llvm-lit-actual.py + cp "${MONOREPO_ROOT}"/.ci/lit-wrapper.py "${RUNTIMES_BUILD_DIR}"/bin/llvm-lit echo "--- ninja runtimes C++03" @@ -104,7 +118,11 @@ if [[ "${runtimes}" != "" ]]; then -D CMAKE_BUILD_TYPE=RelWithDebInfo \ -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ -D LIBCXX_TEST_PARAMS="std=c++26" \ - -D LIBCXXABI_TEST_PARAMS="std=c++26" + -D LIBCXXABI_TEST_PARAMS="std=c++26" \ + -D LLVM_LIT_ARGS="$llvm_lit_args" + + mv "${RUNTIMES_BUILD_DIR}"/bin/llvm-lit "${RUNTIMES_BUILD_DIR}"/bin/llvm-lit-actual.py + cp "${MONOREPO_ROOT}"/.ci/lit-wrapper.py "${RUNTIMES_BUILD_DIR}"/bin/llvm-lit echo "--- ninja runtimes C++26" @@ -121,7 +139,11 @@ if [[ "${runtimes}" != "" ]]; then -D CMAKE_BUILD_TYPE=RelWithDebInfo \ -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ -D LIBCXX_TEST_PARAMS="enable_modules=clang" \ - -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" + -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \ + -D LLVM_LIT_ARGS="$llvm_lit_args" + + mv "${RUNTIMES_BUILD_DIR}"/bin/llvm-lit "${RUNTIMES_BUILD_DIR}"/bin/llvm-lit-actual.py + cp "${MONOREPO_ROOT}"/.ci/lit-wrapper.py "${RUNTIMES_BUILD_DIR}"/bin/llvm-lit echo "--- ninja runtimes clang modules" diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh index 91e719c52d436..624d83cc8a63c 100755 --- a/.ci/monolithic-windows.sh +++ b/.ci/monolithic-windows.sh @@ -27,16 +27,20 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then fi sccache --zero-stats -function show-stats { +function at-exit { + python "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":windows: Windows x64 Test Results" \ + "windows-x64-test-results" "${BUILD_DIR}"/test-results.*.xml + mkdir -p artifacts sccache --show-stats >> artifacts/sccache_stats.txt } -trap show-stats EXIT +trap at-exit EXIT projects="${1}" targets="${2}" echo "--- cmake" +pip install junitparser==3.2.0 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt # The CMAKE_*_LINKER_FLAGS to disable the manifest come from research @@ -64,6 +68,10 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_PARALLEL_COMPILE_JOBS=16 \ -D LLVM_PARALLEL_LINK_JOBS=4 +# Configure installs llvm-lit into bin. Replace this with the wrapper script. +mv "${BUILD_DIR}"/bin/llvm-lit.py "${BUILD_DIR}"/bin/llvm-lit-actual.py +cp "${MONOREPO_ROOT}"/.ci/lit-wrapper.py "${BUILD_DIR}"/bin/llvm-lit.py + echo "--- ninja" # Targets are not escaped as they are passed as separate arguments. ninja -C "${BUILD_DIR}" -k 0 ${targets} diff --git a/clang/README.md b/clang/README.md index b98182d8a3f68..94b1e1a7a0743 100644 --- a/clang/README.md +++ b/clang/README.md @@ -23,3 +23,5 @@ If you're interested in more (including how to build Clang) it is best to read t * If you find a bug in Clang, please file it in the LLVM bug tracker: https://github.com/llvm/llvm-project/issues + +test change \ No newline at end of file diff --git a/clang/test/Driver/cl-pch.c b/clang/test/Driver/cl-pch.c index 36d83a11242dc..1a9527458ebf4 100644 --- a/clang/test/Driver/cl-pch.c +++ b/clang/test/Driver/cl-pch.c @@ -25,7 +25,7 @@ // CHECK-YCTP-SAME: -o // CHECK-YCTP-SAME: pchfile.pch // CHECK-YCTP-SAME: -x -// CHECK-YCTP-SAME: c++-header +// CHECK-YCTP-SAME: c++-header -- this will fail tests on windows! // Except if a later /TC changes it back. // RUN: %clang_cl -Werror /Yc%S/Inputs/pchfile.h /FI%S/Inputs/pchfile.h /c /Fo%t/pchfile.obj /Fp%t/pchfile.pch -v -- %s 2>&1 \ diff --git a/llvm/README.txt b/llvm/README.txt index b9b71a3b6daff..ba60b8ffdd072 100644 --- a/llvm/README.txt +++ b/llvm/README.txt @@ -15,3 +15,5 @@ documentation setup. If you are writing a package for LLVM, see docs/Packaging.rst for our suggestions. + +test change \ No newline at end of file