diff --git a/.ci/generate-buildkite-pipeline-premerge b/.ci/generate-buildkite-pipeline-premerge index 7676ff716c418..363efe9ae5fb5 100755 --- a/.ci/generate-buildkite-pipeline-premerge +++ b/.ci/generate-buildkite-pipeline-premerge @@ -272,7 +272,7 @@ if [[ "${linux_projects}" != "" ]]; then artifact_paths: - 'artifacts/**/*' - '*_result.json' - - 'build/test-results.xml' + - 'build/*-test-results.xml' agents: ${LINUX_AGENTS} retry: automatic: @@ -289,27 +289,27 @@ if [[ "${linux_projects}" != "" ]]; then EOF fi -if [[ "${windows_projects}" != "" ]]; then - cat < + + + + """ + ) + ) + ], + ), + ("", None), + ) + + def test_no_failures(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + + """ + ) + ) + ], + ), + ( + dedent( + """\ + # Foo + + * 1 test passed""" + ), + "success", + ), + ) + + def test_report_single_file_single_testsuite(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + + + + + + + """ + ) + ) + ], + ), + ( + dedent( + """\ + # Foo + + * 1 test passed + * 1 test skipped + * 2 tests failed + + ## Failed tests + (click to see output) + + ### Bar +
+ Bar/test_3/test_3 + + ``` + Output goes here + ``` +
+
+ Bar/test_4/test_4 + + ``` + Other output goes here + ``` +
""" + ), + "error", + ), + ) + + MULTI_SUITE_OUTPUT = ( + dedent( + """\ + # ABC and DEF + + * 1 test passed + * 1 test skipped + * 2 tests failed + + ## Failed tests + (click to see output) + + ### ABC +
+ ABC/test_2/test_2 + + ``` + ABC/test_2 output goes here + ``` +
+ + ### DEF +
+ DEF/test_2/test_2 + + ``` + DEF/test_2 output goes here + ``` +
""" + ), + "error", + ) + + def test_report_single_file_multiple_testsuites(self): + self.assertEqual( + _generate_report( + "ABC and DEF", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + + + + + + + + + """ + ) + ) + ], + ), + self.MULTI_SUITE_OUTPUT, + ) + + def test_report_multiple_files_multiple_testsuites(self): + self.assertEqual( + _generate_report( + "ABC and DEF", + [ + junit_from_xml( + dedent( + """\ + + + + + + + + + """ + ) + ), + junit_from_xml( + dedent( + """\ + + + + + + + + + + + """ + ) + ), + ], + ), + self.MULTI_SUITE_OUTPUT, + ) + + +def _generate_report(title, junit_objects): + style = None + + if not junit_objects: + return ("", style) + + failures = {} + tests_run = 0 + tests_skipped = 0 + tests_failed = 0 + + for results in junit_objects: + for testsuite in results: + tests_run += testsuite.tests + tests_skipped += testsuite.skipped + tests_failed += testsuite.failures + + for test in testsuite: + if ( + not test.is_passed + and test.result + and isinstance(test.result[0], Failure) + ): + if failures.get(testsuite.name) is None: + failures[testsuite.name] = [] + failures[testsuite.name].append( + (test.classname + "/" + test.name, test.result[0].text) + ) + + if not tests_run: + return ("", style) + + style = "error" if tests_failed else "success" + report = [f"# {title}", ""] + + tests_passed = tests_run - tests_skipped - tests_failed + + def plural(num_tests): + return "test" if num_tests == 1 else "tests" + + if tests_passed: + report.append(f"* {tests_passed} {plural(tests_passed)} passed") + if tests_skipped: + report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped") + if tests_failed: + report.append(f"* {tests_failed} {plural(tests_failed)} failed") + + if failures: + report.extend(["", "## Failed tests", "(click to see output)"]) + for testsuite_name, failures in failures.items(): + report.extend(["", f"### {testsuite_name}"]) + for name, output in failures: + report.extend( + [ + "
", + f"{name}", + "", + "```", + output, + "```", + "
", + ] + ) + + return "\n".join(report), style + + +def generate_report(title, junit_files): + return _generate_report(title, [JUnitXml.fromfile(p) for p in junit_files]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "title", help="Title of the test report, without Markdown formatting." + ) + parser.add_argument("context", help="Annotation context to write to.") + parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*") + args = parser.parse_args() + + report, style = generate_report(args.title, args.junit_files) + check_call( + [ + "buildkite-agent", + "annotate", + "--context", + args.context, + "--style", + style, + report, + ] + ) diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh index b78dc59432b65..c9ad32a46e286 100755 --- a/.ci/monolithic-linux.sh +++ b/.ci/monolithic-linux.sh @@ -28,18 +28,47 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then ccache --clear fi -function show-stats { +function at-exit { + python3 "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":linux: Linux x64 Test Results" \ + "linux-x64-test-results" "${BUILD_DIR}"/*-test-results.xml + mkdir -p artifacts ccache --print-stats > artifacts/ccache_stats.txt } -trap show-stats EXIT +trap at-exit EXIT + +# TODO: separate file for import into Windows script? +function ninja-targets { + # $1 is the ninja arguments to use + # $2 is is an optional postfix to add to the target name when renaming result files. + # $3 is the list of targets + set +e + err_code=0 + for target in $3; do + ninja $1 ${target} + new_err_code=$? + if [[ $new_err_code -ne 0 ]]; then + err_code=${new_err_code} + fi + mv "${BUILD_DIR}/test-results.xml" "${BUILD_DIR}/${target}${2}-test-results.xml" + done + + if [[ $err_code -ne 0 ]]; then + exit $err_code + fi + + set -e +} projects="${1}" targets="${2}" +lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" + echo "--- cmake" pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt pip install -q -r "${MONOREPO_ROOT}"/lldb/test/requirements.txt +pip install -q junitparser==3.2.0 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_ENABLE_PROJECTS="${projects}" \ -G Ninja \ @@ -47,7 +76,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D LLVM_ENABLE_ASSERTIONS=ON \ -D LLVM_BUILD_EXAMPLES=ON \ -D COMPILER_RT_BUILD_LIBFUZZER=OFF \ - -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --timeout=1200 --time-tests" \ + -D LLVM_LIT_ARGS="${lit_args}" \ -D LLVM_ENABLE_LLD=ON \ -D CMAKE_CXX_FLAGS=-gmlt \ -D LLVM_CCACHE_BUILD=ON \ @@ -55,8 +84,8 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \ -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" echo "--- ninja" -# Targets are not escaped as they are passed as separate arguments. -ninja -C "${BUILD_DIR}" -k 0 ${targets} + +ninja-targets "-C "${BUILD_DIR}" -k 0" "" "$targets" runtimes="${3}" runtime_targets="${4}" @@ -87,11 +116,14 @@ if [[ "${runtimes}" != "" ]]; then -D CMAKE_BUILD_TYPE=RelWithDebInfo \ -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ -D LIBCXX_TEST_PARAMS="std=c++03" \ - -D LIBCXXABI_TEST_PARAMS="std=c++03" + -D LIBCXXABI_TEST_PARAMS="std=c++03" \ + -D LLVM_LIT_ARGS="${lit_args}" echo "--- ninja runtimes C++03" - ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets} + # TODO: there's no way to tell a failure here apart from a failure of the same + # test in the other build mode. + ninja-targets "-vC "${RUNTIMES_BUILD_DIR}"" "-cxx03" "${runtime_targets}" echo "--- cmake runtimes C++26" @@ -104,11 +136,12 @@ if [[ "${runtimes}" != "" ]]; then -D CMAKE_BUILD_TYPE=RelWithDebInfo \ -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ -D LIBCXX_TEST_PARAMS="std=c++26" \ - -D LIBCXXABI_TEST_PARAMS="std=c++26" + -D LIBCXXABI_TEST_PARAMS="std=c++26" \ + -D LLVM_LIT_ARGS="${lit_args}" echo "--- ninja runtimes C++26" - ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets} + ninja-targets "-vC "${RUNTIMES_BUILD_DIR}"" "-cxx26" "${runtime_targets}" echo "--- cmake runtimes clang modules" @@ -121,9 +154,10 @@ if [[ "${runtimes}" != "" ]]; then -D CMAKE_BUILD_TYPE=RelWithDebInfo \ -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \ -D LIBCXX_TEST_PARAMS="enable_modules=clang" \ - -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" + -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \ + -D LLVM_LIT_ARGS="${lit_args}" echo "--- ninja runtimes clang modules" - - ninja -vC "${RUNTIMES_BUILD_DIR}" ${runtime_targets} + + ninja-targets "-vC "${RUNTIMES_BUILD_DIR}"" "-modules" "${runtime_targets}" fi diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh index 91e719c52d436..daac89b4a632d 100755 --- a/.ci/monolithic-windows.sh +++ b/.ci/monolithic-windows.sh @@ -27,16 +27,20 @@ if [[ -n "${CLEAR_CACHE:-}" ]]; then fi sccache --zero-stats -function show-stats { +function at-exit { + python "${MONOREPO_ROOT}"/.ci/generate_test_report.py ":windows: Windows x64 Test Results" \ + "windows-x64-test-results" "${BUILD_DIR}"/test-results*.xml + mkdir -p artifacts sccache --show-stats >> artifacts/sccache_stats.txt } -trap show-stats EXIT +trap at-exit EXIT projects="${1}" targets="${2}" echo "--- cmake" +pip install junitparser==3.2.0 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt # The CMAKE_*_LINKER_FLAGS to disable the manifest come from research diff --git a/clang/README.md b/clang/README.md index b98182d8a3f68..94b1e1a7a0743 100644 --- a/clang/README.md +++ b/clang/README.md @@ -23,3 +23,5 @@ If you're interested in more (including how to build Clang) it is best to read t * If you find a bug in Clang, please file it in the LLVM bug tracker: https://github.com/llvm/llvm-project/issues + +test change \ No newline at end of file diff --git a/clang/test/Driver/cl-pch.c b/clang/test/Driver/cl-pch.c index 36d83a11242dc..1a9527458ebf4 100644 --- a/clang/test/Driver/cl-pch.c +++ b/clang/test/Driver/cl-pch.c @@ -25,7 +25,7 @@ // CHECK-YCTP-SAME: -o // CHECK-YCTP-SAME: pchfile.pch // CHECK-YCTP-SAME: -x -// CHECK-YCTP-SAME: c++-header +// CHECK-YCTP-SAME: c++-header -- this will fail tests on windows! // Except if a later /TC changes it back. // RUN: %clang_cl -Werror /Yc%S/Inputs/pchfile.h /FI%S/Inputs/pchfile.h /c /Fo%t/pchfile.obj /Fp%t/pchfile.pch -v -- %s 2>&1 \ diff --git a/llvm/README.txt b/llvm/README.txt index b9b71a3b6daff..ba60b8ffdd072 100644 --- a/llvm/README.txt +++ b/llvm/README.txt @@ -15,3 +15,5 @@ documentation setup. If you are writing a package for LLVM, see docs/Packaging.rst for our suggestions. + +test change \ No newline at end of file