Merge remote-tracking branch 'origin/main' into lv-find-min-max-index

Author: fhahn

.ci/generate_test_report_lib.py

@@ -98,6 +98,23 @@ def _format_ninja_failures(ninja_failures: list[tuple[str, str]]) -> list[str]:
         )
     return output
 
+def get_failures(junit_objects) -> dict[str, list[tuple[str, str]]]:
+    failures = {}
+    for results in junit_objects:
+        for testsuite in results:
+            for test in testsuite:
+                if (
+                    not test.is_passed
+                    and test.result
+                    and isinstance(test.result[0], Failure)
+                ):
+                    if failures.get(testsuite.name) is None:
+                        failures[testsuite.name] = []
+                    failures[testsuite.name].append(
+                        (test.classname + "/" + test.name, test.result[0].text)
+                    )
+    return failures
+
 
 # Set size_limit to limit the byte size of the report. The default is 1MB as this
 # is the most that can be put into an annotation. If the generated report exceeds
@@ -113,7 +130,7 @@ def generate_report(
     size_limit=1024 * 1024,
     list_failures=True,
 ):
-    failures = {}
+    failures = get_failures(junit_objects)
     tests_run = 0
     tests_skipped = 0
     tests_failed = 0
@@ -124,18 +141,6 @@ def generate_report(
             tests_skipped += testsuite.skipped
             tests_failed += testsuite.failures
 
-            for test in testsuite:
-                if (
-                    not test.is_passed
-                    and test.result
-                    and isinstance(test.result[0], Failure)
-                ):
-                    if failures.get(testsuite.name) is None:
-                        failures[testsuite.name] = []
-                    failures[testsuite.name].append(
-                        (test.classname + "/" + test.name, test.result[0].text)
-                    )
-
     report = [f"# {title}", ""]
 
     if tests_run == 0:
@@ -258,7 +263,7 @@ def plural(num_tests):
     return report
 
 
-def generate_report_from_files(title, return_code, build_log_files):
+def load_info_from_files(build_log_files):
     junit_files = [
         junit_file for junit_file in build_log_files if junit_file.endswith(".xml")
     ]
@@ -271,6 +276,9 @@ def generate_report_from_files(title, return_code, build_log_files):
             ninja_logs.append(
                 [log_line.strip() for log_line in ninja_log_file_handle.readlines()]
             )
-    return generate_report(
-        title, return_code, [JUnitXml.fromfile(p) for p in junit_files], ninja_logs
-    )
+    return [JUnitXml.fromfile(p) for p in junit_files], ninja_logs
+
+
+def generate_report_from_files(title, return_code, build_log_files):
+    junit_objects, ninja_logs = load_info_from_files(build_log_files)
+    return generate_report(title, return_code, junit_objects, ninja_logs)

.ci/metrics/metrics.py

@@ -40,6 +40,7 @@
 GITHUB_JOB_TO_TRACK = {
     "github_llvm_premerge_checks": {
         "Build and Test Linux": "premerge_linux",
+        "Build and Test Linux AArch64": "premerge_linux_aarch64",
         "Build and Test Windows": "premerge_windows",
     },
     "github_libcxx_premerge_checks": {

.ci/monolithic-linux.sh

@@ -66,11 +66,13 @@ start-group "ninja"
 
 # Targets are not escaped as they are passed as separate arguments.
 ninja -C "${BUILD_DIR}" -k 0 ${targets} |& tee ninja.log
+cp ${BUILD_DIR}/.ninja_log ninja.ninja_log
 
 if [[ "${runtime_targets}" != "" ]]; then
   start-group "ninja Runtimes"
 
   ninja -C "${BUILD_DIR}" ${runtime_targets} |& tee ninja_runtimes.log
+  cp ${BUILD_DIR}/.ninja_log ninja_runtimes.ninja_log
 fi
 
 # Compiling runtimes with just-built Clang and running their tests
@@ -87,6 +89,7 @@ if [[ "${runtime_targets_needs_reconfig}" != "" ]]; then
 
   ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig} \
     |& tee ninja_runtimes_needs_reconfig1.log
+  cp ${BUILD_DIR}/.ninja_log ninja_runtimes_needs_reconig.ninja_log
 
   start-group "CMake Runtimes Clang Modules"
 
@@ -99,4 +102,5 @@ if [[ "${runtime_targets_needs_reconfig}" != "" ]]; then
 
   ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig} \
     |& tee ninja_runtimes_needs_reconfig2.log
+  cp ${BUILD_DIR}/.ninja_log ninja_runtimes_needs_reconfig2.ninja_log
 fi

.ci/monolithic-windows.sh

@@ -23,8 +23,8 @@ runtimes_targets="${4}"
 start-group "CMake"
 pip install -q -r "${MONOREPO_ROOT}"/.ci/all_requirements.txt
 
-export CC=cl
-export CXX=cl
+export CC=C:/clang/clang-msvc/bin/clang-cl.exe
+export CXX=C:/clang/clang-msvc/bin/clang-cl.exe
 export LD=link
 
 # The CMAKE_*_LINKER_FLAGS to disable the manifest come from research
@@ -49,15 +49,18 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D CMAKE_EXE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_MODULE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_SHARED_LINKER_FLAGS="/MANIFEST:NO" \
+      -D CMAKE_CXX_FLAGS="-Wno-c++98-compat -Wno-c++14-compat -Wno-unsafe-buffer-usage -Wno-old-style-cast" \
       -D LLVM_ENABLE_RUNTIMES="${runtimes}"
 
 start-group "ninja"
 
 # Targets are not escaped as they are passed as separate arguments.
 ninja -C "${BUILD_DIR}" -k 0 ${targets} |& tee ninja.log
+cp ${BUILD_DIR}/.ninja_log ninja.ninja_log
 
 if [[ "${runtime_targets}" != "" ]]; then
   start-group "ninja runtimes"
 
   ninja -C "${BUILD_DIR}" -k 0 ${runtimes_targets} |& tee ninja_runtimes.log
+  cp ${BUILD_DIR}/.ninja_log ninja_runtimes.ninja_log
 fi

.ci/premerge_advisor_explain.py

@@ -0,0 +1,63 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Script for getting explanations from the premerge advisor."""
+
+import argparse
+import os
+import platform
+import sys
+
+import requests
+
+import generate_test_report_lib
+
+PREMERGE_ADVISOR_URL = (
+    "http://premerge-advisor.premerge-advisor.svc.cluster.local:5000/explain"
+)
+
+
+def main(commit_sha: str, build_log_files: list[str]):
+    junit_objects, ninja_logs = generate_test_report_lib.load_info_from_files(
+        build_log_files
+    )
+    test_failures = generate_test_report_lib.get_failures(junit_objects)
+    current_platform = f"{platform.system()}-{platform.machine()}".lower()
+    explanation_request = {
+        "base_commit_sha": commit_sha,
+        "platform": current_platform,
+        "failures": [],
+    }
+    if test_failures:
+        for _, failures in test_failures.items():
+            for name, failure_messsage in failures:
+                explanation_request["failures"].append(
+                    {"name": name, "message": failure_messsage}
+                )
+    else:
+        ninja_failures = generate_test_report_lib.find_failure_in_ninja_logs(ninja_logs)
+        for name, failure_message in ninja_failures:
+            explanation_request["failures"].append(
+                {"name": name, "message": failure_message}
+            )
+    advisor_response = requests.get(PREMERGE_ADVISOR_URL, json=explanation_request)
+    if advisor_response.status_code == 200:
+        print(advisor_response.json())
+    else:
+        print(advisor_response.reason)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("commit_sha", help="The base commit SHA for the test.")
+    parser.add_argument(
+        "build_log_files", help="Paths to JUnit report files and ninja logs.", nargs="*"
+    )
+    args = parser.parse_args()
+
+    # Skip looking for results on AArch64 for now because the premerge advisor
+    # service is not available on AWS currently.
+    if platform.machine() == "arm64":
+        sys.exit(0)
+
+    main(args.commit_sha, args.build_log_files)

.ci/premerge_advisor_upload.py

@@ -0,0 +1,65 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Script for uploading results to the premerge advisor."""
+
+import argparse
+import os
+import platform
+import sys
+
+import requests
+
+import generate_test_report_lib
+
+# These are IP addresses of the two premerge advisor instances. They should
+# eventually be updated to domain names.
+PREMERGE_ADVISOR_URLS = [
+    "http://34.82.126.63:5000/upload",
+    "http://136.114.125.23:5000/upload",
+]
+
+
+def main(commit_sha, workflow_run_number, build_log_files):
+    junit_objects, ninja_logs = generate_test_report_lib.load_info_from_files(
+        build_log_files
+    )
+    test_failures = generate_test_report_lib.get_failures(junit_objects)
+    source = "pull_request" if "GITHUB_ACTIONS" in os.environ else "postcommit"
+    current_platform = f"{platform.system()}-{platform.machine()}".lower()
+    failure_info = {
+        "source_type": source,
+        "base_commit_sha": commit_sha,
+        "source_id": workflow_run_number,
+        "failures": [],
+        "platform": current_platform,
+    }
+    if test_failures:
+        for _, failures in test_failures.items():
+            for name, failure_message in failures:
+                failure_info["failures"].append(
+                    {"name": name, "message": failure_message}
+                )
+    else:
+        ninja_failures = generate_test_report_lib.find_failure_in_ninja_logs(ninja_logs)
+        for name, failure_message in ninja_failures:
+            failure_info["failures"].append({"name": name, "message": failure_message})
+    for premerge_advisor_url in PREMERGE_ADVISOR_URLS:
+        requests.post(premerge_advisor_url, json=failure_info)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("commit_sha", help="The base commit SHA for the test.")
+    parser.add_argument("workflow_run_number", help="The run number from GHA.")
+    parser.add_argument(
+        "build_log_files", help="Paths to JUnit report files and ninja logs.", nargs="*"
+    )
+    args = parser.parse_args()
+
+    # Skip uploading results on AArch64 for now because the premerge advisor
+    # service is not available on AWS currently.
+    if platform.machine() == "arm64":
+        sys.exit(0)
+
+    main(args.commit_sha, args.workflow_run_number, args.build_log_files)

.ci/utils.sh

@@ -26,7 +26,7 @@ function at-exit {
   mkdir -p artifacts
   sccache --show-stats
   sccache --show-stats >> artifacts/sccache_stats.txt
-  cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
+  cp "${MONOREPO_ROOT}"/*.ninja_log artifacts/ || :
   cp "${MONOREPO_ROOT}"/*.log artifacts/ || :
   cp "${BUILD_DIR}"/test-results.*.xml artifacts/ || :
 
@@ -38,6 +38,21 @@ function at-exit {
       $retcode "${BUILD_DIR}"/test-results.*.xml "${MONOREPO_ROOT}"/ninja*.log \
       >> $GITHUB_STEP_SUMMARY
   fi
+
+  if [[ "$retcode" != "0" ]]; then
+    if [[ "$GITHUB_ACTIONS" != "" ]]; then
+      python "${MONOREPO_ROOT}"/.ci/premerge_advisor_explain.py \
+        $(git rev-parse HEAD~1) "${BUILD_DIR}"/test-results.*.xml \
+        "${MONOREPO_ROOT}"/ninja*.log
+      python "${MONOREPO_ROOT}"/.ci/premerge_advisor_upload.py \
+        $(git rev-parse HEAD~1) $GITHUB_RUN_NUMBER \
+        "${BUILD_DIR}"/test-results.*.xml "${MONOREPO_ROOT}"/ninja*.log
+    else
+      python "${MONOREPO_ROOT}"/.ci/premerge_advisor_upload.py \
+        $(git rev-parse HEAD) $BUILDBOT_BUILDNUMBER \
+        "${BUILD_DIR}"/test-results.*.xml "${MONOREPO_ROOT}"/ninja*.log
+    fi
+  fi
 }
 trap at-exit EXIT
 

.github/CODEOWNERS

@@ -53,6 +53,13 @@
 /mlir/include/mlir/Interfaces/DestinationStyleOpInterface.* @matthias-springer
 /mlir/lib/Interfaces/DestinationStyleOpInterface.* @matthias-springer
 
+# AMDGPU and ROCDL dialects in MLIR.
+/mlir/include/mlir/Dialect/AMDGPU @krzysz00 @kuhar
+/mlir/lib/Dialect/AMDGPU @krzysz00 @kuhar
+/mlir/lib/Conversion/*AMDGPU* @krzysz00 @kuhar
+/mlir/lib/Conversion/*ToROCDL @krzysz00 @kuhar
+/mlir/include/mlir/Dialect/LLVMIR/ROCDL* @krzysz00 @kuhar
+
 # Bufferization Dialect in MLIR.
 /mlir/include/mlir/Dialect/Bufferization @matthias-springer
 /mlir/lib/Dialect/Bufferization @matthias-springer
@@ -131,6 +138,7 @@
 /mlir/test/python/ @ftynse @makslevental @stellaraccident @rolfmorel
 /mlir/python/ @ftynse @makslevental @stellaraccident @rolfmorel
 /mlir/lib/Bindings/Python @makslevental @rolfmorel
+/mlir/include/Bindings/Python @makslevental @rolfmorel
 
 # MLIR Mem2Reg/SROA
 /mlir/**/Transforms/Mem2Reg.* @moxinilian

.github/workflows/containers/github-action-ci-tooling/Dockerfile

@@ -1,14 +1,19 @@
 ARG LLVM_VERSION=21.1.0
+# FIXME: Use "${LLVM_VERSION%%.*}" instead of "LLVM_VERSION_MAJOR" once we update runners to Ubuntu-26.04 with Buildah >= 1.37
+ARG LLVM_VERSION_MAJOR=21
 
 FROM docker.io/library/ubuntu:24.04 AS llvm-downloader
 ARG LLVM_VERSION
+ARG LLVM_VERSION_MAJOR
 
 RUN apt-get update && \
     apt-get install -y wget xz-utils && \
     wget -O llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/LLVM-${LLVM_VERSION}-Linux-X64.tar.xz && \
     mkdir -p /llvm-extract && \
     tar -xvJf llvm.tar.xz -C /llvm-extract \
         # Only unpack these tools to save space on Github runner.
+        LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-${LLVM_VERSION_MAJOR} \
+        LLVM-${LLVM_VERSION}-Linux-X64/lib/clang/${LLVM_VERSION_MAJOR}/include \
         LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-tidy \
         LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-format \
         LLVM-${LLVM_VERSION}-Linux-X64/bin/git-clang-format && \
@@ -32,6 +37,14 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Create a new user with id 1001 as that is the user id that
+# Github Actions uses to perform the checkout action.
+RUN useradd gha -u 1001 -m -s /bin/bash
+RUN adduser gha sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+# Don't set USER gha right away because we still need to install packages
+# as root in 'ci-container-code-format' and 'ci-container-code-lint' containers
+
 
 FROM base AS ci-container-code-format
 ARG LLVM_VERSION
@@ -46,17 +59,36 @@ ENV PATH=${LLVM_SYSROOT}/bin:${PATH}
 COPY llvm/utils/git/requirements_formatting.txt requirements_formatting.txt
 RUN pip install -r requirements_formatting.txt --break-system-packages && \
     rm requirements_formatting.txt
+USER gha
+WORKDIR /home/gha
 
 
 FROM base AS ci-container-code-lint
 ARG LLVM_VERSION
+ARG LLVM_VERSION_MAJOR
 
-COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-tidy ${LLVM_SYSROOT}/bin/
+COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-tidy \
+                            /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-${LLVM_VERSION_MAJOR} \
+                            ${LLVM_SYSROOT}/bin/
+COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/lib/clang/${LLVM_VERSION_MAJOR}/include \
+                            ${LLVM_SYSROOT}/lib/clang/${LLVM_VERSION_MAJOR}/include
 COPY clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py ${LLVM_SYSROOT}/bin/clang-tidy-diff.py
 
+RUN ln -s ${LLVM_SYSROOT}/bin/clang-${LLVM_VERSION_MAJOR} ${LLVM_SYSROOT}/bin/clang && \
+    ln -s ${LLVM_SYSROOT}/bin/clang ${LLVM_SYSROOT}/bin/clang++
+
 ENV PATH=${LLVM_SYSROOT}/bin:${PATH}
 
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    cmake \
+    ninja-build && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
 # Install dependencies for 'pr-code-lint.yml' job
 COPY llvm/utils/git/requirements_linting.txt requirements_linting.txt
 RUN pip install -r requirements_linting.txt --break-system-packages && \
     rm requirements_linting.txt
+USER gha
+WORKDIR /home/gha