Merge branch 'main' into clang-tidy-doc-fix-02

Author: zeyi2

.ci/all_requirements.txt

@@ -4,20 +4,90 @@
 """Script for getting explanations from the premerge advisor."""
 
 import argparse
-import os
 import platform
 import sys
+import json
+
+# TODO(boomanaiden154): Remove the optional call once we can require Python
+# 3.10.
+from typing import Optional
 
 import requests
+import github
+import github.PullRequest
 
 import generate_test_report_lib
 
 PREMERGE_ADVISOR_URL = (
     "http://premerge-advisor.premerge-advisor.svc.cluster.local:5000/explain"
 )
+COMMENT_TAG = "<!--PREMERGE ADVISOR COMMENT: {platform}-->"
+
+
+def get_comment_id(platform: str, pr: github.PullRequest.PullRequest) -> Optional[int]:
+    platform_comment_tag = COMMENT_TAG.format(platform=platform)
+    for comment in pr.as_issue().get_comments():
+        if platform_comment_tag in comment.body:
+            return comment.id
+    return None
+
+
+def get_comment(
+    github_token: str,
+    pr_number: int,
+    body: str,
+) -> dict[str, str]:
+    repo = github.Github(github_token).get_repo("llvm/llvm-project")
+    pr = repo.get_issue(pr_number).as_pull_request()
+    comment = {"body": body}
+    comment_id = get_comment_id(platform.system(), pr)
+    if comment_id:
+        comment["id"] = comment_id
+    return comment
 
 
-def main(commit_sha: str, build_log_files: list[str]):
+def main(
+    commit_sha: str,
+    build_log_files: list[str],
+    github_token: str,
+    pr_number: int,
+    return_code: int,
+):
+    """The main entrypoint for the script.
+
+    This function parses failures from files, requests information from the
+    premerge advisor, and may write a Github comment depending upon the output.
+    There are four different scenarios:
+    1. There has never been a previous failure and the job passes - We do not
+       create a comment. We write out an empty file to the comment path so the
+       issue-write workflow knows not to create anything.
+    2. There has never been a previous failure and the job fails - We create a
+       new comment containing the failure information and any possible premerge
+       advisor findings.
+    3. There has been a previous failure and the job passes - We update the
+       existing comment by passing its ID and a passed message to the
+       issue-write workflow.
+    4. There has been a previous failure and the job fails - We update the
+       existing comment in the same manner as above, but generate the comment
+       as if we have a failure.
+
+    Args:
+      commit_sha: The base commit SHA for this PR run.
+      build_log_files: The list of JUnit XML files and ninja logs.
+      github_token: The token to use to access the Github API.
+      pr_number: The number of the PR associated with this run.
+      return_code: The numerical return code of ninja/CMake.
+    """
+    if return_code == 0:
+        with open("comment", "w") as comment_file_handle:
+            comment = get_comment(
+                github_token,
+                pr_number,
+                ":white_check_mark: With the latest revision this PR passed "
+                "the premerge checks.",
+            )
+            if "id" in comment:
+                json.dump([comment], comment_file_handle)
     junit_objects, ninja_logs = generate_test_report_lib.load_info_from_files(
         build_log_files
     )
@@ -45,13 +115,31 @@ def main(commit_sha: str, build_log_files: list[str]):
     )
     if advisor_response.status_code == 200:
         print(advisor_response.json())
+        comments = [
+            get_comment(
+                github_token,
+                pr_number,
+                generate_test_report_lib.generate_report(
+                    generate_test_report_lib.compute_platform_title(),
+                    return_code,
+                    junit_objects,
+                    ninja_logs,
+                    failure_explanations_list=advisor_response.json(),
+                ),
+            )
+        ]
+        with open("comment", "w") as comment_file_handle:
+            json.dump(comments, comment_file_handle)
     else:
         print(advisor_response.reason)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("commit_sha", help="The base commit SHA for the test.")
+    parser.add_argument("return_code", help="The build's return code", type=int)
+    parser.add_argument("github_token", help="Github authentication token", type=str)
+    parser.add_argument("pr_number", help="The PR number", type=int)
     parser.add_argument(
         "build_log_files", help="Paths to JUnit report files and ninja logs.", nargs="*"
     )
@@ -62,4 +150,10 @@ def main(commit_sha: str, build_log_files: list[str]):
     if platform.machine() == "arm64":
         sys.exit(0)
 
-    main(args.commit_sha, args.build_log_files)
+    main(
+        args.commit_sha,
+        args.build_log_files,
+        args.github_token,
+        args.pr_number,
+        args.return_code,
+    )

.ci/premerge_advisor_explain.py

@@ -1,2 +1,3 @@
 junitparser==3.2.0
 google-cloud-storage==3.3.0
+PyGithub==2.8.1

.ci/requirements.txt

@@ -33,17 +33,18 @@ function at-exit {
   # If building fails there will be no results files.
   shopt -s nullglob
 
-  if [[ "$GITHUB_STEP_SUMMARY" != "" ]]; then
+  if [[ "$GITHUB_ACTIONS" != "" ]]; then
     python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py \
       $retcode "${BUILD_DIR}"/test-results.*.xml "${MONOREPO_ROOT}"/ninja*.log \
       >> $GITHUB_STEP_SUMMARY
+    python "${MONOREPO_ROOT}"/.ci/premerge_advisor_explain.py \
+      $(git rev-parse HEAD~1) $retcode "${GITHUB_TOKEN}" \
+      $GITHUB_PR_NUMBER "${BUILD_DIR}"/test-results.*.xml \
+      "${MONOREPO_ROOT}"/ninja*.log
   fi
 
   if [[ "$retcode" != "0" ]]; then
     if [[ "$GITHUB_ACTIONS" != "" ]]; then
-      python "${MONOREPO_ROOT}"/.ci/premerge_advisor_explain.py \
-        $(git rev-parse HEAD~1) "${BUILD_DIR}"/test-results.*.xml \
-        "${MONOREPO_ROOT}"/ninja*.log
       python "${MONOREPO_ROOT}"/.ci/premerge_advisor_upload.py \
         $(git rev-parse HEAD~1) $GITHUB_RUN_NUMBER \
         "${BUILD_DIR}"/test-results.*.xml "${MONOREPO_ROOT}"/ninja*.log

.ci/utils.sh

@@ -1,3 +1,4 @@
+HeaderFilterRegex: ''
 Checks: >
   -*,
   clang-diagnostic-*,

.clang-tidy

@@ -52,4 +52,6 @@ jobs:
         working-directory: utils/bazel
         run: |
           bazelisk test --config=ci --sandbox_base="" \
+            --remote_cache=https://storage.googleapis.com/$CACHE_GCS_BUCKET-bazel \
+            --google_default_credentials \
             @llvm-project//llvm/unittests:adt_tests

.github/workflows/bazel-checks.yml

@@ -36,6 +36,9 @@ jobs:
             test-command: 'cd $HOME && clang-format --version | grep version && git-clang-format -h | grep usage && black --version | grep black'
           - container-name: lint
             test-command: 'cd $HOME && clang-tidy --version | grep version && clang-tidy-diff.py -h | grep usage'
+          - container-name: abi-tests
+            test-command: 'cd $HOME && abi-compliance-checker --help'
+            target: abi-tests
     steps:
       - name: Checkout LLVM
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -52,7 +55,7 @@ jobs:
         with:
           container-name: ci-ubuntu-24.04-${{ matrix.container-name }}
           dockerfile: .github/workflows/containers/github-action-ci-tooling/Dockerfile
-          target: ci-container-code-${{ matrix.container-name }}
+          target: ci-container-${{ matrix.target || format('code-{0}', matrix.container-name) }}
           test-command: ${{ matrix.test-command }}
 
   push-ci-container:

.github/workflows/build-ci-container-tooling.yml

@@ -47,6 +47,28 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 # as root in 'ci-container-code-format' and 'ci-container-code-lint' containers
 
 
+FROM base AS ci-container-build-tools
+ARG LLVM_VERSION
+ARG LLVM_VERSION_MAJOR
+
+COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-${LLVM_VERSION_MAJOR} \
+                            ${LLVM_SYSROOT}/bin/
+COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/lib/clang/${LLVM_VERSION_MAJOR}/include \
+                            ${LLVM_SYSROOT}/lib/clang/${LLVM_VERSION_MAJOR}/include
+RUN ln -s ${LLVM_SYSROOT}/bin/clang-${LLVM_VERSION_MAJOR} ${LLVM_SYSROOT}/bin/clang && \
+    ln -s ${LLVM_SYSROOT}/bin/clang ${LLVM_SYSROOT}/bin/clang++
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    cmake \
+    ninja-build && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV CC=${LLVM_SYSROOT}/bin/clang
+ENV CXX=${LLVM_SYSROOT}/bin/clang++
+
+
 FROM base AS ci-container-code-format
 ARG LLVM_VERSION
 
@@ -63,31 +85,38 @@ USER gha
 WORKDIR /home/gha
 
 
-FROM base AS ci-container-code-lint
+FROM ci-container-build-tools AS ci-container-code-lint
 ARG LLVM_VERSION
 ARG LLVM_VERSION_MAJOR
 
 COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-tidy \
-                            /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/bin/clang-${LLVM_VERSION_MAJOR} \
                             ${LLVM_SYSROOT}/bin/
-COPY --from=llvm-downloader /llvm-extract/LLVM-${LLVM_VERSION}-Linux-X64/lib/clang/${LLVM_VERSION_MAJOR}/include \
-                            ${LLVM_SYSROOT}/lib/clang/${LLVM_VERSION_MAJOR}/include
 COPY clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py ${LLVM_SYSROOT}/bin/clang-tidy-diff.py
 
-RUN ln -s ${LLVM_SYSROOT}/bin/clang-${LLVM_VERSION_MAJOR} ${LLVM_SYSROOT}/bin/clang && \
-    ln -s ${LLVM_SYSROOT}/bin/clang ${LLVM_SYSROOT}/bin/clang++
+# Install dependencies for 'pr-code-lint.yml' job
+COPY llvm/utils/git/requirements_linting.txt requirements_linting.txt
+RUN pip install -r requirements_linting.txt --break-system-packages && \
+    rm requirements_linting.txt
+USER gha
+WORKDIR /home/gha
+
 
+FROM ci-container-build-tools as ci-container-abi-tests
 
 RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    cmake \
-    ninja-build && \
+    abi-compliance-checker \
+    abi-dumper \
+    autoconf \
+    parallel \
+    pkg-config && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# Install dependencies for 'pr-code-lint.yml' job
-COPY llvm/utils/git/requirements_linting.txt requirements_linting.txt
-RUN pip install -r requirements_linting.txt --break-system-packages && \
-    rm requirements_linting.txt
-USER gha
-WORKDIR /home/gha
+RUN git clone https://github.com/universal-ctags/ctags.git && \
+    cd ctags && \
+    ./autogen.sh && \
+    ./configure && \
+    sudo make install && \
+    rm -Rf ../ctags
+

.github/workflows/containers/github-action-ci-tooling/Dockerfile

@@ -7,6 +7,7 @@ on:
       - "Check for private emails used in PRs"
       - "PR Request Release Note"
       - "Code lint"
+      - "CI Checks"
     types:
       - completed
 

.github/workflows/issue-write.yml

@@ -84,6 +84,8 @@ jobs:
     if: github.repository_owner == 'llvm'
     needs: abi-dump-setup
     runs-on: ubuntu-24.04
+    container:
+      image: "ghcr.io/llvm/ci-ubuntu-24.04-abi-tests@sha256:f80125c0f767e29b8616210c0fd5cea2cd1f4fb6f2ca86d89f6016b6329b8d7f" #ghcr.io/llvm/ci-ubuntu-24.04-abi-tests:9524b37c503f
     strategy:
       matrix:
         name:
@@ -101,17 +103,6 @@ jobs:
     steps:
       - name: Install Ninja
         uses: llvm/actions/install-ninja@42d80571b13f4599bbefbc7189728b64723c7f78 # main
-      - name: Install abi-compliance-checker
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y abi-dumper autoconf pkg-config
-      - name: Install universal-ctags
-        run: |
-          git clone https://github.com/universal-ctags/ctags.git
-          cd ctags
-          ./autogen.sh
-          ./configure
-          sudo make install
       - name: Download source code
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
@@ -139,6 +130,8 @@ jobs:
   abi-compare:
     if: github.repository_owner == 'llvm'
     runs-on: ubuntu-24.04
+    container:
+      image: "ghcr.io/llvm/ci-ubuntu-24.04-abi-tests@sha256:f80125c0f767e29b8616210c0fd5cea2cd1f4fb6f2ca86d89f6016b6329b8d7f" #ghcr.io/llvm/ci-ubuntu-24.04-abi-tests:9524b37c503f
     needs:
       - abi-dump-setup
       - abi-dump
@@ -154,10 +147,6 @@ jobs:
           name: build-latest
           path: build-latest
 
-      - name: Install abi-compliance-checker
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y abi-compliance-checker
       - name: Compare ABI
         run: |
           for lib in ${{ needs.abi-dump-setup.outputs.ABI_LIBS }}; do