Merge branch 'main' into vplan-dyncast-vpirmetadata

Author: fhahn

.ci/all_requirements.txt

@@ -32,8 +32,6 @@ export LD=link
 # see https://github.com/llvm/llvm-project/pull/82393 and
 # https://discourse.llvm.org/t/rfc-future-of-windows-pre-commit-ci/76840/40
 # for further information.
-# We limit the number of parallel compile jobs to 24 control memory
-# consumption and improve build reliability.
 cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
       -G Ninja \
@@ -49,7 +47,6 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D CMAKE_EXE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_MODULE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_SHARED_LINKER_FLAGS="/MANIFEST:NO" \
-      -D CMAKE_CXX_FLAGS="-Wno-c++98-compat -Wno-c++14-compat -Wno-unsafe-buffer-usage -Wno-old-style-cast" \
       -D LLVM_ENABLE_RUNTIMES="${runtimes}"
 
 start-group "ninja"

.ci/monolithic-windows.sh

@@ -4,20 +4,90 @@
 """Script for getting explanations from the premerge advisor."""
 
 import argparse
-import os
 import platform
 import sys
+import json
+
+# TODO(boomanaiden154): Remove the optional call once we can require Python
+# 3.10.
+from typing import Optional
 
 import requests
+import github
+import github.PullRequest
 
 import generate_test_report_lib
 
 PREMERGE_ADVISOR_URL = (
     "http://premerge-advisor.premerge-advisor.svc.cluster.local:5000/explain"
 )
+COMMENT_TAG = "<!--PREMERGE ADVISOR COMMENT: {platform}-->"
+
+
+def get_comment_id(platform: str, pr: github.PullRequest.PullRequest) -> Optional[int]:
+    platform_comment_tag = COMMENT_TAG.format(platform=platform)
+    for comment in pr.as_issue().get_comments():
+        if platform_comment_tag in comment.body:
+            return comment.id
+    return None
+
+
+def get_comment(
+    github_token: str,
+    pr_number: int,
+    body: str,
+) -> dict[str, str]:
+    repo = github.Github(github_token).get_repo("llvm/llvm-project")
+    pr = repo.get_issue(pr_number).as_pull_request()
+    comment = {"body": body}
+    comment_id = get_comment_id(platform.system(), pr)
+    if comment_id:
+        comment["id"] = comment_id
+    return comment
 
 
-def main(commit_sha: str, build_log_files: list[str]):
+def main(
+    commit_sha: str,
+    build_log_files: list[str],
+    github_token: str,
+    pr_number: int,
+    return_code: int,
+):
+    """The main entrypoint for the script.
+
+    This function parses failures from files, requests information from the
+    premerge advisor, and may write a Github comment depending upon the output.
+    There are four different scenarios:
+    1. There has never been a previous failure and the job passes - We do not
+       create a comment. We write out an empty file to the comment path so the
+       issue-write workflow knows not to create anything.
+    2. There has never been a previous failure and the job fails - We create a
+       new comment containing the failure information and any possible premerge
+       advisor findings.
+    3. There has been a previous failure and the job passes - We update the
+       existing comment by passing its ID and a passed message to the
+       issue-write workflow.
+    4. There has been a previous failure and the job fails - We update the
+       existing comment in the same manner as above, but generate the comment
+       as if we have a failure.
+
+    Args:
+      commit_sha: The base commit SHA for this PR run.
+      build_log_files: The list of JUnit XML files and ninja logs.
+      github_token: The token to use to access the Github API.
+      pr_number: The number of the PR associated with this run.
+      return_code: The numerical return code of ninja/CMake.
+    """
+    if return_code == 0:
+        with open("comment", "w") as comment_file_handle:
+            comment = get_comment(
+                github_token,
+                pr_number,
+                ":white_check_mark: With the latest revision this PR passed "
+                "the premerge checks.",
+            )
+            if "id" in comment:
+                json.dump([comment], comment_file_handle)
     junit_objects, ninja_logs = generate_test_report_lib.load_info_from_files(
         build_log_files
     )
@@ -45,13 +115,31 @@ def main(commit_sha: str, build_log_files: list[str]):
     )
     if advisor_response.status_code == 200:
         print(advisor_response.json())
+        comments = [
+            get_comment(
+                github_token,
+                pr_number,
+                generate_test_report_lib.generate_report(
+                    generate_test_report_lib.compute_platform_title(),
+                    return_code,
+                    junit_objects,
+                    ninja_logs,
+                    failure_explanations_list=advisor_response.json(),
+                ),
+            )
+        ]
+        with open("comment", "w") as comment_file_handle:
+            json.dump(comments, comment_file_handle)
     else:
         print(advisor_response.reason)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("commit_sha", help="The base commit SHA for the test.")
+    parser.add_argument("return_code", help="The build's return code", type=int)
+    parser.add_argument("github_token", help="Github authentication token", type=str)
+    parser.add_argument("pr_number", help="The PR number", type=int)
     parser.add_argument(
         "build_log_files", help="Paths to JUnit report files and ninja logs.", nargs="*"
     )
@@ -62,4 +150,10 @@ def main(commit_sha: str, build_log_files: list[str]):
     if platform.machine() == "arm64":
         sys.exit(0)
 
-    main(args.commit_sha, args.build_log_files)
+    main(
+        args.commit_sha,
+        args.build_log_files,
+        args.github_token,
+        args.pr_number,
+        args.return_code,
+    )

.ci/premerge_advisor_explain.py

@@ -1,2 +1,3 @@
 junitparser==3.2.0
 google-cloud-storage==3.3.0
+PyGithub==2.8.1

.ci/requirements.txt

@@ -33,17 +33,18 @@ function at-exit {
   # If building fails there will be no results files.
   shopt -s nullglob
 
-  if [[ "$GITHUB_STEP_SUMMARY" != "" ]]; then
+  if [[ "$GITHUB_ACTIONS" != "" ]]; then
     python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py \
       $retcode "${BUILD_DIR}"/test-results.*.xml "${MONOREPO_ROOT}"/ninja*.log \
       >> $GITHUB_STEP_SUMMARY
+    python "${MONOREPO_ROOT}"/.ci/premerge_advisor_explain.py \
+      $(git rev-parse HEAD~1) $retcode "${GITHUB_TOKEN}" \
+      $GITHUB_PR_NUMBER "${BUILD_DIR}"/test-results.*.xml \
+      "${MONOREPO_ROOT}"/ninja*.log
   fi
 
   if [[ "$retcode" != "0" ]]; then
     if [[ "$GITHUB_ACTIONS" != "" ]]; then
-      python "${MONOREPO_ROOT}"/.ci/premerge_advisor_explain.py \
-        $(git rev-parse HEAD~1) "${BUILD_DIR}"/test-results.*.xml \
-        "${MONOREPO_ROOT}"/ninja*.log
       python "${MONOREPO_ROOT}"/.ci/premerge_advisor_upload.py \
         $(git rev-parse HEAD~1) $GITHUB_RUN_NUMBER \
         "${BUILD_DIR}"/test-results.*.xml "${MONOREPO_ROOT}"/ninja*.log

.ci/utils.sh

@@ -1,3 +1,4 @@
+HeaderFilterRegex: ''
 Checks: >
   -*,
   clang-diagnostic-*,

.clang-tidy

@@ -0,0 +1,95 @@
+name: Build Container
+description: >-
+  Build and test a container using the standard llvm naming scheme for containers.
+
+inputs:
+  tag:
+    description: >-
+      The tag to use for this container.
+    required: false
+  container-name:
+    description: >-
+      The name for the container.
+    required: true
+  dockerfile:
+    description: >-
+      Path to docker file.
+    required: false
+  target:
+    description: >-
+      The container target to build 'passed to podman via ---target option'
+    required: false
+  context:
+    description: >-
+      Path to context for the container build.
+    required: false
+  test-command:
+    description: >-
+      Test command to run to ensure the container is working correctly.
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    # podman is not installed by default on the ARM64 images.
+    - name: Install Podman
+      if: runner.arch == 'ARM64'
+      shell: bash
+      run: |
+        sudo apt-get install podman
+
+    - name: Build Container
+      shell: bash
+      env:
+        INPUT_TAG: ${{inputs.tag }}
+        INPUT_CONTAINER_NAME: ${{ inputs.container-name }}
+        INPUT_TARGET: ${{ inputs.target }}
+        INPUT_DOCKERFILE: ${{ inputs.dockerfile }}
+        INPUT_CONTEXT: ${{ inputs.context }}
+      id: build
+      run: |
+        env
+        tag="${INPUT_TAG:-$(git rev-parse --short=12 HEAD)}"
+
+        case "$RUNNER_ARCH" in
+          ARM64)
+            container_arch="arm64v8"
+            ;;
+          *)
+            container_arch="amd64"
+            ;;
+        esac
+
+        container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/$container_arch/$INPUT_CONTAINER_NAME:$tag"
+        container_filename="$(echo $container_name  | sed -e 's/\//-/g' -e 's/:/-/g').tar"
+        if [ -n "$INPUT_TARGET" ]; then
+          podman_options="$podman_options --target $INPUT_TARGET"
+        fi
+        if [ -n "$INPUT_DOCKERFILE" ]; then
+          podman_options="$podman_options -f $INPUT_DOCKERFILE"
+        fi
+        podman_options="$podman_options ${INPUT_CONTEXT:-.}"
+        echo "Podman Options: $podman_options"
+
+        podman build -t $container_name $podman_options
+
+        podman save $container_name > $container_filename
+
+        echo "container-full-name=$container_name" >> $GITHUB_OUTPUT
+
+    - name: Create container artifact
+      uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+      with:
+        name: ${{ inputs.container-name }}-${{ runner.arch }}
+        path: "*.tar"
+        retention-days: 14
+
+    - name: Test container
+      shell: bash
+      if: inputs.test-command
+      env:
+        INPUT_TEST_COMMAND: ${{ inputs.test-command }}
+        CONTAINER_FULL_NAME: ${{ steps.build.outputs.container-full-name }}
+      run: |
+        podman run --pull=never --rm -it $CONTAINER_FULL_NAME /usr/bin/bash -x -c "$INPUT_TEST_COMMAND"
+

.github/actions/build-container/action.yml

@@ -0,0 +1,44 @@
+name: Push Container
+description: >-
+  Download all container artifacts for this job and push them to the GitHub registry.
+
+inputs:
+  token:
+    description: >-
+      Token to use to authenticate with the container registry.
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download container
+      uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
+
+    - name: Push Container
+      env:
+        GITHUB_TOKEN: ${{ inputs.token }}
+      shell: bash
+      run: |
+        function push_container {
+          image_name=$1
+          latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g')
+          podman tag $image_name $latest_name
+          echo "Pushing $image_name ..."
+          podman push --compression-format=zstd $image_name
+          echo "Pushing $latest_name ..."
+          podman push --compression-format=zstd $latest_name
+        }
+
+        podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
+        for f in $(find . -iname '*.tar'); do
+          image_name=$(podman load -q -i $f | sed 's/Loaded image: //g')
+          push_container $image_name
+
+          if echo $image_name | grep '/amd64/'; then
+            # For amd64, create an alias with the arch component removed.
+            # This matches the convention used on dockerhub.
+            default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name))
+            podman tag $image_name $default_image_name
+            push_container $default_image_name
+          fi
+        done

.github/actions/push-container/action.yml

@@ -8,5 +8,12 @@
   "minimumReleaseAge": "3 days",
   "assignees": ["boomanaiden154"],
   "ignorePaths": [".github/workflows/containers/**"],
-  "groupName": "[Github] Update GHA Dependencies"
+  "groupName": "[Github] Update GHA Dependencies",
+  "packageRules": [
+    {
+      "matchPackageNames": ["windows", "macos"],
+      "matchManagers": ["github-actions"],
+      "enabled": false
+    }
+  ]
 }

.github/renovate.json

@@ -30,3 +30,32 @@ jobs:
       - name: Run Buildifier
         run: |
           buildifier --mode=check $(find ./utils/bazel -name *BUILD*)
+  
+  bazel-build:
+    name: "Bazel Build/Test"
+    # Only run on US Central workers so we only have to keep one cache warm as
+    # the cache buckets are per cluster.
+    runs-on:
+      group: llvm-premerge-cluster-us-central
+      labels: llvm-premerge-linux-runners
+    if: github.repository == 'llvm/llvm-project'
+    steps:
+      - name: Fetch LLVM sources
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        # TODO(boomanaiden154): We should use a purpose built container for this. Move
+        # over when we have fixed the issues with using custom containers with Github
+        # ARC in GKE.
+      - name: Setup System Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libmpfr-dev libpfm4-dev m4 libedit-dev
+          sudo curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.27.0/bazelisk-amd64.deb > /tmp/bazelisk.deb
+          sudo apt-get install -y /tmp/bazelisk.deb
+          rm /tmp/bazelisk.deb
+      - name: Build/Test
+        working-directory: utils/bazel
+        run: |
+          bazelisk test --config=ci --sandbox_base="" \
+            --remote_cache=https://storage.googleapis.com/$CACHE_GCS_BUCKET-bazel \
+            --google_default_credentials \
+            @llvm-project//... //...