Merge branch 'main' of https://github.com/llvm/llvm-project into aligned_accessor

Author: dalg24

.ci/metrics/metrics.py

@@ -3,6 +3,7 @@
 import os
 from dataclasses import dataclass
 import sys
+import logging
 
 import github
 from github import Github
@@ -24,6 +25,7 @@ class JobMetrics:
     status: int
     created_at_ns: int
     workflow_id: int
+    workflow_name: str
 
 
 @dataclass
@@ -43,40 +45,60 @@ def get_sampled_workflow_metrics(github_repo: github.Repository):
       Returns a list of GaugeMetric objects, containing the relevant metrics about
       the workflow
     """
+    queued_job_counts = {}
+    running_job_counts = {}
 
     # Other states are available (pending, waiting, etc), but the meaning
     # is not documented (See #70540).
     # "queued" seems to be the info we want.
-    queued_workflow_count = len(
-        [
-            x
-            for x in github_repo.get_workflow_runs(status="queued")
-            if x.name in WORKFLOWS_TO_TRACK
-        ]
-    )
-    running_workflow_count = len(
-        [
-            x
-            for x in github_repo.get_workflow_runs(status="in_progress")
-            if x.name in WORKFLOWS_TO_TRACK
-        ]
-    )
+    for queued_workflow in github_repo.get_workflow_runs(status="queued"):
+        if queued_workflow.name not in WORKFLOWS_TO_TRACK:
+            continue
+        for queued_workflow_job in queued_workflow.jobs():
+            job_name = queued_workflow_job.name
+            # Workflows marked as queued can potentially only have some jobs
+            # queued, so make sure to also count jobs currently in progress.
+            if queued_workflow_job.status == "queued":
+                if job_name not in queued_job_counts:
+                    queued_job_counts[job_name] = 1
+                else:
+                    queued_job_counts[job_name] += 1
+            elif queued_workflow_job.status == "in_progress":
+                if job_name not in running_job_counts:
+                    running_job_counts[job_name] = 1
+                else:
+                    running_job_counts[job_name] += 1
+
+    for running_workflow in github_repo.get_workflow_runs(status="in_progress"):
+        if running_workflow.name not in WORKFLOWS_TO_TRACK:
+            continue
+        for running_workflow_job in running_workflow.jobs():
+            job_name = running_workflow_job.name
+            if running_workflow_job.status != "in_progress":
+                continue
+
+            if job_name not in running_job_counts:
+                running_job_counts[job_name] = 1
+            else:
+                running_job_counts[job_name] += 1
 
     workflow_metrics = []
-    workflow_metrics.append(
-        GaugeMetric(
-            "workflow_queue_size",
-            queued_workflow_count,
-            time.time_ns(),
+    for queued_job in queued_job_counts:
+        workflow_metrics.append(
+            GaugeMetric(
+                f"workflow_queue_size_{queued_job}",
+                queued_job_counts[queued_job],
+                time.time_ns(),
+            )
         )
-    )
-    workflow_metrics.append(
-        GaugeMetric(
-            "running_workflow_count",
-            running_workflow_count,
-            time.time_ns(),
+    for running_job in running_job_counts:
+        workflow_metrics.append(
+            GaugeMetric(
+                f"running_workflow_count_{running_job}",
+                running_job_counts[running_job],
+                time.time_ns(),
+            )
         )
-    )
     # Always send a hearbeat metric so we can monitor is this container is still able to log to Grafana.
     workflow_metrics.append(
         GaugeMetric("metrics_container_heartbeat", 1, time.time_ns())
@@ -157,7 +179,7 @@ def get_per_workflow_metrics(
                 # longer in a testing state and we can directly assert the workflow
                 # result.
                 for step in workflow_job.steps:
-                    if step.conclusion != "success":
+                    if step.conclusion != "success" and step.conclusion != "skipped":
                         job_result = 0
                         break
 
@@ -171,6 +193,10 @@ def get_per_workflow_metrics(
             # in nanoseconds.
             created_at_ns = int(created_at.timestamp()) * 10**9
 
+            logging.info(
+                f"Adding a job metric for job {workflow_job.id} in workflow {workflow_run.id}"
+            )
+
             workflow_metrics.append(
                 JobMetrics(
                     workflow_run.name + "-" + workflow_job.name,
@@ -179,6 +205,7 @@ def get_per_workflow_metrics(
                     job_result,
                     created_at_ns,
                     workflow_run.id,
+                    workflow_run.name,
                 )
             )
 
@@ -198,7 +225,7 @@ def upload_metrics(workflow_metrics, metrics_userid, api_key):
     """
 
     if len(workflow_metrics) == 0:
-        print("No metrics found to upload.", file=sys.stderr)
+        logging.info("No metrics found to upload.")
         return
 
     metrics_batch = []
@@ -227,16 +254,12 @@ def upload_metrics(workflow_metrics, metrics_userid, api_key):
     )
 
     if response.status_code < 200 or response.status_code >= 300:
-        print(
-            f"Failed to submit data to Grafana: {response.status_code}", file=sys.stderr
-        )
+        logging.info(f"Failed to submit data to Grafana: {response.status_code}")
 
 
 def main():
     # Authenticate with Github
     auth = Auth.Token(os.environ["GITHUB_TOKEN"])
-    github_object = Github(auth=auth)
-    github_repo = github_object.get_repo("llvm/llvm-project")
 
     grafana_api_key = os.environ["GRAFANA_API_KEY"]
     grafana_metrics_userid = os.environ["GRAFANA_METRICS_USERID"]
@@ -248,24 +271,24 @@ def main():
     # Enter the main loop. Every five minutes we wake up and dump metrics for
     # the relevant jobs.
     while True:
+        github_object = Github(auth=auth)
+        github_repo = github_object.get_repo("llvm/llvm-project")
+
         current_metrics = get_per_workflow_metrics(github_repo, workflows_to_track)
         current_metrics += get_sampled_workflow_metrics(github_repo)
-        # Always send a hearbeat metric so we can monitor is this container is still able to log to Grafana.
-        current_metrics.append(
-            GaugeMetric("metrics_container_heartbeat", 1, time.time_ns())
-        )
 
         upload_metrics(current_metrics, grafana_metrics_userid, grafana_api_key)
-        print(f"Uploaded {len(current_metrics)} metrics", file=sys.stderr)
+        logging.info(f"Uploaded {len(current_metrics)} metrics")
 
         for workflow_metric in reversed(current_metrics):
             if isinstance(workflow_metric, JobMetrics):
                 workflows_to_track[
-                    workflow_metric.job_name
+                    workflow_metric.workflow_name
                 ] = workflow_metric.workflow_id
 
         time.sleep(SCRAPE_INTERVAL_SECONDS)
 
 
 if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
     main()

.github/CODEOWNERS

@@ -131,7 +131,7 @@
 /bolt/ @aaupov @maksfb @rafaelauler @ayermolo @dcci @yota9
 
 # Bazel build system.
-/utils/bazel/ @rupprecht @keith
+/utils/bazel/ @rupprecht @keith @aaronmondal
 
 # InstallAPI and TextAPI
 /llvm/**/TextAPI/ @cyndyishida

.github/new-prs-labeler.yml

@@ -499,6 +499,7 @@ clang:static analyzer:
   - clang/tools/scan-build/**
   - clang/utils/analyzer/**
   - clang/docs/analyzer/**
+  - clang/test/Analysis/**
 
 pgo:
   - llvm/lib/Transforms/Instrumentation/CGProfile.cpp

.github/workflows/build-ci-container-windows.yml

@@ -27,7 +27,7 @@ jobs:
       container-filename: ${{ steps.vars.outputs.container-filename }}
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/workflows/containers/github-action-ci-windows
       - name: Write Variables
@@ -46,7 +46,7 @@ jobs:
         run: |
           docker save  ${{ steps.vars.outputs.container-name-tag }} >  ${{ steps.vars.outputs.container-filename }}
       - name: Upload container image
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: container
           path: ${{ steps.vars.outputs.container-filename }}
@@ -63,7 +63,7 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Download container
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           name: container
       - name: Push Container

.github/workflows/build-ci-container.yml

@@ -20,24 +20,31 @@ on:
 jobs:
   build-ci-container:
     if: github.repository_owner == 'llvm'
-    runs-on: depot-ubuntu-22.04-16
-    outputs:
-      container-name: ${{ steps.vars.outputs.container-name }}
-      container-name-agent: ${{ steps.vars.outputs.container-name-agent }}
-      container-name-tag: ${{ steps.vars.outputs.container-name-tag }}
-      container-name-agent-tag: ${{ steps.vars.outputs.container-name-agent-tag }}
-      container-filename: ${{ steps.vars.outputs.container-filename }}
-      container-agent-filename: ${{ steps.vars.outputs.container-agent-filename }}
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      matrix:
+        include:
+          # The arch names should match the names used on dockerhub.
+          # See https://github.com/docker-library/official-images#architectures-other-than-amd64
+          - arch: amd64
+            runs-on: depot-ubuntu-22.04-16
+          - arch: arm64v8
+            runs-on: depot-ubuntu-22.04-arm-16
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/workflows/containers/github-action-ci/
+      # podman is not installed by default on the ARM64 images.
+      - name: Install Podman
+        if: runner.arch == 'ARM64'
+        run: |
+          sudo apt-get install podman
       - name: Write Variables
         id: vars
         run: |
-          tag=`date +%s`
-          container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/ci-ubuntu-22.04"
+          tag=$(git rev-parse --short=12 HEAD)
+          container_name="ghcr.io/$GITHUB_REPOSITORY_OWNER/${{ matrix.arch }}/ci-ubuntu-22.04"
           echo "container-name=$container_name" >> $GITHUB_OUTPUT
           echo "container-name-agent=$container_name-agent" >> $GITHUB_OUTPUT
           echo "container-name-tag=$container_name:$tag" >> $GITHUB_OUTPUT
@@ -59,9 +66,9 @@ jobs:
           podman save ${{ steps.vars.outputs.container-name-agent-tag }} > ${{ steps.vars.outputs.container-agent-filename }}
 
       - name: Upload container image
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
-          name: container
+          name: container-${{ matrix.arch }}
           path: "*.tar"
           retention-days: 14
 
@@ -83,19 +90,30 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Download container
-        uses: actions/download-artifact@v4
-        with:
-          name: container
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
 
       - name: Push Container
         run: |
-          podman load -i ${{ needs.build-ci-container.outputs.container-filename }}
-          podman tag ${{ needs.build-ci-container.outputs.container-name-tag }} ${{ needs.build-ci-container.outputs.container-name }}:latest
+          function push_container {
+            image_name=$1
+            latest_name=$(echo $image_name | sed 's/:[a-f0-9]\+$/:latest/g')
+            podman tag $image_name $latest_name
+            echo "Pushing $image_name ..."
+            podman push $image_name
+            echo "Pushing $latest_name ..."
+            podman push $latest_name
+          }
+
           podman login -u ${{ github.actor }} -p $GITHUB_TOKEN ghcr.io
-          podman push ${{ needs.build-ci-container.outputs.container-name-tag }}
-          podman push ${{ needs.build-ci-container.outputs.container-name }}:latest
+          for f in $(find . -iname *.tar); do
+            image_name=$(podman load -q -i $f | sed 's/Loaded image: //g')
+            push_container $image_name
 
-          podman load -i ${{ needs.build-ci-container.outputs.container-agent-filename }}
-          podman tag ${{ needs.build-ci-container.outputs.container-name-agent-tag }} ${{ needs.build-ci-container.outputs.container-name-agent }}:latest
-          podman push ${{ needs.build-ci-container.outputs.container-name-agent-tag }}
-          podman push ${{ needs.build-ci-container.outputs.container-name-agent }}:latest
+            if echo $image_name | grep '/amd64/'; then
+              # For amd64, create an alias with the arch component removed.
+              # This matches the convention used on dockerhub.
+              default_image_name=$(echo $(dirname $(dirname $image_name))/$(basename $image_name))
+              podman tag $image_name $default_image_name
+              push_container $default_image_name
+            fi
+          done

.github/workflows/build-metrics-container.yml

@@ -20,14 +20,14 @@ on:
 jobs:
   build-metrics-container:
     if: github.repository_owner == 'llvm'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     outputs:
       container-name: ${{ steps.vars.outputs.container-name }}
       container-name-tag: ${{ steps.vars.outputs.container-name-tag }}
       container-filename: ${{ steps.vars.outputs.container-filename }}
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .ci/metrics/
       - name: Write Variables
@@ -49,7 +49,7 @@ jobs:
         run: |
           podman save  ${{ steps.vars.outputs.container-name-tag }} >  ${{ steps.vars.outputs.container-filename }}
       - name: Upload Container Image
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: container
           path: ${{ steps.vars.outputs.container-filename }}
@@ -66,7 +66,7 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Download Container
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           name: container
       - name: Push Container

.github/workflows/ci-post-commit-analyzer.yml

@@ -44,7 +44,7 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Setup ccache
-        uses: hendrikmuhs/ccache-action@v1
+        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
         with:
           # A full build of llvm, clang, lld, and lldb takes about 250MB
           # of ccache space. There's not much reason to have more than this,

.github/workflows/commit-access-review.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Fetch LLVM sources
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install dependencies
         run: |