Merge branch 'main' into add-unroll-full-option

Author: linuxlonelyeagle

.ci/metrics/metrics.py

@@ -24,6 +24,7 @@ class JobMetrics:
     status: int
     created_at_ns: int
     workflow_id: int
+    workflow_name: str
 
 
 @dataclass
@@ -43,40 +44,60 @@ def get_sampled_workflow_metrics(github_repo: github.Repository):
       Returns a list of GaugeMetric objects, containing the relevant metrics about
       the workflow
     """
+    queued_job_counts = {}
+    running_job_counts = {}
 
     # Other states are available (pending, waiting, etc), but the meaning
     # is not documented (See #70540).
     # "queued" seems to be the info we want.
-    queued_workflow_count = len(
-        [
-            x
-            for x in github_repo.get_workflow_runs(status="queued")
-            if x.name in WORKFLOWS_TO_TRACK
-        ]
-    )
-    running_workflow_count = len(
-        [
-            x
-            for x in github_repo.get_workflow_runs(status="in_progress")
-            if x.name in WORKFLOWS_TO_TRACK
-        ]
-    )
+    for queued_workflow in github_repo.get_workflow_runs(status="queued"):
+        if queued_workflow.name not in WORKFLOWS_TO_TRACK:
+            continue
+        for queued_workflow_job in queued_workflow.jobs():
+            job_name = queued_workflow_job.name
+            # Workflows marked as queued can potentially only have some jobs
+            # queued, so make sure to also count jobs currently in progress.
+            if queued_workflow_job.status == "queued":
+                if job_name not in queued_job_counts:
+                    queued_job_counts[job_name] = 1
+                else:
+                    queued_job_counts[job_name] += 1
+            elif queued_workflow_job.status == "in_progress":
+                if job_name not in running_job_counts:
+                    running_job_counts[job_name] = 1
+                else:
+                    running_job_counts[job_name] += 1
+
+    for running_workflow in github_repo.get_workflow_runs(status="in_progress"):
+        if running_workflow.name not in WORKFLOWS_TO_TRACK:
+            continue
+        for running_workflow_job in running_workflow.jobs():
+            job_name = running_workflow_job.name
+            if running_workflow_job.status != "in_progress":
+                continue
+
+            if job_name not in running_job_counts:
+                running_job_counts[job_name] = 1
+            else:
+                running_job_counts[job_name] += 1
 
     workflow_metrics = []
-    workflow_metrics.append(
-        GaugeMetric(
-            "workflow_queue_size",
-            queued_workflow_count,
-            time.time_ns(),
+    for queued_job in queued_job_counts:
+        workflow_metrics.append(
+            GaugeMetric(
+                f"workflow_queue_size_{queued_job}",
+                queued_job_counts[queued_job],
+                time.time_ns(),
+            )
         )
-    )
-    workflow_metrics.append(
-        GaugeMetric(
-            "running_workflow_count",
-            running_workflow_count,
-            time.time_ns(),
+    for running_job in running_job_counts:
+        workflow_metrics.append(
+            GaugeMetric(
+                f"running_workflow_count_{running_job}",
+                running_job_counts[running_job],
+                time.time_ns(),
+            )
         )
-    )
     # Always send a hearbeat metric so we can monitor is this container is still able to log to Grafana.
     workflow_metrics.append(
         GaugeMetric("metrics_container_heartbeat", 1, time.time_ns())
@@ -157,7 +178,7 @@ def get_per_workflow_metrics(
                 # longer in a testing state and we can directly assert the workflow
                 # result.
                 for step in workflow_job.steps:
-                    if step.conclusion != "success":
+                    if step.conclusion != "success" and step.conclusion != "skipped":
                         job_result = 0
                         break
 
@@ -179,6 +200,7 @@ def get_per_workflow_metrics(
                     job_result,
                     created_at_ns,
                     workflow_run.id,
+                    workflow_run.name,
                 )
             )
 
@@ -235,8 +257,6 @@ def upload_metrics(workflow_metrics, metrics_userid, api_key):
 def main():
     # Authenticate with Github
     auth = Auth.Token(os.environ["GITHUB_TOKEN"])
-    github_object = Github(auth=auth)
-    github_repo = github_object.get_repo("llvm/llvm-project")
 
     grafana_api_key = os.environ["GRAFANA_API_KEY"]
     grafana_metrics_userid = os.environ["GRAFANA_METRICS_USERID"]
@@ -248,20 +268,19 @@ def main():
     # Enter the main loop. Every five minutes we wake up and dump metrics for
     # the relevant jobs.
     while True:
+        github_object = Github(auth=auth)
+        github_repo = github_object.get_repo("llvm/llvm-project")
+
         current_metrics = get_per_workflow_metrics(github_repo, workflows_to_track)
         current_metrics += get_sampled_workflow_metrics(github_repo)
-        # Always send a hearbeat metric so we can monitor is this container is still able to log to Grafana.
-        current_metrics.append(
-            GaugeMetric("metrics_container_heartbeat", 1, time.time_ns())
-        )
 
         upload_metrics(current_metrics, grafana_metrics_userid, grafana_api_key)
         print(f"Uploaded {len(current_metrics)} metrics", file=sys.stderr)
 
         for workflow_metric in reversed(current_metrics):
             if isinstance(workflow_metric, JobMetrics):
                 workflows_to_track[
-                    workflow_metric.job_name
+                    workflow_metric.workflow_name
                 ] = workflow_metric.workflow_id
 
         time.sleep(SCRAPE_INTERVAL_SECONDS)

.github/workflows/build-ci-container-windows.yml

@@ -27,7 +27,7 @@ jobs:
       container-filename: ${{ steps.vars.outputs.container-filename }}
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/workflows/containers/github-action-ci-windows
       - name: Write Variables
@@ -46,7 +46,7 @@ jobs:
         run: |
           docker save  ${{ steps.vars.outputs.container-name-tag }} >  ${{ steps.vars.outputs.container-filename }}
       - name: Upload container image
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: container
           path: ${{ steps.vars.outputs.container-filename }}
@@ -63,7 +63,7 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Download container
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           name: container
       - name: Push Container

.github/workflows/build-ci-container.yml

@@ -32,7 +32,7 @@ jobs:
             runs-on: depot-ubuntu-22.04-arm-16
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           sparse-checkout: .github/workflows/containers/github-action-ci/
       # podman is not installed by default on the ARM64 images.
@@ -66,7 +66,7 @@ jobs:
           podman save ${{ steps.vars.outputs.container-name-agent-tag }} > ${{ steps.vars.outputs.container-agent-filename }}
 
       - name: Upload container image
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
           name: container-${{ matrix.arch }}
           path: "*.tar"
@@ -90,7 +90,7 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Download container
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
 
       - name: Push Container
         run: |

.github/workflows/premerge.yaml

@@ -33,11 +33,11 @@ jobs:
     runs-on: llvm-premerge-linux-runners
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 2
       - name: Setup ccache
-        uses: hendrikmuhs/[email protected].14
+        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
         with:
           max-size: "2000M"
       - name: Build and Test
@@ -94,11 +94,11 @@ jobs:
         shell: bash
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 2
       - name: Setup ccache
-        uses: hendrikmuhs/[email protected].14
+        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
         with:
           variant: "sccache"
           max-size: "2000M"
@@ -144,7 +144,7 @@ jobs:
           call C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64
           bash .ci/monolithic-windows.sh "${{ steps.vars.outputs.windows-projects }}" "${{ steps.vars.outputs.windows-check-targets }}"
 
-  permerge-check-macos:
+  premerge-check-macos:
     runs-on: macos-14
     if: >-
       github.repository_owner == 'llvm' &&
@@ -153,11 +153,11 @@ jobs:
       (github.event_name != 'pull_request' || github.event.action != 'closed')
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 2
       - name: Setup ccache
-        uses: hendrikmuhs/[email protected].14
+        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
         with:
           max-size: "2000M"
       - name: Install Ninja

.github/workflows/release-asset-audit.py

@@ -1,4 +1,5 @@
 import github
+import re
 import sys
 
 _SPECIAL_CASE_BINARIES = {
@@ -16,38 +17,73 @@ def _is_valid(uploader_name, valid_uploaders, asset_name):
     return False
 
 
+def _get_uploaders(release_version):
+    # Until llvm 18, assets were uploaded by community members, the release managers
+    # and the GitHub Actions bot.
+    if release_version <= 18:
+        return set(
+            [
+                "DimitryAndric",
+                "stefanp-ibm",
+                "lei137",
+                "omjavaid",
+                "nicolerabjohn",
+                "amy-kwan",
+                "mandlebug",
+                "zmodem",
+                "androm3da",
+                "tru",
+                "rovka",
+                "rorth",
+                "quinnlp",
+                "kamaub",
+                "abrisco",
+                "jakeegan",
+                "maryammo",
+                "tstellar",
+                "github-actions[bot]",
+            ]
+        )
+    # llvm 19 and beyond, only the release managers, bot and a much smaller
+    # number of community members.
+    elif release_version >= 19:
+        return set(
+            [
+                "zmodem",
+                "omjavaid",
+                "tru",
+                "tstellar",
+                "github-actions[bot]",
+            ]
+        )
+
+
+def _get_major_release_version(release_title):
+    # All release titles are of the form "LLVM X.Y.Z(-rcN)".
+    match = re.match("LLVM ([0-9]+)\.", release_title)
+    if match is None:
+        _write_comment_and_exit_with_error(
+            f'Could not parse release version from release title "{release_title}".'
+        )
+    else:
+        return int(match.groups()[0])
+
+
+def _write_comment_and_exit_with_error(comment):
+    with open("comment", "w") as file:
+        file.write(comment)
+    sys.exit(1)
+
+
 def main():
     token = sys.argv[1]
 
     gh = github.Github(login_or_token=token)
     repo = gh.get_repo("llvm/llvm-project")
 
-    uploaders = set(
-        [
-            "DimitryAndric",
-            "stefanp-ibm",
-            "lei137",
-            "omjavaid",
-            "nicolerabjohn",
-            "amy-kwan",
-            "mandlebug",
-            "zmodem",
-            "androm3da",
-            "tru",
-            "rovka",
-            "rorth",
-            "quinnlp",
-            "kamaub",
-            "abrisco",
-            "jakeegan",
-            "maryammo",
-            "tstellar",
-            "github-actions[bot]",
-        ]
-    )
-
     for release in repo.get_releases():
         print("Release:", release.title)
+        uploaders = _get_uploaders(_get_major_release_version(release.title))
         for asset in release.get_assets():
             created_at = asset.created_at
             updated_at = (
@@ -57,9 +93,9 @@ def main():
                 f"{asset.name} : {asset.uploader.login} [{created_at} {updated_at}] ( {asset.download_count} )"
             )
             if not _is_valid(asset.uploader.login, uploaders, asset.name):
-                with open('comment', 'w') as file:
-                    file.write(f'@{asset.uploader.login} is not a valid uploader.')
-                sys.exit(1)
+                _write_comment_and_exit_with_error(
+                    f"@{asset.uploader.login} is not a valid uploader."
+                )
 
 
 if __name__ == "__main__":

.github/workflows/release-binaries.yml

@@ -133,13 +133,13 @@ jobs:
         # add extra CMake args to disable them.
         # See https://github.com/llvm/llvm-project/issues/99767
         if [ "$RUNNER_OS" = "macOS" ]; then
-          target_cmake_flags="$target_cmake_flags -DBOOTSTRAP_COMPILER_RT_ENABLE_IOS=OFF"
+          target_cmake_flags="$target_cmake_flags -DBOOTSTRAP_BOOTSTRAP_COMPILER_RT_ENABLE_IOS=OFF"
           if [ "$RUNNER_ARCH" = "ARM64" ]; then
             arches=arm64
           else
             arches=x86_64
           fi
-          target_cmake_flags="$target_cmake_flags -DBOOTSTRAP_DARWIN_osx_ARCHS=$arches -DBOOTSTRAP_DARWIN_osx_BUILTIN_ARCHS=$arches"
+          target_cmake_flags="$target_cmake_flags -DBOOTSTRAP_BOOTSTRAP_DARWIN_osx_ARCHS=$arches -DBOOTSTRAP_BOOTSTRAP_DARWIN_osx_BUILTIN_ARCHS=$arches"
         fi
 
         build_flang="true"

.mailmap

@@ -38,6 +38,8 @@ Jianjian GUAN <[email protected]> <[email protected]>
 Jon Roelofs <[email protected]> <[email protected]>
 Jon Roelofs <[email protected]> <[email protected]>
 Jonathan Thackray <[email protected]> <[email protected]>
+klensy <[email protected]>
+klensy <[email protected]> <[email protected]>
 LLVM GN Syncbot <[email protected]>
 Martin Storsjö <[email protected]>
 Med Ismail Bennani <[email protected]> <[email protected]>