Merge remote-tracking branch 'origin/main' into HEAD

Author: tstellar

.ci/all_requirements.txt

@@ -0,0 +1,85 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Caches .lit_test_times.txt files between premerge invocations.
+
+.lit_test_times.txt files are used by lit to order tests to best take advantage
+of parallelism. Having them around and up to date can result in a ~15%
+improvement in test times. This script downloading cached test time files and
+uploading new versions to the GCS buckets used for caching.
+"""
+
+import sys
+import os
+import logging
+import multiprocessing.pool
+import pathlib
+import platform
+import glob
+
+from google.cloud import storage
+
+GCS_PARALLELISM = 100
+
+
+def _get_blob_prefix():
+    return f"lit_timing_{platform.system().lower()}"
+
+
+def _maybe_upload_timing_file(bucket, timing_file_path):
+    if os.path.exists(timing_file_path):
+        timing_file_blob = bucket.blob(_get_blob_prefix() + "/" + timing_file_path)
+        timing_file_blob.upload_from_filename(timing_file_path)
+
+
+def upload_timing_files(storage_client, bucket_name: str):
+    bucket = storage_client.bucket(bucket_name)
+    with multiprocessing.pool.ThreadPool(GCS_PARALLELISM) as thread_pool:
+        futures = []
+        for timing_file_path in glob.glob("**/.lit_test_times.txt", recursive=True):
+            futures.append(
+                thread_pool.apply_async(
+                    _maybe_upload_timing_file, (bucket, timing_file_path)
+                )
+            )
+        for future in futures:
+            future.get()
+    print("Done uploading")
+
+
+def _maybe_download_timing_file(blob):
+    file_name = blob.name.removeprefix(_get_blob_prefix() + "/")
+    pathlib.Path(os.path.dirname(file_name)).mkdir(parents=True, exist_ok=True)
+    blob.download_to_filename(file_name)
+
+
+def download_timing_files(storage_client, bucket_name: str):
+    bucket = storage_client.bucket(bucket_name)
+    blobs = bucket.list_blobs(prefix=_get_blob_prefix())
+    with multiprocessing.pool.ThreadPool(GCS_PARALLELISM) as thread_pool:
+        futures = []
+        for timing_file_blob in blobs:
+            futures.append(
+                thread_pool.apply_async(
+                    _maybe_download_timing_file, (timing_file_blob,)
+                )
+            )
+        for future in futures:
+            future.get()
+    print("Done downloading")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        logging.fatal("Expected usage is cache_lit_timing_files.py <upload/download>")
+        sys.exit(1)
+    action = sys.argv[1]
+    storage_client = storage.Client()
+    bucket_name = os.environ["CACHE_GCS_BUCKET"]
+    if action == "download":
+        download_timing_files(storage_client, bucket_name)
+    elif action == "upload":
+        upload_timing_files(storage_client, bucket_name)
+    else:
+        logging.fatal("Expected usage is cache_lit_timing_files.py <upload/download>")
+        sys.exit(1)

.ci/cache_lit_timing_files.py

@@ -27,6 +27,13 @@ def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
             # We hit the end of the log without finding a build failure, go to
             # the next log.
             return failures
+        # If we are doing a build with LLVM_ENABLE_RUNTIMES, we can have nested
+        # ninja invocations. The sub-ninja will print that a subcommand failed,
+        # and then the outer ninja will list the command that failed. We should
+        # ignore the outer failure.
+        if ninja_log[index - 1].startswith("ninja: build stopped:"):
+            index += 1
+            continue
         # We are trying to parse cases like the following:
         #
         # [4/5] test/4.stamp

.ci/generate_test_report_lib.py

@@ -126,6 +126,39 @@ def test_ninja_log_multiple_failures(self):
             ),
         )
 
+    # Test that we can correctly handle the runtimes build. the LLVM runtimes
+    # build will involve ninja invoking more ninja processes within the
+    # runtimes directory. This means that we see two failures for a failure in
+    # the runtimes build: one from the inner ninja containing the actual action
+    # that failed, and one for the sub ninja invocation that failed.
+    def test_ninja_log_runtimes_failure(self):
+        failures = generate_test_report_lib.find_failure_in_ninja_logs(
+            [
+                [
+                    "[1/5] test/1.stamp",
+                    "[2/5] test/2.stamp",
+                    "FAILED: touch test/2.stamp",
+                    "Wow! This system is really broken!",
+                    "ninja: build stopped: subcommand failed.",
+                    "FAILED: running check-runtime failed.",
+                    "<some random command>",
+                    "ninja: build stopped: subcommand failed.",
+                ]
+            ]
+        )
+        self.assertEqual(len(failures), 1)
+        self.assertEqual(
+            failures[0],
+            (
+                "test/2.stamp",
+                dedent(
+                    """\
+                    FAILED: touch test/2.stamp
+                    Wow! This system is really broken!"""
+                ),
+            ),
+        )
+
     def test_title_only(self):
         self.assertEqual(
             generate_test_report_lib.generate_report("Foo", 0, [], []),

.ci/generate_test_report_lib_test.py

@@ -70,6 +70,7 @@
 # by trial and error).
 GRAFANA_METRIC_MAX_AGE_MN = 120
 
+
 @dataclass
 class JobMetrics:
     job_name: str
@@ -243,6 +244,7 @@ def clean_up_libcxx_job_name(old_name: str) -> str:
     new_name = stage + "_" + remainder
     return new_name
 
+
 def github_get_metrics(
     github_repo: github.Repository, last_workflows_seen_as_completed: set[int]
 ) -> tuple[list[JobMetrics], int]:

.ci/metrics/metrics.py

@@ -409,5 +409,6 @@ def test_clean_up_libcxx_job_name(self):
         out_name4 = metrics.clean_up_libcxx_job_name(bad_name)
         self.assertEqual(out_name4, bad_name)
 
+
 if __name__ == "__main__":
     unittest.main()

.ci/metrics/metrics_test.py

@@ -1 +1,2 @@
 junitparser==3.2.0
+google-cloud-storage==3.3.0

.ci/requirements.txt

@@ -2,7 +2,7 @@ FROM docker.io/library/ubuntu:24.04 as base
 ENV LLVM_SYSROOT=/opt/llvm
 
 FROM base as stage1-toolchain
-ENV LLVM_VERSION=20.1.8
+ENV LLVM_VERSION=21.1.0
 
 RUN apt-get update && \
     apt-get install -y \

.github/workflows/containers/github-action-ci/Dockerfile

@@ -32,6 +32,14 @@ jobs:
     steps:
     - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
+    # The default Docker storage location for GitHub Actions doesn't have
+    # enough disk space, so change it to /mnt, which has more disk space.
+    - name: Change Docker storage location
+      run: |
+        sudo mkdir /mnt/docker
+        echo '{ "data-root": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
+        sudo systemctl restart docker
+
     - name: Build the Linux builder image
       working-directory: libcxx/utils/ci
       run: |
@@ -40,11 +48,11 @@ jobs:
       env:
         TAG: ${{ github.sha }}
 
-    # - name: Build the Android builder image
-    #   working-directory: libcxx/utils/ci
-    #   run: docker compose build android-buildkite-builder
-    #   env:
-    #     TAG: ${{ github.sha }}
+    - name: Build the Android builder image
+      working-directory: libcxx/utils/ci
+      run: docker compose build android-buildkite-builder
+      env:
+        TAG: ${{ github.sha }}
 
     - name: Log in to GitHub Container Registry
       uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
@@ -62,10 +70,10 @@ jobs:
       env:
         TAG: ${{ github.sha }}
 
-    # - name: Push the Android builder image
-    #   if: github.event_name == 'push'
-    #   working-directory: libcxx/utils/ci
-    #   run: |
-    #     docker compose push android-buildkite-builder
-    #   env:
-    #     TAG: ${{ github.sha }}
+    - name: Push the Android builder image
+      if: github.event_name == 'push'
+      working-directory: libcxx/utils/ci
+      run: |
+        docker compose push android-buildkite-builder
+      env:
+        TAG: ${{ github.sha }}

.github/workflows/libcxx-build-containers.yml

@@ -52,10 +52,13 @@ jobs:
           echo "Formatting files:"
           echo "$CHANGED_FILES"
 
+      # The clang format version should always be upgraded to the first version
+      # of a release cycle (x.1.0) or the last version of a release cycle, or
+      # if there have been relevant clang-format backports.
       - name: Install clang-format
         uses: aminya/setup-cpp@17c11551771948abc5752bbf3183482567c7caf0 # v1.1.1
         with:
-          clangformat: 20.1.8
+          clangformat: 21.1.0
 
       - name: Setup Python env
         uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0