Merge branch 'main' into correct-decompress-offloading-api

Author: david-salinas

.ci/all_requirements.txt

@@ -0,0 +1,85 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Caches .lit_test_times.txt files between premerge invocations.
+
+.lit_test_times.txt files are used by lit to order tests to best take advantage
+of parallelism. Having them around and up to date can result in a ~15%
+improvement in test times. This script downloading cached test time files and
+uploading new versions to the GCS buckets used for caching.
+"""
+
+import sys
+import os
+import logging
+import multiprocessing.pool
+import pathlib
+import platform
+import glob
+
+from google.cloud import storage
+
+GCS_PARALLELISM = 100
+
+
+def _get_blob_prefix():
+    return f"lit_timing_{platform.system().lower()}"
+
+
+def _maybe_upload_timing_file(bucket, timing_file_path):
+    if os.path.exists(timing_file_path):
+        timing_file_blob = bucket.blob(_get_blob_prefix() + "/" + timing_file_path)
+        timing_file_blob.upload_from_filename(timing_file_path)
+
+
+def upload_timing_files(storage_client, bucket_name: str):
+    bucket = storage_client.bucket(bucket_name)
+    with multiprocessing.pool.ThreadPool(GCS_PARALLELISM) as thread_pool:
+        futures = []
+        for timing_file_path in glob.glob("**/.lit_test_times.txt", recursive=True):
+            futures.append(
+                thread_pool.apply_async(
+                    _maybe_upload_timing_file, (bucket, timing_file_path)
+                )
+            )
+        for future in futures:
+            future.get()
+    print("Done uploading")
+
+
+def _maybe_download_timing_file(blob):
+    file_name = blob.name.removeprefix(_get_blob_prefix() + "/")
+    pathlib.Path(os.path.dirname(file_name)).mkdir(parents=True, exist_ok=True)
+    blob.download_to_filename(file_name)
+
+
+def download_timing_files(storage_client, bucket_name: str):
+    bucket = storage_client.bucket(bucket_name)
+    blobs = bucket.list_blobs(prefix=_get_blob_prefix())
+    with multiprocessing.pool.ThreadPool(GCS_PARALLELISM) as thread_pool:
+        futures = []
+        for timing_file_blob in blobs:
+            futures.append(
+                thread_pool.apply_async(
+                    _maybe_download_timing_file, (timing_file_blob,)
+                )
+            )
+        for future in futures:
+            future.get()
+    print("Done downloading")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        logging.fatal("Expected usage is cache_lit_timing_files.py <upload/download>")
+        sys.exit(1)
+    action = sys.argv[1]
+    storage_client = storage.Client()
+    bucket_name = os.environ["CACHE_GCS_BUCKET"]
+    if action == "download":
+        download_timing_files(storage_client, bucket_name)
+    elif action == "upload":
+        upload_timing_files(storage_client, bucket_name)
+    else:
+        logging.fatal("Expected usage is cache_lit_timing_files.py <upload/download>")
+        sys.exit(1)

.ci/cache_lit_timing_files.py

@@ -49,8 +49,7 @@
         "flang",
     },
     "lld": {"bolt", "cross-project-tests"},
-    # TODO(issues/132795): LLDB should be enabled on clang changes.
-    "clang": {"clang-tools-extra", "cross-project-tests"},
+    "clang": {"clang-tools-extra", "cross-project-tests", "lldb"},
     "mlir": {"flang"},
     # Test everything if ci scripts are changed.
     ".ci": {
@@ -80,6 +79,7 @@
     "clang": {"compiler-rt"},
     "clang-tools-extra": {"libc"},
     "libc": {"libc"},
+    "compiler-rt": {"compiler-rt"},
     ".ci": {"compiler-rt", "libc"},
 }
 DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG = {
@@ -100,6 +100,9 @@
     "libc",  # No Windows Support.
     "lldb",  # TODO(issues/132800): Needs environment setup.
     "bolt",  # No Windows Support.
+    "libcxx",
+    "libcxxabi",
+    "libunwind",
 }
 
 # These are projects that we should test if the project itself is changed but
@@ -118,6 +121,9 @@
     "lldb",
     "openmp",
     "polly",
+    "libcxx",
+    "libcxxabi",
+    "libunwind",
 }
 
 PROJECT_CHECK_TARGETS = {
@@ -144,6 +150,23 @@
 
 RUNTIMES = {"libcxx", "libcxxabi", "libunwind", "compiler-rt", "libc"}
 
+# Meta projects are projects that need explicit handling but do not reside
+# in their own top level folder. To add a meta project, the start of the path
+# for the metaproject should be mapped to the name of the project below.
+# Multiple paths can map to the same metaproject.
+META_PROJECTS = {
+    ("clang", "lib", "CIR"): "CIR",
+    ("clang", "test", "CIR"): "CIR",
+    ("clang", "include", "clang", "CIR"): "CIR",
+    ("*", "docs"): "docs",
+    ("llvm", "utils", "gn"): "gn",
+    (".github", "workflows", "premerge.yaml"): ".ci",
+    ("third-party",): ".ci",
+}
+
+# Projects that should not run any tests. These need to be metaprojects.
+SKIP_PROJECTS = ["docs", "gn"]
+
 
 def _add_dependencies(projects: Set[str], runtimes: Set[str]) -> Set[str]:
     projects_with_dependents = set(projects)
@@ -236,29 +259,34 @@ def _compute_runtimes_to_build(
     return _exclude_projects(runtimes_to_build, platform)
 
 
+def _path_matches(matcher: tuple[str], file_path: tuple[str]) -> bool:
+    if len(file_path) < len(matcher):
+        return False
+    for match_part, file_part in zip(matcher, file_path):
+        if match_part == "*" or file_part == "*":
+            continue
+        if match_part != file_part:
+            return False
+    return True
+
+
+def _get_modified_projects_for_file(modified_file: str) -> Set[str]:
+    modified_projects = set()
+    path_parts = pathlib.Path(modified_file).parts
+    for meta_project_files in META_PROJECTS.keys():
+        if _path_matches(meta_project_files, path_parts):
+            meta_project = META_PROJECTS[meta_project_files]
+            if meta_project in SKIP_PROJECTS:
+                return set()
+            modified_projects.add(meta_project)
+    modified_projects.add(pathlib.Path(modified_file).parts[0])
+    return modified_projects
+
+
 def _get_modified_projects(modified_files: list[str]) -> Set[str]:
     modified_projects = set()
     for modified_file in modified_files:
-        path_parts = pathlib.Path(modified_file).parts
-        # Exclude files in the docs directory. They do not impact an test
-        # targets and there is a separate workflow used for ensuring the
-        # documentation builds.
-        if len(path_parts) > 2 and path_parts[1] == "docs":
-            continue
-        # Exclude files for the gn build. We do not test it within premerge
-        # and changes occur often enough that they otherwise take up
-        # capacity.
-        if len(path_parts) > 3 and path_parts[:3] == ("llvm", "utils", "gn"):
-            continue
-        # If the file is in the clang/lib/CIR directory, add the CIR project.
-        if len(path_parts) > 3 and (
-            path_parts[:3] == ("clang", "lib", "CIR")
-            or path_parts[:3] == ("clang", "test", "CIR")
-            or path_parts[:4] == ("clang", "include", "clang", "CIR")
-        ):
-            modified_projects.add("CIR")
-            # Fall through to add clang.
-        modified_projects.add(pathlib.Path(modified_file).parts[0])
+        modified_projects.update(_get_modified_projects_for_file(modified_file))
     return modified_projects
 
 
@@ -305,6 +333,7 @@ def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
     current_platform = platform.system()
     if len(sys.argv) == 2:
         current_platform = sys.argv[1]
-    env_variables = get_env_variables(sys.stdin.readlines(), current_platform)
+    changed_files = [line.strip() for line in sys.stdin.readlines()]
+    env_variables = get_env_variables(changed_files, current_platform)
     for env_variable in env_variables:
         print(f"{env_variable}='{env_variables[env_variable]}'")