Merge branch 'main' into common-view

Author: changkhothuychung

.ci/all_requirements.txt

@@ -0,0 +1,94 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Caches .lit_test_times.txt files between premerge invocations.
+
+.lit_test_times.txt files are used by lit to order tests to best take advantage
+of parallelism. Having them around and up to date can result in a ~15%
+improvement in test times. This script downloading cached test time files and
+uploading new versions to the GCS buckets used for caching.
+"""
+
+import sys
+import os
+import logging
+import multiprocessing.pool
+import pathlib
+import glob
+
+from google.cloud import storage
+from google.api_core import exceptions
+
+GCS_PARALLELISM = 100
+
+
+def _maybe_upload_timing_file(bucket, timing_file_path):
+    if os.path.exists(timing_file_path):
+        timing_file_blob = bucket.blob("lit_timing/" + timing_file_path)
+        timing_file_blob.upload_from_filename(timing_file_path)
+
+
+def upload_timing_files(storage_client, bucket_name: str):
+    bucket = storage_client.bucket(bucket_name)
+    with multiprocessing.pool.ThreadPool(GCS_PARALLELISM) as thread_pool:
+        futures = []
+        for timing_file_path in glob.glob("**/.lit_test_times.txt", recursive=True):
+            futures.append(
+                thread_pool.apply_async(
+                    _maybe_upload_timing_file, (bucket, timing_file_path)
+                )
+            )
+        for future in futures:
+            future.get()
+    print("Done uploading")
+
+
+def _maybe_download_timing_file(blob):
+    file_name = blob.name.removeprefix("lit_timing/")
+    pathlib.Path(os.path.dirname(file_name)).mkdir(parents=True, exist_ok=True)
+    blob.download_to_filename(file_name)
+
+
+def download_timing_files(storage_client, bucket_name: str):
+    bucket = storage_client.bucket(bucket_name)
+    try:
+        blobs = bucket.list_blobs(prefix="lit_timing")
+    except exceptions.ClientError as client_error:
+        print(
+            "::warning file=cache_lit_timing_files.py::Failed to list blobs "
+            "in bucket."
+        )
+        sys.exit(0)
+    with multiprocessing.pool.ThreadPool(GCS_PARALLELISM) as thread_pool:
+        futures = []
+        for timing_file_blob in blobs:
+            futures.append(
+                thread_pool.apply_async(
+                    _maybe_download_timing_file, (timing_file_blob,)
+                )
+            )
+        for future in futures:
+            future.wait()
+            if not future.successful():
+                print(
+                    "::warning file=cache_lit_timing_files.py::Failed to "
+                    "download lit timing file."
+                )
+                continue
+    print("Done downloading")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        logging.fatal("Expected usage is cache_lit_timing_files.py <upload/download>")
+        sys.exit(1)
+    action = sys.argv[1]
+    storage_client = storage.Client()
+    bucket_name = os.environ["CACHE_GCS_BUCKET"]
+    if action == "download":
+        download_timing_files(storage_client, bucket_name)
+    elif action == "upload":
+        upload_timing_files(storage_client, bucket_name)
+    else:
+        logging.fatal("Expected usage is cache_lit_timing_files.py <upload/download>")
+        sys.exit(1)

.ci/cache_lit_timing_files.py

@@ -26,6 +26,7 @@
     "libc": {"clang", "lld"},
     "openmp": {"clang", "lld"},
     "flang": {"llvm", "clang"},
+    "flang-rt": {"flang"},
     "lldb": {"llvm", "clang"},
     "libclc": {"llvm", "clang"},
     "lld": {"llvm"},
@@ -49,8 +50,7 @@
         "flang",
     },
     "lld": {"bolt", "cross-project-tests"},
-    # TODO(issues/132795): LLDB should be enabled on clang changes.
-    "clang": {"clang-tools-extra", "cross-project-tests"},
+    "clang": {"clang-tools-extra", "cross-project-tests", "lldb"},
     "mlir": {"flang"},
     # Test everything if ci scripts are changed.
     ".ci": {
@@ -81,7 +81,9 @@
     "clang-tools-extra": {"libc"},
     "libc": {"libc"},
     "compiler-rt": {"compiler-rt"},
-    ".ci": {"compiler-rt", "libc"},
+    "flang": {"flang-rt"},
+    "flang-rt": {"flang-rt"},
+    ".ci": {"compiler-rt", "libc", "flang-rt"},
 }
 DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG = {
     "llvm": {"libcxx", "libcxxabi", "libunwind"},
@@ -96,14 +98,14 @@
 
 EXCLUDE_WINDOWS = {
     "cross-project-tests",  # TODO(issues/132797): Tests are failing.
-    "compiler-rt",  # TODO(issues/132798): Tests take excessive time.
     "openmp",  # TODO(issues/132799): Does not detect perl installation.
     "libc",  # No Windows Support.
     "lldb",  # TODO(issues/132800): Needs environment setup.
     "bolt",  # No Windows Support.
     "libcxx",
     "libcxxabi",
     "libunwind",
+    "flang-rt",
 }
 
 # These are projects that we should test if the project itself is changed but
@@ -141,15 +143,17 @@
     "bolt": "check-bolt",
     "lld": "check-lld",
     "flang": "check-flang",
+    "flang-rt": "check-flang-rt",
     "libc": "check-libc",
     "lld": "check-lld",
     "lldb": "check-lldb",
     "mlir": "check-mlir",
     "openmp": "check-openmp",
     "polly": "check-polly",
+    "lit": "check-lit",
 }
 
-RUNTIMES = {"libcxx", "libcxxabi", "libunwind", "compiler-rt", "libc"}
+RUNTIMES = {"libcxx", "libcxxabi", "libunwind", "compiler-rt", "libc", "flang-rt"}
 
 # Meta projects are projects that need explicit handling but do not reside
 # in their own top level folder. To add a meta project, the start of the path
@@ -163,8 +167,12 @@
     ("llvm", "utils", "gn"): "gn",
     (".github", "workflows", "premerge.yaml"): ".ci",
     ("third-party",): ".ci",
+    ("llvm", "utils", "lit"): "lit",
 }
 
+# Projects that should run tests but cannot be explicitly built.
+SKIP_BUILD_PROJECTS = ["CIR", "lit"]
+
 # Projects that should not run any tests. These need to be metaprojects.
 SKIP_PROJECTS = ["docs", "gn"]
 
@@ -312,7 +320,9 @@ def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
     # clang build, but it requires an explicit option to enable. We set that
     # option here, and remove it from the projects_to_build list.
     enable_cir = "ON" if "CIR" in projects_to_build else "OFF"
-    projects_to_build.discard("CIR")
+    # Remove any metaprojects from the list of projects to build.
+    for project in SKIP_BUILD_PROJECTS:
+        projects_to_build.discard(project)
 
     # We use a semicolon to separate the projects/runtimes as they get passed
     # to the CMake invocation and thus we need to use the CMake list separator
@@ -334,6 +344,7 @@ def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
     current_platform = platform.system()
     if len(sys.argv) == 2:
         current_platform = sys.argv[1]
-    env_variables = get_env_variables(sys.stdin.readlines(), current_platform)
+    changed_files = [line.strip() for line in sys.stdin.readlines()]
+    env_variables = get_env_variables(changed_files, current_platform)
     for env_variable in env_variables:
         print(f"{env_variable}='{env_variables[env_variable]}'")

.ci/compute_projects.py

@@ -83,11 +83,11 @@ def test_clang(self):
         )
         self.assertEqual(
             env_variables["projects_to_build"],
-            "clang;clang-tools-extra;lld;llvm",
+            "clang;clang-tools-extra;lld;lldb;llvm",
         )
         self.assertEqual(
             env_variables["project_check_targets"],
-            "check-clang check-clang-tools",
+            "check-clang check-clang-tools check-lldb",
         )
         self.assertEqual(
             env_variables["runtimes_to_build"], "compiler-rt;libcxx;libcxxabi;libunwind"
@@ -110,15 +110,15 @@ def test_clang_windows(self):
             ["clang/CMakeLists.txt"], "Windows"
         )
         self.assertEqual(
-            env_variables["projects_to_build"], "clang;clang-tools-extra;llvm"
+            env_variables["projects_to_build"], "clang;clang-tools-extra;lld;llvm"
         )
         self.assertEqual(
             env_variables["project_check_targets"], "check-clang check-clang-tools"
         )
-        self.assertEqual(env_variables["runtimes_to_build"], "")
+        self.assertEqual(env_variables["runtimes_to_build"], "compiler-rt")
         self.assertEqual(
             env_variables["runtimes_check_targets"],
-            "",
+            "check-compiler-rt",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets_needs_reconfig"],
@@ -158,11 +158,11 @@ def test_cir(self):
         )
         self.assertEqual(
             env_variables["projects_to_build"],
-            "clang;clang-tools-extra;lld;llvm;mlir",
+            "clang;clang-tools-extra;lld;lldb;llvm;mlir",
         )
         self.assertEqual(
             env_variables["project_check_targets"],
-            "check-clang check-clang-cir check-clang-tools",
+            "check-clang check-clang-cir check-clang-tools check-lldb",
         )
         self.assertEqual(
             env_variables["runtimes_to_build"], "compiler-rt;libcxx;libcxxabi;libunwind"
@@ -216,8 +216,8 @@ def test_flang(self):
         )
         self.assertEqual(env_variables["projects_to_build"], "clang;flang;llvm")
         self.assertEqual(env_variables["project_check_targets"], "check-flang")
-        self.assertEqual(env_variables["runtimes_to_build"], "")
-        self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_to_build"], "flang-rt")
+        self.assertEqual(env_variables["runtimes_check_targets"], "check-flang-rt")
         self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
         self.assertEqual(env_variables["enable_cir"], "OFF")
 
@@ -293,11 +293,11 @@ def test_ci(self):
         )
         self.assertEqual(
             env_variables["runtimes_to_build"],
-            "compiler-rt;libc;libcxx;libcxxabi;libunwind",
+            "compiler-rt;flang-rt;libc;libcxx;libcxxabi;libunwind",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
-            "check-compiler-rt check-libc",
+            "check-compiler-rt check-flang-rt check-libc",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets_needs_reconfig"],
@@ -318,11 +318,11 @@ def test_windows_ci(self):
         )
         self.assertEqual(
             env_variables["runtimes_to_build"],
-            "",
+            "compiler-rt",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
-            "",
+            "check-compiler-rt",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets_needs_reconfig"],
@@ -367,11 +367,11 @@ def test_premerge_workflow(self):
         )
         self.assertEqual(
             env_variables["runtimes_to_build"],
-            "compiler-rt;libc;libcxx;libcxxabi;libunwind",
+            "compiler-rt;flang-rt;libc;libcxx;libcxxabi;libunwind",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
-            "check-compiler-rt check-libc",
+            "check-compiler-rt check-flang-rt check-libc",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets_needs_reconfig"],
@@ -402,11 +402,35 @@ def test_third_party_benchmark(self):
         )
         self.assertEqual(
             env_variables["runtimes_to_build"],
-            "compiler-rt;libc;libcxx;libcxxabi;libunwind",
+            "compiler-rt;flang-rt;libc;libcxx;libcxxabi;libunwind",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets"],
+            "check-compiler-rt check-flang-rt check-libc",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
+            "check-cxx check-cxxabi check-unwind",
+        )
+
+    def test_lit(self):
+        env_variables = compute_projects.get_env_variables(
+            ["llvm/utils/lit/CMakeLists.txt"], "Linux"
+        )
+        self.assertEqual(
+            env_variables["projects_to_build"],
+            "bolt;clang;clang-tools-extra;flang;lld;lldb;llvm;mlir;polly",
+        )
+        self.assertEqual(
+            env_variables["project_check_targets"],
+            "check-bolt check-clang check-clang-tools check-flang check-lit check-lld check-lldb check-llvm check-mlir check-polly",
+        )
+        self.assertEqual(
+            env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
-            "check-compiler-rt check-libc",
+            "",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets_needs_reconfig"],

.ci/compute_projects_test.py

@@ -8,10 +8,15 @@
 
 import generate_test_report_lib
 
-PLATFORM_TITLES = {
-    "Windows": ":window: Windows x64 Test Results",
-    "Linux": ":penguin: Linux x64 Test Results",
-}
+def compute_platform_title() -> str:
+    logo = ":window:" if platform.system() == "Windows" else ":penguin:"
+    # On Linux the machine value is x86_64 on Windows it is AMD64.
+    if platform.machine() == "x86_64" or platform.machine() == "AMD64":
+        arch = "x64"
+    else:
+        arch = platform.machine()
+    return f"{logo} {platform.system()} {arch} Test Results"
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -22,7 +27,7 @@
     args = parser.parse_args()
 
     report = generate_test_report_lib.generate_report_from_files(
-        PLATFORM_TITLES[platform.system()], args.return_code, args.build_test_logs
+        compute_platform_title(), args.return_code, args.build_test_logs
     )
 
     print(report)