Reapply "[CI] Migrate to runtimes build" (#143612)

This reverts commit 6f62979.

The reapplies commit 80ea5f4.

That commit was reverted because it was causing compiler-rt test
failures due to tysan not having its dependencies set up properly within
CMake. That situation has since been rectified in
3cef099.

Reviewers: lnihlen, rnk, gburgessiv, cmtice

Reviewed By: rnk, cmtice

Pull Request: #144033

Author: boomanaiden154

.ci/compute_projects.py

@@ -49,8 +49,7 @@
     },
     "lld": {"bolt", "cross-project-tests"},
     # TODO(issues/132795): LLDB should be enabled on clang changes.
-    "clang": {"clang-tools-extra", "compiler-rt", "cross-project-tests"},
-    "clang-tools-extra": {"libc"},
+    "clang": {"clang-tools-extra", "cross-project-tests"},
     "mlir": {"flang"},
     # Test everything if ci scripts are changed.
     # FIXME: Figure out what is missing and add here.
@@ -64,7 +63,15 @@
 
 # This mapping describes runtimes that should be tested when the key project is
 # touched.
-DEPENDENT_RUNTIMES_TO_TEST = {"clang": {"libcxx", "libcxxabi", "libunwind"}}
+DEPENDENT_RUNTIMES_TO_TEST = {
+    "clang": {"compiler-rt"},
+    "clang-tools-extra": {"libc"},
+}
+DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG = {
+    "llvm": {"libcxx", "libcxxabi", "libunwind"},
+    "clang": {"libcxx", "libcxxabi", "libunwind"},
+    ".ci": {"libcxx", "libcxxabi", "libunwind"},
+}
 
 EXCLUDE_LINUX = {
     "cross-project-tests",  # TODO(issues/132796): Tests are failing.
@@ -93,9 +100,6 @@
     "cross-project-tests",
     "flang",
     "libc",
-    "libcxx",
-    "libcxxabi",
-    "libunwind",
     "lldb",
     "openmp",
     "polly",
@@ -122,10 +126,10 @@
     "polly": "check-polly",
 }
 
-RUNTIMES = {"libcxx", "libcxxabi", "libunwind"}
+RUNTIMES = {"libcxx", "libcxxabi", "libunwind", "compiler-rt", "libc"}
 
 
-def _add_dependencies(projects: Set[str]) -> Set[str]:
+def _add_dependencies(projects: Set[str], runtimes: Set[str]) -> Set[str]:
     projects_with_dependents = set(projects)
     current_projects_count = 0
     while current_projects_count != len(projects_with_dependents):
@@ -134,9 +138,25 @@ def _add_dependencies(projects: Set[str]) -> Set[str]:
             if project not in PROJECT_DEPENDENCIES:
                 continue
             projects_with_dependents.update(PROJECT_DEPENDENCIES[project])
+    for runtime in runtimes:
+        if runtime not in PROJECT_DEPENDENCIES:
+            continue
+        projects_with_dependents.update(PROJECT_DEPENDENCIES[runtime])
     return projects_with_dependents
 
 
+def _exclude_projects(current_projects: Set[str], platform: str) -> Set[str]:
+    if platform == "Linux":
+        to_exclude = EXCLUDE_LINUX
+    elif platform == "Windows":
+        to_exclude = EXCLUDE_WINDOWS
+    elif platform == "Darwin":
+        to_exclude = EXCLUDE_MAC
+    else:
+        raise ValueError(f"Unexpected platform: {platform}")
+    return current_projects.difference(to_exclude)
+
+
 def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
     projects_to_test = set()
     for modified_project in modified_projects:
@@ -154,25 +174,14 @@ def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set
             ):
                 continue
             projects_to_test.add(dependent_project)
-    if platform == "Linux":
-        for to_exclude in EXCLUDE_LINUX:
-            if to_exclude in projects_to_test:
-                projects_to_test.remove(to_exclude)
-    elif platform == "Windows":
-        for to_exclude in EXCLUDE_WINDOWS:
-            if to_exclude in projects_to_test:
-                projects_to_test.remove(to_exclude)
-    elif platform == "Darwin":
-        for to_exclude in EXCLUDE_MAC:
-            if to_exclude in projects_to_test:
-                projects_to_test.remove(to_exclude)
-    else:
-        raise ValueError("Unexpected platform.")
+    projects_to_test = _exclude_projects(projects_to_test, platform)
     return projects_to_test
 
 
-def _compute_projects_to_build(projects_to_test: Set[str]) -> Set[str]:
-    return _add_dependencies(projects_to_test)
+def _compute_projects_to_build(
+    projects_to_test: Set[str], runtimes: Set[str]
+) -> Set[str]:
+    return _add_dependencies(projects_to_test, runtimes)
 
 
 def _compute_project_check_targets(projects_to_test: Set[str]) -> Set[str]:
@@ -184,24 +193,36 @@ def _compute_project_check_targets(projects_to_test: Set[str]) -> Set[str]:
     return check_targets
 
 
-def _compute_runtimes_to_test(projects_to_test: Set[str]) -> Set[str]:
+def _compute_runtimes_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
     runtimes_to_test = set()
-    for project_to_test in projects_to_test:
-        if project_to_test in DEPENDENT_RUNTIMES_TO_TEST:
-            runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_TEST[project_to_test])
-        if project_to_test in DEPENDENT_RUNTIMES_TO_BUILD:
-            runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_BUILD[project_to_test])
-    return runtimes_to_test
+    for modified_project in modified_projects:
+        if modified_project not in DEPENDENT_RUNTIMES_TO_TEST:
+            continue
+        runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_TEST[modified_project])
+    return _exclude_projects(runtimes_to_test, platform)
 
 
-def _compute_runtime_check_targets(projects_to_test: Set[str]) -> Set[str]:
-    check_targets = set()
-    for project_to_test in projects_to_test:
-        if project_to_test not in DEPENDENT_RUNTIMES_TO_TEST:
+def _compute_runtimes_to_test_needs_reconfig(
+    modified_projects: Set[str], platform: str
+) -> Set[str]:
+    runtimes_to_test = set()
+    for modified_project in modified_projects:
+        if modified_project not in DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG:
             continue
-        for runtime_to_test in DEPENDENT_RUNTIMES_TO_TEST[project_to_test]:
-            check_targets.add(PROJECT_CHECK_TARGETS[runtime_to_test])
-    return check_targets
+        runtimes_to_test.update(
+            DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG[modified_project]
+        )
+    return _exclude_projects(runtimes_to_test, platform)
+
+
+def _compute_runtimes_to_build(
+    runtimes_to_test: Set[str], modified_projects: Set[str], platform: str
+) -> Set[str]:
+    runtimes_to_build = set(runtimes_to_test)
+    for modified_project in modified_projects:
+        if modified_project in DEPENDENT_RUNTIMES_TO_BUILD:
+            runtimes_to_build.update(DEPENDENT_RUNTIMES_TO_BUILD[modified_project])
+    return _exclude_projects(runtimes_to_build, platform)
 
 
 def _get_modified_projects(modified_files: list[str]) -> Set[str]:
@@ -225,10 +246,19 @@ def _get_modified_projects(modified_files: list[str]) -> Set[str]:
 def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
     modified_projects = _get_modified_projects(modified_files)
     projects_to_test = _compute_projects_to_test(modified_projects, platform)
-    projects_to_build = _compute_projects_to_build(projects_to_test)
+    runtimes_to_test = _compute_runtimes_to_test(modified_projects, platform)
+    runtimes_to_test_needs_reconfig = _compute_runtimes_to_test_needs_reconfig(
+        modified_projects, platform
+    )
+    runtimes_to_build = _compute_runtimes_to_build(
+        runtimes_to_test | runtimes_to_test_needs_reconfig, modified_projects, platform
+    )
+    projects_to_build = _compute_projects_to_build(projects_to_test, runtimes_to_build)
     projects_check_targets = _compute_project_check_targets(projects_to_test)
-    runtimes_to_build = _compute_runtimes_to_test(projects_to_test)
-    runtimes_check_targets = _compute_runtime_check_targets(projects_to_test)
+    runtimes_check_targets = _compute_project_check_targets(runtimes_to_test)
+    runtimes_check_targets_needs_reconfig = _compute_project_check_targets(
+        runtimes_to_test_needs_reconfig
+    )
     # We use a semicolon to separate the projects/runtimes as they get passed
     # to the CMake invocation and thus we need to use the CMake list separator
     # (;). We use spaces to separate the check targets as they end up getting
@@ -238,6 +268,9 @@ def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
         "project_check_targets": " ".join(sorted(projects_check_targets)),
         "runtimes_to_build": ";".join(sorted(runtimes_to_build)),
         "runtimes_check_targets": " ".join(sorted(runtimes_check_targets)),
+        "runtimes_check_targets_needs_reconfig": " ".join(
+            sorted(runtimes_check_targets_needs_reconfig)
+        ),
     }
 
 

.ci/compute_projects_test.py

@@ -26,6 +26,10 @@ def test_llvm(self):
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -46,6 +50,10 @@ def test_llvm_windows(self):
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -66,6 +74,10 @@ def test_llvm_mac(self):
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -75,17 +87,21 @@ def test_clang(self):
         )
         self.assertEqual(
             env_variables["projects_to_build"],
-            "clang;clang-tools-extra;compiler-rt;lld;llvm",
+            "clang;clang-tools-extra;lld;llvm",
         )
         self.assertEqual(
             env_variables["project_check_targets"],
-            "check-clang check-clang-tools check-compiler-rt",
+            "check-clang check-clang-tools",
         )
         self.assertEqual(
-            env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
+            env_variables["runtimes_to_build"], "compiler-rt;libcxx;libcxxabi;libunwind"
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "check-compiler-rt",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -104,6 +120,10 @@ def test_clang_windows(self):
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -115,6 +135,7 @@ def test_bolt(self):
         self.assertEqual(env_variables["project_check_targets"], "check-bolt")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_lldb(self):
         env_variables = compute_projects.get_env_variables(
@@ -124,6 +145,7 @@ def test_lldb(self):
         self.assertEqual(env_variables["project_check_targets"], "check-lldb")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_mlir(self):
         env_variables = compute_projects.get_env_variables(
@@ -135,6 +157,7 @@ def test_mlir(self):
         )
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_flang(self):
         env_variables = compute_projects.get_env_variables(
@@ -144,6 +167,7 @@ def test_flang(self):
         self.assertEqual(env_variables["project_check_targets"], "check-flang")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_invalid_subproject(self):
         env_variables = compute_projects.get_env_variables(
@@ -153,13 +177,15 @@ def test_invalid_subproject(self):
         self.assertEqual(env_variables["project_check_targets"], "")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_top_level_file(self):
         env_variables = compute_projects.get_env_variables(["README.md"], "Linux")
         self.assertEqual(env_variables["projects_to_build"], "")
         self.assertEqual(env_variables["project_check_targets"], "")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_exclude_runtiems_in_projects(self):
         env_variables = compute_projects.get_env_variables(
@@ -169,6 +195,7 @@ def test_exclude_runtiems_in_projects(self):
         self.assertEqual(env_variables["project_check_targets"], "")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_exclude_docs(self):
         env_variables = compute_projects.get_env_variables(
@@ -178,6 +205,7 @@ def test_exclude_docs(self):
         self.assertEqual(env_variables["project_check_targets"], "")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_exclude_gn(self):
         env_variables = compute_projects.get_env_variables(
@@ -187,6 +215,7 @@ def test_exclude_gn(self):
         self.assertEqual(env_variables["project_check_targets"], "")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_ci(self):
         env_variables = compute_projects.get_env_variables(
@@ -198,10 +227,15 @@ def test_ci(self):
             "check-clang check-lld check-lldb check-llvm",
         )
         self.assertEqual(
-            env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
+            env_variables["runtimes_to_build"],
+            "libcxx;libcxxabi;libunwind",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -215,6 +249,19 @@ def test_lldb(self):
             env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
         )
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
+
+    def test_clang_tools_extra(self):
+        env_variables = compute_projects.get_env_variables(
+            ["clang-tools-extra/CMakeLists.txt"], "Linux"
+        )
+        self.assertEqual(
+            env_variables["projects_to_build"], "clang;clang-tools-extra;lld;llvm"
+        )
+        self.assertEqual(env_variables["project_check_targets"], "check-clang-tools")
+        self.assertEqual(env_variables["runtimes_to_build"], "libc")
+        self.assertEqual(env_variables["runtimes_check_targets"], "check-libc")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
 
 if __name__ == "__main__":