Merge branch 'main' into nfc_same_elt

Author: hiraditya

.ci/compute-projects.sh

@@ -52,6 +52,9 @@
     "clang": {"clang-tools-extra", "compiler-rt", "cross-project-tests"},
     "clang-tools-extra": {"libc"},
     "mlir": {"flang"},
+    # Test everything if ci scripts are changed.
+    # FIXME: Figure out what is missing and add here.
+    ".ci": {"llvm", "clang", "lld", "lldb"},
 }
 
 DEPENDENT_RUNTIMES_TO_TEST = {"clang": {"libcxx", "libcxxabi", "libunwind"}}
@@ -112,6 +115,8 @@
     "polly": "check-polly",
 }
 
+RUNTIMES = {"libcxx", "libcxxabi", "libunwind"}
+
 
 def _add_dependencies(projects: Set[str]) -> Set[str]:
     projects_with_dependents = set(projects)
@@ -128,10 +133,11 @@ def _add_dependencies(projects: Set[str]) -> Set[str]:
 def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
     projects_to_test = set()
     for modified_project in modified_projects:
-        # Skip all projects where we cannot run tests.
-        if modified_project not in PROJECT_CHECK_TARGETS:
+        if modified_project in RUNTIMES:
             continue
-        projects_to_test.add(modified_project)
+        # Skip all projects where we cannot run tests.
+        if modified_project in PROJECT_CHECK_TARGETS:
+            projects_to_test.add(modified_project)
         if modified_project not in DEPENDENTS_TO_TEST:
             continue
         for dependent_project in DEPENDENTS_TO_TEST[modified_project]:
@@ -190,6 +196,17 @@ def _compute_runtime_check_targets(runtimes_to_test: Set[str]) -> Set[str]:
 def _get_modified_projects(modified_files: list[str]) -> Set[str]:
     modified_projects = set()
     for modified_file in modified_files:
+        path_parts = pathlib.Path(modified_file).parts
+        # Exclude files in the docs directory. They do not impact an test
+        # targets and there is a separate workflow used for ensuring the
+        # documentation builds.
+        if len(path_parts) > 2 and path_parts[1] == "docs":
+            continue
+        # Exclude files for the gn build. We do not test it within premerge
+        # and changes occur often enough that they otherwise take up
+        # capacity.
+        if len(path_parts) > 3 and path_parts[:3] == ("llvm", "utils", "gn"):
+            continue
         modified_projects.add(pathlib.Path(modified_file).parts[0])
     return modified_projects
 

.ci/compute_projects.py

@@ -161,6 +161,46 @@ def test_top_level_file(self):
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
 
+    def test_exclude_runtiems_in_projects(self):
+        env_variables = compute_projects.get_env_variables(
+            [".ci/compute_projects.py", "libcxx/CMakeLists.txt"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"], "")
+        self.assertEqual(env_variables["project_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_to_build"], "")
+        self.assertEqual(env_variables["runtimes_check_targets"], "")
+
+    def test_exclude_docs(self):
+        env_variables = compute_projects.get_env_variables(
+            ["llvm/docs/CIBestPractices.rst"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"], "")
+        self.assertEqual(env_variables["project_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_to_build"], "")
+        self.assertEqual(env_variables["runtimes_check_targets"], "")
+
+    def test_exclude_gn(self):
+        env_variables = compute_projects.get_env_variables(
+            ["llvm/utils/gn/build/BUILD.gn"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"], "")
+        self.assertEqual(env_variables["project_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_to_build"], "")
+        self.assertEqual(env_variables["runtimes_check_targets"], "")
+
+     def test_ci(self):
+        env_variables = compute_projects.get_env_variables(
+            [".ci/compute_projects.py"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"],
+                         "clang;lld;llvm;lldb")
+        self.assertEqual(env_variables["project_check_targets"], "check-clang
+                         check-lld check-llvm check-lldb")
+        self.assertEqual(env_variables["runtimes_to_build"],
+                         "libcxx;libcxxabi;libunwind")
+        self.assertEqual(env_variables["runtimes_check_targets"], "check-cxx
+                         check-cxxabi check-unwind")
+
 
 if __name__ == "__main__":
     unittest.main()

.ci/compute_projects_test.py

@@ -52,8 +52,6 @@ modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
 echo "Directories modified:" >&2
 echo "$modified_dirs" >&2
 
-. ./.ci/compute-projects.sh
-
 # Project specific pipelines.
 
 # If libc++ or one of the runtimes directories changed.
@@ -73,20 +71,16 @@ fi
 # needs while letting them run on the infrastructure provided by LLVM.
 
 # Figure out which projects need to be built on each platform
-all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
-modified_projects="$(keep-modified-projects ${all_projects})"
-
-linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
-linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
-linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
+source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Linux)
+linux_projects=${projects_to_build}
+linux_check_targets=${project_check_targets}
+linux_runtimes=${runtimes_to_build}
+linux_runtime_check_targets=${runtimes_check_targets}
 
-linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
-linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
-linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
+source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Windows)
+windows_projects=${projects_to_build}
+windows_check_targets=${project_check_targets}
 
-windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
-windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
-windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 
 # Generate the appropriate pipeline
 if [[ "${linux_projects}" != "" ]]; then