[CI] Move CI over to new project computation script

This patch migrates the CI over to the new compute_projects.py script
for calculating what projects need to be tested based on a change to
LLVM.

Reviewers: lnihlen, ldionne, tstellar, Endilll, joker-eph, Keenuts

Reviewed By: Keenuts, tstellar

Pull Request: #132642

Author: boomanaiden154

.ci/compute-projects.sh

@@ -52,8 +52,6 @@ modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
 echo "Directories modified:" >&2
 echo "$modified_dirs" >&2
 
-. ./.ci/compute-projects.sh
-
 # Project specific pipelines.
 
 # If libc++ or one of the runtimes directories changed.
@@ -73,20 +71,16 @@ fi
 # needs while letting them run on the infrastructure provided by LLVM.
 
 # Figure out which projects need to be built on each platform
-all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
-modified_projects="$(keep-modified-projects ${all_projects})"
-
-linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
-linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
-linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
+source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Linux)
+linux_projects=${projects_to_build}
+linux_check_targets=${project_check_targets}
+linux_runtimes=${runtimes_to_build}
+linux_runtime_check_targets=${runtimes_check_targets}
 
-linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
-linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
-linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
+source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Windows)
+windows_projects=${projects_to_build}
+windows_check_targets=${project_check_targets}
 
-windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
-windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
-windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 
 # Generate the appropriate pipeline
 if [[ "${linux_projects}" != "" ]]; then

.ci/generate-buildkite-pipeline-premerge

@@ -49,39 +49,22 @@ jobs:
         run: |
           git config --global --add safe.directory '*'
 
-          modified_files=$(git diff --name-only HEAD~1...HEAD)
-          modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u)
+          source <(git diff --name-only HEAD~1...HEAD | python3 .ci/compute_projects.py)
 
-          echo $modified_files
-          echo $modified_dirs
-
-          . ./.ci/compute-projects.sh
-
-          all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
-          modified_projects="$(keep-modified-projects ${all_projects})"
-
-          linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
-          linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
-          linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
-
-          linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
-          linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
-          linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
-
-          if [[ "${linux_projects}" == "" ]]; then
+          if [[ "${projects_to_build}" == "" ]]; then
             echo "No projects to build"
             exit 0
           fi
 
-          echo "Building projects: ${linux_projects}"
-          echo "Running project checks targets: ${linux_check_targets}"
-          echo "Building runtimes: ${linux_runtimes}"
-          echo "Running runtimes checks targets: ${linux_runtime_check_targets}"
+          echo "Building projects: ${projects_to_build}"
+          echo "Running project checks targets: ${project_check_targets}"
+          echo "Building runtimes: ${runtimes_to_build}"
+          echo "Running runtimes checks targets: ${runtimes_check_targets}"
 
           export CC=/opt/llvm/bin/clang
           export CXX=/opt/llvm/bin/clang++
 
-          ./.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})"
+          ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}"
 
   premerge-checks-windows:
     name: Windows Premerge Checks (Test Only - Please Ignore Results)
@@ -105,30 +88,17 @@ jobs:
       - name: Compute Projects
         id: vars
         run: |
-          modified_files=$(git diff --name-only HEAD~1...HEAD)
-          modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort | uniq)
-
-          echo $modified_files
-          echo $modified_dirs
-
-          . ./.ci/compute-projects.sh
+          source <(git diff --name-only HEAD~1...HEAD | python .ci/compute_projects.py)
 
-          all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
-          modified_projects="$(keep-modified-projects ${all_projects})"
-
-          windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
-          windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ' ')
-          windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ';')
-
-          if [[ "${windows_projects}" == "" ]]; then
+          if [[ "${projects_to_build}" == "" ]]; then
             echo "No projects to build"
           fi
 
-          echo "Building projects: ${windows_projects}"
-          echo "Running project checks targets: ${windows_check_targets}"
+          echo "Building projects: ${projects_to_build}"
+          echo "Running project checks targets: ${project_check_targets}"
 
-          echo "windows-projects=${windows_projects}" >> $GITHUB_OUTPUT
-          echo "windows-check-targets=${windows_check_targets}" >> $GITHUB_OUTPUT
+          echo "windows-projects=${projects_to_build}" >> $GITHUB_OUTPUT
+          echo "windows-check-targets=${project_check_targets}" >> $GITHUB_OUTPUT
       - name: Build and Test
         # Mark the job as a success even if the step fails so that people do
         # not get notified while the new premerge pipeline is in an
@@ -165,45 +135,23 @@ jobs:
         uses: llvm/actions/install-ninja@main
       - name: Build and Test
         run: |
-          modified_files=$(git diff --name-only HEAD~1...HEAD)
-          modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u)
-
-          echo $modified_files
-          echo $modified_dirs
-
-          . ./.ci/compute-projects.sh
-
-          all_projects="clang clang-tools-extra lld lldb llvm mlir"
-          modified_projects="$(keep-modified-projects ${all_projects})"
-
-          # We have to disable the runtimes builds due to https://github.com/llvm/llvm-project/issues/90568
-          # and the lldb tests depend on libcxx, so we need to skip them.
-          mac_check_targets=$(check-targets ${modified_projects} | sort | uniq | tr '\n' ' ' | sed -e 's/check-lldb //g')
-          mac_projects=$(add-dependencies ${modified_projects} | sort | uniq | tr '\n' ' ')
-
-          mac_runtimes_to_test=$(compute-runtimes-to-test ${modified_projects})
-          mac_runtime_check_targets=$(check-targets ${mac_runtimes_to_test} | sort | uniq | tr '\n' ' ')
-          mac_runtimes=$(echo ${mac_runtimes_to_test} | tr ' ' '\n' | sort | uniq | tr '\n' ' ')
+          source <(git diff --name-only HEAD~2..HEAD | python3 .ci/compute_projects.py)
 
-          if [[ "${mac_projects}" == "" ]]; then
+          if [[ "${projects_to_build}" == "" ]]; then
             echo "No projects to build"
             exit 0
           fi
 
-          echo "Projects to test: ${modified_projects}"
-          echo "Runtimes to test: ${mac_runtimes_to_test}"
-          echo "Building projects: ${mac_projects}"
-          echo "Running project checks targets: ${mac_check_targets}"
-          echo "Building runtimes: ${mac_runtimes}"
-          echo "Running runtimes checks targets: ${mac_runtime_check_targets}"
+          echo "Building projects: ${projects_to_build}"
+          echo "Running project checks targets: ${project_check_targets}"
 
           # -DLLVM_DISABLE_ASSEMBLY_FILES=ON is for
           # https://github.com/llvm/llvm-project/issues/81967
           # Disable sharding in lit so that the LIT_XFAIL environment var works.
           cmake -G Ninja \
                 -B build \
                 -S llvm \
-                -DLLVM_ENABLE_PROJECTS="$(echo ${mac_projects} | tr ' ' ';')" \
+                -DLLVM_ENABLE_PROJECTS="${projects_to_build}" \
                 -DLLVM_DISABLE_ASSEMBLY_FILES=ON \
                 -DCMAKE_BUILD_TYPE=Release \
                 -DLLDB_INCLUDE_TESTS=OFF \
@@ -212,4 +160,4 @@ jobs:
                 -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 
           # The libcxx tests fail, so we are skipping the runtime targets.
-          ninja -C build  $mac_check_targets
+          ninja -C build ${project_check_targets}