[CI] Pull in the latest mutes from base branch for PRs at runtime (#117587)

Author: brianseeders

.buildkite/hooks/pre-command

@@ -47,6 +47,8 @@ export GRADLE_BUILD_CACHE_PASSWORD
 BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
 export BUILDKITE_API_TOKEN
 
+export GH_TOKEN="$VAULT_GITHUB_TOKEN"
+
 if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
   data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots)
 
@@ -117,3 +119,5 @@ if [[ -f /etc/os-release ]] && grep -q '"Amazon Linux 2"' /etc/os-release; then
   echo "$(hostname -i | cut -d' ' -f 2)  $(hostname -f)." | sudo tee /etc/dnsmasq.hosts
   sudo systemctl restart dnsmasq.service
 fi
+
+.buildkite/scripts/get-latest-test-mutes.sh

.buildkite/hooks/pre-command.bat

@@ -15,9 +15,12 @@ set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER%
 set COMPOSE_HTTP_TIMEOUT=120
 set JOB_BRANCH=%BUILDKITE_BRANCH%
 
+set GH_TOKEN=%VAULT_GITHUB_TOKEN%
+
 set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
 set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
 
 bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &"
+bash.exe -c "bash .buildkite/scripts/get-latest-test-mutes.sh"
 
 exit /b 0

.buildkite/scripts/get-latest-test-mutes.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [[ ! "${BUILDKITE_PULL_REQUEST:-}" || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
+  exit 0
+fi
+
+testMuteBranch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
+testMuteFile="$(mktemp)"
+
+# If this PR contains changes to muted-tests.yml, we disable this functionality
+# Otherwise, we wouldn't be able to test unmutes
+if [[ ! $(gh pr diff "$BUILDKITE_PULL_REQUEST" --name-only | grep 'muted-tests.yml') ]]; then
+  gh api -H 'Accept: application/vnd.github.v3.raw' "repos/elastic/elasticsearch/contents/muted-tests.yml?ref=$testMuteBranch" > "$testMuteFile"
+
+  if [[ -s "$testMuteFile" ]]; then
+    mkdir -p ~/.gradle
+    # This is using gradle.properties instead of an env var so that it's easily compatible with the Windows pre-command hook
+    echo "org.gradle.project.org.elasticsearch.additional.muted.tests=$testMuteFile" >> ~/.gradle/gradle.properties
+  fi
+fi

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java

@@ -28,10 +28,12 @@
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Set;
 
 public abstract class MutedTestsBuildService implements BuildService<MutedTestsBuildService.Params> {
-    private final List<String> excludePatterns = new ArrayList<>();
+    private final Set<String> excludePatterns = new LinkedHashSet<>();
     private final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
 
     public MutedTestsBuildService() {
@@ -43,23 +45,23 @@ public MutedTestsBuildService() {
         }
     }
 
-    public List<String> getExcludePatterns() {
+    public Set<String> getExcludePatterns() {
         return excludePatterns;
     }
 
-    private List<String> buildExcludePatterns(File file) {
+    private Set<String> buildExcludePatterns(File file) {
         List<MutedTest> mutedTests;
 
         try (InputStream is = new BufferedInputStream(new FileInputStream(file))) {
             mutedTests = objectMapper.readValue(is, MutedTests.class).getTests();
             if (mutedTests == null) {
-                return Collections.emptyList();
+                return Collections.emptySet();
             }
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
 
-        List<String> excludes = new ArrayList<>();
+        Set<String> excludes = new LinkedHashSet<>();
         if (mutedTests.isEmpty() == false) {
             for (MutedTestsBuildService.MutedTest mutedTest : mutedTests) {
                 if (mutedTest.getClassName() != null && mutedTest.getMethods().isEmpty() == false) {