Merge branch 'main' into debertaTokenizerNormalizeFix

Author: maxhniebergall

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.plugins.PluginsLoader;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.DocReader;
@@ -76,8 +77,7 @@ public class ScriptScoreBenchmark {
     private final PluginsService pluginsService = new PluginsService(
         Settings.EMPTY,
         null,
-        null,
-        Path.of(System.getProperty("plugins.dir"))
+        new PluginsLoader(null, Path.of(System.getProperty("plugins.dir")))
     );
     private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
 

branches.json

@@ -7,6 +7,9 @@
     {
       "branch": "8.16"
     },
+    {
+      "branch": "8.17"
+    },
     {
       "branch": "8.x"
     },

build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy

@@ -15,16 +15,12 @@ import org.elasticsearch.gradle.internal.test.AntFixture
 import org.gradle.api.file.FileSystemOperations
 import org.gradle.api.file.ProjectLayout
 import org.gradle.api.provider.ProviderFactory
-import org.gradle.api.tasks.Internal
 import org.gradle.process.ExecOperations
 
 import javax.inject.Inject
 
 abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
 
-    @Internal
-    AntFixture fixture
-
     @Inject
     AntFixtureStop(ProjectLayout projectLayout,
                    ExecOperations execOperations,
@@ -34,12 +30,12 @@ abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
     }
 
     void setFixture(AntFixture fixture) {
-        assert this.fixture == null
-        this.fixture = fixture;
-        final Object pid = "${-> this.fixture.pid}"
-        onlyIf("pidFile exists") { fixture.pidFile.exists() }
+        def pidFile = fixture.pidFile
+        def fixtureName = fixture.name
+        final Object pid = "${-> Integer.parseInt(pidFile.getText('UTF-8').trim())}"
+        onlyIf("pidFile exists") { pidFile.exists() }
         doFirst {
-            logger.info("Shutting down ${fixture.name} with pid ${pid}")
+            logger.info("Shutting down ${fixtureName} with pid ${pid}")
         }
 
         if (OS.current() == OS.WINDOWS) {
@@ -51,9 +47,8 @@ abstract class AntFixtureStop extends LoggedExec implements FixtureStop {
         }
         doLast {
             fileSystemOperations.delete {
-                it.delete(fixture.pidFile)
+                it.delete(pidFile)
             }
         }
-        this.fixture = fixture
     }
 }

build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/AntTask.groovy

@@ -29,11 +29,6 @@ import java.nio.charset.Charset
  */
 public abstract class AntTask extends DefaultTask {
 
-    /**
-     * A buffer that will contain the output of the ant code run,
-     * if the output was not already written directly to stdout.
-     */
-    public final ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream()
 
     @Inject
     protected FileSystemOperations getFileSystemOperations() {
@@ -57,6 +52,11 @@ public abstract class AntTask extends DefaultTask {
 
         // otherwise groovy replaces System.out, and you have no chance to debug
         // ant.saveStreams = false
+        /**
+         * A buffer that will contain the output of the ant code run,
+         * if the output was not already written directly to stdout.
+         */
+        ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream()
 
         final int outputLevel = logger.isDebugEnabled() ? Project.MSG_DEBUG : Project.MSG_INFO
         final PrintStream stream = useStdout() ? System.out : new PrintStream(outputBuffer, true, Charset.defaultCharset().name())

build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/test/AntFixture.groovy

@@ -10,22 +10,37 @@
 package org.elasticsearch.gradle.internal.test
 
 import org.elasticsearch.gradle.OS
+
 import org.elasticsearch.gradle.internal.AntFixtureStop
 import org.elasticsearch.gradle.internal.AntTask
+import org.elasticsearch.gradle.testclusters.TestClusterInfo
+import org.elasticsearch.gradle.testclusters.TestClusterValueSource
+import org.elasticsearch.gradle.testclusters.TestClustersRegistry
 import org.gradle.api.GradleException
+import org.gradle.api.file.ProjectLayout
+import org.gradle.api.provider.Property
+import org.gradle.api.provider.Provider
+import org.gradle.api.provider.ProviderFactory
+import org.gradle.api.provider.ValueSource
+import org.gradle.api.provider.ValueSourceParameters
+import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.TaskProvider
 
+import javax.inject.Inject
+
 /**
  * A fixture for integration tests which runs in a separate process launched by Ant.
  */
-class AntFixture extends AntTask implements Fixture {
+class AntFixture extends AntTask {
 
     /** The path to the executable that starts the fixture. */
     @Internal
     String executable
 
     private final List<Object> arguments = new ArrayList<>()
+    private ProjectLayout projectLayout
+    private final ProviderFactory providerFactory
 
     void args(Object... args) {
         arguments.addAll(args)
@@ -69,19 +84,14 @@ class AntFixture extends AntTask implements Fixture {
         return tmpFile.exists()
     }
 
-    private final TaskProvider<AntFixtureStop> stopTask
-
-    AntFixture() {
-        stopTask = createStopTask()
+    @Inject
+    AntFixture(ProjectLayout projectLayout, ProviderFactory providerFactory) {
+        this.providerFactory = providerFactory
+        this.projectLayout = projectLayout;
+        TaskProvider<AntFixtureStop> stopTask = createStopTask()
         finalizedBy(stopTask)
     }
 
-    @Override
-    @Internal
-    TaskProvider<AntFixtureStop> getStopTask() {
-        return stopTask
-    }
-
     @Override
     protected void runAnt(AntBuilder ant) {
         // reset everything
@@ -231,7 +241,7 @@ class AntFixture extends AntTask implements Fixture {
      */
     @Internal
     protected File getBaseDir() {
-        return new File(project.buildDir, "fixtures/${name}")
+        return new File(projectLayout.getBuildDirectory().getAsFile().get(), "fixtures/${name}")
     }
 
     /** Returns the working directory for the process. Defaults to "cwd" inside baseDir. */
@@ -242,7 +252,7 @@ class AntFixture extends AntTask implements Fixture {
 
     /** Returns the file the process writes its pid to. Defaults to "pid" inside baseDir. */
     @Internal
-    protected File getPidFile() {
+    File getPidFile() {
         return new File(baseDir, 'pid')
     }
 
@@ -264,6 +274,12 @@ class AntFixture extends AntTask implements Fixture {
         return portsFile.readLines("UTF-8").get(0)
     }
 
+    @Internal
+    Provider<String> getAddressAndPortProvider() {
+        File thePortFile = portsFile
+        return providerFactory.provider(() -> thePortFile.readLines("UTF-8").get(0))
+    }
+
     /** Returns a file that wraps around the actual command when {@code spawn == true}. */
     @Internal
     protected File getWrapperScript() {
@@ -281,4 +297,22 @@ class AntFixture extends AntTask implements Fixture {
     protected File getRunLog() {
         return new File(cwd, 'run.log')
     }
+
+    @Internal
+    Provider<AntFixtureValueSource> getAddressAndPortSource() {
+        return providerFactory.of(AntFixtureValueSource.class, spec -> {
+            spec.getParameters().getPortFile().set(portsFile);
+        });
+    }
+
+    static abstract class AntFixtureValueSource implements ValueSource<String, AntFixtureValueSource.Parameters> {
+        @Override
+        String obtain() {
+            return getParameters().getPortFile().map { it.readLines("UTF-8").get(0) }.get()
+        }
+
+        interface Parameters extends ValueSourceParameters {
+            Property<File> getPortFile();
+        }
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java

@@ -26,6 +26,7 @@
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.file.Directory;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.file.RelativePath;
 import org.gradle.api.internal.file.FileOperations;
@@ -244,10 +245,11 @@ public void apply(Project project) {
         yamlRestCompatTestTask.configure(testTask -> {
             testTask.systemProperty("tests.restCompat", true);
             // Use test runner and classpath from "normal" yaml source set
+            FileCollection outputFileCollection = yamlCompatTestSourceSet.getOutput();
             testTask.setTestClassesDirs(
                 yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs())
             );
-            testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getOutput().isEmpty() == false);
+            testTask.onlyIf("Compatibility tests are available", t -> outputFileCollection.isEmpty() == false);
             testTask.setClasspath(
                 yamlCompatTestSourceSet.getRuntimeClasspath()
                     // remove the "normal" api and tests

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -76,6 +76,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
     private final LinkedHashMap<String, Predicate<TestClusterConfiguration>> waitConditions = new LinkedHashMap<>();
     private final transient Project project;
     private final Provider<ReaperService> reaper;
+    private final Provider<TestClustersRegistry> testClustersRegistryProvider;
     private final FileSystemOperations fileSystemOperations;
     private final ArchiveOperations archiveOperations;
     private final ExecOperations execOperations;
@@ -87,11 +88,14 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
 
     private boolean shared = false;
 
+    private int claims = 0;
+
     public ElasticsearchCluster(
         String path,
         String clusterName,
         Project project,
         Provider<ReaperService> reaper,
+        Provider<TestClustersRegistry> testClustersRegistryProvider,
         FileSystemOperations fileSystemOperations,
         ArchiveOperations archiveOperations,
         ExecOperations execOperations,
@@ -104,6 +108,7 @@ public ElasticsearchCluster(
         this.clusterName = clusterName;
         this.project = project;
         this.reaper = reaper;
+        this.testClustersRegistryProvider = testClustersRegistryProvider;
         this.fileSystemOperations = fileSystemOperations;
         this.archiveOperations = archiveOperations;
         this.execOperations = execOperations;
@@ -120,6 +125,7 @@ public ElasticsearchCluster(
                 clusterName + "-0",
                 project,
                 reaper,
+                testClustersRegistryProvider,
                 fileSystemOperations,
                 archiveOperations,
                 execOperations,
@@ -177,6 +183,7 @@ public void setNumberOfNodes(int numberOfNodes) {
                     clusterName + "-" + i,
                     project,
                     reaper,
+                    testClustersRegistryProvider,
                     fileSystemOperations,
                     archiveOperations,
                     execOperations,
@@ -408,6 +415,7 @@ public void setPreserveDataDir(boolean preserveDataDir) {
     public void freeze() {
         nodes.forEach(ElasticsearchNode::freeze);
         configurationFrozen.set(true);
+        nodes.whenObjectAdded(node -> { throw new IllegalStateException("Cannot add nodes to test cluster after is has been frozen"); });
     }
 
     private void checkFrozen() {
@@ -663,4 +671,11 @@ public String toString() {
         return "cluster{" + path + ":" + clusterName + "}";
     }
 
+    int addClaim() {
+        return ++this.claims;
+    }
+
+    int removeClaim() {
+        return --this.claims;
+    }
 }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -124,6 +124,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final String name;
     transient private final Project project;
     private final Provider<ReaperService> reaperServiceProvider;
+    private final Provider<TestClustersRegistry> testClustersRegistryProvider;
+
     private final FileSystemOperations fileSystemOperations;
     private final ArchiveOperations archiveOperations;
     private final ExecOperations execOperations;
@@ -164,7 +166,6 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final List<ElasticsearchDistribution> distributions = new ArrayList<>();
     private int currentDistro = 0;
     private TestDistribution testDistribution;
-    private volatile Process esProcess;
     private Function<String, String> nameCustomization = s -> s;
     private boolean isWorkingDirConfigured = false;
     private String httpPort = "0";
@@ -179,6 +180,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         String name,
         Project project,
         Provider<ReaperService> reaperServiceProvider,
+        Provider<TestClustersRegistry> testClustersRegistryProvider,
         FileSystemOperations fileSystemOperations,
         ArchiveOperations archiveOperations,
         ExecOperations execOperations,
@@ -191,6 +193,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         this.name = name;
         this.project = project;
         this.reaperServiceProvider = reaperServiceProvider;
+        this.testClustersRegistryProvider = testClustersRegistryProvider;
         this.fileSystemOperations = fileSystemOperations;
         this.archiveOperations = archiveOperations;
         this.execOperations = execOperations;
@@ -892,11 +895,13 @@ private void startElasticsearchProcess() {
             }
         }
         LOGGER.info("Running `{}` in `{}` for {} env: {}", command, workingDir, this, environment);
+        Process esProcess;
         try {
             esProcess = processBuilder.start();
         } catch (IOException e) {
             throw new TestClustersException("Failed to start ES process for " + this, e);
         }
+        testClustersRegistryProvider.get().storeProcess(id(), esProcess);
         reaperServiceProvider.get().registerPid(toString(), esProcess.pid());
     }
 
@@ -982,6 +987,7 @@ public synchronized void stop(boolean tailLogs) {
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
+        Process esProcess = testClustersRegistryProvider.get().getProcess(id());
         if (esProcess == null && tailLogs) {
             // This is a special case. If start() throws an exception the plugin will still call stop
             // Another exception here would eat the orriginal.
@@ -1574,6 +1580,7 @@ public List<FeatureFlag> getFeatureFlags() {
     @Override
     @Internal
     public boolean isProcessAlive() {
+        Process esProcess = testClustersRegistryProvider.get().getProcess(id());
         requireNonNull(esProcess, "Can't wait for `" + this + "` as it's not started. Does the task have `useCluster` ?");
         return esProcess.isAlive();
     }
@@ -1602,6 +1609,10 @@ public int hashCode() {
 
     @Override
     public String toString() {
+        return id() + " (" + System.identityHashCode(this) + ")";
+    }
+
+    private String id() {
         return "node{" + path + ":" + name + "}";
     }
 
@@ -1702,7 +1713,7 @@ public CharSequence[] getArgs() {
         }
     }
 
-    private record FeatureFlag(String feature, Version from, Version until) {
+    public record FeatureFlag(String feature, Version from, Version until) {
 
         @Input
         public String getFeature() {