Merge branch 'main' into can_match_running_on_transport_thread

Author: chrisparrinello

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -51,7 +51,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
     def "resolves expanded bwc versions from source"() {
         given:
         internalBuild()
-        bwcMajor1ProjectSetup()
+        bwcProjectSetup("major1")
         buildFile << """
             apply plugin: 'elasticsearch.internal-distribution-download'
 
@@ -81,7 +81,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
     def "fails on resolving bwc versions with no bundled jdk"() {
         given:
         internalBuild()
-        bwcMajor1ProjectSetup()
+        bwcProjectSetup("major1")
         buildFile << """
             apply plugin: 'elasticsearch.internal-distribution-download'
 
@@ -105,12 +105,47 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
                 "without a bundled JDK is not supported.")
     }
 
-    private void bwcMajor1ProjectSetup() {
+    def "resolves detached version from source"() {
+        given:
+        internalBuild()
+        bwcProjectSetup("main")
+        buildFile << """
+            apply plugin: 'elasticsearch.internal-distribution-download'
+
+            System.setProperty("tests.bwc.main.version", "9.2.0")
+
+            elasticsearch_distributions {
+              test_distro {
+                  version = "9.2.0"
+                  type = "archive"
+                  platform = "linux"
+                  architecture = Architecture.current();
+                  detachedVersion = true
+              }
+            }
+
+            tasks.register("setupDistro", Sync) {
+                from(elasticsearch_distributions.test_distro)
+                into("build/distro")
+            }
+        """
+
+        when:
+        def result = gradleRunner("setupDistro").build()
+
+        then:
+        result.task(":distribution:bwc:main:buildBwcExpandedTask").outcome == TaskOutcome.SUCCESS
+        result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
+        assertExtractedDistroIsCreated("distribution/bwc/main/build/install/elastic-distro",
+            'bwc-marker.txt')
+    }
+
+    private void bwcProjectSetup(String bwcProjectName) {
         settingsFile << """
-        include ':distribution:bwc:major1'
+        include ':distribution:bwc:$bwcProjectName'
         """
-        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "major1")
-        new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=major1"
+        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", bwcProjectName)
+        new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=$bwcProjectName"
         new File(bwcSubProjectFolder, 'build.gradle') << """
             apply plugin:'base'
 

build-tools-internal/src/main/groovy/elasticsearch.bc-upgrade-test.gradle

@@ -11,8 +11,18 @@ import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 tasks.register("bcUpgradeTest", StandaloneRestIntegTestTask) {
-  // We use a phony version here as the real version is provided via `tests.bwc.main.version` system property
-  usesBwcDistribution(Version.fromString("0.0.0"))
-  systemProperty("tests.old_cluster_version", "0.0.0")
-  onlyIf("tests.bwc.main.version system property exists") { System.getProperty("tests.bwc.main.version") != null }
+  project.gradle.taskGraph.whenReady { allTasks ->
+    if (allTasks.hasTask(getPath())) {
+      if (System.getProperty("tests.bwc.refspec.main") == null) {
+        throw new GradleException("You must set the `tests.bwc.refspec.main` system property to run bcUpgradeTest")
+      }
+      if (System.getProperty("tests.bwc.main.version") == null) {
+        throw new GradleException("You must set the `tests.bwc.main.version` system property to run bcUpgradeTest")
+      }
+    }
+  }
+  if (System.getProperty("tests.bwc.refspec.main") != null && System.getProperty("tests.bwc.main.version") != null) {
+    usesBwcDistributionFromRef(System.getProperty("tests.bwc.refspec.main"), Version.fromString(System.getProperty("tests.bwc.main.version")))
+    systemProperty("tests.old_cluster_version", System.getProperty("tests.bwc.main.version"))
+  }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -74,7 +74,7 @@ public void apply(Project project) {
      */
     private void registerInternalDistributionResolutions(List<DistributionResolution> resolutions, Provider<BwcVersions> bwcVersions) {
         resolutions.add(new DistributionResolution("local-build", (project, distribution) -> {
-            if (isCurrentVersion(distribution)) {
+            if (isCurrentVersion(distribution) && distribution.isDetachedVersion() == false) {
                 // non-external project, so depend on local build
                 return new ProjectBasedDistributionDependency(
                     config -> projectDependency(project.getDependencies(), distributionProjectPath(distribution), config)
@@ -86,7 +86,7 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
         resolutions.add(new DistributionResolution("bwc", (project, distribution) -> {
             BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.get()
                 .unreleasedInfo(Version.fromString(distribution.getVersion()));
-            if (unreleasedInfo != null) {
+            if (unreleasedInfo != null && distribution.isDetachedVersion() == false) {
                 if (distribution.getBundledJdk() == false) {
                     throw new GradleException(
                         "Configuring a snapshot bwc distribution ('"
@@ -103,21 +103,17 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
             return null;
         }));
 
-        // Distribution resolution for "override" versions. This allows for building from source for any version, including the current
+        // Distribution resolution for "detached" versions. This allows for building from source for any version, including the current
         // version of existing released versions from a commit form the main branch. This is done by passing certain system properties, ex:
         //
         // -Dtests.bwc.refspec.main=deadbeef -Dtests.bwc.main.version=9.0.0
         //
-        // The 'test.bwc.main.version' property should map to the version returned by the commit referenced in 'tests.bwc.refspec.main'.
-        resolutions.add(new DistributionResolution("override", (project, distribution) -> {
-            String versionProperty = System.getProperty("tests.bwc.main.version");
-            // We use this phony version as a placeholder for the real version
-            if (distribution.getVersion().equals("0.0.0")) {
-                if (versionProperty == null) {
-                    throw new GradleException("System property 'tests.bwc.main.version' expected for building bwc version.");
-                }
+        // The distribution version set by 'test.bwc.main.version' property should map to the version returned by the commit
+        // referenced in 'tests.bwc.refspec.main'.
+        resolutions.add(new DistributionResolution("detached", (project, distribution) -> {
+            if (distribution.isDetachedVersion()) {
                 BwcVersions.UnreleasedVersionInfo unreleasedVersionInfo = new BwcVersions.UnreleasedVersionInfo(
-                    Version.fromString(versionProperty),
+                    Version.fromString(distribution.getVersion()),
                     "main",
                     ":distribution:bwc:main"
                 );

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java

@@ -11,9 +11,7 @@
 
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
-import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
-import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 
@@ -33,17 +31,5 @@ public void apply(Project project) {
             version -> (version.equals(VersionProperties.getElasticsearchVersion()) && buildParams.getSnapshotBuild() == false)
                 || buildParams.getBwcVersions().unreleasedInfo(version) == null
         );
-
-        NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
-            .getExtensions()
-            .getByName(TestClustersPlugin.EXTENSION_NAME);
-        // Limit the number of allocated processors for all nodes to 2 in the cluster by default.
-        // This is to ensure that the tests run consistently across different environments.
-        String processorCount = shouldConfigureTestClustersWithOneProcessor() ? "1" : "2";
-        testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.setting("node.processors", processorCount));
-    }
-
-    private boolean shouldConfigureTestClustersWithOneProcessor() {
-        return Boolean.parseBoolean(System.getProperty("tests.configure_test_clusters_with_one_processor", "false"));
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -106,13 +106,15 @@ public void apply(Project project) {
         ElasticsearchDistribution defaultDistro = createDistribution(
             project,
             DEFAULT_REST_INTEG_TEST_DISTRO,
-            VersionProperties.getElasticsearch()
+            VersionProperties.getElasticsearch(),
+            false
         );
         ElasticsearchDistribution integTestDistro = createDistribution(
             project,
             INTEG_TEST_REST_INTEG_TEST_DISTRO,
             VersionProperties.getElasticsearch(),
-            ElasticsearchDistributionTypes.INTEG_TEST_ZIP
+            ElasticsearchDistributionTypes.INTEG_TEST_ZIP,
+            false
         );
 
         // Create configures for module and plugin dependencies
@@ -238,21 +240,11 @@ public Void call(Object... args) {
                     }
 
                     Version version = (Version) args[0];
-                    boolean isReleased = bwcVersions.unreleasedInfo(version) == null && version.toString().equals("0.0.0") == false;
+                    boolean isReleased = bwcVersions.unreleasedInfo(version) == null;
                     String versionString = version.toString();
-                    ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
-
-                    if (jdkIsIncompatibleWithOS(Version.fromString(versionString))) {
-                        var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
-                        var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
-                            spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
-                            spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
-                        });
-                        task.environment(
-                            "ES_FALLBACK_JAVA_HOME",
-                            fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
-                        );
-                    }
+                    ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString, false);
+
+                    handleJdkIncompatibleWithOS(version, project, task);
                     task.dependsOn(bwcDistro);
                     registerDistributionInputs(task, bwcDistro);
 
@@ -261,16 +253,63 @@ public Void call(Object... args) {
                         providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath())
                     );
 
-                    if (version.getMajor() > 0 && version.before(bwcVersions.getMinimumWireCompatibleVersion())) {
+                    if (version.before(bwcVersions.getMinimumWireCompatibleVersion())) {
                         // If we are upgrade testing older versions we also need to upgrade to 7.last
                         this.call(bwcVersions.getMinimumWireCompatibleVersion());
                     }
                     return null;
                 }
             });
+
+            task.getExtensions().getExtraProperties().set("usesBwcDistributionFromRef", new Closure<Void>(task) {
+                @Override
+                public Void call(Object... args) {
+                    if (args.length != 2 || args[0] instanceof String == false || args[1] instanceof Version == false) {
+                        throw new IllegalArgumentException("Expected arguments (String refSpec, org.elasticsearch.gradle.Version version)");
+                    }
+
+                    String refSpec = (String) args[0];
+                    Version version = (Version) args[1];
+                    boolean isDetachedVersion = true;
+                    String versionString = version.toString();
+
+                    ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + refSpec, versionString, isDetachedVersion);
+                    handleJdkIncompatibleWithOS(version, project, task);
+
+                    task.dependsOn(bwcDistro);
+                    registerDistributionInputs(task, bwcDistro);
+
+                    nonInputSystemProperties.systemProperty(
+                        BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX + versionString,
+                        providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath())
+                    );
+                    return null;
+                }
+            });
         });
     }
 
+    /**
+     * Older distributions ship with openjdk versions that are not compatible with newer kernels of ubuntu 24.04 and later
+     * Therefore we pass explicitly the runtime java to use the adoptium jdk that is maintained longer and compatible
+     * with newer kernels.
+     * 8.10.4 is the last version shipped with jdk < 21. We configure these cluster to run with jdk 17 adoptium as 17 was
+     * the last LTS release before 21
+     */
+    private static void handleJdkIncompatibleWithOS(Version version, Project project, StandaloneRestIntegTestTask task) {
+        if (jdkIsIncompatibleWithOS(version)) {
+            var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
+            var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
+                spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
+                spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
+            });
+            task.environment(
+                "ES_FALLBACK_JAVA_HOME",
+                fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
+            );
+        }
+    }
+
     private void copyDependencies(Project project, DependencySet dependencies, Configuration configuration) {
         configuration.getDependencies()
             .stream()
@@ -279,16 +318,23 @@ private void copyDependencies(Project project, DependencySet dependencies, Confi
             .forEach(dependencies::add);
     }
 
-    private ElasticsearchDistribution createDistribution(Project project, String name, String version) {
-        return createDistribution(project, name, version, null);
+    private ElasticsearchDistribution createDistribution(Project project, String name, String version, boolean detachedVersion) {
+        return createDistribution(project, name, version, null, detachedVersion);
     }
 
-    private ElasticsearchDistribution createDistribution(Project project, String name, String version, ElasticsearchDistributionType type) {
+    private ElasticsearchDistribution createDistribution(
+        Project project,
+        String name,
+        String version,
+        ElasticsearchDistributionType type,
+        boolean detachedVersion
+    ) {
         NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
         ElasticsearchDistribution maybeDistro = distributions.findByName(name);
         if (maybeDistro == null) {
             return distributions.create(name, distro -> {
                 distro.setVersion(version);
+                distro.setDetachedVersion(detachedVersion);
                 distro.setArchitecture(Architecture.current());
                 if (type != null) {
                     distro.setType(type);

build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -49,6 +49,7 @@ public String toString() {
 
     private final Property<Architecture> architecture;
     private final Property<String> version;
+    private final Property<Boolean> detachedVersion;
     private final Property<ElasticsearchDistributionType> type;
     private final Property<Platform> platform;
     private final Property<Boolean> bundledJdk;
@@ -69,6 +70,7 @@ public String toString() {
         this.configuration = fileConfiguration;
         this.architecture = objectFactory.property(Architecture.class);
         this.version = objectFactory.property(String.class).convention(VersionProperties.getElasticsearch());
+        this.detachedVersion = objectFactory.property(Boolean.class).convention(false);
         this.type = objectFactory.property(ElasticsearchDistributionType.class);
         this.type.convention(ElasticsearchDistributionTypes.ARCHIVE);
         this.platform = objectFactory.property(Platform.class);
@@ -91,6 +93,19 @@ public void setVersion(String version) {
         this.version.set(version);
     }
 
+    /**
+     * Informs if the version is not tied to any Elasticsearch release and is a custom build.
+     * This is true when the distribution is not from HEAD but also not any known released version.
+     * In that case the detached source build needs to be prepared by `usedBwcDistributionFromRef(ref, version)`.
+     */
+    public boolean isDetachedVersion() {
+        return detachedVersion.get();
+    }
+
+    public void setDetachedVersion(boolean detachedVersion) {
+        this.detachedVersion.set(detachedVersion);
+    }
+
     public Platform getPlatform() {
         return platform.getOrNull();
     }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -1404,6 +1404,15 @@ private void createConfiguration() {
             baseConfig.put("cluster.service.slow_master_task_logging_threshold", "5s");
         }
 
+        // Limit the number of allocated processors for all nodes in the cluster by default.
+        // This is to ensure that the tests run consistently across different environments.
+        String processorCount = shouldConfigureTestClustersWithOneProcessor() ? "1" : "2";
+        if (getVersion().onOrAfter("7.6.0")) {
+            baseConfig.put("node.processors", processorCount);
+        } else {
+            baseConfig.put("processors", processorCount);
+        }
+
         baseConfig.put("action.destructive_requires_name", "false");
 
         HashSet<String> overriden = new HashSet<>(baseConfig.keySet());
@@ -1789,4 +1798,8 @@ private static class LinkCreationException extends UncheckedIOException {
             super(message, cause);
         }
     }
+
+    private boolean shouldConfigureTestClustersWithOneProcessor() {
+        return Boolean.parseBoolean(System.getProperty("tests.configure_test_clusters_with_one_processor", "false"));
+    }
 }

docs/changelog/132962.yaml

@@ -0,0 +1,5 @@
+pr: 132962
+summary: Don't store keyword multi fields when they trip `ignore_above`
+area: Mapping
+type: bug
+issues: []

docs/changelog/134604.yaml

@@ -0,0 +1,5 @@
+pr: 134604
+summary: Adds certificate identity field to cross-cluster API keys
+area: Security
+type: enhancement
+issues: []

docs/changelog/135714.yaml

@@ -0,0 +1,5 @@
+pr: 135714
+summary: Improve performance when a single-valued field filter gets pushed down. (SingleValueMatchQuery)
+area: ES|QL
+type: enhancement
+issues: []