Merge branch 'main' into charlotte-elasticsearch-settings-689

Author: charlotte-hoblik

build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle

@@ -47,7 +47,7 @@ plugins.withType(InternalJavaRestTestPlugin) {
   tasks.withType(StandaloneRestIntegTestTask).configureEach {
     testClassesDirs = sourceSets.javaRestTest.output.classesDirs
     classpath = sourceSets.javaRestTest.runtimeClasspath
-    usesDefaultDistribution()
+    usesDefaultDistribution("BWC tests require full distribution for now")
   }
 }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -17,10 +17,7 @@
 import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
-import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
-import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
-import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 
@@ -81,29 +78,6 @@ public void doCall() {
         if (isModule == false || isXPackModule) {
             addNoticeGeneration(project, extension);
         }
-        project.afterEvaluate(p -> {
-            @SuppressWarnings("unchecked")
-            NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
-                .getExtensions()
-                .getByName(TestClustersPlugin.EXTENSION_NAME);
-            p.getExtensions().getByType(PluginPropertiesExtension.class).getExtendedPlugins().forEach(pluginName -> {
-                // Auto add any dependent modules
-                findModulePath(project, pluginName).ifPresent(
-                    path -> testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.module(path))
-                );
-            });
-        });
-    }
-
-    Optional<String> findModulePath(Project project, String pluginName) {
-        return project.getRootProject()
-            .getAllprojects()
-            .stream()
-            .filter(p -> GradleUtils.isModuleProject(p.getPath()))
-            .filter(p -> p.getPlugins().hasPlugin(PluginBuildPlugin.class))
-            .filter(p -> p.getExtensions().getByType(PluginPropertiesExtension.class).getName().equals(pluginName))
-            .findFirst()
-            .map(Project::getPath);
     }
 
     /**

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -49,8 +49,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-ec2");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-gce");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:mapper-annotated-text");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:mapper-murmur3");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:repository-hdfs");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:store-smb");
         map.put(LegacyRestTestBasePlugin.class, ":qa:ccs-rolling-upgrade-remote-cluster");
         map.put(LegacyRestTestBasePlugin.class, ":qa:mixed-cluster");
@@ -74,11 +72,7 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:logstash");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-constant-keyword");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-unsigned-long");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-version");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:vector-tile");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:wildcard");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:mixed-tier-cluster");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:repository-old-versions");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -198,6 +198,11 @@ public void apply(Project project) {
             task.getExtensions().getExtraProperties().set("usesDefaultDistribution", new Closure<Void>(task) {
                 @Override
                 public Void call(Object... args) {
+                    if (reasonForUsageProvided(args) == false) {
+                        throw new IllegalArgumentException(
+                            "Reason for using `usesDefaultDistribution` required.\nUse usesDefaultDistribution(\"reason why default distro is required here\")."
+                        );
+                    }
                     task.dependsOn(defaultDistro);
                     registerDistributionInputs(task, defaultDistro);
 
@@ -212,6 +217,10 @@ public Void call(Object... args) {
 
                     return null;
                 }
+
+                private static boolean reasonForUsageProvided(Object[] args) {
+                    return args.length == 1 && args[0] instanceof String && ((String) args[0]).isBlank() == false;
+                }
             });
 
             // Add `usesBwcDistribution(version)` extension method to test tasks to indicate they require a BWC distribution

docs/changelog/122062.yaml

@@ -0,0 +1,54 @@
+pr: 122062
+summary: Upgrade `discovery-ec2` to AWS SDK v2
+area: Discovery-Plugins
+type: breaking
+issues: []
+breaking:
+  title: Upgrade `discovery-ec2` to AWS SDK v2
+  area: Cluster and node setting
+  details: >-
+
+    In earlier versions of {es} the `discovery-ec2` plugin was based on the AWS
+    SDK v1. AWS will withdraw support for this SDK before the end of the life
+    of {es} {minor-version} so we must migrate to the newer AWS SDK v2.
+
+    Unfortunately there are several differences between the two AWS SDK
+    versions which may require you to adjust your system configuration when
+    upgrading to {es} {minor-version} or later. These differences include, but
+    may not be limited to, the following items.
+
+    * AWS SDK v2 does not support the EC2 IMDSv1 protocol.
+
+    * AWS SDK v2 does not support the `aws.secretKey` or
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties.
+
+    * AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so
+      the `discovery.ec2.protocol` setting is no longer effective.
+
+    * AWS SDK v2 does not accept an access key without a secret key or vice
+      versa.
+
+  impact: >-
+
+    If you use the `discovery-ec2` plugin, test your upgrade thoroughly before
+    upgrading any production workloads.
+
+    Adapt your configuration to the new SDK functionality. This includes, but
+    may not be limited to, the following items.
+
+    * If you use IMDS to determine the availability zone of a node or to obtain
+      credentials for accessing the EC2 API, ensure that it supports the IMDSv2
+      protocol.
+
+    * If applicable, discontinue use of the `aws.secretKey` and
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties.
+
+    * If applicable, specify that you wish to use the insecure HTTP protocol to
+      access the EC2 API by setting `discovery.ec2.endpoint` to a URL which
+      starts with `http://`.
+
+    * Either supply both an access key and a secret key using the keystore
+      settings `discovery.ec2.access_key` and `discovery.ec2.secret_key`, or
+      configure neither of these settings.
+
+  notable: true

docs/changelog/122638.yaml

@@ -0,0 +1,7 @@
+pr: 122638
+summary: Expose `input_type` option at root level for `text_embedding` task type in
+  Perform Inference API
+area: Machine Learning
+type: enhancement
+issues:
+ - 117856

docs/changelog/124784.yaml

@@ -0,0 +1,6 @@
+pr: 124784
+summary: Merge template mappings properly during validation
+area: Mapping
+type: bug
+issues:
+ - 123372

docs/changelog/124841.yaml

@@ -0,0 +1,5 @@
+pr: 124841
+summary: Pass `IndexReshardingMetadata` over the wire
+area: Distributed
+type: bug
+issues: []

docs/changelog/124918.yaml

@@ -0,0 +1,5 @@
+pr: 124918
+summary: Fix EQL double invoking listener
+area: EQL
+type: bug
+issues: []

docs/changelog/124936.yaml

@@ -0,0 +1,5 @@
+pr: 124936
+summary: Indicate when errors represent timeouts
+area: Infra/REST API
+type: enhancement
+issues: []