Merge pull request ESQL-608 from elastic/main

🤖 ESQL: Merge upstream

Author: elasticsearchmachine

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java

@@ -117,6 +117,7 @@ public void execute(Task task) {
             maybeSkipTask(dockerSupport, buildFixture);
 
             ComposeExtension composeExtension = project.getExtensions().getByType(ComposeExtension.class);
+            composeExtension.setProjectName(project.getName());
             composeExtension.getUseComposeFiles().addAll(Collections.singletonList(DOCKER_COMPOSE_YML));
             composeExtension.getRemoveContainers().set(true);
             composeExtension.getCaptureContainersOutput()

distribution/docker/build.gradle

@@ -8,7 +8,6 @@ import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin
 import org.elasticsearch.gradle.internal.docker.DockerSupportService
 import org.elasticsearch.gradle.internal.docker.ShellRetry
 import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter
-import org.elasticsearch.gradle.internal.dra.DraResolvePlugin
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.util.GradleUtils
 
@@ -75,11 +74,12 @@ configurations {
   log4jConfig
   tini
   allPlugins
-  filebeat
-  metricbeat
+  filebeat_aarch64
+  filebeat_x86_64
+  metricbeat_aarch64
+  metricbeat_x86_64
 }
 
-String beatsArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'x86_64'
 String tiniArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'amd64'
 
 dependencies {
@@ -88,8 +88,10 @@ dependencies {
   log4jConfig project(path: ":distribution", configuration: 'log4jConfig')
   tini "krallin:tini:0.19.0:${tiniArch}"
   allPlugins project(path: ':plugins', configuration: 'allPlugins')
-  filebeat "beats:filebeat:${VersionProperties.elasticsearch}:linux-${beatsArch}@tar.gz"
-  metricbeat "beats:metricbeat:${VersionProperties.elasticsearch}:linux-${beatsArch}@tar.gz"
+  filebeat_aarch64 "beats:filebeat:${VersionProperties.elasticsearch}:[email protected]"
+  filebeat_x86_64 "beats:filebeat:${VersionProperties.elasticsearch}:[email protected]"
+  metricbeat_aarch64 "beats:metricbeat:${VersionProperties.elasticsearch}:[email protected]"
+  metricbeat_x86_64 "beats:metricbeat:${VersionProperties.elasticsearch}:[email protected]"
 }
 
 ext.expansions = { Architecture architecture, DockerBase base ->
@@ -273,8 +275,8 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
         boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
 
         if (includeBeats) {
-          from configurations.filebeat
-          from configurations.metricbeat
+          from configurations.getByName("filebeat_${architecture.classifier}")
+          from configurations.getByName("metricbeat_${architecture.classifier}")
         }
         // For some reason, the artifact name can differ depending on what repository we used.
         rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
@@ -548,5 +550,5 @@ subprojects { Project subProject ->
 
 tasks.named('resolveAllDependencies') {
   // Don't try and resolve filebeat or metricbeat snapshots as they may not always be available
-  configs = configurations.matching { it.name.endsWith('beat') == false }
+  configs = configurations.matching { it.name.contains('beat') == false }
 }

docs/changelog/92856.yaml

@@ -0,0 +1,6 @@
+pr: 92856
+summary: Don't announce ready until file settings are applied
+area: Infra/Core
+type: bug
+issues:
+ - 92812

docs/reference/ml/anomaly-detection/apis/get-record.asciidoc

@@ -135,6 +135,11 @@ reduced. If the bucket contains fewer samples than expected, the score is reduce
 `lower_confidence_bound`::::
 (Optional, double) Lower bound of the 95% confidence interval.
 
+`multimodal_distribution`::::
+(Optional, boolean) Indicates whether the bucket values' probability distribution has 
+several modes. When there are multiple modes, the typical value may not be the most 
+likely.
+
 `multi_bucket_impact`::::
 (Optional, integer) Impact of the deviation between actual and typical values in the 
 past 12 buckets.

libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/ClassReadersTests.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.plugin.scanner;
 
+import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.compiler.InMemoryJavaCompiler;
 import org.elasticsearch.test.jar.JarUtils;
@@ -39,6 +40,9 @@ public void testModuleInfoIsNotReturnedAsAClassFromJar() throws IOException {
 
         List<ClassReader> classReaders = ClassReaders.ofPaths(Stream.of(jar));
         org.hamcrest.MatcherAssert.assertThat(classReaders, Matchers.empty());
+
+        // aggressively delete the jar dir, so that any leaked filed handles fail this specific test on windows
+        IOUtils.rm(tmp);
     }
 
     public void testTwoClassesInAStreamFromJar() throws IOException {
@@ -57,6 +61,9 @@ public class B {}
         List<ClassReader> classReaders = ClassReaders.ofPaths(Stream.of(jar));
         List<String> collect = classReaders.stream().map(cr -> cr.getClassName()).collect(Collectors.toList());
         org.hamcrest.MatcherAssert.assertThat(collect, Matchers.containsInAnyOrder("p/A", "p/B"));
+
+        // aggressively delete the jar dir, so that any leaked filed handles fail this specific test on windows
+        IOUtils.rm(tmp);
     }
 
     public void testStreamOfJarsAndIndividualClasses() throws IOException {

libs/plugin-scanner/src/test/java/org/elasticsearch/plugin/scanner/NamedComponentScannerTests.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.plugin.scanner;
 
+import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.plugin.scanner.test_model.ExtensibleClass;
 import org.elasticsearch.plugin.scanner.test_model.ExtensibleInterface;
 import org.elasticsearch.plugin.scanner.test_model.TestNamedComponent;
@@ -100,6 +101,9 @@ public class B implements ExtensibleInterface{}
                 )
             )
         );
+
+        // aggressively delete the jar dir, so that any leaked filed handles fail this specific test on windows
+        IOUtils.rm(tmp);
     }
 
     public void testNamedComponentsCanExtednCommonSuperClass() throws IOException {
@@ -174,6 +178,9 @@ public class B implements CustomExtensibleInterface{}
                 )
             )
         );
+
+        // aggressively delete the jar dir, so that any leaked filed handles fail this specific test on windows
+        IOUtils.rm(tmp);
     }
 
     public void testWriteToFile() throws IOException {

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -806,6 +806,7 @@ public void testRandomDataSteamAliasesUpdate() throws Exception {
         String alias = randomAlphaOfLength(4);
         String[] dataStreams = Arrays.stream(generateRandomStringArray(16, 4, false, false))
             .map(s -> "log-" + s.toLowerCase(Locale.ROOT))
+            .distinct()
             .toArray(String[]::new);
         for (String dataStream : dataStreams) {
             CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStream);

qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java

@@ -18,29 +18,20 @@
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.junit.ClassRule;
-import org.junit.rules.RuleChain;
-import org.junit.rules.TemporaryFolder;
-import org.junit.rules.TestRule;
 
 @TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell
 public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
-    private static TemporaryFolder repoDirectory = new TemporaryFolder();
-
-    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .nodes(2)
         .module("mapper-extras")
         .module("ingest-common")
-        .setting("path.repo", () -> repoDirectory.getRoot().getPath())
         // The first node does not have the ingest role so we're sure ingest requests are forwarded:
         .node(0, n -> n.setting("node.roles", "[master,data,ml,remote_cluster_client,transform]"))
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .build();
 
-    @ClassRule
-    // Ensure the shared repo dir is created before cluster start
-    public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster);
-
     public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }

server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java

@@ -356,6 +356,7 @@ public AllocationCommand getCancelPrimaryCommand() {
         return null;
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/93010")
     public void testSearchRouting() throws InterruptedException {
 
         var routingTableWatcher = new RoutingTableWatcher();

server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java

@@ -20,7 +20,6 @@
 import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.plugins.Plugin;
@@ -51,11 +50,6 @@
 
 public class IngestFileSettingsIT extends ESIntegTestCase {
 
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
-        return applyWorkaroundForIssue92812(super.nodeSettings(nodeOrdinal, otherSettings));
-    }
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(CustomIngestTestPlugin.class);
@@ -107,7 +101,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
                  "ingest_pipelines": {
                    "my_ingest_pipeline": {
                        "description": "_description",
-                       "processors": [
+                       "processors":
                           {
                             "foo" : {
                               "field": "pipeline",
@@ -210,11 +204,11 @@ public void clusterChanged(ClusterChangedEvent event) {
                     clusterService.removeListener(this);
                     metadataVersion.set(event.state().metadata().version());
                     savedClusterState.countDown();
-                    assertEquals(ReservedStateErrorMetadata.ErrorKind.VALIDATION, reservedState.errorMetadata().errorKind());
+                    assertEquals(ReservedStateErrorMetadata.ErrorKind.PARSING, reservedState.errorMetadata().errorKind());
                     assertThat(reservedState.errorMetadata().errors(), allOf(notNullValue(), hasSize(1)));
                     assertThat(
                         reservedState.errorMetadata().errors().get(0),
-                        containsString("org.elasticsearch.ElasticsearchParseException: No processor type exists with name [foo]")
+                        containsString("org.elasticsearch.xcontent.XContentParseException: [17:16] [reserved_state_chunk] failed")
                     );
                 }
             }