Merge remote-tracking branch 'upstream/main' into add-base64-encoded-float32-support

Author: benwtrent

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java

@@ -9,8 +9,11 @@
 
 package org.elasticsearch.gradle.testclusters;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.util.LRUMap;
+import com.fasterxml.jackson.databind.util.LookupCache;
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 import com.sun.net.httpserver.HttpServer;
@@ -28,7 +31,9 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.InetSocketAddress;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -48,6 +53,8 @@
 public class MockApmServer {
     private static final Logger logger = Logging.getLogger(MockApmServer.class);
     private static final org.slf4j.Logger log = LoggerFactory.getLogger(MockApmServer.class);
+    private static final LookupCache<String, String> transactionCache = new LRUMap(16, 16);
+
     private final Pattern metricFilter;
     private final Pattern transactionFilter;
     private final Pattern transactionExcludesFilter;
@@ -136,22 +143,28 @@ private void logFiltered(InputStream body) throws IOException {
             ObjectMapper mapper = new ObjectMapper();
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(body))) {
                 String line;
-                String tier = null;
-                String node = null;
+                String nodeMetadata = null;
+
+                List<JsonNode> spans = new ArrayList<>();
 
                 while ((line = reader.readLine()) != null) {
                     var jsonNode = mapper.readTree(line);
 
                     if (jsonNode.has("metadata")) {
-                        node = jsonNode.path("metadata").path("service").path("node").path("configured_name").asText(null);
-                        tier = jsonNode.path("metadata").path("labels").path("node_tier").asText(null);
+                        nodeMetadata = jsonNode.path("metadata").path("service").path("node").path("configured_name").asText(null);
+                        var tier = jsonNode.path("metadata").path("labels").path("node_tier").asText(null);
+                        nodeMetadata += tier != null ? "/" + tier : "";
+
                     } else if (transactionFilter != null && jsonNode.has("transaction")) {
                         var transaction = jsonNode.get("transaction");
                         var name = transaction.get("name").asText();
                         if (transactionFilter.matcher(name).matches()
                             && (transactionExcludesFilter == null || transactionExcludesFilter.matcher(name).matches() == false)) {
-                            logger.lifecycle("Transaction [{}/{}]: {}", node, tier, transaction);
+                            transactionCache.put(transaction.get("id").asText(), name);
+                            logger.lifecycle("Transaction {} [{}]: {}", name, nodeMetadata, transaction);
                         }
+                    } else if (jsonNode.has("span")) {
+                        spans.add(jsonNode.get("span")); // make sure to record all transactions first
                     } else if (metricFilter != null && jsonNode.has("metricset")) {
                         var metricset = jsonNode.get("metricset");
                         var samples = (ObjectNode) metricset.get("samples");
@@ -161,10 +174,20 @@ private void logFiltered(InputStream body) throws IOException {
                             }
                         }
                         if (samples.isEmpty() == false) {
-                            logger.lifecycle("Metricset [{}/{}]", node, tier, metricset);
+                            logger.lifecycle("Metricset [{}]: {}", nodeMetadata, metricset);
                         }
                     }
                 }
+
+                // emit only spans for previously matched transactions using the transaction cache
+                for (var span : spans) {
+                    var name = span.get("name").asText();
+                    var transactionId = span.get("transaction_id").asText();
+                    var transactionName = transactionCache.get(transactionId);
+                    if (transactionName != null) {
+                        logger.lifecycle("Span {} of {} [{}]: {}", name, transactionName, nodeMetadata, span);
+                    }
+                }
             }
         }
     }

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java

@@ -81,7 +81,10 @@ class APMJvmOptions {
         "metrics_interval", "120s",
         "breakdown_metrics", "false",
         "central_config", "false",
-        "transaction_sample_rate", "0.2"
+        "transaction_sample_rate", "0.2",
+        // Don't collect stacktraces for spans, typically these are of little use as
+        // always pointing to APMTracer.stopTrace invoked from TaskManager
+        "stack_trace_limit", "0"
         );
     // end::noformat
 

docs/changelog/136951.yaml

@@ -0,0 +1,5 @@
+pr: 136951
+summary: Support different downsampling methods through ILM
+area: "ILM+SLM"
+type: enhancement
+issues: []

docs/changelog/137244.yaml

@@ -0,0 +1,6 @@
+pr: 137244
+summary: Preserve deployments with zero allocations during assignment planning
+area: Machine Learning
+type: bug
+issues:
+ - 137134

docs/reference/elasticsearch/index-lifecycle-actions/ilm-downsample.md

@@ -19,9 +19,13 @@ To use the `downsample` action in the `hot` phase, the `rollover` action **must*
 
 `fixed_interval`
 :   (Required, string) The [fixed time interval](docs-content://manage-data/lifecycle/rollup/understanding-groups.md#rollup-understanding-group-intervals) into which the data will be downsampled.
+
 `force_merge_index` {applies_to}`stack: ga 9.3`
 :   (Optional, boolean) When true, the downsampled index will be [force merged](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-forcemerge) to one [segment](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-segments). Defaults to `true`.
 
+`sampling_method` {applies_to}`stack: ga 9.3`
+:   (Optional, string) The sampling method that will be used to sample metrics; there are two methods available `aggregate` and
+the `last_value`. Defaults to `aggregate`.
 
 ## Example [ilm-downsample-ex]
 

muted-tests.yml

@@ -501,9 +501,6 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=indices.validate_query/20_query_string/validate_query with query_string parameters}
   issue: https://github.com/elastic/elasticsearch/issues/137391
-- class: org.elasticsearch.xpack.downsample.ILMDownsampleDisruptionIT
-  method: testILMDownsampleRollingRestart
-  issue: https://github.com/elastic/elasticsearch/issues/136585
 
 # Examples:
 #

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java

@@ -96,7 +96,6 @@ public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         // geohex_grid requires gold license
         .setting("xpack.license.self_generated.type", "trial")
         .feature(FeatureFlag.TIME_SERIES_MODE)
-        .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
         .feature(FeatureFlag.SYNTHETIC_VECTORS);
 
     private static ElasticsearchCluster remoteCluster = ElasticsearchCluster.local()

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java

@@ -95,7 +95,6 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .setting("xpack.security.remote_cluster_server.ssl.enabled", "false")
         .setting("xpack.security.remote_cluster_client.ssl.enabled", "false")
         .feature(FeatureFlag.TIME_SERIES_MODE)
-        .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
         .feature(FeatureFlag.SYNTHETIC_VECTORS)
         .user("test_admin", "x-pack-test-password");
 

qa/mixed-cluster/build.gradle

@@ -89,7 +89,6 @@ buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
         setting 'health.master_history.no_master_transitions_threshold', '10'
       }
       requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
-      requiresFeature 'sub_objects_auto', Version.fromString("8.16.0")
       if (bwcVersion.before(Version.fromString("8.18.0"))) {
         jvmArgs '-da:org.elasticsearch.index.mapper.DocumentMapper'
         jvmArgs '-da:org.elasticsearch.index.mapper.MapperService'

qa/packaging/src/test/java/org/elasticsearch/packaging/test/LinuxSystemTests.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.packaging.test;
+
+import org.elasticsearch.packaging.util.Platforms;
+import org.elasticsearch.packaging.util.ProcessInfo;
+import org.junit.BeforeClass;
+
+import java.util.List;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.assumeTrue;
+import static org.hamcrest.Matchers.equalTo;
+
+// tests for how the linux distro interacts with the OS
+public class LinuxSystemTests extends PackagingTestCase {
+
+    @BeforeClass
+    public static void ensureLinux() {
+        assumeTrue(Platforms.LINUX);
+    }
+
+    public void test10Install() throws Exception {
+        install();
+    }
+
+    public void test20CoredumpFilter() throws Exception {
+        startElasticsearch();
+
+        // find the Elasticsearch process
+        int esPid = -1;
+        List<ProcessInfo> procs = ProcessInfo.getProcessInfo(sh, "java");
+        for (ProcessInfo proc : procs) {
+            if (proc.commandLine().contains("org.elasticsearch.bootstrap.Elasticsearch")) {
+                esPid = proc.pid();
+            }
+        }
+        if (esPid == -1) {
+            fail("Could not find Elasticsearch process, existing processes:\n" + procs);
+        }
+
+        // check the coredump filter was set correctly
+        String coredumpFilter = sh.run("cat /proc/" + esPid + "/coredump_filter").stdout().trim();
+        assertThat(coredumpFilter, equalTo("00000023"));
+    }
+
+}