Merge branch 'main' into foldable

Author: julian-elastic

docs/reference/elasticsearch/jvm-settings.md

@@ -87,10 +87,18 @@ To override the default heap size, set the minimum and maximum heap size setting
 
 The heap size should be based on the available RAM:
 
-* Set `Xms` and `Xmx` to no more than 50% of your total memory. {{es}} requires memory for purposes other than the JVM heap. For example, {{es}} uses off-heap buffers for efficient network communication and relies on the operating system’s filesystem cache for efficient access to files. The JVM itself also requires some memory. It’s normal for {{es}} to use more memory than the limit configured with the `Xmx` setting.
+* Set `Xms` and `Xmx` to no more than 50% of the total memory available to each {{es}} node. {{es}} requires memory for purposes other than the JVM heap. For example, {{es}} uses off-heap buffers for efficient network communication and relies on the operating system’s filesystem cache for efficient access to files. The JVM itself also requires some memory. It’s normal for {{es}} to use more memory than the limit configured with the `Xmx` setting.
 
     ::::{note}
-    When running in a container, such as [Docker](docs-content://deploy-manage/deploy/self-managed/install-elasticsearch-with-docker.md), total memory is defined as the amount of memory visible to the container, not the total system memory on the host.
+    When running in a container, such as [Docker](docs-content://deploy-manage/deploy/self-managed/install-elasticsearch-with-docker.md), the total memory available to {{es}} means the amount of memory available within the container, not the total system memory on the host.
+
+    If you are running multiple {{es}} nodes on the same host, or in the same container, the total of all the nodes' heap sizes should not exceed 50% of the total available memory.
+
+    Account for the memory usage of other processes running on the same host, or in the same container, when computing the total memory available to {{es}}.
+
+    The 50% guideline is intended as a safe upper bound on the heap size. You may find that heap sizes smaller than this maximum offer better performance, for instance by allowing your operating system to use a larger filesystem cache.
+
+    If you set the heap size too large, {{es}} may perform poorly and nodes may be terminated by the operating system.
     ::::
 
 * Set `Xms` and `Xmx` to no more than the threshold for compressed ordinary object pointers (oops). The exact threshold varies but 26GB is safe on most systems and can be as large as 30GB on some systems. To verify you are under the threshold, check the {{es}} log for an entry like this:

modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java

@@ -38,6 +38,13 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
         .setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN)
         .build();
 
+    public static final SystemIndexDescriptor ONECHAT_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()
+        .setIndexPattern(".chat-*")
+        .setDescription("Onechat system index")
+        .setType(Type.EXTERNAL_UNMANAGED)
+        .setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN)
+        .build();
+
     public static final SystemIndexDescriptor APM_AGENT_CONFIG_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()
         .setIndexPattern(".apm-agent-configuration*")
         .setDescription("system index for APM agent configuration")
@@ -57,6 +64,7 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
         return List.of(
             KIBANA_INDEX_DESCRIPTOR,
             REPORTING_INDEX_DESCRIPTOR,
+            ONECHAT_INDEX_DESCRIPTOR,
             APM_AGENT_CONFIG_INDEX_DESCRIPTOR,
             APM_CUSTOM_LINK_INDEX_DESCRIPTOR
         );

modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java

@@ -20,7 +20,7 @@ public class KibanaPluginTests extends ESTestCase {
     public void testKibanaIndexNames() {
         assertThat(
             new KibanaPlugin().getSystemIndexDescriptors(Settings.EMPTY).stream().map(SystemIndexDescriptor::getIndexPattern).toList(),
-            contains(".kibana_*", ".reporting-*", ".apm-agent-configuration*", ".apm-custom-link*")
+            contains(".kibana_*", ".reporting-*", ".chat-*", ".apm-agent-configuration*", ".apm-custom-link*")
         );
     }
 }

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java

@@ -830,7 +830,13 @@ void run(BytesReference expected, BytesReference updated, ActionListener<Optiona
 
                 .<Void>newForked(l -> ensureOtherUploadsComplete(uploadId, uploadIndex, currentUploads, l))
 
-                // Step 4: Read the current register value.
+                // Step 4: Read the current register value. Note that getRegister only has read-after-write semantics but that's ok here as:
+                // - all earlier uploads are now complete,
+                // - our upload is not completing yet, and
+                // - later uploads can only be completing if they have already aborted ours.
+                // Thus if our operation ultimately succeeds then there cannot have been any concurrent writes in flight, so this read
+                // cannot have observed a stale value, whereas if our operation ultimately fails then it doesn't matter what this read
+                // observes.
 
                 .<OptionalBytesReference>andThen(l -> getRegister(purpose, rawKey, l))
 

muted-tests.yml

@@ -475,15 +475,24 @@ tests:
 - class: org.elasticsearch.xpack.search.CrossClusterAsyncSearchIT
   method: testCancellationViaTimeoutWithAllowPartialResultsSetToFalse
   issue: https://github.com/elastic/elasticsearch/issues/131248
-- class: org.elasticsearch.xpack.downsample.DownsampleIT
-  method: testAggMetricInEsqlTSAfterDownsampling
-  issue: https://github.com/elastic/elasticsearch/issues/131500
 - class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
   method: test
   issue: https://github.com/elastic/elasticsearch/issues/131508
 - class: org.elasticsearch.action.admin.cluster.node.tasks.CancellableTasksIT
   method: testRemoveBanParentsOnDisconnect
   issue: https://github.com/elastic/elasticsearch/issues/131562
+- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithPartialResultsIT
+  method: testPartialResults
+  issue: https://github.com/elastic/elasticsearch/issues/131481
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test010Install
+  issue: https://github.com/elastic/elasticsearch/issues/131376
+- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
+  method: test {p0=search/40_indices_boost/Indices boost with alias}
+  issue: https://github.com/elastic/elasticsearch/issues/131598
+- class: org.elasticsearch.compute.lucene.read.SortedSetOrdinalsBuilderTests
+  method: testReader
+  issue: https://github.com/elastic/elasticsearch/issues/131573
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java

@@ -60,26 +60,30 @@ public void testElectOnlyBetweenMasterNodes() throws Exception {
 
         logger.info("--> start master node (1)");
         final String masterNodeName = internalCluster().startMasterOnlyNode();
-        awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNodeName);
-        awaitMasterNode(internalCluster().getMasterName(), masterNodeName);
+        for (var nodeName : internalCluster().getNodeNames()) {
+            awaitMasterNode(nodeName, masterNodeName);
+        }
 
         logger.info("--> start master node (2)");
         final String nextMasterEligableNodeName = internalCluster().startMasterOnlyNode();
-        awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNodeName);
-        awaitMasterNode(internalCluster().getMasterName(), masterNodeName);
+        for (var nodeName : internalCluster().getNodeNames()) {
+            awaitMasterNode(nodeName, masterNodeName);
+        }
 
         logger.info("--> closing master node (1)");
         client().execute(
             TransportAddVotingConfigExclusionsAction.TYPE,
             new AddVotingConfigExclusionsRequest(TEST_REQUEST_TIMEOUT, masterNodeName)
         ).get();
         // removing the master from the voting configuration immediately triggers the master to step down
-        awaitMasterNode(internalCluster().getNonMasterNodeName(), nextMasterEligableNodeName);
-        awaitMasterNode(internalCluster().getMasterName(), nextMasterEligableNodeName);
+        for (var nodeName : internalCluster().getNodeNames()) {
+            awaitMasterNode(nodeName, nextMasterEligableNodeName);
+        }
 
         internalCluster().stopNode(masterNodeName);
-        awaitMasterNode(internalCluster().getNonMasterNodeName(), nextMasterEligableNodeName);
-        awaitMasterNode(internalCluster().getMasterName(), nextMasterEligableNodeName);
+        for (var nodeName : internalCluster().getNodeNames()) {
+            awaitMasterNode(nodeName, nextMasterEligableNodeName);
+        }
     }
 
     public void testAliasFilterValidation() {

server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.NodeUsageStatsForThreadPools;
 import org.elasticsearch.cluster.NodeUsageStatsForThreadPoolsCollector;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
 import org.elasticsearch.cluster.routing.RecoverySource;
@@ -104,6 +105,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiLettersOfLength;
@@ -355,6 +357,62 @@ public void testNodeWriteLoadsArePresent() {
         }
     }
 
+    public void testShardWriteLoadsArePresent() {
+        // Create some indices and some write-load
+        final int numIndices = randomIntBetween(1, 5);
+        final String indexPrefix = randomIdentifier();
+        IntStream.range(0, numIndices).forEach(i -> {
+            final String indexName = indexPrefix + "_" + i;
+            createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 3)).build());
+            IntStream.range(0, randomIntBetween(1, 500))
+                .forEach(j -> prepareIndex(indexName).setSource("foo", randomIdentifier(), "bar", randomIdentifier()).get());
+        });
+
+        final InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class);
+
+        // Not collecting stats yet because allocation write load stats collection is disabled by default.
+        {
+            ClusterInfoServiceUtils.refresh(clusterInfoService);
+            final Map<ShardId, Double> shardWriteLoads = clusterInfoService.getClusterInfo().getShardWriteLoads();
+            assertNotNull(shardWriteLoads);
+            assertTrue(shardWriteLoads.isEmpty());
+        }
+
+        // Turn on collection of write-load stats.
+        updateClusterSettings(
+            Settings.builder()
+                .put(
+                    WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey(),
+                    WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED
+                )
+                .build()
+        );
+
+        try {
+            // Force a ClusterInfo refresh to run collection of the write-load stats.
+            ClusterInfoServiceUtils.refresh(clusterInfoService);
+            final Map<ShardId, Double> shardWriteLoads = clusterInfoService.getClusterInfo().getShardWriteLoads();
+
+            // Verify that each shard has write-load reported.
+            final ClusterState state = getInstanceFromNode(ClusterService.class).state();
+            assertEquals(state.projectState(ProjectId.DEFAULT).metadata().getTotalNumberOfShards(), shardWriteLoads.size());
+            double maximumLoadRecorded = 0;
+            for (IndexMetadata indexMetadata : state.projectState(ProjectId.DEFAULT).metadata()) {
+                for (int i = 0; i < indexMetadata.getNumberOfShards(); i++) {
+                    final ShardId shardId = new ShardId(indexMetadata.getIndex(), i);
+                    assertTrue(shardWriteLoads.containsKey(shardId));
+                    maximumLoadRecorded = Math.max(shardWriteLoads.get(shardId), maximumLoadRecorded);
+                }
+            }
+            // And that at least one is greater than zero
+            assertThat(maximumLoadRecorded, greaterThan(0.0));
+        } finally {
+            updateClusterSettings(
+                Settings.builder().putNull(WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey()).build()
+            );
+        }
+    }
+
     public void testIndexCanChangeCustomDataPath() throws Exception {
         final String index = "test-custom-data-path";
         final Path sharedDataPath = getInstanceFromNode(Environment.class).sharedDataDir().resolve(randomAsciiLettersOfLength(10));

server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.search.sort;
 
+import org.apache.http.util.EntityUtils;
 import org.apache.lucene.tests.util.TestUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.UnicodeUtil;
@@ -20,6 +21,9 @@
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
@@ -36,6 +40,7 @@
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.junit.annotations.TestIssueLogging;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
@@ -84,6 +89,12 @@
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
+@TestIssueLogging(
+    issueUrl = "https://github.com/elastic/elasticsearch/issues/129445",
+    value = "org.elasticsearch.action.search.SearchQueryThenFetchAsyncAction:DEBUG,"
+        + "org.elasticsearch.action.search.SearchPhaseController:DEBUG,"
+        + "org.elasticsearch.search:TRACE"
+)
 public class FieldSortIT extends ESIntegTestCase {
     public static class CustomScriptPlugin extends MockScriptPlugin {
         @Override
@@ -112,6 +123,10 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(InternalSettingsPlugin.class, CustomScriptPlugin.class);
     }
 
+    protected boolean addMockHttpTransport() {
+        return false;
+    }
+
     public void testIssue8226() {
         int numIndices = between(5, 10);
         final boolean useMapping = randomBoolean();
@@ -2145,7 +2160,7 @@ public void testLongSortOptimizationCorrectResults() {
         );
     }
 
-    public void testSortMixedFieldTypes() {
+    public void testSortMixedFieldTypes() throws IOException {
         assertAcked(
             prepareCreate("index_long").setMapping("foo", "type=long"),
             prepareCreate("index_integer").setMapping("foo", "type=integer"),
@@ -2159,6 +2174,16 @@ public void testSortMixedFieldTypes() {
         prepareIndex("index_keyword").setId("1").setSource("foo", "123").get();
         refresh();
 
+        // for debugging, we try to see where the documents are located
+        try (RestClient restClient = createRestClient()) {
+            Request checkShardsRequest = new Request(
+                "GET",
+                "/_cat/shards/index_long,index_double,index_keyword?format=json&h=index,node,shard,prirep,state,docs,index"
+            );
+            Response response = restClient.performRequest(checkShardsRequest);
+            logger.info("FieldSortIT#testSortMixedFieldTypes document distribution: " + EntityUtils.toString(response.getEntity()));
+        }
+
         { // mixing long and integer types is ok, as we convert integer sort to long sort
             assertNoFailures(prepareSearch("index_long", "index_integer").addSort(new FieldSortBuilder("foo")).setSize(10));
         }

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -344,6 +344,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_AZURE_AI_STUDIO_RERANK_ADDED = def(9_123_0_00);
     public static final TransportVersion PROJECT_STATE_REGISTRY_ENTRY = def(9_124_0_00);
     public static final TransportVersion ML_INFERENCE_LLAMA_ADDED = def(9_125_0_00);
+    public static final TransportVersion SHARD_WRITE_LOAD_IN_CLUSTER_INFO = def(9_126_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.action.search;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Sort;
@@ -72,6 +74,8 @@
 
 public final class SearchPhaseController {
 
+    private static final Logger logger = LogManager.getLogger(SearchPhaseController.class);
+
     private final BiFunction<
         Supplier<Boolean>,
         AggregatorFactories.Builder,
@@ -153,17 +157,22 @@ static TopDocs mergeTopDocs(List<TopDocs> results, int topN, int from) {
             return topDocs;
         }
         final TopDocs mergedTopDocs;
-        if (topDocs instanceof TopFieldGroups firstTopDocs) {
-            final Sort sort = validateSameSortTypesAndMaybeRewrite(results, firstTopDocs.fields);
-            TopFieldGroups[] shardTopDocs = topDocsList.toArray(new TopFieldGroups[0]);
-            mergedTopDocs = TopFieldGroups.merge(sort, from, topN, shardTopDocs, false);
-        } else if (topDocs instanceof TopFieldDocs firstTopDocs) {
-            TopFieldDocs[] shardTopDocs = topDocsList.toArray(new TopFieldDocs[0]);
-            final Sort sort = validateSameSortTypesAndMaybeRewrite(results, firstTopDocs.fields);
-            mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs);
-        } else {
-            final TopDocs[] shardTopDocs = topDocsList.toArray(new TopDocs[0]);
-            mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs);
+        try {
+            if (topDocs instanceof TopFieldGroups firstTopDocs) {
+                final Sort sort = validateSameSortTypesAndMaybeRewrite(results, firstTopDocs.fields);
+                TopFieldGroups[] shardTopDocs = topDocsList.toArray(new TopFieldGroups[0]);
+                mergedTopDocs = TopFieldGroups.merge(sort, from, topN, shardTopDocs, false);
+            } else if (topDocs instanceof TopFieldDocs firstTopDocs) {
+                TopFieldDocs[] shardTopDocs = topDocsList.toArray(new TopFieldDocs[0]);
+                final Sort sort = validateSameSortTypesAndMaybeRewrite(results, firstTopDocs.fields);
+                mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs);
+            } else {
+                final TopDocs[] shardTopDocs = topDocsList.toArray(new TopDocs[0]);
+                mergedTopDocs = TopDocs.merge(from, topN, shardTopDocs);
+            }
+        } catch (IllegalArgumentException e) {
+            logger.debug("Failed to merge top docs: ", e);
+            throw e;
         }
         return mergedTopDocs;
     }