Merge branch 'main' into esql-ccs-simplified-grammar

Author: pawankartik-elastic

.buildkite/pipelines/intake.template.yml

@@ -72,8 +72,6 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: "{{matrix.BWC_VERSION}}"
-  - label: bc-upgrade
-    command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
   - group: lucene-compat
     steps:
       - label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

.buildkite/pipelines/intake.yml

@@ -72,11 +72,6 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: "{{matrix.BWC_VERSION}}"
-  - label: bc-upgrade
-    command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
-      memory: "4G"
   - group: lucene-compat
     steps:
       - label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

.buildkite/scripts/run-bc-upgrade-tests.sh

@@ -0,0 +1,5 @@
+pr: 129074
+summary: "[apm-data] Set `event.dataset` if empty for logs"
+area: Data streams
+type: bug
+issues: []

docs/changelog/129074.yaml

@@ -33,7 +33,6 @@
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.IndexVersion;
-import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -71,6 +70,8 @@
 import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.index.mapper.TextFieldMapper.Builder.multiFieldsNotStoredByDefaultIndexVersionCheck;
+
 /**
  * A {@link FieldMapper} for full-text fields that only indexes
  * {@link IndexOptions#DOCS} and runs positional queries by looking at the
@@ -140,7 +141,7 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) {
         public MatchOnlyTextFieldMapper build(MapperBuilderContext context) {
             MatchOnlyTextFieldType tft = buildFieldType(context);
             final boolean storeSource;
-            if (indexCreatedVersion.onOrAfter(IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED)) {
+            if (multiFieldsNotStoredByDefaultIndexVersionCheck(indexCreatedVersion)) {
                 storeSource = context.isSourceSynthetic()
                     && withinMultiField == false
                     && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false;

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -574,6 +574,18 @@ tests:
 - class: org.elasticsearch.xpack.logsdb.qa.StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT
   method: testMatchAllQuery
   issue: https://github.com/elastic/elasticsearch/issues/129529
+- class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceTests
+  method: testIORateIsAdjustedForAllRunningMergeTasks
+  issue: https://github.com/elastic/elasticsearch/issues/129531
+- class: org.elasticsearch.upgrades.IndexingIT
+  method: testIndexing
+  issue: https://github.com/elastic/elasticsearch/issues/129533
+- class: org.elasticsearch.upgrades.MlJobSnapshotUpgradeIT
+  method: testSnapshotUpgrader
+  issue: https://github.com/elastic/elasticsearch/issues/98560
+- class: org.elasticsearch.upgrades.QueryableBuiltInRolesUpgradeIT
+  method: testBuiltInRolesSyncedOnClusterUpgrade
+  issue: https://github.com/elastic/elasticsearch/issues/129534
 
 # Examples:
 #

muted-tests.yml

@@ -19,9 +19,9 @@
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterInfoServiceUtils;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.EstimatedHeapUsage;
+import org.elasticsearch.cluster.EstimatedHeapUsageCollector;
 import org.elasticsearch.cluster.InternalClusterInfoService;
-import org.elasticsearch.cluster.ShardHeapUsage;
-import org.elasticsearch.cluster.ShardHeapUsageCollector;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
@@ -123,7 +123,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(InternalSettingsPlugin.class, BogusShardHeapUsagePlugin.class);
+        return pluginList(InternalSettingsPlugin.class, BogusEstimatedHeapUsagePlugin.class);
     }
 
     public void testLockTryingToDelete() throws Exception {
@@ -264,31 +264,31 @@ public void testExpectedShardSizeIsPresent() throws InterruptedException {
     public void testHeapUsageEstimateIsPresent() {
         InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class);
         ClusterInfoServiceUtils.refresh(clusterInfoService);
-        Map<String, ShardHeapUsage> shardHeapUsages = clusterInfoService.getClusterInfo().getShardHeapUsages();
-        assertNotNull(shardHeapUsages);
+        Map<String, EstimatedHeapUsage> estimatedHeapUsages = clusterInfoService.getClusterInfo().getEstimatedHeapUsages();
+        assertNotNull(estimatedHeapUsages);
         // Not collecting yet because it is disabled
-        assertTrue(shardHeapUsages.isEmpty());
+        assertTrue(estimatedHeapUsages.isEmpty());
 
-        // Enable collection for shard heap usages
+        // Enable collection for estimated heap usages
         updateClusterSettings(
             Settings.builder()
-                .put(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_SHARD_HEAP_THRESHOLD_DECIDER_ENABLED.getKey(), true)
+                .put(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_ESTIMATED_HEAP_THRESHOLD_DECIDER_ENABLED.getKey(), true)
                 .build()
         );
         try {
             ClusterInfoServiceUtils.refresh(clusterInfoService);
             ClusterState state = getInstanceFromNode(ClusterService.class).state();
-            shardHeapUsages = clusterInfoService.getClusterInfo().getShardHeapUsages();
-            assertEquals(state.nodes().size(), shardHeapUsages.size());
+            estimatedHeapUsages = clusterInfoService.getClusterInfo().getEstimatedHeapUsages();
+            assertEquals(state.nodes().size(), estimatedHeapUsages.size());
             for (DiscoveryNode node : state.nodes()) {
-                assertTrue(shardHeapUsages.containsKey(node.getId()));
-                ShardHeapUsage shardHeapUsage = shardHeapUsages.get(node.getId());
-                assertThat(shardHeapUsage.estimatedFreeBytes(), lessThanOrEqualTo(shardHeapUsage.totalBytes()));
+                assertTrue(estimatedHeapUsages.containsKey(node.getId()));
+                EstimatedHeapUsage estimatedHeapUsage = estimatedHeapUsages.get(node.getId());
+                assertThat(estimatedHeapUsage.estimatedFreeBytes(), lessThanOrEqualTo(estimatedHeapUsage.totalBytes()));
             }
         } finally {
             updateClusterSettings(
                 Settings.builder()
-                    .putNull(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_SHARD_HEAP_THRESHOLD_DECIDER_ENABLED.getKey())
+                    .putNull(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_ESTIMATED_HEAP_THRESHOLD_DECIDER_ENABLED.getKey())
                     .build()
             );
         }
@@ -838,11 +838,11 @@ private static void assertAllIndicesRemovedAndDeletionCompleted(Iterable<Indices
         }
     }
 
-    public static class BogusShardShardHeapUsageCollector implements ShardHeapUsageCollector {
+    public static class BogusEstimatedEstimatedHeapUsageCollector implements EstimatedHeapUsageCollector {
 
-        private final BogusShardHeapUsagePlugin plugin;
+        private final BogusEstimatedHeapUsagePlugin plugin;
 
-        public BogusShardShardHeapUsageCollector(BogusShardHeapUsagePlugin plugin) {
+        public BogusEstimatedEstimatedHeapUsageCollector(BogusEstimatedHeapUsagePlugin plugin) {
             this.plugin = plugin;
         }
 
@@ -859,7 +859,7 @@ public void collectClusterHeapUsage(ActionListener<Map<String, Long>> listener)
         }
     }
 
-    public static class BogusShardHeapUsagePlugin extends Plugin implements ClusterPlugin {
+    public static class BogusEstimatedHeapUsagePlugin extends Plugin implements ClusterPlugin {
 
         private final SetOnce<ClusterService> clusterService = new SetOnce<>();
 

server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java

@@ -7,4 +7,4 @@
 # License v3.0 only", or the "Server Side Public License, v 1".
 #
 
-org.elasticsearch.index.shard.IndexShardIT$BogusShardShardHeapUsageCollector
+org.elasticsearch.index.shard.IndexShardIT$BogusEstimatedEstimatedHeapUsageCollector

server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageCollector renamed to server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.EstimatedHeapUsageCollector

@@ -57,7 +57,7 @@ public class ClusterInfo implements ChunkedToXContent, Writeable {
     final Map<ShardId, Long> shardDataSetSizes;
     final Map<NodeAndShard, String> dataPath;
     final Map<NodeAndPath, ReservedSpace> reservedSpace;
-    final Map<String, ShardHeapUsage> shardHeapUsages;
+    final Map<String, EstimatedHeapUsage> estimatedHeapUsages;
 
     protected ClusterInfo() {
         this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of());
@@ -72,7 +72,7 @@ protected ClusterInfo() {
      * @param shardDataSetSizes a shard id to data set size in bytes mapping per shard
      * @param dataPath the shard routing to datapath mapping
      * @param reservedSpace reserved space per shard broken down by node and data path
-     * @param shardHeapUsages shard heap usage broken down by node
+     * @param estimatedHeapUsages estimated heap usage broken down by node
      * @see #shardIdentifierFromRouting
      */
     public ClusterInfo(
@@ -82,15 +82,15 @@ public ClusterInfo(
         Map<ShardId, Long> shardDataSetSizes,
         Map<NodeAndShard, String> dataPath,
         Map<NodeAndPath, ReservedSpace> reservedSpace,
-        Map<String, ShardHeapUsage> shardHeapUsages
+        Map<String, EstimatedHeapUsage> estimatedHeapUsages
     ) {
         this.leastAvailableSpaceUsage = Map.copyOf(leastAvailableSpaceUsage);
         this.mostAvailableSpaceUsage = Map.copyOf(mostAvailableSpaceUsage);
         this.shardSizes = Map.copyOf(shardSizes);
         this.shardDataSetSizes = Map.copyOf(shardDataSetSizes);
         this.dataPath = Map.copyOf(dataPath);
         this.reservedSpace = Map.copyOf(reservedSpace);
-        this.shardHeapUsages = Map.copyOf(shardHeapUsages);
+        this.estimatedHeapUsages = Map.copyOf(estimatedHeapUsages);
     }
 
     public ClusterInfo(StreamInput in) throws IOException {
@@ -103,9 +103,9 @@ public ClusterInfo(StreamInput in) throws IOException {
             : in.readImmutableMap(nested -> NodeAndShard.from(new ShardRouting(nested)), StreamInput::readString);
         this.reservedSpace = in.readImmutableMap(NodeAndPath::new, ReservedSpace::new);
         if (in.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) {
-            this.shardHeapUsages = in.readImmutableMap(ShardHeapUsage::new);
+            this.estimatedHeapUsages = in.readImmutableMap(EstimatedHeapUsage::new);
         } else {
-            this.shardHeapUsages = Map.of();
+            this.estimatedHeapUsages = Map.of();
         }
     }
 
@@ -122,7 +122,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeMap(this.reservedSpace);
         if (out.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) {
-            out.writeMap(this.shardHeapUsages, StreamOutput::writeWriteable);
+            out.writeMap(this.estimatedHeapUsages, StreamOutput::writeWriteable);
         }
     }
 
@@ -204,7 +204,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                 return builder.endObject(); // NodeAndPath
             }),
             endArray() // end "reserved_sizes"
-            // NOTE: We don't serialize shardHeapUsages at this stage, to avoid
+            // NOTE: We don't serialize estimatedHeapUsages at this stage, to avoid
             // committing to API payloads until the feature is settled
         );
     }
@@ -216,8 +216,8 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
      * Also note that the map may not be complete, it may contain none, or a subset of the nodes in
      * the cluster at any time. It may also contain entries for nodes that have since left the cluster.
      */
-    public Map<String, ShardHeapUsage> getShardHeapUsages() {
-        return shardHeapUsages;
+    public Map<String, EstimatedHeapUsage> getEstimatedHeapUsages() {
+        return estimatedHeapUsages;
     }
 
     /**

server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java

@@ -33,7 +33,7 @@ public class ClusterInfoSimulator {
     private final CopyOnFirstWriteMap<String, Long> shardSizes;
     private final Map<ShardId, Long> shardDataSetSizes;
     private final Map<NodeAndShard, String> dataPath;
-    private final Map<String, ShardHeapUsage> shardHeapUsages;
+    private final Map<String, EstimatedHeapUsage> estimatedHeapUsages;
 
     public ClusterInfoSimulator(RoutingAllocation allocation) {
         this.allocation = allocation;
@@ -42,7 +42,7 @@ public ClusterInfoSimulator(RoutingAllocation allocation) {
         this.shardSizes = new CopyOnFirstWriteMap<>(allocation.clusterInfo().shardSizes);
         this.shardDataSetSizes = Map.copyOf(allocation.clusterInfo().shardDataSetSizes);
         this.dataPath = Map.copyOf(allocation.clusterInfo().dataPath);
-        this.shardHeapUsages = allocation.clusterInfo().getShardHeapUsages();
+        this.estimatedHeapUsages = allocation.clusterInfo().getEstimatedHeapUsages();
     }
 
     /**
@@ -156,7 +156,7 @@ public ClusterInfo getClusterInfo() {
             shardDataSetSizes,
             dataPath,
             Map.of(),
-            shardHeapUsages
+            estimatedHeapUsages
         );
     }
 }