Merge branch 'main' into bc-upgrade/bc-commit-hash-script

Author: ldematte

docs/changelog/129074.yaml

@@ -0,0 +1,5 @@
+pr: 129074
+summary: "[apm-data] Set `event.dataset` if empty for logs"
+area: Data streams
+type: bug
+issues: []

muted-tests.yml

@@ -577,6 +577,15 @@ tests:
 - class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceTests
   method: testIORateIsAdjustedForAllRunningMergeTasks
   issue: https://github.com/elastic/elasticsearch/issues/129531
+- class: org.elasticsearch.upgrades.IndexingIT
+  method: testIndexing
+  issue: https://github.com/elastic/elasticsearch/issues/129533
+- class: org.elasticsearch.upgrades.MlJobSnapshotUpgradeIT
+  method: testSnapshotUpgrader
+  issue: https://github.com/elastic/elasticsearch/issues/98560
+- class: org.elasticsearch.upgrades.QueryableBuiltInRolesUpgradeIT
+  method: testBuiltInRolesSyncedOnClusterUpgrade
+  issue: https://github.com/elastic/elasticsearch/issues/129534
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java

@@ -19,9 +19,9 @@
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterInfoServiceUtils;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.EstimatedHeapUsage;
+import org.elasticsearch.cluster.EstimatedHeapUsageCollector;
 import org.elasticsearch.cluster.InternalClusterInfoService;
-import org.elasticsearch.cluster.ShardHeapUsage;
-import org.elasticsearch.cluster.ShardHeapUsageCollector;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
@@ -123,7 +123,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(InternalSettingsPlugin.class, BogusShardHeapUsagePlugin.class);
+        return pluginList(InternalSettingsPlugin.class, BogusEstimatedHeapUsagePlugin.class);
     }
 
     public void testLockTryingToDelete() throws Exception {
@@ -264,31 +264,31 @@ public void testExpectedShardSizeIsPresent() throws InterruptedException {
     public void testHeapUsageEstimateIsPresent() {
         InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class);
         ClusterInfoServiceUtils.refresh(clusterInfoService);
-        Map<String, ShardHeapUsage> shardHeapUsages = clusterInfoService.getClusterInfo().getShardHeapUsages();
-        assertNotNull(shardHeapUsages);
+        Map<String, EstimatedHeapUsage> estimatedHeapUsages = clusterInfoService.getClusterInfo().getEstimatedHeapUsages();
+        assertNotNull(estimatedHeapUsages);
         // Not collecting yet because it is disabled
-        assertTrue(shardHeapUsages.isEmpty());
+        assertTrue(estimatedHeapUsages.isEmpty());
 
-        // Enable collection for shard heap usages
+        // Enable collection for estimated heap usages
         updateClusterSettings(
             Settings.builder()
-                .put(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_SHARD_HEAP_THRESHOLD_DECIDER_ENABLED.getKey(), true)
+                .put(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_ESTIMATED_HEAP_THRESHOLD_DECIDER_ENABLED.getKey(), true)
                 .build()
         );
         try {
             ClusterInfoServiceUtils.refresh(clusterInfoService);
             ClusterState state = getInstanceFromNode(ClusterService.class).state();
-            shardHeapUsages = clusterInfoService.getClusterInfo().getShardHeapUsages();
-            assertEquals(state.nodes().size(), shardHeapUsages.size());
+            estimatedHeapUsages = clusterInfoService.getClusterInfo().getEstimatedHeapUsages();
+            assertEquals(state.nodes().size(), estimatedHeapUsages.size());
             for (DiscoveryNode node : state.nodes()) {
-                assertTrue(shardHeapUsages.containsKey(node.getId()));
-                ShardHeapUsage shardHeapUsage = shardHeapUsages.get(node.getId());
-                assertThat(shardHeapUsage.estimatedFreeBytes(), lessThanOrEqualTo(shardHeapUsage.totalBytes()));
+                assertTrue(estimatedHeapUsages.containsKey(node.getId()));
+                EstimatedHeapUsage estimatedHeapUsage = estimatedHeapUsages.get(node.getId());
+                assertThat(estimatedHeapUsage.estimatedFreeBytes(), lessThanOrEqualTo(estimatedHeapUsage.totalBytes()));
             }
         } finally {
             updateClusterSettings(
                 Settings.builder()
-                    .putNull(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_SHARD_HEAP_THRESHOLD_DECIDER_ENABLED.getKey())
+                    .putNull(InternalClusterInfoService.CLUSTER_ROUTING_ALLOCATION_ESTIMATED_HEAP_THRESHOLD_DECIDER_ENABLED.getKey())
                     .build()
             );
         }
@@ -838,11 +838,11 @@ private static void assertAllIndicesRemovedAndDeletionCompleted(Iterable<Indices
         }
     }
 
-    public static class BogusShardShardHeapUsageCollector implements ShardHeapUsageCollector {
+    public static class BogusEstimatedEstimatedHeapUsageCollector implements EstimatedHeapUsageCollector {
 
-        private final BogusShardHeapUsagePlugin plugin;
+        private final BogusEstimatedHeapUsagePlugin plugin;
 
-        public BogusShardShardHeapUsageCollector(BogusShardHeapUsagePlugin plugin) {
+        public BogusEstimatedEstimatedHeapUsageCollector(BogusEstimatedHeapUsagePlugin plugin) {
             this.plugin = plugin;
         }
 
@@ -859,7 +859,7 @@ public void collectClusterHeapUsage(ActionListener<Map<String, Long>> listener)
         }
     }
 
-    public static class BogusShardHeapUsagePlugin extends Plugin implements ClusterPlugin {
+    public static class BogusEstimatedHeapUsagePlugin extends Plugin implements ClusterPlugin {
 
         private final SetOnce<ClusterService> clusterService = new SetOnce<>();
 

server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.ShardHeapUsageCollector renamed to server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.cluster.EstimatedHeapUsageCollector

@@ -7,4 +7,4 @@
 # License v3.0 only", or the "Server Side Public License, v 1".
 #
 
-org.elasticsearch.index.shard.IndexShardIT$BogusShardShardHeapUsageCollector
+org.elasticsearch.index.shard.IndexShardIT$BogusEstimatedEstimatedHeapUsageCollector

server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java

@@ -57,7 +57,7 @@ public class ClusterInfo implements ChunkedToXContent, Writeable {
     final Map<ShardId, Long> shardDataSetSizes;
     final Map<NodeAndShard, String> dataPath;
     final Map<NodeAndPath, ReservedSpace> reservedSpace;
-    final Map<String, ShardHeapUsage> shardHeapUsages;
+    final Map<String, EstimatedHeapUsage> estimatedHeapUsages;
 
     protected ClusterInfo() {
         this(Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of(), Map.of());
@@ -72,7 +72,7 @@ protected ClusterInfo() {
      * @param shardDataSetSizes a shard id to data set size in bytes mapping per shard
      * @param dataPath the shard routing to datapath mapping
      * @param reservedSpace reserved space per shard broken down by node and data path
-     * @param shardHeapUsages shard heap usage broken down by node
+     * @param estimatedHeapUsages estimated heap usage broken down by node
      * @see #shardIdentifierFromRouting
      */
     public ClusterInfo(
@@ -82,15 +82,15 @@ public ClusterInfo(
         Map<ShardId, Long> shardDataSetSizes,
         Map<NodeAndShard, String> dataPath,
         Map<NodeAndPath, ReservedSpace> reservedSpace,
-        Map<String, ShardHeapUsage> shardHeapUsages
+        Map<String, EstimatedHeapUsage> estimatedHeapUsages
     ) {
         this.leastAvailableSpaceUsage = Map.copyOf(leastAvailableSpaceUsage);
         this.mostAvailableSpaceUsage = Map.copyOf(mostAvailableSpaceUsage);
         this.shardSizes = Map.copyOf(shardSizes);
         this.shardDataSetSizes = Map.copyOf(shardDataSetSizes);
         this.dataPath = Map.copyOf(dataPath);
         this.reservedSpace = Map.copyOf(reservedSpace);
-        this.shardHeapUsages = Map.copyOf(shardHeapUsages);
+        this.estimatedHeapUsages = Map.copyOf(estimatedHeapUsages);
     }
 
     public ClusterInfo(StreamInput in) throws IOException {
@@ -103,9 +103,9 @@ public ClusterInfo(StreamInput in) throws IOException {
             : in.readImmutableMap(nested -> NodeAndShard.from(new ShardRouting(nested)), StreamInput::readString);
         this.reservedSpace = in.readImmutableMap(NodeAndPath::new, ReservedSpace::new);
         if (in.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) {
-            this.shardHeapUsages = in.readImmutableMap(ShardHeapUsage::new);
+            this.estimatedHeapUsages = in.readImmutableMap(EstimatedHeapUsage::new);
         } else {
-            this.shardHeapUsages = Map.of();
+            this.estimatedHeapUsages = Map.of();
         }
     }
 
@@ -122,7 +122,7 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         out.writeMap(this.reservedSpace);
         if (out.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) {
-            out.writeMap(this.shardHeapUsages, StreamOutput::writeWriteable);
+            out.writeMap(this.estimatedHeapUsages, StreamOutput::writeWriteable);
         }
     }
 
@@ -204,7 +204,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
                 return builder.endObject(); // NodeAndPath
             }),
             endArray() // end "reserved_sizes"
-            // NOTE: We don't serialize shardHeapUsages at this stage, to avoid
+            // NOTE: We don't serialize estimatedHeapUsages at this stage, to avoid
             // committing to API payloads until the feature is settled
         );
     }
@@ -216,8 +216,8 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params
      * Also note that the map may not be complete, it may contain none, or a subset of the nodes in
      * the cluster at any time. It may also contain entries for nodes that have since left the cluster.
      */
-    public Map<String, ShardHeapUsage> getShardHeapUsages() {
-        return shardHeapUsages;
+    public Map<String, EstimatedHeapUsage> getEstimatedHeapUsages() {
+        return estimatedHeapUsages;
     }
 
     /**

server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java

@@ -33,7 +33,7 @@ public class ClusterInfoSimulator {
     private final CopyOnFirstWriteMap<String, Long> shardSizes;
     private final Map<ShardId, Long> shardDataSetSizes;
     private final Map<NodeAndShard, String> dataPath;
-    private final Map<String, ShardHeapUsage> shardHeapUsages;
+    private final Map<String, EstimatedHeapUsage> estimatedHeapUsages;
 
     public ClusterInfoSimulator(RoutingAllocation allocation) {
         this.allocation = allocation;
@@ -42,7 +42,7 @@ public ClusterInfoSimulator(RoutingAllocation allocation) {
         this.shardSizes = new CopyOnFirstWriteMap<>(allocation.clusterInfo().shardSizes);
         this.shardDataSetSizes = Map.copyOf(allocation.clusterInfo().shardDataSetSizes);
         this.dataPath = Map.copyOf(allocation.clusterInfo().dataPath);
-        this.shardHeapUsages = allocation.clusterInfo().getShardHeapUsages();
+        this.estimatedHeapUsages = allocation.clusterInfo().getEstimatedHeapUsages();
     }
 
     /**
@@ -156,7 +156,7 @@ public ClusterInfo getClusterInfo() {
             shardDataSetSizes,
             dataPath,
             Map.of(),
-            shardHeapUsages
+            estimatedHeapUsages
         );
     }
 }

server/src/main/java/org/elasticsearch/cluster/ShardHeapUsage.java renamed to server/src/main/java/org/elasticsearch/cluster/EstimatedHeapUsage.java

@@ -18,14 +18,14 @@
 /**
  * Record representing an estimate of the heap used by allocated shards and ongoing merges on a particular node
  */
-public record ShardHeapUsage(String nodeId, long totalBytes, long estimatedUsageBytes) implements Writeable {
+public record EstimatedHeapUsage(String nodeId, long totalBytes, long estimatedUsageBytes) implements Writeable {
 
-    public ShardHeapUsage {
+    public EstimatedHeapUsage {
         assert totalBytes >= 0;
         assert estimatedUsageBytes >= 0;
     }
 
-    public ShardHeapUsage(StreamInput in) throws IOException {
+    public EstimatedHeapUsage(StreamInput in) throws IOException {
         this(in.readString(), in.readVLong(), in.readVLong());
     }
 

server/src/main/java/org/elasticsearch/cluster/ShardHeapUsageCollector.java renamed to server/src/main/java/org/elasticsearch/cluster/EstimatedHeapUsageCollector.java

@@ -14,21 +14,21 @@
 import java.util.Map;
 
 /**
- * Collect the shard heap usage for each node in the cluster.
+ * Collect the estimated heap usage for each node in the cluster.
  * <p>
  * Results are returned as a map of node ID to estimated heap usage in bytes
  *
- * @see ShardHeapUsage
+ * @see EstimatedHeapUsage
  */
-public interface ShardHeapUsageCollector {
+public interface EstimatedHeapUsageCollector {
 
     /**
-     * This will be used when there is no ShardHeapUsageCollector available
+     * This will be used when there is no EstimatedHeapUsageCollector available
      */
-    ShardHeapUsageCollector EMPTY = listener -> listener.onResponse(Map.of());
+    EstimatedHeapUsageCollector EMPTY = listener -> listener.onResponse(Map.of());
 
     /**
-     * Collect the shard heap usage for every node in the cluster
+     * Collect the estimated heap usage for every node in the cluster
      *
      * @param listener The listener which will receive the results
      */