Merge branch 'main' into get-alias-action-reduce-alloc

Author: JeremyDahlgren

docs/reference/elasticsearch/index-settings/slow-log.md

@@ -216,7 +216,7 @@ If you aren’t sure how to start investigating traffic issues, consider enablin
 
 Slow log thresholds being met does not guarantee cluster performance issues. In the event that symptoms are noticed, slow logs can provide helpful data to diagnose upstream traffic patterns or sources to resolve client-side issues. For example, you can use data included in `X-Opaque-ID`, the `_source` request body, or `user.*` fields to identify the source of your issue. This is similar to troubleshooting [live expensive tasks](docs-content://troubleshoot/elasticsearch/task-queue-backlog.md).
 
-If you’re experiencing search performance issues, then you might also consider investigating searches flagged for their query durations using the [profile API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-profile.html). You can then use the profiled query to investigate optimization options using the [query profiler](docs-content://explore-analyze/query-filter/tools/search-profiler.md). This type of investigation should usually take place in a non-production environment.
+If you’re experiencing search performance issues, then you might also consider investigating searches flagged for their query durations using the [profile API](/reference/elasticsearch/rest-apis/search-profile.md). You can then use the profiled query to investigate optimization options using the [query profiler](docs-content://explore-analyze/query-filter/tools/search-profiler.md). This type of investigation should usually take place in a non-production environment.
 
 Slow logging checks each event against the reporting threshold when the event is complete. This means that it can’t report if events trigger [circuit breaker errors](docs-content://troubleshoot/elasticsearch/circuit-breaker-errors.md). If suspect circuit breaker errors, then you should also consider enabling [audit logging](docs-content://deploy-manage/security/logging-configuration/enabling-audit-logs.md), which logs events before they are executed.
 

docs/reference/elasticsearch/rest-apis/reciprocal-rank-fusion.md

@@ -108,7 +108,7 @@ The `rrf` retriever supports:
 * [suggesters](/reference/elasticsearch/rest-apis/search-suggesters.md)
 * [highlighting](/reference/elasticsearch/rest-apis/highlighting.md)
 * [collapse](/reference/elasticsearch/rest-apis/collapse-search-results.md)
-* [profiling](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-profile.html#profiling-queries)
+* [profiling](/reference/elasticsearch/rest-apis/search-profile.md)
 
 The `rrf` retriever does not currently support:
 

docs/reference/elasticsearch/rest-apis/search-profile.md

@@ -98,6 +98,7 @@ toc:
             - file: rest-apis/create-index-from-source.md
             - file: rest-apis/shard-request-cache.md
             - file: rest-apis/search-suggesters.md
+            - file: rest-apis/search-profile.md
   - file: mapping-reference/index.md
     children:
       - file: mapping-reference/document-metadata-fields.md

docs/reference/elasticsearch/toc.yml

@@ -405,6 +405,12 @@ tests:
 - class: org.elasticsearch.search.SearchWithRejectionsIT
   method: testOpenContextsAfterRejections
   issue: https://github.com/elastic/elasticsearch/issues/126340
+- class: org.elasticsearch.xpack.ilm.ClusterStateWaitThresholdBreachTests
+  method: testWaitInShrunkShardsAllocatedExceedsThreshold
+  issue: https://github.com/elastic/elasticsearch/issues/126348
+- class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT
+  method: testSearchableSnapshotTotalShardsPerNode
+  issue: https://github.com/elastic/elasticsearch/issues/126354
 
 # Examples:
 #

muted-tests.yml

@@ -12,6 +12,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
@@ -20,7 +21,6 @@
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Objects;
 
 /**
@@ -207,16 +207,18 @@ public static IndexReshardingMetadata newSplitByMultiple(int shardCount, int mul
         return new IndexReshardingMetadata(IndexReshardingState.Split.newSplitByMultiple(shardCount, multiple));
     }
 
-    public IndexReshardingMetadata transitionSplitTargetToHandoff(ShardId shardId) {
+    public static boolean isSplitTarget(ShardId shardId, @Nullable IndexReshardingMetadata reshardingMetadata) {
+        return reshardingMetadata != null && reshardingMetadata.isSplit() && reshardingMetadata.getSplit().isTargetShard(shardId.id());
+    }
+
+    public IndexReshardingMetadata transitionSplitTargetToNewState(
+        ShardId shardId,
+        IndexReshardingState.Split.TargetShardState newTargetState
+    ) {
         assert state instanceof IndexReshardingState.Split;
-        IndexReshardingState.Split splitState = (IndexReshardingState.Split) state;
-        IndexReshardingState.Split.TargetShardState[] newTargets = Arrays.copyOf(
-            splitState.targetShards(),
-            splitState.targetShards().length
-        );
-        int i = shardId.getId() - state.shardCountBefore();
-        newTargets[i] = IndexReshardingState.Split.TargetShardState.HANDOFF;
-        return new IndexReshardingMetadata(new IndexReshardingState.Split(splitState.sourceShards(), newTargets));
+        IndexReshardingState.Split.Builder builder = new IndexReshardingState.Split.Builder((IndexReshardingState.Split) state);
+        builder.setTargetShardState(shardId.getId(), newTargetState);
+        return new IndexReshardingMetadata(builder.build());
     }
 
     /**

server/src/main/java/org/elasticsearch/cluster/metadata/IndexReshardingMetadata.java

@@ -23,6 +23,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
+import java.util.stream.Stream;
 
 /**
  * IndexReshardingState is an abstract class holding the persistent state of a generic resharding operation. It contains
@@ -366,6 +367,10 @@ public TargetShardState getTargetShardState(int shardNum) {
             return targetShards[targetShardNum];
         }
 
+        public boolean targetStateAtLeast(int shardNum, TargetShardState targetShardState) {
+            return getTargetShardState(shardNum).ordinal() >= targetShardState.ordinal();
+        }
+
         /**
          * Check whether this metadata represents an incomplete split
          * @return true if the split is incomplete (not all source shards are DONE)
@@ -380,6 +385,10 @@ public boolean inProgress() {
             return false;
         }
 
+        public Stream<TargetShardState> targetStates() {
+            return Arrays.stream(targetShards);
+        }
+
         /**
          * Check whether all target shards for the given source shard are done.
          * @param shardNum a source shard index greater than or equal to 0 and less than the original shard count

server/src/main/java/org/elasticsearch/cluster/metadata/IndexReshardingState.java

@@ -50,7 +50,6 @@
 import java.util.function.IntFunction;
 
 import static java.util.Map.entry;
-import static org.elasticsearch.TransportVersions.V_8_11_X;
 
 /**
  * A stream from another node to this node. Technically, it can also be streamed from a byte array but that is mostly for testing.
@@ -813,13 +812,10 @@ public final void writeOptionalInstant(@Nullable Instant instant) throws IOExcep
         entry(GenericNamedWriteable.class, (o, v) -> {
             // Note that we do not rely on the checks in VersionCheckingStreamOutput because that only applies to CCS
             final var genericNamedWriteable = (GenericNamedWriteable) v;
-            TransportVersion minSupportedVersion = genericNamedWriteable.getMinimalSupportedVersion();
-            assert minSupportedVersion.onOrAfter(V_8_11_X) : "[GenericNamedWriteable] requires [" + V_8_11_X + "]";
-            if (o.getTransportVersion().before(minSupportedVersion)) {
+            if (genericNamedWriteable.supportsVersion(o.getTransportVersion()) == false) {
                 final var message = Strings.format(
-                    "[%s] requires minimal transport version [%s] and cannot be sent using transport version [%s]",
+                    "[%s] doesn't support serialization with transport version [%s]",
                     genericNamedWriteable.getWriteableName(),
-                    minSupportedVersion,
                     o.getTransportVersion()
                 );
                 assert false : message;

server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java

@@ -21,13 +21,15 @@ public void testSplit() {
         var split = metadata.getSplit();
 
         // starting state is as expected
-        assert metadata.shardCountBefore() == numShards;
-        assert metadata.shardCountAfter() == numShards * multiple;
+        assertEquals(numShards, metadata.shardCountBefore());
+        assertEquals(numShards * multiple, metadata.shardCountAfter());
         for (int i = 0; i < numShards; i++) {
-            assert split.getSourceShardState(i) == IndexReshardingState.Split.SourceShardState.SOURCE;
+            assertSame(IndexReshardingState.Split.SourceShardState.SOURCE, split.getSourceShardState(i));
+            assertFalse(split.isTargetShard(i));
         }
         for (int i = numShards; i < numShards * multiple; i++) {
-            assert split.getTargetShardState(i) == IndexReshardingState.Split.TargetShardState.CLONE;
+            assertSame(IndexReshardingState.Split.TargetShardState.CLONE, split.getTargetShardState(i));
+            assertTrue(split.isTargetShard(i));
         }
 
         // advance split state randomly and expect to terminate
@@ -62,10 +64,12 @@ public void testSplit() {
         }
 
         for (int i = 0; i < numShards; i++) {
-            assert split.getSourceShardState(i) == IndexReshardingState.Split.SourceShardState.DONE;
+            assertSame(IndexReshardingState.Split.SourceShardState.DONE, split.getSourceShardState(i));
+            assertFalse(split.isTargetShard(i));
         }
         for (int i = numShards; i < numShards * multiple; i++) {
-            assert split.getTargetShardState(i) == IndexReshardingState.Split.TargetShardState.DONE;
+            assertSame(IndexReshardingState.Split.TargetShardState.DONE, split.getTargetShardState(i));
+            assertTrue(split.isTargetShard(i));
         }
     }
 }

server/src/test/java/org/elasticsearch/cluster/metadata/IndexReshardingMetadataTests.java

@@ -85,7 +85,7 @@ public void testSerializationFailsWithOlderVersion() {
             output.setTransportVersion(older);
             assertThat(
                 expectThrows(Throwable.class, () -> output.writeGenericValue(testInstance)).getMessage(),
-                containsString("[GeoBoundingBox] requires minimal transport version")
+                containsString("[GeoBoundingBox] doesn't support serialization with transport version")
             );
         }
     }