Merge branch 'main' into markjhoy/add_sparse_vector_token_pruning_index_options

Author: markjhoy

docs/changelog/128635.yaml

@@ -0,0 +1,6 @@
+pr: 128635
+summary: Add `state` query param to Get snapshots API
+area: Snapshot/Restore
+type: enhancement
+issues:
+ - 97446

docs/changelog/129455.yaml

@@ -0,0 +1,6 @@
+pr: 129455
+summary: Prevent ILM from processing shrunken index before its execution state is copied over
+area: ILM+SLM
+type: bug
+issues:
+ - 109206

muted-tests.yml

@@ -562,6 +562,12 @@ tests:
 - class: org.elasticsearch.xpack.security.PermissionsIT
   method: testWhenUserLimitedByOnlyAliasOfIndexCanWriteToIndexWhichWasRolledoverByILMPolicy
   issue: https://github.com/elastic/elasticsearch/issues/129481
+- class: org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT
+  method: testFullPolicy
+  issue: https://github.com/elastic/elasticsearch/issues/129510
+- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
+  method: test {knn-function.KnnSearchWithKOption SYNC}
+  issue: https://github.com/elastic/elasticsearch/issues/129512
 
 # Examples:
 #

qa/vector/build.gradle

@@ -42,9 +42,12 @@ tasks.register("checkVec", JavaExec) {
   systemProperty "es.logger.out", "console"
   systemProperty "es.logger.level", "INFO"  // Change to DEBUG if needed
   systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../../libs/native/libraries/build/platform/").toString())
-
+  jvmArgs '-Xms4g', '-Xmx4g', '-Djava.util.concurrent.ForkJoinPool.common.parallelism=8', '-XX:+UnlockDiagnosticVMOptions', '-XX:+DebugNonSafepoints', '-XX:+HeapDumpOnOutOfMemoryError'
   if (buildParams.getRuntimeJavaVersion().map { it.majorVersion.toInteger() }.get() >= 21) {
-    jvmArgs '-Xms4g', '-Xmx4g', '--add-modules=jdk.incubator.vector', '--enable-native-access=ALL-UNNAMED', '-Djava.util.concurrent.ForkJoinPool.common.parallelism=8', '-XX:+UnlockDiagnosticVMOptions', '-XX:+DebugNonSafepoints', '-XX:+HeapDumpOnOutOfMemoryError'
+    jvmArgs '--add-modules=jdk.incubator.vector', '--enable-native-access=ALL-UNNAMED'
+  }
+  if (System.getenv("DO_PROFILING") != null) {
+    jvmArgs '-XX:StartFlightRecording=dumponexit=true,maxsize=250M,filename=knn.jfr,settings=profile.jfc'
   }
 }
 

rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json

@@ -85,6 +85,10 @@
       "verbose":{
         "type":"boolean",
         "description":"Whether to show verbose snapshot info or only show the basic info found in the repository index blob"
+      },
+      "state": {
+        "type": "list",
+        "description": "Filter snapshots by a comma-separated list of states. Valid state values are 'SUCCESS', 'IN_PROGRESS', 'FAILED', 'PARTIAL', or 'INCOMPATIBLE'."
       }
     }
   }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.get/10_basic.yml

@@ -303,3 +303,72 @@ setup:
       snapshot.delete:
         repository: test_repo_get_1
         snapshot: test_snapshot_no_repo_name
+
+---
+"Get snapshot using state parameter":
+  - requires:
+      cluster_features: "snapshots.get.state_parameter"
+      test_runner_features: capabilities
+      capabilities:
+        - method: GET
+          path: /_snapshot/{repository}/{snapshot}
+          parameters: [ state ]
+      reason: "state parameter was introduced in 9.1"
+
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+
+  - do:
+      snapshot.create:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_state_param
+        wait_for_completion: true
+
+  - do:
+      snapshot.get:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_state_param
+        state: SUCCESS
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: test_snapshot_with_state_param }
+  - match: { snapshots.0.state: SUCCESS }
+
+  - do:
+      snapshot.get:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_state_param
+        state: SUCCESS,PARTIAL
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: test_snapshot_with_state_param }
+  - match: { snapshots.0.state: SUCCESS }
+
+  - do:
+      snapshot.get:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_state_param
+        state: FAILED
+
+  - is_true: snapshots
+  - length: { snapshots: 0 }
+
+  - do:
+      catch: bad_request
+      snapshot.get:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_state_param
+        state: FOO
+
+  - match: { error.type: "illegal_argument_exception" }
+  - match: { error.reason: "No enum constant org.elasticsearch.snapshots.SnapshotState.FOO" }
+
+  - do:
+      snapshot.delete:
+        repository: test_repo_get_1
+        snapshot: test_snapshot_with_state_param

server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java

@@ -55,13 +55,16 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
@@ -635,6 +638,63 @@ public void testRetrievingSnapshotsWhenRepositoryIsMissing() throws Exception {
         expectThrows(RepositoryMissingException.class, multiRepoFuture::actionGet);
     }
 
+    public void testFilterByState() throws Exception {
+        final String repoName = "test-repo";
+        final Path repoPath = randomRepoPath();
+        createRepository(repoName, "mock", repoPath);
+
+        // Create a successful snapshot
+        createFullSnapshot(repoName, "snapshot-success");
+
+        final Function<EnumSet<SnapshotState>, List<SnapshotInfo>> getSnapshotsForStates = (states) -> {
+            return clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repoName).setStates(states).get().getSnapshots();
+        };
+
+        // Fetch snapshots with state=SUCCESS
+        var snapshots = getSnapshotsForStates.apply(EnumSet.of(SnapshotState.SUCCESS));
+        assertThat(snapshots, hasSize(1));
+        assertThat(snapshots.getFirst().state(), is(SnapshotState.SUCCESS));
+
+        // Create a snapshot in progress
+        blockAllDataNodes(repoName);
+        startFullSnapshot(repoName, "snapshot-in-progress");
+        awaitNumberOfSnapshotsInProgress(1);
+
+        // Fetch snapshots with state=IN_PROGRESS
+        snapshots = getSnapshotsForStates.apply(EnumSet.of(SnapshotState.IN_PROGRESS));
+        assertThat(snapshots, hasSize(1));
+        assertThat(snapshots.getFirst().state(), is(SnapshotState.IN_PROGRESS));
+
+        // Fetch snapshots with multiple states (SUCCESS, IN_PROGRESS)
+        snapshots = getSnapshotsForStates.apply(EnumSet.of(SnapshotState.SUCCESS, SnapshotState.IN_PROGRESS));
+        assertThat(snapshots, hasSize(2));
+        var states = snapshots.stream().map(SnapshotInfo::state).collect(Collectors.toSet());
+        assertTrue(states.contains(SnapshotState.SUCCESS));
+        assertTrue(states.contains(SnapshotState.IN_PROGRESS));
+
+        // Fetch all snapshots (without state)
+        snapshots = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repoName).get().getSnapshots();
+        assertThat(snapshots, hasSize(2));
+
+        // Fetch snapshots with an invalid state
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> getSnapshotsForStates.apply(EnumSet.of(SnapshotState.valueOf("FOO")))
+        );
+        assertThat(e.getMessage(), is("No enum constant org.elasticsearch.snapshots.SnapshotState.FOO"));
+
+        // Allow the IN_PROGRESS snapshot to finish, then verify GET using SUCCESS has results and IN_PROGRESS does not.
+        unblockAllDataNodes(repoName);
+        awaitNumberOfSnapshotsInProgress(0);
+        snapshots = clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repoName).get().getSnapshots();
+        assertThat(snapshots, hasSize(2));
+        states = snapshots.stream().map(SnapshotInfo::state).collect(Collectors.toSet());
+        assertThat(states, hasSize(1));
+        assertTrue(states.contains(SnapshotState.SUCCESS));
+        snapshots = getSnapshotsForStates.apply(EnumSet.of(SnapshotState.IN_PROGRESS));
+        assertThat(snapshots, hasSize(0));
+    }
+
     public void testRetrievingSnapshotsWhenRepositoryIsUnreadable() throws Exception {
         final String repoName = randomIdentifier();
         final Path repoPath = randomRepoPath();
@@ -956,6 +1016,12 @@ public void testAllFeatures() {
         // INDICES and by SHARDS. The actual sorting behaviour for these cases is tested elsewhere, here we're just checking that sorting
         // interacts correctly with the other parameters to the API.
 
+        final EnumSet<SnapshotState> states = EnumSet.copyOf(randomNonEmptySubsetOf(Arrays.asList(SnapshotState.values())));
+        // Note: The selected state(s) may not match any existing snapshots.
+        // The actual filtering behaviour for such cases is tested in the dedicated test.
+        // Here we're just checking that states interacts correctly with the other parameters to the API.
+        snapshotInfoPredicate = snapshotInfoPredicate.and(si -> states.contains(si.state()));
+
         // compute the ordered sequence of snapshots which match the repository/snapshot name filters and SLM policy filter
         final var selectedSnapshots = snapshotInfos.stream()
             .filter(snapshotInfoPredicate)
@@ -967,7 +1033,8 @@ public void testAllFeatures() {
         )
             // apply sorting params
             .sort(sortKey)
-            .order(order);
+            .order(order)
+            .states(states);
 
         // sometimes use ?from_sort_value to skip some items; note that snapshots skipped in this way are subtracted from
         // GetSnapshotsResponse.totalCount whereas snapshots skipped by ?after and ?offset are not
@@ -1054,7 +1121,8 @@ public void testAllFeatures() {
                 .sort(sortKey)
                 .order(order)
                 .size(nextSize)
-                .after(SnapshotSortKey.decodeAfterQueryParam(nextRequestAfter));
+                .after(SnapshotSortKey.decodeAfterQueryParam(nextRequestAfter))
+                .states(states);
             final GetSnapshotsResponse nextResponse = safeAwait(l -> client().execute(TransportGetSnapshotsAction.TYPE, nextRequest, l));
 
             assertEquals(

server/src/main/java/module-info.java

@@ -425,6 +425,7 @@
             org.elasticsearch.action.bulk.BulkFeatures,
             org.elasticsearch.features.InfrastructureFeatures,
             org.elasticsearch.rest.action.admin.cluster.ClusterRerouteFeatures,
+            org.elasticsearch.rest.action.admin.cluster.GetSnapshotsFeatures,
             org.elasticsearch.index.mapper.MapperFeatures,
             org.elasticsearch.index.IndexFeatures,
             org.elasticsearch.search.SearchFeatures,

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -300,7 +300,11 @@ static TransportVersion def(int id) {
     public static final TransportVersion NONE_CHUNKING_STRATEGY = def(9_097_0_00);
     public static final TransportVersion PROJECT_DELETION_GLOBAL_BLOCK = def(9_098_0_00);
     public static final TransportVersion SECURITY_CLOUD_API_KEY_REALM_AND_TYPE = def(9_099_0_00);
+<<<<<<< markjhoy/add_sparse_vector_token_pruning_index_options
     public static final TransportVersion SPARSE_VECTOR_FIELD_PRUNING_OPTIONS = def(9_100_0_00);
+=======
+    public static final TransportVersion STATE_PARAM_GET_SNAPSHOT = def(9_100_0_00);
+>>>>>>> main
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/ActionModule.java

@@ -864,7 +864,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
         registerHandler.accept(new RestDeleteRepositoryAction());
         registerHandler.accept(new RestVerifyRepositoryAction());
         registerHandler.accept(new RestCleanupRepositoryAction());
-        registerHandler.accept(new RestGetSnapshotsAction());
+        registerHandler.accept(new RestGetSnapshotsAction(clusterSupportsFeature));
         registerHandler.accept(new RestCreateSnapshotAction());
         registerHandler.accept(new RestCloneSnapshotAction());
         registerHandler.accept(new RestRestoreSnapshotAction());