Fix project lookup in ILM task executor (#129506)

Since #129366 we obtain a `ProjectState` by using the `Index` to look up
the project from the cluster state. However, when an index is deleted,
that statement fails. Since we have no guaranteed way of determining
which project a deleted index was from, we need to explicitly pass the
`ProjectID` in the `IndexLifecycleClusterStateUpdateTask`s.

Incidentally, we have to make parts of the `IndexLifecycleService` and
`IndexLifecycleRunner` project-aware - but they'll likely need a
follow-up PR to finish them.

Fixes #129489
Fixes #129490
Fixes #129491
Fixes #129510

Author: nielsbauman

muted-tests.yml

@@ -547,21 +547,9 @@ tests:
 - class: org.elasticsearch.xpack.profiling.action.GetStatusActionIT
   method: testWaitsUntilResourcesAreCreated
   issue: https://github.com/elastic/elasticsearch/issues/129486
-- class: org.elasticsearch.xpack.ilm.TimeseriesMoveToStepIT
-  method: testMoveToRolloverStep
-  issue: https://github.com/elastic/elasticsearch/issues/129489
-- class: org.elasticsearch.xpack.ilm.TimeseriesMoveToStepIT
-  method: testMoveToAllocateStep
-  issue: https://github.com/elastic/elasticsearch/issues/129490
-- class: org.elasticsearch.xpack.ilm.actions.ShrinkActionIT
-  method: testShrinkDuringSnapshot
-  issue: https://github.com/elastic/elasticsearch/issues/129491
 - class: org.elasticsearch.xpack.security.PermissionsIT
   method: testWhenUserLimitedByOnlyAliasOfIndexCanWriteToIndexWhichWasRolledoverByILMPolicy
   issue: https://github.com/elastic/elasticsearch/issues/129481
-- class: org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT
-  method: testFullPolicy
-  issue: https://github.com/elastic/elasticsearch/issues/129510
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {knn-function.KnnSearchWithKOption SYNC}
   issue: https://github.com/elastic/elasticsearch/issues/129512

x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java

@@ -141,15 +141,17 @@ public void testWaitInShrunkShardsAllocatedExceedsThreshold() throws Exception {
         clusterService.submitUnbatchedStateUpdateTask("testing-move-to-step-to-manipulate-step-time", new ClusterStateUpdateTask() {
             @Override
             public ClusterState execute(ClusterState currentState) throws Exception {
+                final var projectState = currentState.projectState();
                 return new MoveToNextStepUpdateTask(
+                    projectState.projectId(),
                     managedIndexMetadata.getIndex(),
                     policy,
                     currentStepKey,
                     currentStepKey,
                     nowWayBackInThePastSupplier,
                     indexLifecycleService.getPolicyRegistry(),
                     state -> {}
-                ).execute(currentState.projectState());
+                ).execute(projectState);
             }
 
             @Override

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ExecuteStepsUpdateTask.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.ProjectState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.LifecycleExecutionState;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xcontent.ToXContentObject;
@@ -45,14 +46,15 @@ public class ExecuteStepsUpdateTask extends IndexLifecycleClusterStateUpdateTask
     private Exception failure = null;
 
     public ExecuteStepsUpdateTask(
+        ProjectId projectId,
         String policy,
         Index index,
         Step startStep,
         PolicyStepsRegistry policyStepsRegistry,
         IndexLifecycleRunner lifecycleRunner,
         LongSupplier nowSupplier
     ) {
-        super(index, startStep.getKey());
+        super(projectId, index, startStep.getKey());
         this.policy = policy;
         this.startStep = startStep;
         this.policyStepsRegistry = policyStepsRegistry;
@@ -234,7 +236,7 @@ public void onClusterStateProcessed(ProjectState newState) {
                 // After the cluster state has been processed and we have moved
                 // to a new step, we need to conditionally execute the step iff
                 // it is an `AsyncAction` so that it is executed exactly once.
-                lifecycleRunner.maybeRunAsyncAction(newState.cluster(), indexMetadata, policy, nextStepKey);
+                lifecycleRunner.maybeRunAsyncAction(newState, indexMetadata, policy, nextStepKey);
             }
         }
         assert indexToStepKeysForAsyncActions.size() <= 1 : "we expect a maximum of one single spawned index currently";
@@ -253,7 +255,7 @@ public void onClusterStateProcessed(ProjectState newState) {
                             nextStep
                         );
                         final String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexMeta.getSettings());
-                        lifecycleRunner.maybeRunAsyncAction(newState.cluster(), indexMeta, policyName, nextStep);
+                        lifecycleRunner.maybeRunAsyncAction(newState, indexMeta, policyName, nextStep);
                     }
                 }
             }

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleClusterStateUpdateTask.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.ProjectState;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xpack.core.ilm.Step;
@@ -24,11 +25,13 @@ public abstract class IndexLifecycleClusterStateUpdateTask implements ClusterSta
 
     private final ListenableFuture<Void> listener = new ListenableFuture<>();
 
+    /** We need to store the project ID along with the index because an index might get deleted, but we still want to run the step */
+    protected final ProjectId projectId;
     protected final Index index;
-
     protected final Step.StepKey currentStepKey;
 
-    protected IndexLifecycleClusterStateUpdateTask(Index index, Step.StepKey currentStepKey) {
+    protected IndexLifecycleClusterStateUpdateTask(ProjectId projectId, Index index, Step.StepKey currentStepKey) {
+        this.projectId = projectId;
         this.index = index;
         this.currentStepKey = currentStepKey;
     }