Prevent default inference model to update the cluster state when deleting (#125369)

The Elastic inference service removes the default models at startup if the node cannot access EIS.
Since #125242 we don't store default models in the cluster state but we still try to delete them.
This change ensures that we don't try to update the cluster state when a default model is deleted
since the delete is not performed on the master node and default models are never stored in the cluster state.

Author: jimczi

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java

@@ -46,6 +46,7 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -627,8 +628,8 @@ public void storeModel(Model model, ActionListener<Boolean> listener, TimeValue
         storeModel(model, true, listener, timeout);
     }
 
-    private void storeModel(Model model, boolean addToClusterState, ActionListener<Boolean> listener, TimeValue timeout) {
-        ActionListener<BulkResponse> bulkResponseActionListener = getStoreIndexListener(model, addToClusterState, listener, timeout);
+    private void storeModel(Model model, boolean updateClusterState, ActionListener<Boolean> listener, TimeValue timeout) {
+        ActionListener<BulkResponse> bulkResponseActionListener = getStoreIndexListener(model, updateClusterState, listener, timeout);
 
         IndexRequest configRequest = createIndexRequest(
             Model.documentId(model.getConfigurations().getInferenceEntityId()),
@@ -653,7 +654,7 @@ private void storeModel(Model model, boolean addToClusterState, ActionListener<B
 
     private ActionListener<BulkResponse> getStoreIndexListener(
         Model model,
-        boolean addToClusterState,
+        boolean updateClusterState,
         ActionListener<Boolean> listener,
         TimeValue timeout
     ) {
@@ -680,7 +681,7 @@ private ActionListener<BulkResponse> getStoreIndexListener(
             BulkItemResponse.Failure failure = getFirstBulkFailure(bulkItemResponses);
 
             if (failure == null) {
-                if (addToClusterState) {
+                if (updateClusterState) {
                     var storeListener = getStoreMetadataListener(inferenceEntityId, listener);
                     try {
                         var projectId = clusterService.state().projectState().projectId();
@@ -781,14 +782,19 @@ public synchronized void removeDefaultConfigs(Set<String> inferenceEntityIds, Ac
         }
 
         defaultConfigIds.keySet().removeAll(inferenceEntityIds);
-        deleteModels(inferenceEntityIds, listener);
+        // default models are not stored in the cluster state.
+        deleteModels(inferenceEntityIds, false, listener);
     }
 
     public void deleteModel(String inferenceEntityId, ActionListener<Boolean> listener) {
         deleteModels(Set.of(inferenceEntityId), listener);
     }
 
     public void deleteModels(Set<String> inferenceEntityIds, ActionListener<Boolean> listener) {
+        deleteModels(inferenceEntityIds, true, listener);
+    }
+
+    private void deleteModels(Set<String> inferenceEntityIds, boolean updateClusterState, ActionListener<Boolean> listener) {
         var lockedInferenceIds = new HashSet<>(inferenceEntityIds);
         lockedInferenceIds.retainAll(preventDeletionLock);
 
@@ -807,16 +813,25 @@ public void deleteModels(Set<String> inferenceEntityIds, ActionListener<Boolean>
         }
 
         var request = createDeleteRequest(inferenceEntityIds);
-        client.execute(DeleteByQueryAction.INSTANCE, request, getDeleteModelClusterStateListener(inferenceEntityIds, listener));
+        client.execute(
+            DeleteByQueryAction.INSTANCE,
+            request,
+            getDeleteModelClusterStateListener(inferenceEntityIds, updateClusterState, listener)
+        );
     }
 
     private ActionListener<BulkByScrollResponse> getDeleteModelClusterStateListener(
         Set<String> inferenceEntityIds,
+        boolean updateClusterState,
         ActionListener<Boolean> listener
     ) {
         return new ActionListener<>() {
             @Override
             public void onResponse(BulkByScrollResponse bulkByScrollResponse) {
+                if (updateClusterState == false) {
+                    listener.onResponse(Boolean.TRUE);
+                    return;
+                }
                 var clusterStateListener = new ActionListener<AcknowledgedResponse>() {
                     @Override
                     public void onResponse(AcknowledgedResponse acknowledgedResponse) {
@@ -924,6 +939,11 @@ public void clusterChanged(ClusterChangedEvent event) {
             return;
         }
 
+        // wait for the cluster state to be recovered
+        if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
+            return;
+        }
+
         if (event.state().metadata().projects().size() > 1) {
             // TODO: Add support to handle multi-projects
             return;