[ML] Improve log message for model snapshot update (#134541) (#134904)

Since the 9.0 upgrade, we have introduced a solution by setting large .ml-anomalies to read only during the upgrade assistant. However, there is an edge case that will cause the obsolete model snapshot to fail to upgrade.

Step to reproduce:

setup a 7.17 cluster
create and start an anomaly detection job
upgrade to 8.18 or 8.19
go to upgrade assistant
change .ml-anomalies-shared to read only
then try upgrade obsolete model snapshot
Observed:

upgrade model snapshot will fail.
This PR extends the warning message explaining the steps to resolve the issue.

(cherry picked from commit c47522c)

# Conflicts:
#	server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
#	server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt

Author: valeriy42

server/src/main/java/org/elasticsearch/common/ReferenceDocs.java

@@ -84,6 +84,7 @@ public enum ReferenceDocs {
     ALLOCATION_EXPLAIN_MAX_RETRY,
     SECURE_SETTINGS,
     CLUSTER_SHARD_LIMIT,
+    DELETE_INDEX_BLOCK,
     // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner
     ;
 

server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt

@@ -45,4 +45,5 @@ CIRCUIT_BREAKER_ERRORS                                          troubleshoot/ela
 ALLOCATION_EXPLAIN_NO_COPIES                                    troubleshoot/elasticsearch/diagnose-unassigned-shards#no-shard-copy
 ALLOCATION_EXPLAIN_MAX_RETRY                                    troubleshoot/elasticsearch/diagnose-unassigned-shards#maximum-retries-exceeded
 SECURE_SETTINGS                                                 deploy-manage/security/secure-settings
-CLUSTER_SHARD_LIMIT                                             reference/elasticsearch/configuration-reference/miscellaneous-cluster-settings#cluster-shard-limit
+CLUSTER_SHARD_LIMIT                                             reference/elasticsearch/configuration-reference/miscellaneous-cluster-settings#cluster-shard-limit
+DELETE_INDEX_BLOCK                                              api/doc/elasticsearch/operation/operation-indices-remove-block

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.CheckedSupplier;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -196,15 +197,21 @@ private void deleteOlderSnapshotDoc(SearchResponse searchResponse, Consumer<Exce
                     );
                 }
             }, e -> {
-                logger.warn(
-                    () -> format(
+                logger.warn(() -> {
+                    String baseMessage = format(
                         "[%s] [%s] failed to delete old snapshot [%s] result document",
                         jobId,
                         snapshotId,
                         ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName()
-                    ),
-                    e
-                );
+                    );
+                    if (e instanceof org.elasticsearch.cluster.block.ClusterBlockException) {
+                        return baseMessage
+                            + ". Remove the write block from the results index to delete the model snapshot. See "
+                            + ReferenceDocs.DELETE_INDEX_BLOCK
+                            + " for details.";
+                    }
+                    return baseMessage;
+                }, e);
             }), () -> runAfter.accept(null)));
     }