From 14d0507e1cb6c47471979844ea18709809e53d1b Mon Sep 17 00:00:00 2001
From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com>
Date: Mon, 15 Sep 2025 12:20:14 +0200
Subject: [PATCH] [ML] Improve log message for model snapshot update (#134541)

Since the 9.0 upgrade, we have introduced a solution by setting large .ml-anomalies to read only during the upgrade assistant. However, there is an edge case that will cause the obsolete model snapshot to fail to upgrade.

Step to reproduce:

setup a 7.17 cluster
create and start an anomaly detection job
upgrade to 8.18 or 8.19
go to upgrade assistant
change .ml-anomalies-shared to read only
then try upgrade obsolete model snapshot
Observed:

upgrade model snapshot will fail.
This PR extends the warning message explaining the steps to resolve the issue.

(cherry picked from commit c47522ca31a88a38dbb9d75cead41ae596a573ce)

# Conflicts:
#	server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
#	server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt
---
 .../org/elasticsearch/common/ReferenceDocs.java |  1 +
 .../common/reference-docs-links.txt             |  3 ++-
 .../autodetect/JobModelSnapshotUpgrader.java    | 17 ++++++++++++-----
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
index bf8451b81c55d..912f8c8fbe88c 100644
--- a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
+++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java
@@ -84,6 +84,7 @@ public enum ReferenceDocs {
     ALLOCATION_EXPLAIN_MAX_RETRY,
     SECURE_SETTINGS,
     CLUSTER_SHARD_LIMIT,
+    DELETE_INDEX_BLOCK,
     // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner
     ;
 
diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt
index ae99ef33a3716..8f97a935a5e80 100644
--- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt
+++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt
@@ -45,4 +45,5 @@ CIRCUIT_BREAKER_ERRORS                                          troubleshoot/ela
 ALLOCATION_EXPLAIN_NO_COPIES                                    troubleshoot/elasticsearch/diagnose-unassigned-shards#no-shard-copy
 ALLOCATION_EXPLAIN_MAX_RETRY                                    troubleshoot/elasticsearch/diagnose-unassigned-shards#maximum-retries-exceeded
 SECURE_SETTINGS                                                 deploy-manage/security/secure-settings
-CLUSTER_SHARD_LIMIT                                             reference/elasticsearch/configuration-reference/miscellaneous-cluster-settings#cluster-shard-limit
\ No newline at end of file
+CLUSTER_SHARD_LIMIT                                             reference/elasticsearch/configuration-reference/miscellaneous-cluster-settings#cluster-shard-limit
+DELETE_INDEX_BLOCK                                              api/doc/elasticsearch/operation/operation-indices-remove-block
\ No newline at end of file
diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java
index d42eb8f748b51..d270e00f78e12 100644
--- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java
+++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/JobModelSnapshotUpgrader.java
@@ -17,6 +17,7 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.CheckedSupplier;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -196,15 +197,21 @@ private void deleteOlderSnapshotDoc(SearchResponse searchResponse, Consumer<Exce
                     );
                 }
             }, e -> {
-                logger.warn(
-                    () -> format(
+                logger.warn(() -> {
+                    String baseMessage = format(
                         "[%s] [%s] failed to delete old snapshot [%s] result document",
                         jobId,
                         snapshotId,
                         ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName()
-                    ),
-                    e
-                );
+                    );
+                    if (e instanceof org.elasticsearch.cluster.block.ClusterBlockException) {
+                        return baseMessage
+                            + ". Remove the write block from the results index to delete the model snapshot. See "
+                            + ReferenceDocs.DELETE_INDEX_BLOCK
+                            + " for details.";
+                    }
+                    return baseMessage;
+                }, e);
             }), () -> runAfter.accept(null)));
     }