Summarize cause when replacing no shard available exception in search

When a shard fails during search, the error handling logic looks at the root
cause and in case it is in the availability bucket, the exception is replaced
with a more generic NoShardAvailableException without a shard_id and cause,
to prevent spamming the coord node with many of the same errors.

This is quite lossy, and makes it very hard to track actual problems if they arise,
especially as the message used for the new exception does not include the cause in
any way. This commit reuses the toString of both the exception and its cause, which
is going to concatenate their class name and message in a single reason field.

Author: javanna

server/src/main/java/org/elasticsearch/action/NoShardAvailableActionException.java

@@ -23,8 +23,8 @@ public final class NoShardAvailableActionException extends ElasticsearchExceptio
     // It isn't necessary to serialize this field over the wire as the empty stack trace is serialized instead.
     private final boolean onShardFailureWrapper;
 
-    public static NoShardAvailableActionException forOnShardFailureWrapper(String msg) {
-        return new NoShardAvailableActionException(null, msg, null, true);
+    public static NoShardAvailableActionException forOnShardFailureWrapper(Exception e, Throwable cause) {
+        return new NoShardAvailableActionException(null, e + ", cause: " + cause, null, true);
     }
 
     public NoShardAvailableActionException(ShardId shardId) {

server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java

@@ -442,10 +442,11 @@ protected void onShardGroupFailure(int shardIndex, SearchShardTarget shardTarget
      * @param e the failure reason
      */
     void onShardFailure(final int shardIndex, SearchShardTarget shardTarget, Exception e) {
-        if (TransportActions.isShardNotAvailableException(e)) {
+        Throwable cause = ExceptionsHelper.unwrapCause(e);
+        if (TransportActions.isShardNotAvailableException(cause)) {
             // Groups shard not available exceptions under a generic exception that returns a SERVICE_UNAVAILABLE(503)
             // temporary error.
-            e = NoShardAvailableActionException.forOnShardFailureWrapper(e.getMessage());
+            e = NoShardAvailableActionException.forOnShardFailureWrapper(e, cause);
         }
         // we don't aggregate shard on failures due to the internal cancellation,
         // but do keep the header counts right

server/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.action.search;
 
+import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.ParsingException;
@@ -116,17 +117,22 @@ public void testToXContent() throws IOException {
     public void testToXContentForNoShardAvailable() throws IOException {
         ShardId shardId = new ShardId(new Index("indexName", "indexUuid"), 123);
         ShardSearchFailure failure = new ShardSearchFailure(
-            NoShardAvailableActionException.forOnShardFailureWrapper("shard unassigned"),
+            NoShardAvailableActionException.forOnShardFailureWrapper(new AlreadyClosedException("shard unassigned"), null),
             new SearchShardTarget("nodeId", shardId, null)
         );
         BytesReference xContent = toXContent(failure, XContentType.JSON, randomBoolean());
-        assertEquals(XContentHelper.stripWhitespace("""
-            {
-              "shard": 123,
-              "index": "indexName",
-              "node": "nodeId",
-              "reason":{"type":"no_shard_available_action_exception","reason":"shard unassigned"}
-            }"""), xContent.utf8ToString());
+        assertEquals(
+            XContentHelper.stripWhitespace(
+                """
+                    {
+                      "shard": 123,
+                      "index": "indexName",
+                      "node": "nodeId",
+                      "reason":{"type":"no_shard_available_action_exception","reason":"org.apache.lucene.store.AlreadyClosedException: shard unassigned, cause: null"}
+                    }"""
+            ),
+            xContent.utf8ToString()
+        );
     }
 
     public void testToXContentWithClusterAlias() throws IOException {
@@ -156,7 +162,10 @@ public void testSerialization() throws IOException {
                 testItem = createTestItem(randomAlphaOfLength(12));
             } else {
                 SearchShardTarget target = randomShardTarget(randomAlphaOfLength(12));
-                testItem = new ShardSearchFailure(NoShardAvailableActionException.forOnShardFailureWrapper("unavailable"), target);
+                testItem = new ShardSearchFailure(
+                    NoShardAvailableActionException.forOnShardFailureWrapper(new AlreadyClosedException("unavailable"), null),
+                    target
+                );
             }
             ShardSearchFailure deserializedInstance = copyWriteable(
                 testItem,

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSender.java

@@ -202,8 +202,9 @@ interface NodeListener {
 
     private static Exception unwrapFailure(Exception e) {
         e = e instanceof TransportException te ? FailureCollector.unwrapTransportException(te) : e;
-        if (TransportActions.isShardNotAvailableException(e)) {
-            return NoShardAvailableActionException.forOnShardFailureWrapper(e.getMessage());
+        Throwable cause = ExceptionsHelper.unwrapCause(e);
+        if (TransportActions.isShardNotAvailableException(cause)) {
+            return NoShardAvailableActionException.forOnShardFailureWrapper(e, cause);
         } else {
             return e;
         }