Mention network handler logging in docs (#100118) (#100126)

Mentions the `InboundHandler` (and `OutboundHandler`) as potential
sources of useful log messages when tracking down a network threading
bug.

Author: DaveCTurner

docs/reference/modules/network/threading.asciidoc

@@ -107,3 +107,12 @@ However, this API itself sends network messages so may not work correctly if
 the `transport_worker` threads are too busy. It is more reliable to use
 `jstack` to obtain stack dumps or use Java Flight Recorder to obtain a
 profiling trace. These tools are independent of any work the JVM is performing.
+
+It may also be possible to identify some reasons for delays from the server
+logs, particularly looking at warnings from
+`org.elasticsearch.transport.InboundHandler` and
+`org.elasticsearch.transport.OutboundHandler`. Warnings about long processing
+times from the `InboundHandler` are particularly indicative of incorrect
+threading behaviour, whereas the transmission time reported by the
+`OutboundHandler` includes time spent waiting for network congestion and the
+`transport_worker` thread is free to do other work during this time.

docs/reference/troubleshooting/network-timeouts.asciidoc

@@ -47,5 +47,7 @@ since it doesn't require any JVM threads.
 The threads involved in discovery and cluster membership are mainly
 `transport_worker` and `cluster_coordination` threads, for which there should
 never be a long wait. There may also be evidence of long waits for threads in
-the {es} logs. See <<modules-network-threading-model>> for more information.
+the {es} logs, particularly looking at warning logs from
+`org.elasticsearch.transport.InboundHandler`. See
+<<modules-network-threading-model>> for more information.
 end::troubleshooting-network-timeouts-threads[]

server/src/test/java/org/elasticsearch/transport/InboundHandlerTests.java

@@ -236,7 +236,7 @@ public void testClosesChannelOnErrorInHandshake() throws Exception {
         mockAppender.addExpectation(
             new MockLogAppender.SeenEventExpectation(
                 "expected message",
-                InboundHandler.class.getCanonicalName(),
+                EXPECTED_LOGGER_NAME,
                 Level.WARN,
                 "error processing handshake version"
             )
@@ -273,6 +273,12 @@ public void testClosesChannelOnErrorInHandshake() throws Exception {
         }
     }
 
+    /**
+     * This logger is mentioned in the docs by name, so we cannot rename it without adjusting the docs. Thus we fix the expected logger
+     * name in this string constant rather than using {@code InboundHandler.class.getCanonicalName()}.
+     */
+    private static final String EXPECTED_LOGGER_NAME = "org.elasticsearch.transport.InboundHandler";
+
     public void testLogsSlowInboundProcessing() throws Exception {
         final MockLogAppender mockAppender = new MockLogAppender();
         mockAppender.start();
@@ -284,12 +290,7 @@ public void testLogsSlowInboundProcessing() throws Exception {
             final TransportVersion remoteVersion = TransportVersion.current();
 
             mockAppender.addExpectation(
-                new MockLogAppender.SeenEventExpectation(
-                    "expected slow request",
-                    InboundHandler.class.getCanonicalName(),
-                    Level.WARN,
-                    "handling request "
-                )
+                new MockLogAppender.SeenEventExpectation("expected slow request", EXPECTED_LOGGER_NAME, Level.WARN, "handling request ")
             );
 
             final long requestId = randomNonNegativeLong();
@@ -316,12 +317,7 @@ public void testLogsSlowInboundProcessing() throws Exception {
             mockAppender.assertAllExpectationsMatched();
 
             mockAppender.addExpectation(
-                new MockLogAppender.SeenEventExpectation(
-                    "expected slow response",
-                    InboundHandler.class.getCanonicalName(),
-                    Level.WARN,
-                    "handling response "
-                )
+                new MockLogAppender.SeenEventExpectation("expected slow response", EXPECTED_LOGGER_NAME, Level.WARN, "handling response ")
             );
 
             final long responseId = randomNonNegativeLong();

server/src/test/java/org/elasticsearch/transport/OutboundHandlerTests.java

@@ -317,16 +317,17 @@ public void onResponseSent(long requestId, String action, Exception error) {
         assertEquals("header_value", header.getHeaders().v1().get("header"));
     }
 
+    /**
+     * This logger is mentioned in the docs by name, so we cannot rename it without adjusting the docs. Thus we fix the expected logger
+     * name in this string constant rather than using {@code OutboundHandler.class.getCanonicalName()}.
+     */
+    private static final String EXPECTED_LOGGER_NAME = "org.elasticsearch.transport.OutboundHandler";
+
     public void testSlowLogOutboundMessage() throws Exception {
         final MockLogAppender mockAppender = new MockLogAppender();
         mockAppender.start();
         mockAppender.addExpectation(
-            new MockLogAppender.SeenEventExpectation(
-                "expected message",
-                OutboundHandler.class.getCanonicalName(),
-                Level.WARN,
-                "sending transport message "
-            )
+            new MockLogAppender.SeenEventExpectation("expected message", EXPECTED_LOGGER_NAME, Level.WARN, "sending transport message ")
         );
         final Logger outboundHandlerLogger = LogManager.getLogger(OutboundHandler.class);
         Loggers.addAppender(outboundHandlerLogger, mockAppender);