Fix tests + back to using dispatchBadRequest to emit correct headers and trace information

Author: ldematte

server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java

@@ -38,7 +38,6 @@
 import org.elasticsearch.core.RefCounted;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -488,7 +487,7 @@ void dispatchRequest(final RestRequest restRequest, final RestChannel channel, f
                     populatePerRequestThreadContext(restRequest, threadContext);
                 } catch (Exception e) {
                     try {
-                        channel.sendResponse(new RestResponse(channel, e));
+                        dispatcher.dispatchBadRequest(channel, threadContext, e);
                     } catch (Exception inner) {
                         inner.addSuppressed(e);
                         logger.error(() -> "failed to send failure response for uri [" + restRequest.uri() + "]", inner);

server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java

@@ -355,8 +355,6 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
             }
 
         };
-        // the set of headers to copy
-        Set<RestHeaderDefinition> headers = Set.of(new RestHeaderDefinition(Task.TRACE_PARENT_HTTP_HEADER, false));
         // sample request headers to test with
         Map<String, List<String>> restHeaders = new HashMap<>();
         restHeaders.put(Task.TRACE_PARENT_HTTP_HEADER, Collections.singletonList(traceParentValue));
@@ -397,7 +395,7 @@ public HttpStats stats() {
 
                 @Override
                 protected void populatePerRequestThreadContext(RestRequest restRequest, ThreadContext threadContext) {
-                    getFakeActionModule(headers).copyRequestHeadersToThreadContext(restRequest.getHttpRequest(), threadContext);
+                    getFakeActionModule(Set.of()).copyRequestHeadersToThreadContext(restRequest.getHttpRequest(), threadContext);
                 }
             }
         ) {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4HttpServerTransportTests.java

@@ -370,7 +370,21 @@ public void testHttpHeaderAuthnBypassHeaderValidator() throws Exception {
                 new NetworkService(List.of()),
                 testThreadPool,
                 xContentRegistry(),
-                new NullDispatcher(),
+                new HttpServerTransport.Dispatcher() {
+                    @Override
+                    public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
+                        fail("Request should not be dispatched");
+                    }
+
+                    @Override
+                    public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, Throwable cause) {
+                        try {
+                            channel.sendResponse(new RestResponse(channel, (Exception) cause));
+                        } catch (IOException e) {
+                            fail(e, "Unexpected exception dispatching bad request");
+                        }
+                    }
+                },
                 randomClusterSettings(),
                 new SharedGroupFactory(settings),
                 Tracer.NOOP,