Improve handling of empty response (elastic#126393)

Today `ActionResponse$Empty` implements `ToXContentObject`, but yields
no bytes of content when serialized which creates an invalid JSON
response. This commit removes the bogus interface and adjusts the
affected REST APIs to send a `text/plain` response instead.

Backport of elastic#125562 to `8.x`

Author: DaveCTurner

docs/changelog/125562.yaml

@@ -0,0 +1,6 @@
+pr: 125562
+summary: Improve handling of empty response
+area: Infra/REST API
+type: bug
+issues:
+ - 57639

modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java

@@ -17,22 +17,24 @@
 import org.elasticsearch.test.rest.RestActionTestCase;
 import org.elasticsearch.xcontent.XContentType;
 import org.junit.Before;
-import org.mockito.Mockito;
 
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.rest.RestResponseUtils.setUpXContentMock;
+import static org.mockito.Mockito.mock;
+
 public final class RestMultiSearchTemplateActionTests extends RestActionTestCase {
     final List<String> contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7));
 
     @Before
     public void setUpAction() {
         controller().registerHandler(new RestMultiSearchTemplateAction(Settings.EMPTY));
         // todo how to workaround this? we get AssertionError without this
-        verifyingClient.setExecuteVerifier((actionType, request) -> Mockito.mock(MultiSearchTemplateResponse.class));
-        verifyingClient.setExecuteLocallyVerifier((actionType, request) -> Mockito.mock(MultiSearchTemplateResponse.class));
+        verifyingClient.setExecuteVerifier((actionType, request) -> setUpXContentMock(mock(MultiSearchTemplateResponse.class)));
+        verifyingClient.setExecuteLocallyVerifier((actionType, request) -> setUpXContentMock(mock(MultiSearchTemplateResponse.class)));
     }
 
     public void testTypeInPath() {

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_balance/10_basic.yml

@@ -180,6 +180,11 @@ setup:
   - requires:
       cluster_features: ["gte_v8.8.0"]
       reason: "reset API added in in 8.8.0"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: DELETE
+          path: /_internal/desired_balance
+          capabilities: [ plain_text_empty_response ]
 
   - do:
       _internal.delete_desired_balance: { }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/10_basic.yml

@@ -3,6 +3,11 @@ setup:
   - requires:
       cluster_features: ["gte_v8.13.0"]
       reason: "API added in in 8.1.0 but modified in 8.13 (node_version field removed)"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: DELETE
+          path: /_internal/desired_nodes
+          capabilities: [ plain_text_empty_response ]
 ---
 teardown:
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/11_old_format.yml

@@ -6,6 +6,11 @@ setup:
   - requires:
       cluster_features: ["gte_v8.3.0"]
       reason: "API added in in 8.1.0 but modified in 8.3"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: DELETE
+          path: /_internal/desired_nodes
+          capabilities: [ plain_text_empty_response ]
 ---
 teardown:
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/20_dry_run.yml

@@ -3,6 +3,11 @@ setup:
   - requires:
       cluster_features: ["gte_v8.4.0"]
       reason: "Support for the dry run option was added in in 8.4.0"
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: DELETE
+          path: /_internal/desired_nodes
+          capabilities: [ plain_text_empty_response ]
 ---
 teardown:
   - do:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.voting_config_exclusions/10_basic.yml

@@ -1,3 +1,16 @@
+setup:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: POST
+          path: /_cluster/voting_config_exclusions
+          capabilities: [ plain_text_empty_response ]
+        - method: DELETE
+          path: /_cluster/voting_config_exclusions
+          capabilities: [ plain_text_empty_response ]
+      reason: needs these capabilities
+
+---
 teardown:
   - do:
       cluster.delete_voting_config_exclusions: {}

server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/VotingConfigurationIT.java

@@ -9,22 +9,24 @@
 package org.elasticsearch.cluster.coordination;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
-import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import java.util.concurrent.ExecutionException;
 
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
@@ -38,18 +40,39 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singletonList(MockTransportService.TestPlugin.class);
     }
 
-    public void testAbdicateAfterVotingConfigExclusionAdded() throws ExecutionException, InterruptedException {
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false; // enable HTTP
+    }
+
+    public void testAbdicateAfterVotingConfigExclusionAdded() throws IOException {
         internalCluster().setBootstrapMasterNodeIndex(0);
         internalCluster().startNodes(2);
         final String originalMaster = internalCluster().getMasterName();
+        final var restClient = getRestClient();
 
         logger.info("--> excluding master node {}", originalMaster);
-        client().execute(
-            TransportAddVotingConfigExclusionsAction.TYPE,
-            new AddVotingConfigExclusionsRequest(TEST_REQUEST_TIMEOUT, originalMaster)
-        ).get();
+        final var excludeRequest = new Request("POST", "/_cluster/voting_config_exclusions");
+        excludeRequest.addParameter("node_names", originalMaster);
+        assertEmptyResponse(restClient.performRequest(excludeRequest));
+
         clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get();
         assertNotEquals(originalMaster, internalCluster().getMasterName());
+
+        final var clearRequest = new Request("DELETE", "/_cluster/voting_config_exclusions");
+        clearRequest.addParameter("wait_for_removal", "false");
+        assertEmptyResponse(restClient.performRequest(clearRequest));
+
+        assertThat(
+            internalCluster().getInstance(ClusterService.class).state().metadata().coordinationMetadata().getVotingConfigExclusions(),
+            empty()
+        );
+    }
+
+    private void assertEmptyResponse(Response response) throws IOException {
+        assertEquals("text/plain; charset=UTF-8", response.getHeader("content-type"));
+        assertEquals(0, response.getEntity().getContentLength());
+        assertEquals(0, response.getEntity().getContent().readAllBytes().length);
     }
 
     public void testElectsNodeNotInVotingConfiguration() throws Exception {

server/src/main/java/org/elasticsearch/action/ActionResponse.java

@@ -11,11 +11,10 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.rest.action.EmptyResponseListener;
 import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.xcontent.ToXContentObject;
-import org.elasticsearch.xcontent.XContentBuilder;
-
-import java.io.IOException;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContent;
 
 /**
  * Base class for responses to action requests.
@@ -24,24 +23,25 @@ public abstract class ActionResponse extends TransportResponse {
 
     public ActionResponse() {}
 
-    public ActionResponse(StreamInput in) throws IOException {
-        super(in);
-    }
+    public ActionResponse(StreamInput in) {}
+
+    /**
+     * A response with no payload. This is deliberately not an implementation of {@link ToXContent} or similar because an empty response
+     * has no valid {@link XContent} representation. Use {@link EmptyResponseListener} to convert this to a valid (plain-text) REST
+     * response instead.
+     */
+    public static final class Empty extends ActionResponse {
+
+        private Empty() { /* singleton */ }
 
-    public static final class Empty extends ActionResponse implements ToXContentObject {
         public static final ActionResponse.Empty INSTANCE = new ActionResponse.Empty();
 
         @Override
         public String toString() {
-            return "EmptyActionResponse{}";
+            return "ActionResponse.Empty{}";
         }
 
         @Override
         public void writeTo(StreamOutput out) {}
-
-        @Override
-        public XContentBuilder toXContent(final XContentBuilder builder, final Params params) {
-            return builder;
-        }
     }
 }

server/src/main/java/org/elasticsearch/rest/action/EmptyResponseListener.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.rest.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+
+/**
+ * A listener which converts a successful {@link ActionResponse.Empty} action response into a {@code 200 OK} REST response with empty body.
+ */
+public final class EmptyResponseListener extends RestResponseListener<ActionResponse.Empty> {
+    public EmptyResponseListener(RestChannel channel) {
+        super(channel);
+    }
+
+    @Override
+    public RestResponse buildResponse(ActionResponse.Empty ignored) throws Exception {
+        // Content-type header is not required for an empty body but some clients may expect it; the empty body is a valid text/plain entity
+        // so we use that here.
+        return new RestResponse(RestStatus.OK, RestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
+    }
+
+    /**
+     * Capability name for APIs that previously would return an invalid zero-byte {@code application/json} response so that the YAML test
+     * runner can avoid those APIs.
+     */
+    public static final String PLAIN_TEXT_EMPTY_RESPONSE_CAPABILITY_NAME = "plain_text_empty_response";
+}