Support ListObjectsV2 in S3HttpHandler (#126189)

`ListObjects` and `ListObjectsV2` only really differ in their approach
to pagination, but today `S3HttpHandler` does not simulate pagination
anyway so we can use the same handling code for both APIs. The only
practical difference is that the v2 SDK requires the `<IsTruncated>`
element in a `ListObjectsV2` response, but this element is permitted in
both APIs so we add it here.

Author: DaveCTurner

test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java

@@ -207,10 +207,6 @@ public void handle(final HttpExchange exchange) throws IOException {
                 exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1);
 
             } else if (request.isListObjectsRequest()) {
-                if (request.queryParameters().containsKey("list-type")) {
-                    throw new AssertionError("Test must be adapted for GET Bucket (List Objects) Version 2");
-                }
-
                 final StringBuilder list = new StringBuilder();
                 list.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
                 list.append("<ListBucketResult>");
@@ -223,6 +219,9 @@ public void handle(final HttpExchange exchange) throws IOException {
                 if (delimiter != null) {
                     list.append("<Delimiter>").append(delimiter).append("</Delimiter>");
                 }
+                // Would be good to test pagination here (the only real difference between ListObjects and ListObjectsV2) but for now
+                // we return all the results at once.
+                list.append("<IsTruncated>false</IsTruncated>");
                 for (Map.Entry<String, BytesReference> blob : blobs.entrySet()) {
                     if (prefix != null && blob.getKey().startsWith("/" + bucket + "/" + prefix) == false) {
                         continue;

test/fixtures/s3-fixture/src/test/java/fixture/s3/S3HttpHandlerTests.java

@@ -14,6 +14,7 @@
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpPrincipal;
 
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -28,6 +29,7 @@
 import java.net.InetSocketAddress;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
@@ -45,63 +47,81 @@ public void testRejectsBadUri() {
         );
     }
 
+    private static void assertListObjectsResponse(
+        S3HttpHandler handler,
+        @Nullable String prefix,
+        @Nullable String delimiter,
+        String expectedResponse
+    ) {
+        final var queryParts = new ArrayList<String>(3);
+        if (prefix != null) {
+            queryParts.add("prefix=" + prefix);
+        }
+        if (delimiter != null) {
+            queryParts.add("delimiter=" + delimiter);
+        }
+        if (randomBoolean()) {
+            // test both ListObjects and ListObjectsV2 - they only differ in terms of pagination but S3HttpHandler doesn't do that
+            queryParts.add("list-type=2");
+        }
+        Randomness.shuffle(queryParts);
+
+        final var requestUri = "/bucket" + randomFrom("", "/") + (queryParts.isEmpty() ? "" : "?" + String.join("&", queryParts));
+        assertEquals("GET " + requestUri, new TestHttpResponse(RestStatus.OK, expectedResponse), handleRequest(handler, "GET", requestUri));
+    }
+
     public void testSimpleObjectOperations() {
         final var handler = new S3HttpHandler("bucket", "path");
 
         assertEquals(RestStatus.NOT_FOUND, handleRequest(handler, "GET", "/bucket/path/blob").status());
 
-        assertEquals(
-            new TestHttpResponse(RestStatus.OK, """
-                <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix></ListBucketResult>"""),
-            handleRequest(handler, "GET", "/bucket/?prefix=")
-        );
+        assertListObjectsResponse(handler, "", null, """
+            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
+            </ListBucketResult>""");
 
         final var body = randomAlphaOfLength(50);
         assertEquals(RestStatus.OK, handleRequest(handler, "PUT", "/bucket/path/blob", body).status());
         assertEquals(new TestHttpResponse(RestStatus.OK, body), handleRequest(handler, "GET", "/bucket/path/blob"));
 
-        assertEquals(new TestHttpResponse(RestStatus.OK, """
-            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix>\
+        assertListObjectsResponse(handler, "", null, """
+            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
             <Contents><Key>path/blob</Key><Size>50</Size></Contents>\
-            </ListBucketResult>"""), handleRequest(handler, "GET", "/bucket/?prefix="));
+            </ListBucketResult>""");
 
-        assertEquals(new TestHttpResponse(RestStatus.OK, """
-            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix>path/</Prefix>\
+        assertListObjectsResponse(handler, "path/", null, """
+            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix>path/</Prefix><IsTruncated>false</IsTruncated>\
             <Contents><Key>path/blob</Key><Size>50</Size></Contents>\
-            </ListBucketResult>"""), handleRequest(handler, "GET", "/bucket/?prefix=path/"));
+            </ListBucketResult>""");
 
-        assertEquals(
-            new TestHttpResponse(RestStatus.OK, """
-                <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix>path/other</Prefix></ListBucketResult>"""),
-            handleRequest(handler, "GET", "/bucket/?prefix=path/other")
-        );
+        assertListObjectsResponse(handler, "path/other", null, """
+            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix>path/other</Prefix><IsTruncated>false</IsTruncated>\
+            </ListBucketResult>""");
 
         assertEquals(RestStatus.OK, handleRequest(handler, "PUT", "/bucket/path/subpath1/blob", randomAlphaOfLength(50)).status());
         assertEquals(RestStatus.OK, handleRequest(handler, "PUT", "/bucket/path/subpath2/blob", randomAlphaOfLength(50)).status());
-        assertEquals(new TestHttpResponse(RestStatus.OK, """
-            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix>path/</Prefix>\
-            <Delimiter>/</Delimiter><Contents><Key>path/blob</Key><Size>50</Size></Contents>\
+        assertListObjectsResponse(handler, "path/", "/", """
+            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult>\
+            <Prefix>path/</Prefix><Delimiter>/</Delimiter><IsTruncated>false</IsTruncated>\
+            <Contents><Key>path/blob</Key><Size>50</Size></Contents>\
             <CommonPrefixes><Prefix>path/subpath1/</Prefix></CommonPrefixes>\
             <CommonPrefixes><Prefix>path/subpath2/</Prefix></CommonPrefixes>\
-            </ListBucketResult>"""), handleRequest(handler, "GET", "/bucket/?delimiter=/&prefix=path/"));
+            </ListBucketResult>""");
 
         assertEquals(RestStatus.OK, handleRequest(handler, "DELETE", "/bucket/path/blob").status());
         assertEquals(RestStatus.NO_CONTENT, handleRequest(handler, "DELETE", "/bucket/path/blob").status());
 
-        assertEquals(new TestHttpResponse(RestStatus.OK, """
-            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix>\
+        assertListObjectsResponse(handler, "", null, """
+            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
             <Contents><Key>path/subpath1/blob</Key><Size>50</Size></Contents>\
             <Contents><Key>path/subpath2/blob</Key><Size>50</Size></Contents>\
-            </ListBucketResult>"""), handleRequest(handler, "GET", "/bucket/?prefix="));
+            </ListBucketResult>""");
 
         assertEquals(RestStatus.OK, handleRequest(handler, "DELETE", "/bucket/path/subpath1/blob").status());
         assertEquals(RestStatus.OK, handleRequest(handler, "DELETE", "/bucket/path/subpath2/blob").status());
 
-        assertEquals(
-            new TestHttpResponse(RestStatus.OK, """
-                <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix></ListBucketResult>"""),
-            handleRequest(handler, "GET", "/bucket/?prefix=")
-        );
+        assertListObjectsResponse(handler, "", null, """
+            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
+            </ListBucketResult>""");
     }
 
     public void testGetWithBytesRange() {
@@ -216,10 +236,10 @@ public void testSingleMultipartUpload() {
                 </CompleteMultipartUpload>""", part1Etag, part2Etag))
         );
 
-        assertEquals(new TestHttpResponse(RestStatus.OK, """
-            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix>\
+        assertListObjectsResponse(handler, "", null, """
+            <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
             <Contents><Key>path/blob</Key><Size>100</Size></Contents>\
-            </ListBucketResult>"""), handleRequest(handler, "GET", "/bucket/?prefix="));
+            </ListBucketResult>""");
 
         assertEquals(new TestHttpResponse(RestStatus.OK, part1 + part2), handleRequest(handler, "GET", "/bucket/path/blob"));