Return ETag from S3 fixture GetObject API (#138514)

AWS S3 uses the `ETag` header to identify the object contents in various
API responses. `S3HttpHandler` doesn't today return this header from its
`GetObject` API, and its APIs which do return an `ETag` do not properly
conform to its spec (particularly, they are not surrounded by `"`
characters). This commit adds the missing response header to the
`GetObject` API, fixes its format, and uses SHA256 rather than MD5 to
compute the result.

Author: DaveCTurner

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java

@@ -32,7 +32,6 @@
 import org.elasticsearch.common.blobstore.OptionalBytesReference;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
@@ -430,7 +429,7 @@ public void testWriteLargeBlob() throws Exception {
                 assertThat(contentLength, anyOf(equalTo(lastPartSize), equalTo(bufferSize.getBytes())));
 
                 if (countDownUploads.decrementAndGet() % 2 == 0) {
-                    exchange.getResponseHeaders().add("ETag", getBase16MD5Digest(bytes));
+                    exchange.getResponseHeaders().add("ETag", S3HttpHandler.getEtagFromContents(bytes));
                     exchange.sendResponseHeaders(HttpStatus.SC_OK, -1);
                     exchange.close();
                     return;
@@ -529,7 +528,7 @@ public void testWriteLargeBlobStreaming() throws Exception {
 
                 if (counterUploads.incrementAndGet() % 2 == 0) {
                     bytesReceived.addAndGet(bytes.length());
-                    exchange.getResponseHeaders().add("ETag", getBase16MD5Digest(bytes));
+                    exchange.getResponseHeaders().add("ETag", S3HttpHandler.getEtagFromContents(bytes));
                     exchange.sendResponseHeaders(HttpStatus.SC_OK, -1);
                     exchange.close();
                     return;
@@ -1382,21 +1381,6 @@ public void handle(HttpExchange exchange) throws IOException {
         );
     }
 
-    private static String getBase16MD5Digest(BytesReference bytesReference) {
-        return MessageDigests.toHexString(MessageDigests.digest(bytesReference, MessageDigests.md5()));
-    }
-
-    public void testGetBase16MD5Digest() {
-        // from Wikipedia, see also org.elasticsearch.common.hash.MessageDigestsTests.testMd5
-        assertBase16MD5Digest("", "d41d8cd98f00b204e9800998ecf8427e");
-        assertBase16MD5Digest("The quick brown fox jumps over the lazy dog", "9e107d9d372bb6826bd81d3542a419d6");
-        assertBase16MD5Digest("The quick brown fox jumps over the lazy dog.", "e4d909c290d0fb1ca068ffaddf22cbd0");
-    }
-
-    private static void assertBase16MD5Digest(String input, String expectedDigestString) {
-        assertEquals(expectedDigestString, getBase16MD5Digest(new BytesArray(input)));
-    }
-
     @Override
     protected Matcher<Integer> getMaxRetriesMatcher(int maxRetries) {
         // some attempts make meaningful progress and do not count towards the max retry limit

test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java

@@ -84,6 +84,8 @@ public S3HttpHandler(final String bucket, @Nullable final String basePath) {
      */
     private static final Set<String> METHODS_HAVING_NO_REQUEST_BODY = Set.of("GET", "HEAD", "DELETE");
 
+    private static final String SHA_256_ETAG_PREFIX = "es-test-sha-256-";
+
     @Override
     public void handle(final HttpExchange exchange) throws IOException {
         // Remove custom query parameters before processing the request. This simulates how S3 ignores them.
@@ -322,6 +324,9 @@ public void handle(final HttpExchange exchange) throws IOException {
                     exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1);
                     return;
                 }
+
+                exchange.getResponseHeaders().add("ETag", getEtagFromContents(blob));
+
                 final String rangeHeader = exchange.getRequestHeaders().getFirst("Range");
                 if (rangeHeader == null) {
                     exchange.getResponseHeaders().add("Content-Type", "application/octet-stream");
@@ -413,6 +418,15 @@ private boolean updateBlobContents(HttpExchange exchange, String path, BytesRefe
         }
     }
 
+    /**
+     * Etags are opaque identifiers for the contents of an object.
+     *
+     * @see <a href="https://en.wikipedia.org/wiki/HTTP_ETag">HTTP ETag on Wikipedia</a>.
+     */
+    public static String getEtagFromContents(BytesReference blobContents) {
+        return '"' + SHA_256_ETAG_PREFIX + MessageDigests.toHexString(MessageDigests.digest(blobContents, MessageDigests.sha256())) + '"';
+    }
+
     public Map<String, BytesReference> blobs() {
         return blobs;
     }
@@ -490,7 +504,7 @@ private static Tuple<String, BytesReference> parseRequestBody(final HttpExchange
                     );
                 }
             }
-            return Tuple.tuple(MessageDigests.toHexString(MessageDigests.digest(bytesReference, MessageDigests.md5())), bytesReference);
+            return Tuple.tuple(getEtagFromContents(bytesReference), bytesReference);
         } catch (Exception e) {
             logger.error("exception in parseRequestBody", e);
             exchange.sendResponseHeaders(500, 0);

test/fixtures/s3-fixture/src/test/java/fixture/s3/S3HttpHandlerTests.java

@@ -83,9 +83,12 @@ public void testSimpleObjectOperations() {
             <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
             </ListBucketResult>""");
 
-        final var body = randomAlphaOfLength(50);
+        final var body = new BytesArray(randomAlphaOfLength(50).getBytes(StandardCharsets.UTF_8));
         assertEquals(RestStatus.OK, handleRequest(handler, "PUT", "/bucket/path/blob", body).status());
-        assertEquals(new TestHttpResponse(RestStatus.OK, body), handleRequest(handler, "GET", "/bucket/path/blob"));
+        assertEquals(
+            new TestHttpResponse(RestStatus.OK, body, addETag(S3HttpHandler.getEtagFromContents(body), TestHttpExchange.EMPTY_HEADERS)),
+            handleRequest(handler, "GET", "/bucket/path/blob")
+        );
 
         assertListObjectsResponse(handler, "", null, """
             <?xml version="1.0" encoding="UTF-8"?><ListBucketResult><Prefix></Prefix><IsTruncated>false</IsTruncated>\
@@ -135,39 +138,53 @@ public void testGetWithBytesRange() {
         final var blobBytes = randomBytesReference(256);
         assertEquals(RestStatus.OK, handleRequest(handler, "PUT", blobPath, blobBytes).status());
 
+        final var expectedEtag = S3HttpHandler.getEtagFromContents(blobBytes);
+
         assertEquals(
             "No Range",
-            new TestHttpResponse(RestStatus.OK, blobBytes, TestHttpExchange.EMPTY_HEADERS),
+            new TestHttpResponse(RestStatus.OK, blobBytes, addETag(expectedEtag, TestHttpExchange.EMPTY_HEADERS)),
             handleRequest(handler, "GET", blobPath)
         );
 
         var end = blobBytes.length() - 1;
         assertEquals(
             "Exact Range: bytes=0-" + end,
-            new TestHttpResponse(RestStatus.PARTIAL_CONTENT, blobBytes, contentRangeHeader(0, end, blobBytes.length())),
+            new TestHttpResponse(
+                RestStatus.PARTIAL_CONTENT,
+                blobBytes,
+                addETag(expectedEtag, contentRangeHeader(0, end, blobBytes.length()))
+            ),
             handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, bytesRangeHeader(0, end))
         );
 
         end = randomIntBetween(blobBytes.length() - 1, Integer.MAX_VALUE);
         assertEquals(
             "Larger Range: bytes=0-" + end,
-            new TestHttpResponse(RestStatus.PARTIAL_CONTENT, blobBytes, contentRangeHeader(0, blobBytes.length() - 1, blobBytes.length())),
+            new TestHttpResponse(
+                RestStatus.PARTIAL_CONTENT,
+                blobBytes,
+                addETag(expectedEtag, contentRangeHeader(0, blobBytes.length() - 1, blobBytes.length()))
+            ),
             handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, bytesRangeHeader(0, end))
         );
 
         var start = randomIntBetween(blobBytes.length(), Integer.MAX_VALUE - 1);
         end = randomIntBetween(start, Integer.MAX_VALUE);
         assertEquals(
             "Invalid Range: bytes=" + start + '-' + end,
-            new TestHttpResponse(RestStatus.REQUESTED_RANGE_NOT_SATISFIED, BytesArray.EMPTY, TestHttpExchange.EMPTY_HEADERS),
+            new TestHttpResponse(
+                RestStatus.REQUESTED_RANGE_NOT_SATISFIED,
+                BytesArray.EMPTY,
+                addETag(expectedEtag, TestHttpExchange.EMPTY_HEADERS)
+            ),
             handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, bytesRangeHeader(start, end))
         );
 
         start = randomIntBetween(2, Integer.MAX_VALUE - 1);
         end = randomIntBetween(0, start - 1);
         assertEquals(
             "Weird Valid Range: bytes=" + start + '-' + end,
-            new TestHttpResponse(RestStatus.OK, blobBytes, TestHttpExchange.EMPTY_HEADERS),
+            new TestHttpResponse(RestStatus.OK, blobBytes, addETag(expectedEtag, TestHttpExchange.EMPTY_HEADERS)),
             handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, bytesRangeHeader(start, end))
         );
 
@@ -179,7 +196,7 @@ public void testGetWithBytesRange() {
             new TestHttpResponse(
                 RestStatus.PARTIAL_CONTENT,
                 blobBytes.slice(start, length),
-                contentRangeHeader(start, end, blobBytes.length())
+                addETag(expectedEtag, contentRangeHeader(start, end, blobBytes.length()))
             ),
             handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, bytesRangeHeader(start, end))
         );
@@ -245,7 +262,15 @@ public void testSingleMultipartUpload() {
             <Contents><Key>path/blob</Key><Size>100</Size></Contents>\
             </ListBucketResult>""");
 
-        assertEquals(new TestHttpResponse(RestStatus.OK, part1 + part2), handleRequest(handler, "GET", "/bucket/path/blob"));
+        final var expectedContents = new BytesArray((part1 + part2).getBytes(StandardCharsets.UTF_8));
+        assertEquals(
+            new TestHttpResponse(
+                RestStatus.OK,
+                expectedContents,
+                addETag(S3HttpHandler.getEtagFromContents(expectedContents), TestHttpExchange.EMPTY_HEADERS)
+            ),
+            handleRequest(handler, "GET", "/bucket/path/blob")
+        );
 
         assertEquals(new TestHttpResponse(RestStatus.OK, """
             <?xml version='1.0' encoding='UTF-8'?>\
@@ -416,7 +441,11 @@ public void testPreventObjectOverwrite() throws InterruptedException {
         });
 
         assertEquals(
-            new TestHttpResponse(RestStatus.OK, successfulTasks.getFirst().body, TestHttpExchange.EMPTY_HEADERS),
+            new TestHttpResponse(
+                RestStatus.OK,
+                successfulTasks.getFirst().body,
+                addETag(S3HttpHandler.getEtagFromContents(successfulTasks.getFirst().body), TestHttpExchange.EMPTY_HEADERS)
+            ),
             handleRequest(handler, "GET", "/bucket/path/blob")
         );
     }
@@ -459,6 +488,20 @@ private static TestWriteTask createMultipartUploadTask(S3HttpHandler handler) {
         return multipartUploadTask;
     }
 
+    public void testGetETagFromContents() {
+        // empty-string value from Wikipedia, see also org.elasticsearch.common.hash.MessageDigestsTests.testSha256
+        assertETag("", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+        assertETag("The quick brown fox jumps over the lazy dog", "d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592");
+        assertETag("The quick brown fox jumps over the lazy cog", "e4c4d8f3bf76b692de791a173e05321150f7a345b46484fe427f6acc7ecc81be");
+    }
+
+    private static void assertETag(String input, String expectedHash) {
+        assertEquals(
+            "\"es-test-sha-256-" + expectedHash + '"',
+            S3HttpHandler.getEtagFromContents(new BytesArray(input.getBytes(StandardCharsets.UTF_8)))
+        );
+    }
+
     private static class TestWriteTask {
         final BytesReference body;
         final Runnable consumer;
@@ -562,6 +605,12 @@ private static Headers ifNoneMatchHeader() {
         return headers;
     }
 
+    private static Headers addETag(String eTag, Headers headers) {
+        final var newHeaders = new Headers(headers);
+        newHeaders.add("ETag", eTag);
+        return newHeaders;
+    }
+
     private static class TestHttpExchange extends HttpExchange {
 
         private static final Headers EMPTY_HEADERS = new Headers();