Fix GCS tests broken by idempotency token (#126972)

Author: nicktindall

modules/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java

@@ -343,7 +343,15 @@ protected String requestUniqueId(HttpExchange exchange) {
             }
 
             if (exchange.getRequestHeaders().containsKey(IDEMPOTENCY_TOKEN)) {
-                return exchange.getRequestHeaders().getFirst(IDEMPOTENCY_TOKEN);
+                String idempotencyToken = exchange.getRequestHeaders().getFirst(IDEMPOTENCY_TOKEN);
+                // In the event of a resumable retry, the GCS client uses the same idempotency token for
+                // the retry status check and the subsequent retries.
+                // Including the range header allows us to disambiguate between the requests
+                // see https://github.com/googleapis/java-storage/issues/3040
+                if (exchange.getRequestHeaders().containsKey("Content-Range")) {
+                    idempotencyToken += " " + exchange.getRequestHeaders().getFirst("Content-Range");
+                }
+                return idempotencyToken;
             }
 
             final String range = exchange.getRequestHeaders().getFirst("Content-Range");

muted-tests.yml

@@ -405,18 +405,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/inference_crud/Test delete given unused trained model}
   issue: https://github.com/elastic/elasticsearch/issues/126881
-- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
-  method: testRequestStats
-  issue: https://github.com/elastic/elasticsearch/issues/126902
-- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
-  method: testWriteReadLarge
-  issue: https://github.com/elastic/elasticsearch/issues/126903
-- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
-  method: testWriteFileMultipleOfChunkSize
-  issue: https://github.com/elastic/elasticsearch/issues/126908
-- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
-  method: testSnapshotWithLargeSegmentFiles
-  issue: https://github.com/elastic/elasticsearch/issues/126909
 - class: org.elasticsearch.index.engine.CompletionStatsCacheTests
   method: testCompletionStatsCache
   issue: https://github.com/elastic/elasticsearch/issues/126910