Upgrade to latest GCS SDK

docs/changelog/124062.yaml

@@ -0,0 +1,5 @@
+pr: 124062
+summary: Upgrade to repository-gcs to use com.google.cloud:google-cloud-storage-bom:2.50.0
+area: Snapshot/Restore
+type: upgrade
+issues: []

modules/repository-gcs/build.gradle

@@ -18,40 +18,44 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description = 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
-  classname ='org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
+  classname = 'org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
 }
 
 dependencies {
-  api 'com.google.cloud:google-cloud-storage:2.13.1'
-  api 'com.google.cloud:google-cloud-core:2.8.28'
-  api 'com.google.cloud:google-cloud-core-http:2.8.28'
-  runtimeOnly 'com.google.guava:guava:32.0.1-jre'
-  runtimeOnly 'com.google.guava:failureaccess:1.0.1'
-  api "commons-logging:commons-logging:${versions.commonslogging}"
-  api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
-  api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'com.google.api:api-common:2.3.1'
-  api 'com.google.api:gax:2.20.1'
-  api 'org.threeten:threetenbp:1.6.5'
-  api "com.google.protobuf:protobuf-java-util:${versions.protobuf}"
-  api "com.google.protobuf:protobuf-java:${versions.protobuf}"
-  api 'com.google.code.gson:gson:2.10'
-  api 'com.google.api.grpc:proto-google-common-protos:2.9.6'
-  api 'com.google.api.grpc:proto-google-iam-v1:1.6.2'
-  api 'com.google.auth:google-auth-library-credentials:1.11.0'
-  api 'com.google.auth:google-auth-library-oauth2-http:1.11.0'
-  api "com.google.oauth-client:google-oauth-client:${versions.google_oauth_client}"
-  api 'com.google.api-client:google-api-client:2.1.1'
-  api 'com.google.http-client:google-http-client:1.42.3'
-  api 'com.google.http-client:google-http-client-gson:1.42.3'
-  api 'com.google.http-client:google-http-client-appengine:1.42.3'
-  api 'com.google.http-client:google-http-client-jackson2:1.42.3'
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api 'com.google.api:gax-httpjson:0.105.1'
-  api 'io.grpc:grpc-context:1.49.2'
-  api 'io.opencensus:opencensus-api:0.31.1'
-  api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
-  api 'com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0'
+  // dependencies consistent with 'com.google.cloud:google-cloud-storage-bom:2.50.0'
+  implementation 'com.google.cloud:google-cloud-storage:2.50.0'
+  implementation 'com.google.cloud:google-cloud-core:2.53.1'
+  implementation 'com.google.cloud:google-cloud-core-http:2.53.1'
+  runtimeOnly 'com.google.guava:guava:33.4.0-jre'
+  runtimeOnly 'com.google.guava:failureaccess:1.0.2'
+  runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}" // 2.0.16 in bom
+  runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}" // 1.18.0 in bom
+  implementation 'com.google.api:api-common:2.46.1'
+  implementation 'com.google.api:gax:2.63.1'
+  implementation 'org.threeten:threetenbp:1.7.0'
+  runtimeOnly "com.google.protobuf:protobuf-java-util:${versions.protobuf}" // 3.25.5 in bom
+  runtimeOnly "com.google.protobuf:protobuf-java:${versions.protobuf}"
+  runtimeOnly 'com.google.code.gson:gson:2.12.1'
+  runtimeOnly 'com.google.api.grpc:proto-google-common-protos:2.54.1'
+  runtimeOnly 'com.google.api.grpc:proto-google-iam-v1:1.49.1'
+  implementation 'com.google.auth:google-auth-library-credentials:1.33.1'
+  implementation 'com.google.auth:google-auth-library-oauth2-http:1.33.1'
+  runtimeOnly "com.google.oauth-client:google-oauth-client:${versions.google_oauth_client}" // 1.37.0 in bom
+  implementation 'com.google.api-client:google-api-client:2.7.2'
+  implementation 'com.google.http-client:google-http-client:1.46.3'
+  runtimeOnly 'com.google.http-client:google-http-client-gson:1.46.3'
+  runtimeOnly 'com.google.http-client:google-http-client-appengine:1.46.3'
+  runtimeOnly 'com.google.http-client:google-http-client-jackson2:1.46.3'
+  runtimeOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" // 2.18.2 in bom
+  runtimeOnly 'com.google.api:gax-httpjson:2.63.1'
+  runtimeOnly 'io.opencensus:opencensus-api:0.31.1'
+  runtimeOnly 'io.opencensus:opencensus-contrib-http-util:0.31.1'
+  implementation 'com.google.apis:google-api-services-storage:v1-rev20250224-2.0.0'
+  implementation 'org.checkerframework:checker-qual:3.49.0'
+  runtimeOnly 'io.opentelemetry:opentelemetry-api:1.47.0'
+  runtimeOnly 'io.opentelemetry:opentelemetry-context:1.47.0'
+  runtimeOnly 'com.google.api.grpc:proto-google-cloud-storage-v2:2.50.0'
+  runtimeOnly 'io.grpc:grpc-api:1.70.0'
 
   testImplementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   testImplementation "org.apache.httpcomponents:httpcore:${versions.httpcore}"
@@ -62,7 +66,7 @@ dependencies {
 
 restResources {
   restApi {
-    include '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
+    include '_common', 'cluster', 'nodes', 'snapshot', 'indices', 'index', 'bulk', 'count'
   }
 }
 
@@ -123,11 +127,6 @@ tasks.named("thirdPartyAudit").configure {
     'com.google.appengine.api.urlfetch.HTTPResponse',
     'com.google.appengine.api.urlfetch.URLFetchService',
     'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
-    // commons-logging optional dependencies
-    'org.apache.avalon.framework.logger.Logger',
-    'org.apache.log.Hierarchy',
-    'org.apache.log.Logger',
-    'javax.jms.Message',
 
     // optional apache http client dependencies
     'org.apache.http.ConnectionReuseStrategy',
@@ -171,19 +170,59 @@ tasks.named("thirdPartyAudit").configure {
     'org.apache.http.protocol.HttpProcessor',
     'org.apache.http.protocol.HttpRequestExecutor',
 
-    // commons-logging provided dependencies
-    'javax.servlet.ServletContextEvent',
-    'javax.servlet.ServletContextListener'
+    // grpc/proto stuff
+    'com.google.api.gax.grpc.GrpcCallContext',
+    'com.google.api.gax.grpc.GrpcCallSettings',
+    'com.google.api.gax.grpc.GrpcCallSettings$Builder',
+    'com.google.api.gax.grpc.GrpcInterceptorProvider',
+    'com.google.api.gax.grpc.GrpcStatusCode',
+    'com.google.api.gax.grpc.GrpcStubCallableFactory',
+    'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider',
+    'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider$Builder',
+    'com.google.cloud.grpc.GrpcTransportOptions',
+    'com.google.cloud.grpc.GrpcTransportOptions$Builder',
+    'com.google.cloud.opentelemetry.metric.GoogleCloudMetricExporter',
+    'com.google.cloud.opentelemetry.metric.MetricConfiguration',
+    'com.google.cloud.opentelemetry.metric.MetricConfiguration$Builder',
+    'com.google.storage.v2.StorageClient',
+    'com.google.storage.v2.StorageClient$ListBucketsPagedResponse',
+    'com.google.storage.v2.StorageSettings',
+    'com.google.storage.v2.StorageSettings$Builder',
+    'com.google.storage.v2.stub.GrpcStorageStub',
+    'com.google.storage.v2.stub.StorageStubSettings',
+    // opentelemetry implementation stuff
+    'io.grpc.opentelemetry.GrpcOpenTelemetry',
+    'io.grpc.opentelemetry.GrpcOpenTelemetry$Builder',
+    'io.grpc.protobuf.ProtoUtils',
+    'io.opentelemetry.contrib.gcp.resource.GCPResourceProvider',
+    'io.opentelemetry.sdk.OpenTelemetrySdk',
+    'io.opentelemetry.sdk.OpenTelemetrySdkBuilder',
+    'io.opentelemetry.sdk.common.CompletableResultCode',
+    'io.opentelemetry.sdk.common.export.MemoryMode',
+    'io.opentelemetry.sdk.metrics.Aggregation',
+    'io.opentelemetry.sdk.metrics.InstrumentSelector',
+    'io.opentelemetry.sdk.metrics.InstrumentSelectorBuilder',
+    'io.opentelemetry.sdk.metrics.InstrumentType',
+    'io.opentelemetry.sdk.metrics.SdkMeterProvider',
+    'io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder',
+    'io.opentelemetry.sdk.metrics.View',
+    'io.opentelemetry.sdk.metrics.ViewBuilder',
+    'io.opentelemetry.sdk.metrics.data.AggregationTemporality',
+    'io.opentelemetry.sdk.metrics.export.DefaultAggregationSelector',
+    'io.opentelemetry.sdk.metrics.export.MetricExporter',
+    'io.opentelemetry.sdk.metrics.export.PeriodicMetricReader',
+    'io.opentelemetry.sdk.metrics.export.PeriodicMetricReaderBuilder',
+    'io.opentelemetry.sdk.resources.Resource',
   )
 
 
-  if(buildParams.graalVmRuntime == false) {
+  if (buildParams.graalVmRuntime == false) {
     ignoreMissingClasses(
-        'org.graalvm.nativeimage.hosted.Feature',
-        'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess',
-        'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess',
-        'org.graalvm.nativeimage.hosted.Feature$FeatureAccess',
-        'org.graalvm.nativeimage.hosted.RuntimeReflection'
+      'org.graalvm.nativeimage.hosted.Feature',
+      'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess',
+      'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess',
+      'org.graalvm.nativeimage.hosted.Feature$FeatureAccess',
+      'org.graalvm.nativeimage.hosted.RuntimeReflection'
     )
   }
 }
@@ -213,7 +252,7 @@ Map<String, Object> expansions = [
 
 tasks.named("processYamlRestTestResources").configure {
   inputs.properties(expansions)
-  filter("tokens" : expansions, ReplaceTokens.class)
+  filter("tokens": expansions, ReplaceTokens.class)
 }
 
 tasks.named("internalClusterTest").configure {

modules/repository-gcs/licenses/checker-qual-LICENSE.txt

@@ -0,0 +1,22 @@
+Checker Framework qualifiers
+Copyright 2004-present by the Checker Framework developers
+
+MIT License:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

modules/repository-gcs/licenses/log4j-LICENSE.txt → modules/repository-gcs/licenses/opentelemetry-context-LICENSE.txt

@@ -1,4 +1,3 @@
-
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -187,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 1999-2005 The Apache Software Foundation
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -200,3 +199,4 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+

modules/repository-gcs/licenses/slf4j-api-LICENSE.txt

@@ -0,0 +1,21 @@
+Copyright (c) 2004-2014 QOS.ch
+All rights reserved.
+
+Permission is hereby granted, free  of charge, to any person obtaining
+a  copy  of this  software  and  associated  documentation files  (the
+"Software"), to  deal in  the Software without  restriction, including
+without limitation  the rights to  use, copy, modify,  merge, publish,
+distribute,  sublicense, and/or sell  copies of  the Software,  and to
+permit persons to whom the Software  is furnished to do so, subject to
+the following conditions:
+
+The  above  copyright  notice  and  this permission  notice  shall  be
+included in all copies or substantial portions of the Software.
+
+THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
+EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
+MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java

@@ -75,9 +75,9 @@
 class GoogleCloudStorageBlobStore implements BlobStore {
 
     /**
-     * see com.google.cloud.BaseWriteChannel#DEFAULT_CHUNK_SIZE
+     * see {@link com.google.cloud.storage.BaseStorageWriteChannel#chunkSize}
      */
-    static final int SDK_DEFAULT_CHUNK_SIZE = 60 * 256 * 1024;
+    static final int SDK_DEFAULT_CHUNK_SIZE = Math.toIntExact(ByteSizeValue.ofMb(16).getBytes());
 
     private static final Logger logger = LogManager.getLogger(GoogleCloudStorageBlobStore.class);
 
@@ -652,7 +652,17 @@ private static final class WritableBlobChannel implements WritableByteChannel {
         @SuppressForbidden(reason = "channel is based on a socket")
         @Override
         public int write(final ByteBuffer src) throws IOException {
-            return SocketAccess.doPrivilegedIOException(() -> channel.write(src));
+            try {
+                return SocketAccess.doPrivilegedIOException(() -> channel.write(src));
+            } catch (IOException e) {
+                // BaseStorageWriteChannel#write wraps StorageException in an IOException, but BaseStorageWriteChannel#close
+                // does not, if we unwrap StorageExceptions here, it simplifies our retry-on-gone logic
+                final StorageException storageException = (StorageException) ExceptionsHelper.unwrap(e, StorageException.class);
+                if (storageException != null) {
+                    throw storageException;
+                }
+                throw e;
+            }
         }
 
         @Override

modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java

@@ -41,6 +41,7 @@
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.Proxy;
+import java.net.SocketException;
 import java.net.URI;
 import java.net.URL;
 import java.net.UnknownHostException;
@@ -275,6 +276,10 @@ protected StorageRetryStrategy getRetryStrategy() {
                 if (ExceptionsHelper.unwrap(prevThrowable, UnknownHostException.class) != null) {
                     return true;
                 }
+                // Also retry on `SocketException`s
+                if (ExceptionsHelper.unwrap(prevThrowable, SocketException.class) != null) {
+                    return true;
+                }
                 return delegate.shouldRetry(prevThrowable, prevResponse);
             }
         );

modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java

@@ -21,6 +21,7 @@
 import com.sun.net.httpserver.HttpHandler;
 
 import org.apache.http.HttpStatus;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.BackoffPolicy;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
@@ -327,8 +328,7 @@ public void testWriteBlobWithReadTimeouts() {
     }
 
     public void testWriteLargeBlob() throws IOException {
-        // See {@link BaseWriteChannel#DEFAULT_CHUNK_SIZE}
-        final int defaultChunkSize = 60 * 256 * 1024;
+        final int defaultChunkSize = GoogleCloudStorageBlobStore.SDK_DEFAULT_CHUNK_SIZE;
         final int nbChunks = randomIntBetween(3, 5);
         final int lastChunkSize = randomIntBetween(1, defaultChunkSize - 1);
         final int totalChunks = nbChunks + 1;
@@ -412,13 +412,16 @@ public void testWriteLargeBlob() throws IOException {
                     }
                 }
 
-                final String range = exchange.getRequestHeaders().getFirst("Content-Range");
-                assertTrue(Strings.hasLength(range));
+                final String contentRangeHeaderValue = exchange.getRequestHeaders().getFirst("Content-Range");
+                final HttpHeaderParser.ContentRange contentRange = HttpHeaderParser.parseContentRangeHeader(contentRangeHeaderValue);
+                assertNotNull("Invalid content range header: " + contentRangeHeaderValue, contentRange);
 
-                if (range.equals("bytes */*")) {
+                if (contentRange.hasRange() == false) {
+                    // Content-Range: */... is a status check
+                    // https://cloud.google.com/storage/docs/performing-resumable-uploads#status-check
                     final int receivedSoFar = bytesReceived.get();
                     if (receivedSoFar > 0) {
-                        exchange.getResponseHeaders().add("Range", Strings.format("bytes=0-%d", receivedSoFar));
+                        exchange.getResponseHeaders().add("Range", Strings.format("bytes=0-%s", receivedSoFar));
                     }
                     exchange.getResponseHeaders().add("Content-Length", "0");
                     exchange.sendResponseHeaders(308 /* Resume Incomplete */, -1);
@@ -429,27 +432,33 @@ public void testWriteLargeBlob() throws IOException {
 
                     assertThat(Math.toIntExact(requestBody.length()), anyOf(equalTo(defaultChunkSize), equalTo(lastChunkSize)));
 
-                    final HttpHeaderParser.ContentRange contentRange = HttpHeaderParser.parseContentRangeHeader(range);
                     final int rangeStart = Math.toIntExact(contentRange.start());
                     final int rangeEnd = Math.toIntExact(contentRange.end());
                     assertThat(rangeEnd + 1 - rangeStart, equalTo(Math.toIntExact(requestBody.length())));
                     assertThat(new BytesArray(data, rangeStart, rangeEnd - rangeStart + 1), is(requestBody));
                     bytesReceived.updateAndGet(existing -> Math.max(existing, rangeEnd));
 
                     if (contentRange.size() != null) {
+                        exchange.getResponseHeaders().add("x-goog-stored-content-length", String.valueOf(bytesReceived.get() + 1));
                         exchange.sendResponseHeaders(RestStatus.OK.getStatus(), -1);
                         return;
                     } else {
-                        exchange.getResponseHeaders().add("Range", Strings.format("bytes=%d/%d", rangeStart, rangeEnd));
+                        exchange.getResponseHeaders().add("Range", Strings.format("bytes=%s-%s", rangeStart, rangeEnd));
                         exchange.getResponseHeaders().add("Content-Length", "0");
                         exchange.sendResponseHeaders(308 /* Resume Incomplete */, -1);
                         return;
                     }
                 }
+            } else {
+                ExceptionsHelper.maybeDieOnAnotherThread(
+                    new AssertionError("Unexpected request" + exchange.getRequestMethod() + " " + exchange.getRequestURI())
+                );
             }
 
             if (randomBoolean()) {
                 exchange.sendResponseHeaders(HttpStatus.SC_INTERNAL_SERVER_ERROR, -1);
+            } else {
+                logger.warn("Closing connection without response");
             }
         }));