Upgrade to latest GCS SDK

modules/repository-gcs/build.gradle

@@ -18,40 +18,49 @@ apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description = 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
-  classname ='org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
+  classname = 'org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
 }
 
 dependencies {
-  api 'com.google.cloud:google-cloud-storage:2.13.1'
-  api 'com.google.cloud:google-cloud-core:2.8.28'
-  api 'com.google.cloud:google-cloud-core-http:2.8.28'
-  runtimeOnly 'com.google.guava:guava:32.0.1-jre'
-  runtimeOnly 'com.google.guava:failureaccess:1.0.1'
-  api "commons-logging:commons-logging:${versions.commonslogging}"
-  api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
-  api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'com.google.api:api-common:2.3.1'
-  api 'com.google.api:gax:2.20.1'
-  api 'org.threeten:threetenbp:1.6.5'
-  api "com.google.protobuf:protobuf-java-util:${versions.protobuf}"
+  //api platform('com.google.cloud:libraries-bom:26.55.0')
+  api 'com.google.cloud:google-cloud-storage:2.48.2'
+  api 'com.google.cloud:google-cloud-core:2.51.0'
+  api 'com.google.cloud:google-cloud-core-http:2.51.0'
+  runtimeOnly 'com.google.guava:guava:33.4.0-jre'
+  runtimeOnly 'com.google.guava:failureaccess:1.0.2'
+  // api "commons-logging:commons-logging:${versions.commonslogging}"
+  // api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
+  api "commons-codec:commons-codec:${versions.commonscodec}" // should be 1.18.0
+  api 'com.google.api:api-common:2.44.0'
+  api 'com.google.api:gax:2.61.0'
+  api 'org.threeten:threetenbp:1.7.0'
+  api "com.google.protobuf:protobuf-java-util:${versions.protobuf}" // should be 4.29
   api "com.google.protobuf:protobuf-java:${versions.protobuf}"
-  api 'com.google.code.gson:gson:2.10'
-  api 'com.google.api.grpc:proto-google-common-protos:2.9.6'
-  api 'com.google.api.grpc:proto-google-iam-v1:1.6.2'
-  api 'com.google.auth:google-auth-library-credentials:1.11.0'
-  api 'com.google.auth:google-auth-library-oauth2-http:1.11.0'
-  api "com.google.oauth-client:google-oauth-client:${versions.google_oauth_client}"
-  api 'com.google.api-client:google-api-client:2.1.1'
-  api 'com.google.http-client:google-http-client:1.42.3'
-  api 'com.google.http-client:google-http-client-gson:1.42.3'
-  api 'com.google.http-client:google-http-client-appengine:1.42.3'
-  api 'com.google.http-client:google-http-client-jackson2:1.42.3'
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api 'com.google.api:gax-httpjson:0.105.1'
-  api 'io.grpc:grpc-context:1.49.2'
+  api 'com.google.code.gson:gson:2.12.1'
+  api 'com.google.api.grpc:proto-google-common-protos:2.52.0'
+  api 'com.google.api.grpc:proto-google-iam-v1:1.47.0'
+  api 'com.google.auth:google-auth-library-credentials:1.32.1'
+  api 'com.google.auth:google-auth-library-oauth2-http:1.32.1'
+  api "com.google.oauth-client:google-oauth-client:${versions.google_oauth_client}" // should be 1.37.0
+  api 'com.google.api-client:google-api-client:2.7.2'
+  api 'com.google.http-client:google-http-client:1.46.1'
+  api 'com.google.http-client:google-http-client-gson:1.46.1'
+  api 'com.google.http-client:google-http-client-appengine:1.46.1'
+  api 'com.google.http-client:google-http-client-jackson2:1.46.1'
+  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" // should be 2.18.2
+  api 'com.google.api:gax-httpjson:2.61.0'
+  //api 'io.grpc:grpc-context:1.69.0'
   api 'io.opencensus:opencensus-api:0.31.1'
   api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
-  api 'com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0'
+  api 'com.google.apis:google-api-services-storage:v1-rev20241206-2.0.0'
+  implementation 'org.checkerframework:checker-qual:3.49.0'
+  implementation 'io.opentelemetry:opentelemetry-api:1.47.0'
+  implementation 'io.opentelemetry:opentelemetry-context:1.47.0'
+  /// ???
+  implementation 'com.google.api.grpc:proto-google-cloud-storage-v2:2.48.2'
+  //implementation 'com.google.api.grpc:grpc-google-cloud-storage-v2:2.48.2'
+  //implementation 'com.google.api.grpc:gapic-google-cloud-storage-v2:2.48.2'
+  implementation 'io.grpc:grpc-api:1.69.0'
 
   testImplementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   testImplementation "org.apache.httpcomponents:httpcore:${versions.httpcore}"
@@ -62,7 +71,7 @@ dependencies {
 
 restResources {
   restApi {
-    include '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
+    include '_common', 'cluster', 'nodes', 'snapshot', 'indices', 'index', 'bulk', 'count'
   }
 }
 
@@ -123,11 +132,6 @@ tasks.named("thirdPartyAudit").configure {
     'com.google.appengine.api.urlfetch.HTTPResponse',
     'com.google.appengine.api.urlfetch.URLFetchService',
     'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
-    // commons-logging optional dependencies
-    'org.apache.avalon.framework.logger.Logger',
-    'org.apache.log.Hierarchy',
-    'org.apache.log.Logger',
-    'javax.jms.Message',
 
     // optional apache http client dependencies
     'org.apache.http.ConnectionReuseStrategy',
@@ -171,19 +175,58 @@ tasks.named("thirdPartyAudit").configure {
     'org.apache.http.protocol.HttpProcessor',
     'org.apache.http.protocol.HttpRequestExecutor',
 
-    // commons-logging provided dependencies
-    'javax.servlet.ServletContextEvent',
-    'javax.servlet.ServletContextListener'
+    // grpc/proto stuff
+    'com.google.api.gax.grpc.GrpcCallContext',
+    'com.google.api.gax.grpc.GrpcCallSettings',
+    'com.google.api.gax.grpc.GrpcCallSettings$Builder',
+    'com.google.api.gax.grpc.GrpcInterceptorProvider',
+    'com.google.api.gax.grpc.GrpcStatusCode',
+    'com.google.api.gax.grpc.GrpcStubCallableFactory',
+    'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider',
+    'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider$Builder',
+    'com.google.cloud.grpc.GrpcTransportOptions',
+    'com.google.cloud.grpc.GrpcTransportOptions$Builder',
+    'com.google.cloud.opentelemetry.metric.GoogleCloudMetricExporter',
+    'com.google.cloud.opentelemetry.metric.MetricConfiguration',
+    'com.google.cloud.opentelemetry.metric.MetricConfiguration$Builder',
+    'com.google.storage.v2.StorageClient',
+    'com.google.storage.v2.StorageClient$ListBucketsPagedResponse',
+    'com.google.storage.v2.StorageSettings',
+    'com.google.storage.v2.StorageSettings$Builder',
+    'com.google.storage.v2.stub.GrpcStorageStub',
+    'com.google.storage.v2.stub.StorageStubSettings',
+    'io.grpc.opentelemetry.GrpcOpenTelemetry',
+    'io.grpc.opentelemetry.GrpcOpenTelemetry$Builder',
+    'io.grpc.protobuf.ProtoUtils',
+    'io.opentelemetry.contrib.gcp.resource.GCPResourceProvider',
+    'io.opentelemetry.sdk.OpenTelemetrySdk',
+    'io.opentelemetry.sdk.OpenTelemetrySdkBuilder',
+    'io.opentelemetry.sdk.common.CompletableResultCode',
+    'io.opentelemetry.sdk.common.export.MemoryMode',
+    'io.opentelemetry.sdk.metrics.Aggregation',
+    'io.opentelemetry.sdk.metrics.InstrumentSelector',
+    'io.opentelemetry.sdk.metrics.InstrumentSelectorBuilder',
+    'io.opentelemetry.sdk.metrics.InstrumentType',
+    'io.opentelemetry.sdk.metrics.SdkMeterProvider',
+    'io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder',
+    'io.opentelemetry.sdk.metrics.View',
+    'io.opentelemetry.sdk.metrics.ViewBuilder',
+    'io.opentelemetry.sdk.metrics.data.AggregationTemporality',
+    'io.opentelemetry.sdk.metrics.export.DefaultAggregationSelector',
+    'io.opentelemetry.sdk.metrics.export.MetricExporter',
+    'io.opentelemetry.sdk.metrics.export.PeriodicMetricReader',
+    'io.opentelemetry.sdk.metrics.export.PeriodicMetricReaderBuilder',
+    'io.opentelemetry.sdk.resources.Resource',
   )
 
 
-  if(buildParams.graalVmRuntime == false) {
+  if (buildParams.graalVmRuntime == false) {
     ignoreMissingClasses(
-        'org.graalvm.nativeimage.hosted.Feature',
-        'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess',
-        'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess',
-        'org.graalvm.nativeimage.hosted.Feature$FeatureAccess',
-        'org.graalvm.nativeimage.hosted.RuntimeReflection'
+      'org.graalvm.nativeimage.hosted.Feature',
+      'org.graalvm.nativeimage.hosted.Feature$BeforeAnalysisAccess',
+      'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess',
+      'org.graalvm.nativeimage.hosted.Feature$FeatureAccess',
+      'org.graalvm.nativeimage.hosted.RuntimeReflection'
     )
   }
 }
@@ -213,7 +256,7 @@ Map<String, Object> expansions = [
 
 tasks.named("processYamlRestTestResources").configure {
   inputs.properties(expansions)
-  filter("tokens" : expansions, ReplaceTokens.class)
+  filter("tokens": expansions, ReplaceTokens.class)
 }
 
 tasks.named("internalClusterTest").configure {

modules/repository-gcs/licenses/checker-qual-LICENSE.txt

@@ -0,0 +1,22 @@
+Checker Framework qualifiers
+Copyright 2004-present by the Checker Framework developers
+
+MIT License:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

modules/repository-gcs/licenses/log4j-LICENSE.txt → modules/repository-gcs/licenses/opentelemetry-context-LICENSE.txt

@@ -1,4 +1,3 @@
-
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -187,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 1999-2005 The Apache Software Foundation
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -200,3 +199,4 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
+

test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java

@@ -253,7 +253,7 @@ public void handle(final HttpExchange exchange) throws IOException {
                 if (updateResponse.rangeHeader() != null) {
                     exchange.getResponseHeaders().add("Range", updateResponse.rangeHeader().headerString());
                 }
-                exchange.getResponseHeaders().add("Content-Length", "0");
+                exchange.getResponseHeaders().add("x-goog-stored-content-length", String.valueOf(updateResponse.storedContentLength()));
                 exchange.sendResponseHeaders(updateResponse.statusCode(), -1);
             } else {
                 exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1);

test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java

@@ -117,10 +117,16 @@ UpdateResponse updateResumableUpload(String uploadId, HttpHeaderParser.ContentRa
             if (contentRange.hasRange() == false) {
                 // Content-Range: */... is a status check https://cloud.google.com/storage/docs/performing-resumable-uploads#status-check
                 if (existing.completed) {
-                    updateResponse.set(new UpdateResponse(RestStatus.OK.getStatus(), calculateRangeHeader(blobs.get(existing.path))));
+                    updateResponse.set(
+                        new UpdateResponse(
+                            RestStatus.OK.getStatus(),
+                            calculateRangeHeader(blobs.get(existing.path)),
+                            existing.contents.length()
+                        )
+                    );
                 } else {
                     final HttpHeaderParser.Range range = calculateRangeHeader(existing);
-                    updateResponse.set(new UpdateResponse(RESUME_INCOMPLETE, range));
+                    updateResponse.set(new UpdateResponse(RESUME_INCOMPLETE, range, existing.contents.length()));
                 }
                 return existing;
             } else {
@@ -146,11 +152,11 @@ UpdateResponse updateResumableUpload(String uploadId, HttpHeaderParser.ContentRa
                 // We just received the last chunk, update the blob and remove the resumable upload from the map
                 if (contentRange.hasSize() && updatedContent.length() == contentRange.size()) {
                     updateBlob(existing.path(), existing.ifGenerationMatch, updatedContent);
-                    updateResponse.set(new UpdateResponse(RestStatus.OK.getStatus(), null));
+                    updateResponse.set(new UpdateResponse(RestStatus.OK.getStatus(), null, updatedContent.length()));
                     return existing.update(BytesArray.EMPTY, true);
                 }
                 final ResumableUpload updated = existing.update(updatedContent, false);
-                updateResponse.set(new UpdateResponse(RESUME_INCOMPLETE, calculateRangeHeader(updated)));
+                updateResponse.set(new UpdateResponse(RESUME_INCOMPLETE, calculateRangeHeader(updated), updated.contents.length()));
                 return updated;
             }
         });
@@ -166,7 +172,7 @@ private static HttpHeaderParser.Range calculateRangeHeader(BlobVersion blob) {
         return blob.contents.length() > 0 ? new HttpHeaderParser.Range(0, blob.contents.length() - 1) : null;
     }
 
-    record UpdateResponse(int statusCode, HttpHeaderParser.Range rangeHeader) {}
+    record UpdateResponse(int statusCode, HttpHeaderParser.Range rangeHeader, long storedContentLength) {}
 
     void deleteBlob(String path) {
         blobs.remove(path);