HADOOP-19343: Add native support for GCS connector

Closes #7869

Co-authored-by: Chris Nauroth <[email protected]>
Signed-off-by: Shilun Fan <[email protected]>
Signed-off-by: Steve Loughran <[email protected]>
Signed-off-by: Chris Nauroth <[email protected]>
Signed-off-by: Mukund Thakur <[email protected]>
Signed-off-by: Ayush Saxena <[email protected]>
Reviewed-by: Arunkumar Chacko <[email protected]>
Reviewed-by: Cheng Pan <[email protected]>

Author: arunkumarchacko

LICENSE-binary

@@ -229,6 +229,25 @@ com.fasterxml.woodstox:woodstox-core:5.4.0
 com.github.ben-manes.caffeine:caffeine:2.9.3
 com.github.davidmoten:rxjava-extras:0.8.0.17
 com.github.stephenc.jcip:jcip-annotations:1.0-1
+com.google.api:api-common:2.47.2
+com.google.api-client:google-api-client:2.7.2
+com.google.api:gax:2.64.2
+com.google.api:gax-grpc:2.64.2
+com.google.api:gax-httpjson:2.64.2
+com.google.api.grpc:gapic-google-cloud-storage-v2:2.52.0
+com.google.api.grpc:grpc-google-cloud-storage-v2:2.52.0
+com.google.api.grpc:proto-google-cloud-monitoring-v3:3.52.0
+com.google.api.grpc:proto-google-cloud-storage-v2:2.52.0
+com.google.api.grpc:proto-google-common-protos:2.55.2
+com.google.api.grpc:proto-google-iam-v1:1.50.2
+com.google.apis:google-api-services-storage:v1-rev20250420-2.0.0
+com.google.auto.value:auto-value-annotations:1.11.0
+com.google.cloud:google-cloud-core-grpc:2.54.2
+com.google.cloud:google-cloud-core-http:2.54.2
+com.google.cloud:google-cloud-core:2.54.2
+com.google.cloud:google-cloud-monitoring:3.52.0
+com.google.cloud.opentelemetry:exporter-metrics:0.33.0
+com.google.errorprone:error_prone_annotations:2.36.0
 com.google:guice:5.1.0
 com.google:guice-servlet:5.1.0
 com.google.api.grpc:proto-google-common-protos:1.0.0
@@ -237,9 +256,18 @@ com.google.errorprone:error_prone_annotations:2.5.1
 com.google.j2objc:j2objc-annotations:1.3
 com.google.json-simple:json-simple:1.1.1
 com.google.guava:failureaccess:1.0
+com.google.guava:failureaccess:1.0.2
 com.google.guava:guava:20.0
 com.google.guava:guava:32.0.1-jre
+com.google.guava:guava:33.1.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
+com.google.http-client:google-http-client-apache-v2:1.46.3
+com.google.http-client:google-http-client-appengine:1.46.3
+com.google.http-client:google-http-client-gson:1.46.3
+com.google.http-client:google-http-client-jackson2:1.46.3
+com.google.http-client:google-http-client:1.46.3
+com.google.j2objc:j2objc-annotations:3.0.0
+com.google.oauth-client:google-oauth-client:1.37.0
 com.microsoft.azure:azure-storage:7.0.0
 com.nimbusds:nimbus-jose-jwt:9.37.2
 com.zaxxer:HikariCP:4.0.3
@@ -251,13 +279,29 @@ commons-daemon:commons-daemon:1.0.13
 commons-io:commons-io:2.16.1
 commons-net:commons-net:3.9.0
 de.ruedigermoeller:fst:2.50
+io.grpc:grpc-alts:1.70.0
 io.grpc:grpc-api:1.69.0
+io.grpc:grpc-api:1.70.0
+io.grpc:grpc-auth:1.70.0
 io.grpc:grpc-context:1.69.0
+io.grpc:grpc-context:1.70.0
 io.grpc:grpc-core:1.69.0
+io.grpc:grpc-core:1.70.0
+io.grpc:grpc-googleapis:1.70.0
+io.grpc:grpc-grpclb:1.70.0
+io.grpc:grpc-inprocess:1.70.0
 io.grpc:grpc-netty:1.69.0
+io.grpc:grpc-netty-shaded:1.70.0
+io.grpc:grpc-opentelemetry:1.70.0
 io.grpc:grpc-protobuf:1.69.0
+io.grpc:grpc-protobuf:1.70.0
 io.grpc:grpc-protobuf-lite:1.69.0
+io.grpc:grpc-rls:1.70.0
+io.grpc:grpc-services:1.70.0
 io.grpc:grpc-stub:1.69.0
+io.grpc:grpc-stub:1.70.0
+io.grpc:grpc-util:1.70.0
+io.grpc:grpc-xds:1.70.0
 io.netty:netty-all:4.1.118.Final
 io.netty:netty-buffer:4.1.118.Final
 io.netty:netty-codec:4.1.118.Final
@@ -289,7 +333,19 @@ io.netty:netty-transport-native-epoll:4.1.118.Final
 io.netty:netty-transport-native-kqueue:4.1.118.Final
 io.netty:netty-resolver-dns-native-macos:4.1.118.Final
 io.opencensus:opencensus-api:0.12.3
+io.opencensus:opencensus-api:0.31.1
 io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
+io.opencensus:opencensus-contrib-http-util:0.31.1
+io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37.0-alpha
+io.opentelemetry:opentelemetry-api:1.47.0
+io.opentelemetry:opentelemetry-context:1.47.0
+io.opentelemetry:opentelemetry-sdk-common:1.47.0
+io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.47.0
+io.opentelemetry:opentelemetry-sdk:1.47.0
+io.opentelemetry:opentelemetry-sdk-logs:1.47.0
+io.opentelemetry:opentelemetry-sdk-metrics:1.47.0
+io.opentelemetry:opentelemetry-sdk-trace:1.47.0
+io.opentelemetry.semconv:opentelemetry-semconv:1.29.0-alpha
 io.reactivex:rxjava:1.3.8
 io.reactivex:rxjava-string:1.1.1
 io.reactivex:rxnetty:0.4.20
@@ -340,6 +396,7 @@ org.apache.solr:solr-solrj:8.11.2
 org.apache.yetus:audience-annotations:0.5.0
 org.apache.zookeeper:zookeeper:3.8.4
 org.codehaus.jettison:jettison:1.5.4
+org.conscrypt:conscrypt-openjdk-uber:2.5.2
 org.eclipse.jetty:jetty-annotations:9.4.57.v20241219
 org.eclipse.jetty:jetty-http:9.4.57.v20241219
 org.eclipse.jetty:jetty-io:9.4.57.v20241219
@@ -394,18 +451,22 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
 leveldb v1.13
 
+com.google.auth:google-auth-library-credentials:1.33.1
+com.google.auth:google-auth-library-oauth2-http:1.33.1
 com.google.protobuf:protobuf-java:2.5.0
 com.google.protobuf:protobuf-java:3.25.5
 com.google.re2j:re2j:1.1
 com.jcraft:jsch:0.1.55
 com.thoughtworks.paranamer:paranamer:2.3
 jakarta.activation:jakarta.activation-api:1.2.1
+org.checkerframework:checker-qual:3.49.0
 org.fusesource.leveldbjni:leveldbjni-all:1.8
 org.jline:jline:3.9.0
 org.hamcrest:hamcrest-core:1.3
 org.ow2.asm:asm:5.0.4
 org.ow2.asm:asm-commons:6.0
 org.ow2.asm:asm-tree:6.0
+org.threeten:threetenbp:1.7.0
 
 
 MIT License

hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml

@@ -130,5 +130,20 @@
       <artifactId>hadoop-tos</artifactId>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-gcp</artifactId>
+      <scope>compile</scope>
+      <!--
+      Exclude transitive dependencies to prevent dependency convergence
+      problems. hadoop-gcp is a self-contained shaded jar.
+      -->
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
 </project>

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -1285,7 +1285,7 @@
 
 <property>
   <name>fs.viewfs.overload.scheme.target.gs.impl</name>
-  <value>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS</value>
+  <value>org.apache.hadoop.fs.gs.GoogleHadoopFileSystem</value>
   <description>The GoogleHadoopFS/Google Cloud Storage file system for view
    file system overload scheme when child file system and ViewFSOverloadScheme's
    schemes are gs.
@@ -2373,12 +2373,6 @@ The switch to turn S3A auditing on or off.
       otherwise fall back to hadoop.tmp.dir </description>
   </property>
 
-<property>
-  <name>fs.AbstractFileSystem.gs.impl</name>
-  <value>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS</value>
-  <description>The AbstractFileSystem for gs: uris.</description>
-</property>
-
 <property>
   <name>fs.azure.enable.readahead</name>
   <value>true</value>
@@ -4509,4 +4503,196 @@ The switch to turn S3A auditing on or off.
       If the value is less than or equal to 0, the cache is disabled entirely.
     </description>
   </property>
+
+  <property>
+    <name>fs.gs.impl</name>
+    <value>org.apache.hadoop.fs.gs.GoogleHadoopFileSystem</value>
+    <description>The FileSystem for gs: uris.</description>
+  </property>
+
+  <property>
+    <name>fs.AbstractFileSystem.gs.impl</name>
+    <value>org.apache.hadoop.fs.gs.Gs</value>
+    <description>The AbstractFileSystem for gs: uris.</description>
+  </property>
+
+  <property>
+    <name>fs.gs.project.id</name>
+    <description>
+      Google Cloud Project ID with access to Google Cloud Storage buckets.
+      Required only for list buckets and create bucket operations.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.working.dir</name>
+    <value>/</value>
+    <description>
+      The directory relative gs: uris resolve in inside the default bucket.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.rewrite.max.chunk.size</name>
+    <value>512m</value>
+    <description>
+      Maximum size of object chunk that will be rewritten in a single rewrite
+      request when fs.gs.copy.with.rewrite.enable is set to true.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.bucket.delete.enable</name>
+    <value>false</value>
+    <description>
+      If true, recursive delete on a path that refers to a Cloud Storage bucket
+      itself or delete on that path when it is empty will result in deletion of
+      the bucket itself. If false, any operation that normally would have
+      deleted the bucket will be ignored. Setting to false preserves the typical
+      behavior of rm -rf / which translates to deleting everything inside of
+      root, but without clobbering the filesystem authority corresponding to that
+      root path in the process.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.block.size</name>
+    <value>64m</value>
+    <description>
+      The reported block size of the file system. This does not change any
+      behavior of the connector or the underlying Google Cloud Storage objects.
+      However, it will affect the number of splits Hadoop MapReduce uses for a
+      given input.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.create.items.conflict.check.enable</name>
+    <value>true</value>
+    <description>
+      Enables a check that ensures that conflicting directories do not exist when
+      creating files and conflicting files do not exist when creating directories.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.marker.file.pattern</name>
+    <description>
+      If set, files that match specified pattern are copied last during folder
+      rename operation.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.auth.type</name>
+    <value>COMPUTE_ENGINE</value>
+    <description>
+      What type of authentication mechanism to use for Google Cloud Storage
+      access. Valid values: APPLICATION_DEFAULT, COMPUTE_ENGINE,
+      SERVICE_ACCOUNT_JSON_KEYFILE, UNAUTHENTICATED, USER_CREDENTIALS.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.auth.service.account.json.keyfile</name>
+    <description>
+      The path to the JSON keyfile for the service account when fs.gs.auth.type
+      property is set to SERVICE_ACCOUNT_JSON_KEYFILE. The file must exist at
+      the same path on all nodes
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.auth.client.id</name>
+    <description>
+      The OAuth2 client ID.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.auth.client.secret</name>
+    <description>
+      The OAuth2 client secret.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.auth.refresh.token</name>
+    <description>
+      The refresh token.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.inputstream.support.gzip.encoding.enable</name>
+    <value>false</value>
+    <description>
+      If set to false then reading files with GZIP content encoding (HTTP header
+      Content-Encoding: gzip) will result in failure (IOException is thrown).
+
+      This feature is disabled by default because processing of
+      GZIP encoded files is inefficient and error-prone in Hadoop and Spark.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.outputstream.buffer.size</name>
+    <value>8m</value>
+    <description>
+      Write buffer size used by the file system API to send the data to be
+      uploaded to Cloud Storage upload thread via pipes. The various pipe types
+      are documented below.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.outputstream.sync.min.interval</name>
+    <value>0</value>
+    <description>
+      Output stream configuration that controls the minimum interval between
+      consecutive syncs. This allows to avoid getting rate-limited by Google Cloud
+      Storage. Default is 0 - no wait between syncs. Note that hflush() will
+      be no-op if called more frequently than minimum sync interval and hsync()
+      will block until an end of a min sync interval.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.inputstream.fadvise</name>
+    <value>AUTO</value>
+    <description>
+      Tunes reading objects behavior to optimize HTTP GET requests for various use
+      cases. Valid values: SEQUENTIAL, RANDOM, AUTO, AUTO_RANDOM.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.fadvise.request.track.count</name>
+    <value>3</value>
+    <description>
+      Self adaptive fadvise mode uses distance between the served requests to
+      decide the access pattern. This property controls how many such requests
+      need to be tracked. It is used when AUTO_RANDOM is selected.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.inputstream.inplace.seek.limit</name>
+    <value>8m</value>
+    <description>
+      If forward seeks are within this many bytes of the current position, seeks
+      are performed by reading and discarding bytes in-place rather than opening a
+      new underlying stream.
+    </description>
+  </property>
+
+  <property>
+    <name>fs.gs.inputstream.min.range.request.size</name>
+    <value>2m</value>
+    <description>
+      Minimum size in bytes of the read range for Cloud Storage request when
+      opening a new stream to read an object.
+    </description>
+  </property>
+
 </configuration>

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java

@@ -149,6 +149,12 @@ public void initializeMemberVariables() {
     xmlPropsToSkipCompare.add("fs.azure.saskey.usecontainersaskeyforallaccess");
     xmlPropsToSkipCompare.add("fs.azure.user.agent.prefix");
 
+    // GS properties are in a different class
+    // - org.apache.hadoop.fs.gs.GoogleHadoopFileSystemConfiguration
+    xmlPrefixToSkipCompare.add("gs.");
+    xmlPrefixToSkipCompare.add("fs.gs.");
+    xmlPropsToSkipCompare.add("fs.AbstractFileSystem.gs.impl");
+
     // Properties in enable callqueue overflow trigger failover for stateless servers.
     xmlPropsToSkipCompare.add("ipc.[port_number].callqueue.overflow.trigger.failover");
     xmlPropsToSkipCompare.add("ipc.callqueue.overflow.trigger.failover");