bump hadoop hdfs to 3.4.1 (elastic#117263)

This commit bump hadoop hdfs to 3.4.1 (for repository-hdfs) .

Author: jakelandis

gradle/verification-metadata.xml

@@ -1201,6 +1201,11 @@
             <sha256 value="e7cd8951956d349b568b7ccfd4f5b2529a8c113e67c32b028f52ffda371259d9" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="commons-cli" name="commons-cli" version="1.5.0">
+         <artifact name="commons-cli-1.5.0.jar">
+            <sha256 value="bc8bb01fc0fad250385706e20f927ddcff6173f6339b387dc879237752567ac6" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="commons-codec" name="commons-codec" version="1.11">
          <artifact name="commons-codec-1.11.jar">
             <sha256 value="e599d5318e97aa48f42136a2927e6dfa4e8881dff0e6c8e3109ddbbff51d7b7d" origin="Generated by Gradle"/>
@@ -1256,16 +1261,16 @@
             <sha256 value="05662373044f3dff112567b7bb5dfa1174e91e074c0c727b4412788013f49d56" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="commons-io" name="commons-io" version="2.11.0">
-         <artifact name="commons-io-2.11.0.jar">
-            <sha256 value="961b2f6d87dbacc5d54abf45ab7a6e2495f89b75598962d8c723cea9bc210908" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
       <component group="commons-io" name="commons-io" version="2.15.1">
          <artifact name="commons-io-2.15.1.jar">
             <sha256 value="a58af12ee1b68cfd2ebb0c27caef164f084381a00ec81a48cc275fd7ea54e154" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="commons-io" name="commons-io" version="2.16.1">
+         <artifact name="commons-io-2.16.1.jar">
+            <sha256 value="f41f7baacd716896447ace9758621f62c1c6b0a91d89acee488da26fc477c84f" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="commons-io" name="commons-io" version="2.17.0">
          <artifact name="commons-io-2.17.0.jar">
             <sha256 value="4aa4ca48f3dfd30b78220b7881d8cb93eac4093ec94361b6befa9487998a550b" origin="Generated by Gradle"/>
@@ -2372,14 +2377,14 @@
             <sha256 value="a29e54e5c9d03f52ea28a544342140f73870a5bca9a9789988a66c199b01dd8a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.hadoop" name="hadoop-client-api" version="3.3.3">
-         <artifact name="hadoop-client-api-3.3.3.jar">
-            <sha256 value="447292218fb52aac173efeea549bdc7e2392c6445d33ae28c38a4d9b90d1d31a" origin="Generated by Gradle"/>
+      <component group="org.apache.hadoop" name="hadoop-client-api" version="3.4.1">
+         <artifact name="hadoop-client-api-3.4.1.jar">
+            <sha256 value="a964d4daa054c9615bbafb4553efbb140fa7fb9ac6f358a24393f183a5703438" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.hadoop" name="hadoop-client-runtime" version="3.3.3">
-         <artifact name="hadoop-client-runtime-3.3.3.jar">
-            <sha256 value="e2151873292cb48da649e4a51ce43009faf3528af1778aa64919f7514541ca27" origin="Generated by Gradle"/>
+      <component group="org.apache.hadoop" name="hadoop-client-runtime" version="3.4.1">
+         <artifact name="hadoop-client-runtime-3.4.1.jar">
+            <sha256 value="f6a800a159f918670db533606d33560d6c13b7e13f14eda493280ae33b9eeb2f" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.hadoop" name="hadoop-common" version="2.8.5">
@@ -2414,9 +2419,9 @@
             <sha256 value="581e11802400f77010937e1aa2b5ed48b2dca26db054fb9493ffe799e562cd57" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.hadoop" name="hadoop-hdfs" version="3.3.3">
-         <artifact name="hadoop-hdfs-3.3.3.jar">
-            <sha256 value="544a835362ea112476f05d21d6a16b0f20cdcf3467f170cef762cffdac9166c7" origin="Generated by Gradle"/>
+      <component group="org.apache.hadoop" name="hadoop-hdfs" version="3.4.1">
+         <artifact name="hadoop-hdfs-3.4.1.jar">
+            <sha256 value="91d1bcdb554dcf95d2daa12c7c0ef706c64692c21e9edcf2059d4683bedb0cc6" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.hadoop" name="hadoop-hdfs-client" version="2.8.5">

plugins/repository-hdfs/build.gradle

@@ -19,7 +19,7 @@ esplugin {
 }
 
 versions << [
-  'hadoop': '3.3.3'
+  'hadoop': '3.4.1'
 ]
 
 configurations {
@@ -41,9 +41,9 @@ dependencies {
   api "com.google.protobuf:protobuf-java:${versions.protobuf}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
-  api 'commons-cli:commons-cli:1.2'
+  api 'commons-cli:commons-cli:1.5.0'
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'commons-io:commons-io:2.8.0'
+  api 'commons-io:commons-io:2.16.1'
   api 'org.apache.commons:commons-lang3:3.11'
   api 'javax.servlet:javax.servlet-api:3.1.0'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
@@ -57,14 +57,14 @@ dependencies {
   javaRestTestImplementation project(':test:fixtures:krb5kdc-fixture')
   javaRestTestImplementation "org.slf4j:slf4j-api:${versions.slf4j}"
   javaRestTestRuntimeOnly "com.google.guava:guava:16.0.1"
-  javaRestTestRuntimeOnly "commons-cli:commons-cli:1.2"
+  javaRestTestRuntimeOnly "commons-cli:commons-cli:1.5.0"
   javaRestTestRuntimeOnly "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
 
   yamlRestTestCompileOnly(project(':test:fixtures:hdfs-fixture'))
   yamlRestTestImplementation project(':test:fixtures:krb5kdc-fixture')
   yamlRestTestImplementation "org.slf4j:slf4j-api:${versions.slf4j}"
   yamlRestTestRuntimeOnly "com.google.guava:guava:16.0.1"
-  yamlRestTestRuntimeOnly "commons-cli:commons-cli:1.2"
+  yamlRestTestRuntimeOnly "commons-cli:commons-cli:1.5.0"
   yamlRestTestRuntimeOnly "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
 
   hdfsFixture2 project(path: ':test:fixtures:hdfs-fixture', configuration: 'shadowedHdfs2')
@@ -177,7 +177,6 @@ tasks.named("thirdPartyAudit").configure {
     'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
     'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
     'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
-    'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
     'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64',
     'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$1',
     'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$Cell',
@@ -188,6 +187,9 @@ tasks.named("thirdPartyAudit").configure {
     'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil',
     'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$1',
     'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$JvmMemoryAccessor',
-    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor'
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor',
+    'org.apache.hadoop.thirdparty.protobuf.MessageSchema',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$Android64MemoryAccessor'
   )
 }

plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java

@@ -47,7 +47,8 @@ class HdfsSecurityContext {
             // 2) allow hadoop to add credentials to our Subject
             new AuthPermission("modifyPrivateCredentials"),
             // 3) RPC Engine requires this for re-establishing pooled connections over the lifetime of the client
-            new PrivateCredentialPermission("org.apache.hadoop.security.Credentials * \"*\"", "read") };
+            new PrivateCredentialPermission("org.apache.hadoop.security.Credentials * \"*\"", "read"),
+            new RuntimePermission("getClassLoader") };
 
         // If Security is enabled, we need all the following elevated permissions:
         KERBEROS_AUTH_PERMISSIONS = new Permission[] {