[Build] Build hdfs fixture faster and less (#118801)

The building of the shadowed hdfs2 and hdfs3 fixtures takes quite long time due to being 51 and 80mb in size.
By removing non used dependencies from the shadow jar creation we can speed up this significantly.
Also we avoid building hdfs fixture jars now for compile only (resulting in no shadow jar creation for precommit checks)

Author: breskeby

test/fixtures/hdfs-fixture/build.gradle

@@ -10,12 +10,10 @@
 apply plugin: 'elasticsearch.java'
 apply plugin: 'com.gradleup.shadow'
 
+
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 
 configurations {
-//  all {
-//    transitive = true
-//  }
   hdfs2
   hdfs3
   consumable("shadowedHdfs2")
@@ -27,20 +25,76 @@ dependencies {
     transitive false
   }
   compileOnly "junit:junit:${versions.junit}"
-  hdfs2 "org.apache.hadoop:hadoop-minicluster:2.8.5"
-  hdfs3 "org.apache.hadoop:hadoop-minicluster:3.3.1"
 
+  def commonExcludes = [
+    [group: "org.apache.commons", module: "commons-compress"],
+    [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-app"],
+    [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-core"],
+    [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-hs"],
+    [group: "org.apache.hadoop", module: "hadoop-mapreduce-client-jobclient"],
+    [group: "org.apache.hadoop", module: "hadoop-yarn-server-tests"],
+    [group: "org.apache.httpcomponents", module: "httpclient"],
+    [group: "org.apache.zookeeper", module: "zookeeper"],
+    [group: "org.apache.curator", module: "curator-recipes"],
+    [group: "org.apache.curator", module: "curator-client"],
+    [group: "org.apache.curator", module: "curator-framework"],
+    [group: "org.apache.avro", module: "avro"],
+    [group: "log4j", module: "log4j"],
+    [group: "io.netty", module: "netty-all"],
+    [group: "io.netty", module: "netty"],
+    [group: "com.squareup.okhttp", module: "okhttp"],
+    [group: "com.google.guava", module: "guava"],
+    [group: "com.google.code.gson", module: "gson"],
+    [group: "javax.servlet.jsp", module: "jsp-api"],
+    [group: "org.fusesource.leveldbjni", module: "leveldbjni-all"],
+    [group: "commons-cli", module: "commons-cli"],
+    [group: "org.mortbay.jetty", module: "servlet-api"],
+    [group: "commons-logging", module: "commons-logging"],
+    [group: "org.slf4j", module: "slf4j-log4j12"],
+    [group: "commons-codec", module: "commons-codec"],
+    [group: "com.sun.jersey", module: "jersey-core"],
+    [group: "com.sun.jersey", module: "jersey-json"],
+    [group: "com.google.code.findbugs", module: "jsr305"],
+    [group: "com.sun.jersey", module: "jersey-json"],
+    [group: "com.nimbusds", module: "nimbus-jose-jwt"],
+    [group: "com.jcraft", module: "jsch"],
+    [group: "org.slf4j", module: "slf4j-api"],
+  ]
+
+  hdfs2("org.apache.hadoop:hadoop-minicluster:2.8.5") {
+    commonExcludes.each { exclude it }
+    exclude group: "org.apache.commons", module: "commons-math3"
+    exclude group: "xmlenc", module: "xmlenc"
+    exclude group: "net.java.dev.jets3t", module: "jets3t"
+    exclude group: "org.apache.directory.server", module: "apacheds-i18n"
+    exclude group: "xerces", module: "xercesImpl"
+  }
+
+  hdfs3("org.apache.hadoop:hadoop-minicluster:3.3.1") {
+    commonExcludes.each { exclude it }
+    exclude group: "dnsjava", module: "dnsjava"
+    exclude group: "com.google.inject.extensions", module: "guice-servlet"
+    exclude group: "com.google.inject", module: "guice"
+    exclude group: "com.microsoft.sqlserver", module: "mssql-jdbc"
+    exclude group: "com.sun.jersey.contribs", module: "jersey-guice"
+    exclude group: "com.zaxxer", module: "HikariCP-java7"
+    exclude group: "com.sun.jersey", module: "jersey-server"
+    exclude group: "org.bouncycastle", module: "bcpkix-jdk15on"
+    exclude group: "org.bouncycastle", module: "bcprov-jdk15on"
+    exclude group: "org.ehcache", module: "ehcache"
+    exclude group: "org.apache.geronimo.specs", module: "geronimo-jcache_1.0_spec"
+    exclude group: "org.xerial.snappy", module: "snappy-java"
+  }
 }
 
 tasks.named("shadowJar").configure {
   archiveClassifier.set("hdfs3")
   // fix issues with signed jars
-
   relocate("org.apache.hadoop", "fixture.hdfs3.org.apache.hadoop") {
     exclude "org.apache.hadoop.hdfs.protocol.ClientProtocol"
     exclude "org.apache.hadoop.ipc.StandbyException"
   }
-  configurations << project.configurations.hdfs3
+  configurations.add(project.configurations.hdfs3)
 }
 
 def hdfs2Jar = tasks.register("hdfs2jar", ShadowJar) {
@@ -50,26 +104,15 @@ def hdfs2Jar = tasks.register("hdfs2jar", ShadowJar) {
   }
   archiveClassifier.set("hdfs2")
   from sourceSets.main.output
-  configurations << project.configurations.hdfs2
+  configurations.add(project.configurations.hdfs2)
 }
 
 tasks.withType(ShadowJar).configureEach {
   dependencies {
-//    exclude(dependency('commons-io:commons-io:2.8.0'))
     exclude(dependency("com.carrotsearch.randomizedtesting:randomizedtesting-runner:.*"))
     exclude(dependency("junit:junit:.*"))
-    exclude(dependency("org.slf4j:slf4j-api:.*"))
-    exclude(dependency("com.google.guava:guava:.*"))
-    exclude(dependency("org.apache.commons:commons-compress:.*"))
-    exclude(dependency("commons-logging:commons-logging:.*"))
-    exclude(dependency("commons-codec:commons-codec:.*"))
-    exclude(dependency("org.apache.httpcomponents:httpclient:.*"))
     exclude(dependency("org.apache.httpcomponents:httpcore:.*"))
     exclude(dependency("org.apache.logging.log4j:log4j-1.2-api:.*"))
-    exclude(dependency("log4j:log4j:.*"))
-    exclude(dependency("io.netty:.*:.*"))
-    exclude(dependency("com.nimbusds:nimbus-jose-jwt:.*"))
-    exclude(dependency("commons-cli:commons-cli:1.2"))
     exclude(dependency("net.java.dev.jna:jna:.*"))
     exclude(dependency("org.objenesis:objenesis:.*"))
     exclude(dependency('com.fasterxml.jackson.core:.*:.*'))

x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle

@@ -12,7 +12,8 @@ apply plugin: 'elasticsearch.internal-available-ports'
 dependencies {
   clusterPlugins project(':plugins:repository-hdfs')
   javaRestTestImplementation(testArtifact(project(xpackModule('searchable-snapshots'))))
-  javaRestTestImplementation project(path: ':test:fixtures:hdfs-fixture', configuration:"shadowedHdfs2")
+  javaRestTestCompileOnly project(path: ':test:fixtures:hdfs-fixture')
+  javaRestTestRuntimeOnly project(path: ':test:fixtures:hdfs-fixture', configuration:"shadowedHdfs2")
   javaRestTestImplementation project(':test:fixtures:krb5kdc-fixture')
   javaRestTestRuntimeOnly "com.google.guava:guava:16.0.1"
   javaRestTestRuntimeOnly "commons-cli:commons-cli:1.2"

x-pack/plugin/snapshot-repo-test-kit/qa/hdfs/build.gradle

@@ -10,7 +10,8 @@ apply plugin: 'elasticsearch.rest-resources'
 
 dependencies {
   javaRestTestImplementation testArtifact(project(xpackModule('snapshot-repo-test-kit')))
-  javaRestTestImplementation project(path: ':test:fixtures:hdfs-fixture', configuration:"shadow")
+  javaRestTestCompileOnly project(path: ':test:fixtures:hdfs-fixture')
+  javaRestTestRuntimeOnly project(path: ':test:fixtures:hdfs-fixture', configuration:"shadow")
   javaRestTestImplementation project(':test:fixtures:krb5kdc-fixture')
   javaRestTestImplementation "org.slf4j:slf4j-api:${versions.slf4j}"
   javaRestTestImplementation "org.slf4j:slf4j-simple:${versions.slf4j}"