SOLR-17609: Remove HDFS module (#2923)

Removed the module, and updated the various supporting config files across Solr code base like security policies, logj4 configurations, build tooling and test tooling.   Removed HDFS specific content from Ref Guide.

Existing "blockcache" Java code has been migrated into solr/core under org.apache.solr.blackcache package structure.

Author: epugh

.github/labeler.yml

@@ -125,11 +125,6 @@ module:gcs-repository:
       - any-glob-to-any-file:
           - solr/modules/gcs-repository/**
 
-module:hdfs:
-  - changed-files:
-      - any-glob-to-any-file:
-          - solr/modules/hdfs/**
-
 module:jwt-auth:
   - changed-files:
       - any-glob-to-any-file:
@@ -197,9 +192,7 @@ cat:index:
   - changed-files:
       - any-glob-to-any-file:
           - '**/org/apache/solr/index/**'
-          - '**/org/apache/solr/hdfs/index/**'
           - '**/org/apache/solr/update/**'
-          - '**/org/apache/solr/hdfs/update/**'
           - '**/org/apache/solr/scripting/update/**'
           - '**/org/apache/solr/handler/loader/**'
 

gradle/libs.versions.toml

@@ -32,8 +32,6 @@ apache-commons-math3 = "3.6.1"
 # @keep for version alignment
 apache-commons-text = "1.12.0"
 apache-curator = "5.7.1"
-apache-hadoop = "3.4.0"
-apache-hadoop-thirdparty = "1.2.0"
 apache-httpcomponents-httpclient = "4.5.14"
 apache-httpcomponents-httpcore = "4.4.16"
 apache-httpcomponents-httpmime = "4.5.14"
@@ -243,11 +241,6 @@ apache-curator-client = { module = "org.apache.curator:curator-client", version.
 apache-curator-framework = { module = "org.apache.curator:curator-framework", version.ref = "apache-curator" }
 apache-curator-recipes = { module = "org.apache.curator:curator-recipes", version.ref = "apache-curator" }
 apache-curator-test = { module = "org.apache.curator:curator-test", version.ref = "apache-curator" }
-apache-hadoop-client-api = { module = "org.apache.hadoop:hadoop-client-api", version.ref = "apache-hadoop" }
-apache-hadoop-client-minicluster = { module = "org.apache.hadoop:hadoop-client-minicluster", version.ref = "apache-hadoop" }
-apache-hadoop-client-runtime = { module = "org.apache.hadoop:hadoop-client-runtime", version.ref = "apache-hadoop" }
-apache-hadoop-hdfs = { module = "org.apache.hadoop:hadoop-hdfs", version.ref = "apache-hadoop" }
-apache-hadoop-thirdparty-shadedguava = { module = "org.apache.hadoop.thirdparty:hadoop-shaded-guava", version.ref = "apache-hadoop-thirdparty" }
 apache-httpcomponents-httpclient = { module = "org.apache.httpcomponents:httpclient", version.ref = "apache-httpcomponents-httpclient" }
 apache-httpcomponents-httpcore = { module = "org.apache.httpcomponents:httpcore", version.ref = "apache-httpcomponents-httpcore" }
 apache-httpcomponents-httpmime = { module = "org.apache.httpcomponents:httpmime", version.ref = "apache-httpcomponents-httpmime" }

gradle/solr/solr-forbidden-apis.gradle

@@ -21,6 +21,5 @@
 configure(project(":solr:core")) {
   tasks.matching { it.name == "forbiddenApisMain" || it.name == "forbiddenApisTest" }.all {
     exclude "org/apache/solr/internal/**"
-    exclude "org/apache/hadoop/**"
   }
-}
+}

gradle/testing/randomization/policies/solr-tests.policy

@@ -103,16 +103,8 @@ grant {
   permission java.lang.RuntimePermission "closeClassLoader";
   // needed by HttpSolrClient
   permission java.lang.RuntimePermission "getFileSystemAttributes";
-  // needed by hadoop hdfs (TODO: there is a cleaner way to handle this)
-  permission java.lang.RuntimePermission "loadLibrary.jaas";
-  permission java.lang.RuntimePermission "loadLibrary.jaas_unix";
-  permission java.lang.RuntimePermission "loadLibrary.jaas_nt";
-  // needed by hadoop common RawLocalFileSystem for java nio getOwner
-  permission java.lang.RuntimePermission "accessUserInformation";
-  // needed by hadoop hdfs
-  permission java.lang.RuntimePermission "readFileDescriptor";
-  permission java.lang.RuntimePermission "writeFileDescriptor";
-  // needed by hadoop http
+
+  // needed by mockito http
   permission java.lang.RuntimePermission "getProtectionDomain";
   // SolrProcessMgr to list processes
   permission java.lang.RuntimePermission "manageProcess";
@@ -139,20 +131,11 @@ grant {
   permission javax.management.MBeanServerPermission "releaseMBeanServer";
   permission javax.management.MBeanTrustPermission "register";
 
-  // needed by hadoop hdfs
-  permission javax.security.auth.AuthPermission "getSubject";
-  permission javax.security.auth.AuthPermission "modifyPrincipals";
-  permission javax.security.auth.AuthPermission "doAs";
-  permission javax.security.auth.AuthPermission "modifyPrivateCredentials";
-  permission javax.security.auth.AuthPermission "modifyPublicCredentials";
-  permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
-
   // needed by crossdc
   permission javax.security.auth.AuthPermission "getLoginConfiguration";
   permission javax.security.auth.AuthPermission "setLoginConfiguration";
 
-  // needed by hadoop security
-  permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
+  // needed by benchmark
   permission java.security.SecurityPermission "insertProvider";
 
   // Needed by JWT integration tests & S3 tests
@@ -219,10 +202,7 @@ grant {
 
 // additional permissions based on system properties set by /bin/solr
 // NOTE: if the property is not set, the permission entry is ignored.
-grant {
-  permission java.io.FilePermission "${hadoop.security.credential.provider.path}", "read,write,delete,readlink";
-  permission java.io.FilePermission "${hadoop.security.credential.provider.path}${/}-", "read,write,delete,readlink";
-
+grant {  
   permission java.io.FilePermission "${solr.jetty.keystore}", "read,write,delete,readlink";
   permission java.io.FilePermission "${solr.jetty.keystore}${/}-", "read,write,delete,readlink";
 

gradle/validation/forbidden-apis.gradle

@@ -130,11 +130,6 @@ allprojects { prj ->
       forbiddenApisMain.exclude("**/jmh_generated/**")
     }
 
-    // Disable checks on code copied from Hadoop
-    if (prj.path == ":solr:modules:hdfs") {
-      forbiddenApisTest.exclude("**/org/apache/hadoop/**")
-    }
-
     // We rely on resolved configurations to compute the relevant set of rule
     // files for forbiddenApis. Since we don't want to resolve these configurations until
     // the task is executed, we can't really use them as task inputs properly. This is a

gradle/validation/owasp-dependency-check/exclusions.xml

@@ -76,14 +76,4 @@
     <cpe>cpe:/a:eclipse:jetty</cpe>
     <cpe>cpe:/a:jetty:jetty</cpe>
   </suppress>
-  <suppress>
-    <notes><![CDATA[hadoop-shaded-guava-1.1.1.jar is not Apache Hadoop 1.1.1]]></notes>
-    <packageUrl regex="true">^pkg:maven/org\.apache\.hadoop\.thirdparty/hadoop\-shaded\-guava@.*$</packageUrl>
-    <cpe>cpe:/a:apache:hadoop:1.1.1</cpe>
-  </suppress>
-  <suppress>
-    <notes><![CDATA[hadoop-client-runtime-3.*.jar is not Apache Hadoop 1.1.1]]></notes>
-    <packageUrl regex="true">^pkg:maven/org\.apache\.hadoop/hadoop\-client\-runtime@3.*$</packageUrl>
-    <cpe>cpe:/a:apache:hadoop:1.1.1</cpe>
-  </suppress>
 </suppressions>

gradle/validation/rat-sources.gradle

@@ -106,12 +106,6 @@ allprojects {
                     exclude "src/test-files/META-INF/services/*"
                     break
 
-                case ":solr:modules:hdfs":
-                    exclude "src/test-files/**/*.aff"
-                    exclude "src/test-files/**/*.dic"
-                    exclude "src/test-files/**/*.incl"
-                    break
-
                 case ":solr:modules:langid":
                     exclude "**/langdetect-profiles/*"
                     break

gradle/validation/spotless.gradle

@@ -65,10 +65,6 @@ configure(allprojects) { prj ->
               "src/java/org/apache/solr/parser/Token.java",
               "src/java/org/apache/solr/parser/TokenMgrError.java"
             break
-          case ":solr:modules:hdfs":
-            // Exclude Hadoop copied files to make upgrades easier
-            targetExclude "src/**/org/apache/hadoop/**"
-            break
           case ":solr:solr-ref-guide":
             // Apply to all example Java sources in the ref-guide
             target "modules/**/examples/*.java"

settings.gradle

@@ -49,7 +49,6 @@ include "solr:modules:cross-dc"
 include "solr:modules:opentelemetry"
 include "solr:modules:extraction"
 include "solr:modules:gcs-repository"
-include "solr:modules:hdfs"
 include "solr:modules:jwt-auth"
 include "solr:modules:langid"
 include "solr:modules:llm"

solr/CHANGES.txt

@@ -105,6 +105,8 @@ Deprecation Removals
 
 * SOLR-17540: Removed the Hadoop Auth module, and thus Kerberos authentication and other exotic options. (Eric Pugh)
 
+* SOLR-17609: Removd the HDFS module and therefore support for storing indexes in HDFS.  Existing "blockcache" code preserved by migrating it to core o.a.s.blackcache package.
+
 Dependency Upgrades
 ---------------------
 (No changes)