SOLR-17872: Remove org.apache.http dependency from compile scope (#3498)

Solr no longer uses Apache HttpClient.  Nonetheless SolrJ still includes deprecated SolrClient implementations
 for that (which will be removed eventually), and many tests still use it.  Also, there were server metrics
 relating to HTTP client connection pools that no longer exist.

* solrj-streaming DriverImpl, URI query parsing\
* PKIAuthenticationPlugin: setHeader alternative for tests
* CloudAuthStreamTest: fixed SOLR-14222 ! (how?)
* add httpcomponents to test compile classpath for some modules
* HttpClientBuilderFactory; remove Apache HttpClient method
* Metrics: no more UPDATE.updateShardHandler.maxConnections etc.

Author: dsmiley

solr/CHANGES.txt

@@ -149,6 +149,10 @@ Deprecation Removals
 
 * SOLR-17742: Removed the handleSelect option of <requestDispatcher>.  (Gaurav Tuli, David Smiley)
 
+* SOLR-17872: Solr no longer uses Apache HttpClient.  Nonetheless SolrJ still includes deprecated SolrClient implementations
+ for that (which will be removed eventually), and many tests still use it.  Also, there were server metrics
+ relating to HTTP client connection pools that no longer exist.  (David Smiley)
+
 Dependency Upgrades
 ---------------------
 

solr/api/gradle.lockfile

@@ -105,9 +105,9 @@ org.apache.commons:commons-math3:3.6.1=jarValidation,testRuntimeClasspath
 org.apache.curator:curator-client:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath
 org.apache.curator:curator-framework:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath
 org.apache.curator:curator-test:5.9.0=jarValidation,testRuntimeClasspath
-org.apache.httpcomponents:httpclient:4.5.14=jarValidation,testCompileClasspath,testRuntimeClasspath
-org.apache.httpcomponents:httpcore:4.4.16=jarValidation,testCompileClasspath,testRuntimeClasspath
-org.apache.httpcomponents:httpmime:4.5.14=jarValidation,testCompileClasspath,testRuntimeClasspath
+org.apache.httpcomponents:httpclient:4.5.14=jarValidation,testRuntimeClasspath
+org.apache.httpcomponents:httpcore:4.4.16=jarValidation,testRuntimeClasspath
+org.apache.httpcomponents:httpmime:4.5.14=jarValidation,testRuntimeClasspath
 org.apache.logging.log4j:log4j-api:2.21.0=jarValidation,testRuntimeClasspath
 org.apache.logging.log4j:log4j-core:2.21.0=jarValidation,testRuntimeClasspath
 org.apache.logging.log4j:log4j-slf4j2-impl:2.21.0=jarValidation,testRuntimeClasspath

solr/benchmark/gradle.lockfile

@@ -76,9 +76,9 @@ org.apache.commons:commons-math3:3.6.1=annotationProcessor,compileClasspath,jarV
 org.apache.curator:curator-client:5.9.0=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.curator:curator-framework:5.9.0=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 org.apache.curator:curator-test:5.9.0=jarValidation,runtimeClasspath,testRuntimeClasspath
-org.apache.httpcomponents:httpclient:4.5.14=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.httpcomponents:httpcore:4.4.16=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
-org.apache.httpcomponents:httpmime:4.5.14=compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
+org.apache.httpcomponents:httpclient:4.5.14=jarValidation,runtimeClasspath,testRuntimeClasspath
+org.apache.httpcomponents:httpcore:4.4.16=jarValidation,runtimeClasspath,testRuntimeClasspath
+org.apache.httpcomponents:httpmime:4.5.14=jarValidation,runtimeClasspath,testRuntimeClasspath
 org.apache.logging.log4j:log4j-api:2.21.0=jarValidation,runtimeClasspath,testRuntimeClasspath
 org.apache.logging.log4j:log4j-core:2.21.0=jarValidation,runtimeClasspath,testRuntimeClasspath
 org.apache.logging.log4j:log4j-slf4j2-impl:2.21.0=jarValidation,runtimeClasspath,testRuntimeClasspath

solr/core/build.gradle

@@ -117,9 +117,6 @@ dependencies {
   implementation libs.fasterxml.jackson.dataformat.smile
   implementation libs.fasterxml.jackson.dataformat.cbor
 
-  implementation libs.apache.httpcomponents.httpclient
-  implementation libs.apache.httpcomponents.httpcore
-
   implementation libs.eclipse.jetty.client
   implementation libs.eclipse.jetty.http
   implementation libs.eclipse.jetty.io
@@ -210,4 +207,6 @@ dependencies {
   testRuntimeOnly(libs.mockito.subclass, {
     exclude group: "net.bytebuddy", module: "byte-buddy-agent"
   })
+  testImplementation libs.apache.httpcomponents.httpclient
+  testImplementation libs.apache.httpcomponents.httpcore
 }

solr/core/gradle.lockfile

@@ -88,9 +88,9 @@ org.apache.commons:commons-math3:3.6.1=apiHelper,compileClasspath,jarValidation,
 org.apache.curator:curator-client:5.9.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
 org.apache.curator:curator-framework:5.9.0=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
 org.apache.curator:curator-test:5.9.0=jarValidation,testCompileClasspath,testRuntimeClasspath
-org.apache.httpcomponents:httpclient:4.5.14=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
-org.apache.httpcomponents:httpcore:4.4.16=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
-org.apache.httpcomponents:httpmime:4.5.14=apiHelper,compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
+org.apache.httpcomponents:httpclient:4.5.14=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
+org.apache.httpcomponents:httpcore:4.4.16=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
+org.apache.httpcomponents:httpmime:4.5.14=apiHelper,jarValidation,runtimeClasspath,runtimeLibs,testRuntimeClasspath
 org.apache.logging.log4j:log4j-api:2.21.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
 org.apache.logging.log4j:log4j-core:2.21.0=compileClasspath,jarValidation,runtimeClasspath,runtimeLibs,testCompileClasspath,testRuntimeClasspath
 org.apache.logging.log4j:log4j-slf4j2-impl:2.21.0=jarValidation,runtimeClasspath,runtimeLibs,testRuntimeClasspath

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -55,9 +55,6 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
-import org.apache.http.auth.AuthSchemeProvider;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.config.Lookup;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.store.Directory;
@@ -67,11 +64,6 @@
 import org.apache.solr.api.JerseyResource;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.impl.Http2SolrClient;
-import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
-import org.apache.solr.client.solrj.impl.SolrHttpClientContextBuilder;
-import org.apache.solr.client.solrj.impl.SolrHttpClientContextBuilder.AuthSchemeRegistryProvider;
-import org.apache.solr.client.solrj.impl.SolrHttpClientContextBuilder.CredentialsProviderProvider;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.util.SolrIdentifierValidator;
 import org.apache.solr.cloud.CloudDescriptor;
@@ -148,7 +140,6 @@
 import org.apache.solr.security.AuditLoggerPlugin;
 import org.apache.solr.security.AuthenticationPlugin;
 import org.apache.solr.security.AuthorizationPlugin;
-import org.apache.solr.security.HttpClientBuilderPlugin;
 import org.apache.solr.security.PKIAuthenticationPlugin;
 import org.apache.solr.security.PublicKeyHandler;
 import org.apache.solr.security.SecurityPluginHolder;
@@ -594,58 +585,10 @@ private synchronized void initializeAuthenticationPlugin(
   }
 
   private void setupHttpClientForAuthPlugin(Object authcPlugin) {
-    if (authcPlugin instanceof HttpClientBuilderPlugin builderPlugin) {
-      // Setup HttpClient for internode communication
-      SolrHttpClientBuilder builder =
-          builderPlugin.getHttpClientBuilder(HttpClientUtil.getHttpClientBuilder());
-
-      // The Hadoop Auth Plugin was removed in SOLR-17540, however leaving the below reference
-      // for future readers, as there may be an option to simplify this logic.
-      //
-      // this caused plugins like KerberosPlugin to register its intercepts, but this intercept
-      // logic is also handled by the pki authentication code when it decides to let the plugin
-      // handle auth via its intercept - so you would end up with two intercepts
-      // -->
-      //  shardHandlerFactory.setSecurityBuilder(builderPlugin); // calls setup for the authcPlugin
-      //  updateShardHandler.setSecurityBuilder(builderPlugin);
-      // <--
-
-      // This should not happen here at all - it's only currently required due to its effect on
-      // http1 clients in a test or two incorrectly counting on it for their configuration.
-      // -->
-
-      SolrHttpClientContextBuilder httpClientBuilder = new SolrHttpClientContextBuilder();
-      if (builder.getCredentialsProviderProvider() != null) {
-        httpClientBuilder.setDefaultCredentialsProvider(
-            new CredentialsProviderProvider() {
-
-              @Override
-              public CredentialsProvider getCredentialsProvider() {
-                return builder.getCredentialsProviderProvider().getCredentialsProvider();
-              }
-            });
-      }
-      if (builder.getAuthSchemeRegistryProvider() != null) {
-        httpClientBuilder.setAuthSchemeRegistryProvider(
-            new AuthSchemeRegistryProvider() {
-
-              @Override
-              public Lookup<AuthSchemeProvider> getAuthSchemeRegistry() {
-                return builder.getAuthSchemeRegistryProvider().getAuthSchemeRegistry();
-              }
-            });
-      }
-
-      HttpClientUtil.setHttpClientRequestContextBuilder(httpClientBuilder);
-
-      // <--
-    }
-
     // Always register PKI auth interceptor, which will then delegate the decision of who should
     // secure each request to the configured authentication plugin.
     if (pkiAuthenticationSecurityBuilder != null
         && !pkiAuthenticationSecurityBuilder.isInterceptorRegistered()) {
-      pkiAuthenticationSecurityBuilder.getHttpClientBuilder(HttpClientUtil.getHttpClientBuilder());
       shardHandlerFactory.setSecurityBuilder(pkiAuthenticationSecurityBuilder);
       updateShardHandler.setSecurityBuilder(pkiAuthenticationSecurityBuilder);
       solrClientProvider.setSecurityBuilder(pkiAuthenticationSecurityBuilder);

solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrNodeHandlerMetric.java

@@ -17,7 +17,6 @@
 package org.apache.solr.metrics.prometheus.node;
 
 import com.codahale.metrics.Counter;
-import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Metric;
 import org.apache.solr.metrics.prometheus.SolrMetric;
@@ -27,7 +26,6 @@
 public class SolrNodeHandlerMetric extends SolrNodeMetric {
   public static final String NODE_REQUESTS = "solr_metrics_node_requests";
   public static final String NODE_SECONDS_TOTAL = "solr_metrics_node_requests_time";
-  public static final String NODE_CONNECTIONS = "solr_metrics_node_connections";
 
   public SolrNodeHandlerMetric(Metric dropwizardMetric, String metricName) {
     super(dropwizardMetric, metricName);
@@ -36,7 +34,6 @@ public SolrNodeHandlerMetric(Metric dropwizardMetric, String metricName) {
   /*
    * Metric examples being exported
    * ADMIN./admin/collections.requests
-   * UPDATE.updateShardHandler.maxConnections
    */
 
   @Override
@@ -53,8 +50,6 @@ public void toPrometheus(SolrPrometheusFormatter formatter) {
     if (metricName.endsWith(".totalTime")) {
       labels.remove("type");
       formatter.exportCounter(NODE_SECONDS_TOTAL, (Counter) dropwizardMetric, getLabels());
-    } else if (metricName.endsWith("Connections")) {
-      formatter.exportGauge(NODE_CONNECTIONS, (Gauge<?>) dropwizardMetric, getLabels());
     } else if (dropwizardMetric instanceof Meter) {
       formatter.exportMeter(NODE_REQUESTS, (Meter) dropwizardMetric, getLabels());
     } else if (dropwizardMetric instanceof Counter) {

solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java

@@ -27,8 +27,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import org.apache.http.HttpRequest;
-import org.apache.http.protocol.HttpContext;
 import org.apache.solr.core.SolrInfoBean;
 import org.apache.solr.metrics.SolrMetricsContext;
 import org.eclipse.jetty.client.Request;
@@ -117,25 +115,6 @@ public String getRemoteUser() {
     };
   }
 
-  /**
-   * Override this method to intercept internode requests. This allows your authentication plugin to
-   * decide on per-request basis whether it should handle inter-node requests or delegate to {@link
-   * PKIAuthenticationPlugin}. Return true to indicate that your plugin did handle the request, or
-   * false to signal that PKI plugin should handle it. This method will be called by {@link
-   * PKIAuthenticationPlugin}'s interceptor.
-   *
-   * <p>If not overridden, this method will return true for plugins implementing {@link
-   * HttpClientBuilderPlugin}. This method can be overridden by subclasses e.g. to set HTTP headers,
-   * even if you don't use a clientBuilder.
-   *
-   * @param httpRequest the httpRequest that is about to be sent to another internal Solr node
-   * @param httpContext the context of that request.
-   * @return true if this plugin handled authentication for the request, else false
-   */
-  protected boolean interceptInternodeRequest(HttpRequest httpRequest, HttpContext httpContext) {
-    return this instanceof HttpClientBuilderPlugin;
-  }
-
   /**
    * Override this method to intercept internode requests. This allows your authentication plugin to
    * decide on per-request basis whether it should handle inter-node requests or delegate to {@link

solr/core/src/java/org/apache/solr/security/AuthorizationUtils.java

@@ -30,9 +30,9 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import org.apache.http.HttpStatus;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.core.CoreContainer;
+import org.eclipse.jetty.http.HttpStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -113,7 +113,7 @@ public static AuthorizationFailure authorize(
       return new AuthorizationFailure(
           statusCode, "Unauthorized request, Response code: " + statusCode);
     }
-    if (!(statusCode == HttpStatus.SC_ACCEPTED) && !(statusCode == HttpStatus.SC_OK)) {
+    if (!(statusCode == HttpStatus.ACCEPTED_202) && !(statusCode == HttpStatus.OK_200)) {
       log.warn(
           "ERROR {} during authentication: {}", statusCode, authResponse.getMessage()); // nowarn
       if (shouldAudit(cores, AuditEvent.EventType.ERROR)) {

solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java

@@ -33,29 +33,23 @@
 import java.util.Set;
 import java.util.StringTokenizer;
 import javax.security.auth.Subject;
-import org.apache.http.Header;
-import org.apache.http.HttpHeaders;
-import org.apache.http.HttpRequest;
-import org.apache.http.annotation.Contract;
-import org.apache.http.annotation.ThreadingBehavior;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.protocol.HttpContext;
 import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.SpecProvider;
 import org.apache.solr.common.util.CommandOperation;
 import org.apache.solr.common.util.ValidatingJsonMap;
 import org.eclipse.jetty.client.Request;
+import org.eclipse.jetty.http.HttpField;
+import org.eclipse.jetty.http.HttpHeader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class BasicAuthPlugin extends AuthenticationPlugin
     implements ConfigEditablePlugin, SpecProvider {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
   private AuthenticationProvider authenticationProvider;
-  private static final ThreadLocal<Header> authHeader = new ThreadLocal<>();
+  private static final ThreadLocal<HttpField> authHeader = new ThreadLocal<>();
   private static final String X_REQUESTED_WITH_HEADER = "X-Requested-With";
   private boolean blockUnknown = true;
   private boolean forwardCredentials = false;
@@ -128,7 +122,7 @@ public boolean doAuthenticate(
     boolean isAjaxRequest = isAjaxRequest(request);
 
     if (authHeader != null) {
-      BasicAuthPlugin.authHeader.set(new BasicHeader("Authorization", authHeader));
+      BasicAuthPlugin.authHeader.set(new HttpField("Authorization", authHeader));
       StringTokenizer st = new StringTokenizer(authHeader);
       if (st.hasMoreTokens()) {
         String basic = st.nextToken();
@@ -192,13 +186,14 @@ public boolean doAuthenticate(
    */
   private Map<String, String> getPromptHeaders(boolean isAjaxRequest) {
     Map<String, String> headers = new HashMap<>(authenticationProvider.getPromptHeaders());
+    String WWW_AUTH = HttpHeader.WWW_AUTHENTICATE.asString();
     if (isAjaxRequest
-        && headers.containsKey(HttpHeaders.WWW_AUTHENTICATE)
-        && headers.get(HttpHeaders.WWW_AUTHENTICATE).startsWith("Basic ")) {
-      headers.put(HttpHeaders.WWW_AUTHENTICATE, "x" + headers.get(HttpHeaders.WWW_AUTHENTICATE));
+        && headers.containsKey(WWW_AUTH)
+        && headers.get(WWW_AUTH).startsWith("Basic ")) {
+      headers.put(WWW_AUTH, "x" + headers.get(WWW_AUTH));
       log.debug(
           "Prefixing {} header for Basic Auth with 'x' to prevent browser basic auth popup",
-          HttpHeaders.WWW_AUTHENTICATE);
+          WWW_AUTH);
     }
     return headers;
   }
@@ -219,24 +214,6 @@ public interface AuthenticationProvider extends SpecProvider {
     Map<String, String> getPromptHeaders();
   }
 
-  @Override
-  protected boolean interceptInternodeRequest(HttpRequest httpRequest, HttpContext httpContext) {
-    if (forwardCredentials) {
-      if (httpContext instanceof HttpClientContext httpClientContext) {
-        if (httpClientContext.getUserToken() instanceof BasicAuthUserPrincipal principal) {
-          String userPassBase64 =
-              Base64.getEncoder()
-                  .encodeToString(
-                      (principal.getName() + ":" + principal.getPassword())
-                          .getBytes(StandardCharsets.UTF_8));
-          httpRequest.setHeader(HttpHeaders.AUTHORIZATION, "Basic " + userPassBase64);
-          return true;
-        }
-      }
-    }
-    return false;
-  }
-
   @Override
   protected boolean interceptInternodeRequest(Request request) {
     if (forwardCredentials) {
@@ -248,7 +225,8 @@ protected boolean interceptInternodeRequest(Request request) {
                     (principal.getName() + ":" + principal.getPassword())
                         .getBytes(StandardCharsets.UTF_8));
         request.headers(
-            httpFields -> httpFields.add(HttpHeaders.AUTHORIZATION, "Basic " + userPassBase64));
+            httpFields ->
+                httpFields.add(HttpHeader.AUTHORIZATION.asString(), "Basic " + userPassBase64));
         return true;
       }
     }
@@ -280,7 +258,7 @@ static boolean isAjaxRequest(HttpServletRequest request) {
     return "XMLHttpRequest".equalsIgnoreCase(request.getHeader(X_REQUESTED_WITH_HEADER));
   }
 
-  @Contract(threading = ThreadingBehavior.IMMUTABLE)
+  // IMMUTABLE
   private static class BasicAuthUserPrincipal implements Principal, Serializable {
     private String username;
     private final String password;