Possible semaphore leak when connection is closed

solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java

@@ -218,7 +218,7 @@ public void onBegin(Request request) {
               long delayNanos = System.nanoTime() - start;
               if (delayNanos > DELAY_WARN_THRESHOLD) {
                 long millis = TimeUnit.MILLISECONDS.convert(delayNanos, TimeUnit.NANOSECONDS);
-                log.info("Remote shard request delayed by {} milliseconds", millis);
+                log.info("Remote shard request {} delayed by {} milliseconds", request, millis);
                 if (delayedRequests != null) {
                   delayedRequests.update(millis);
                 }
@@ -454,5 +454,10 @@ public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
             null,
             solrMetricsContext.getMetricRegistry(),
             SolrMetricManager.mkName("httpShardExecutor", expandedScope, "threadPool"));
+    solrMetricsContext.gauge(
+        () -> defaultClient.getAvailablePermits(),
+        false,
+        "httpClientAvailablePermits",
+        expandedScope);
   }
 }

solr/core/src/java/org/apache/solr/servlet/PrometheusMetricsServlet.java

@@ -956,6 +956,12 @@ enum CoreMetric {
         "fcache_docs_hot_local_bytes_used",
         "Bytes used from local fcache-docs-hot cache store (vs backing shared cache store)",
         "ramBytesUsed",
+        PrometheusMetricType.GAUGE),
+    HTTP_CLIENT_AVAILABLE_PERMITS(
+        "QUERY.httpShardHandler.httpClientAvailablePermits",
+        "http_client_available_permits",
+        "Available permits in the HTTP client used by the httpShardHandler",
+        null,
         PrometheusMetricType.GAUGE);
     final String key, metricName, desc, property;
     private final PrometheusMetricType metricType;

solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java

@@ -454,7 +454,8 @@ public CompletableFuture<NamedList<Object>> requestAsync(
       future.completeExceptionally(e);
       return future;
     }
-    mrrv.request
+    final Request request = mrrv.request;
+    request
         .onRequestQueued(asyncTracker.queuedListener)
         .onComplete(asyncTracker.completeListener)
         .send(
@@ -489,6 +490,15 @@ public void onHeaders(Response response) {
               @Override
               public void onFailure(Response response, Throwable failure) {
                 super.onFailure(response, failure);
+                // When transport failures occur (e.g., "session closed"), onFailure is called
+                // but completeListener may not be invoked. Ensure semaphore and phaser
+                // are released. The completeListener is idempotent and will only release once.
+                Request failedRequest = response != null ? response.getRequest() : request;
+                if (failedRequest != null) {
+                  // Create a Result object for the idempotent completeListener
+                  asyncTracker.completeListener.onComplete(
+                      new Result(failedRequest, response, failure));
+                }
                 future.completeExceptionally(
                     new SolrServerException(failure.getMessage(), failure));
               }
@@ -871,7 +881,7 @@ private static class AsyncTracker {
     AsyncTracker() {
       // TODO: what about shared instances?
       phaser = new Phaser(1);
-      available = new Semaphore(MAX_OUTSTANDING_REQUESTS, false);
+      available = new Semaphore(MAX_OUTSTANDING_REQUESTS, true);
       queuedListener =
           request -> {
             phaser.register();
@@ -881,10 +891,23 @@ private static class AsyncTracker {
 
             }
           };
+      // Track completed requests to make release idempotent
+      // This prevents double-release when both onFailure and completeListener are called.
+      // Using WeakHashMap so entries are automatically removed when Request objects are GC'd.
+      // No manual cleanup needed - entries are removed automatically when Requests are no longer
+      // referenced.
+      final java.util.Map<Request, Boolean> completedRequests =
+          java.util.Collections.synchronizedMap(new java.util.WeakHashMap<>());
       completeListener =
           result -> {
-            phaser.arriveAndDeregister();
-            available.release();
+            Request request = result != null ? result.getRequest() : null;
+            // Only release once per request, even if called from both
+            // normal completion and onFailure (e.g., session closed)
+            // Use putIfAbsent to atomically check and add - returns null if key was absent
+            if (request != null && completedRequests.putIfAbsent(request, Boolean.TRUE) == null) {
+              phaser.arriveAndDeregister();
+              available.release();
+            }
           };
     }
 
@@ -898,6 +921,10 @@ public void waitForComplete() {
     }
   }
 
+  public int getAvailablePermits() {
+    return asyncTracker.available.availablePermits();
+  }
+
   public static class Builder
       extends HttpSolrClientBuilderBase<Http2SolrClient.Builder, Http2SolrClient> {