Better recovery after reset on high pool load

Author: rPraml

ebean-datasource/src/main/java/io/ebean/datasource/pool/PooledConnectionQueue.java

@@ -226,18 +226,13 @@ private PooledConnection _obtainConnection() throws InterruptedException, SQLExc
       hitCount++;
       // are other threads already waiting? (they get priority)
       if (waitingThreads == 0) {
-        PooledConnection freeConnection = extractFromFreeList();
-        if (freeConnection != null) {
-          return freeConnection;
+        PooledConnection connection = extractFromFreeList();
+        if (connection != null) {
+          return connection;
         }
-        if (busyList.size() < maxSize) {
-          // grow the connection pool
-          PooledConnection c = pool.createConnectionForQueue(connectionId++);
-          int busySize = registerBusyConnection(c);
-          if (Log.isLoggable(DEBUG)) {
-            Log.debug("DataSource [{0}] grow; id[{1}] busy[{2}] max[{3}]", name, c.name(), busySize, maxSize);
-          }
-          return c;
+        connection = createConnection();
+        if (connection != null) {
+          return connection;
         }
       }
       try {
@@ -257,13 +252,32 @@ private PooledConnection _obtainConnection() throws InterruptedException, SQLExc
     }
   }
 
+  private PooledConnection createConnection() throws SQLException {
+    if (busyList.size() < maxSize) {
+      // grow the connection pool
+      PooledConnection c = pool.createConnectionForQueue(connectionId++);
+      int busySize = registerBusyConnection(c);
+      if (Log.isLoggable(DEBUG)) {
+        Log.debug("DataSource [{0}] grow; id[{1}] busy[{2}] max[{3}]", name, c.name(), busySize, maxSize);
+      }
+      return c;
+    } else {
+      return null;
+    }
+  }
+
   /**
    * Got into a loop waiting for connections to be returned to the pool.
    */
   private PooledConnection _obtainConnectionWaitLoop() throws SQLException, InterruptedException {
     long nanos = MILLIS_TIME_UNIT.toNanos(waitTimeoutMillis);
     for (; ; ) {
       if (nanos <= 0) {
+        // We waited long enough, that a connection was returned, so we try to create a new connection.
+        PooledConnection conn = createConnection();
+        if (conn != null) {
+          return conn;
+        }
         String msg = "Unsuccessfully waited [" + waitTimeoutMillis + "] millis for a connection to be returned."
           + " No connections are free. You need to Increase the max connections of [" + maxSize + "]"
           + " or look for a connection pool leak using datasource.xxx.capturestacktrace=true";

ebean-datasource/src/test/java/io/ebean/datasource/pool/ConnectionPoolRecoverTest.java

@@ -0,0 +1,65 @@
+package io.ebean.datasource.pool;
+
+import io.ebean.datasource.DataSourceBuilder;
+import io.ebean.datasource.DataSourcePool;
+import org.junit.jupiter.api.Test;
+
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ConnectionPoolRecoverTest {
+
+  private int success;
+  private int failure;
+
+  @Test
+  void testHeavyLoadPool() throws Exception {
+    DataSourcePool pool = DataSourceBuilder.create()
+      .url("jdbc:h2:mem:testConnectionPoolFull")
+      .username("sa")
+      .password("sa")
+      .heartbeatFreqSecs(1)
+      .minConnections(1)
+      .maxConnections(1)
+      .trimPoolFreqSecs(1)
+      //     .heartbeatMaxPoolExhaustedCount(1)
+      .failOnStart(false)
+      .build();
+    ExecutorService executorService = Executors.newCachedThreadPool();
+    try {
+      List<Future<?>> futures = new ArrayList<>();
+      for (int i = 0; i < 2; i++) {
+        futures.add(executorService.submit(() -> doSomeWork(pool)));
+      }
+
+      for (Future<?> future : futures) {
+        future.get();
+      }
+      assertThat(success).isGreaterThan(1);
+    } finally {
+      executorService.shutdownNow();
+      pool.shutdown();
+    }
+
+  }
+
+  private void doSomeWork(DataSourcePool pool) {
+    for (int i = 0; i < 20; i++) {
+      System.out.println("Success: " + success + " failure: " + failure);
+      try (Connection conn = pool.getConnection()) {
+        Thread.sleep(2000);
+        success++;
+      } catch (Exception e) {
+        // nop
+        failure++;
+      }
+    }
+  }
+
+}