Add validation on stale connections obtained from idle connections

With Lambda we might prefer this connection validation over periodic validation performed by the heartbeat.

Author: rbygrave

ebean-datasource-api/src/main/java/io/ebean/datasource/DataSourceConfig.java

@@ -719,6 +719,7 @@ public DataSourceConfig validateOnHeartbeat(boolean validateOnHeartbeat) {
   @Override
   public DataSourceBuilder useLambdaCheck(boolean useLambda) {
     this.useLambdaCheck = useLambda;
+    this.validateOnHeartbeat = false;
     return this;
   }
 
@@ -896,4 +897,12 @@ public Properties connectionProperties() {
     }
     return connectionProps;
   }
+
+  public long validateStaleMillis() {
+    if (validateOnHeartbeat) {
+      return 0L;
+    } else {
+      return (maxInactiveTimeSecs + trimPoolFreqSecs) * 1_000L;
+    }
+  }
 }

ebean-datasource/src/main/java/io/ebean/datasource/pool/ConnectionPool.java

@@ -51,6 +51,7 @@ final class ConnectionPool implements DataSourcePool {
   private final boolean readOnly;
   private final boolean failOnStart;
   private final int maxInactiveMillis;
+  private final long validateStaleMillis;
   /**
    * Max age a connection is allowed in millis.
    * A value of 0 means no limit (no trimming based on max age).
@@ -118,6 +119,7 @@ final class ConnectionPool implements DataSourcePool {
     this.heartbeatSql = params.getHeartbeatSql();
     this.validateOnHeartbeat = params.isValidateOnHeartbeat();
     this.trimPoolFreqMillis = 1000L * params.getTrimPoolFreqSecs();
+    this.validateStaleMillis = params.validateStaleMillis();
     this.applicationName = params.getApplicationName();
     this.clientInfo = params.getClientInfo();
     this.queue = new PooledConnectionQueue(this);
@@ -504,6 +506,14 @@ long maxAgeMillis() {
     return maxAgeMillis;
   }
 
+  /**
+   * When obtaining a connection that has been idle for longer than maxInactiveMillis
+   * perform a validation test on the connection before giving it to the application.
+   */
+  long validateStaleMillis() {
+    return validateStaleMillis;
+  }
+
   private boolean testConnection(Connection conn) throws SQLException {
     if (heartbeatSql == null) {
       return conn.isValid(heartbeatTimeoutSeconds);
@@ -529,7 +539,7 @@ boolean validateConnection(PooledConnection conn) {
     try {
       return testConnection(conn);
     } catch (Exception e) {
-      Log.warn("Heartbeat test failed on connection:{0} message: {1}", conn.name(), e.getMessage());
+      Log.warn("Validation test failed on connection:{0} message: {1}", conn.name(), e.getMessage());
       return false;
     }
   }

ebean-datasource/src/main/java/io/ebean/datasource/pool/PooledConnectionQueue.java

@@ -62,6 +62,7 @@ final class PooledConnectionQueue {
    */
   private long lastResetTime;
   private boolean doingShutdown;
+  private final long validateStaleMillis;
 
   PooledConnectionQueue(ConnectionPool pool) {
     this.pool = pool;
@@ -72,6 +73,7 @@ final class PooledConnectionQueue {
     this.waitTimeoutMillis = pool.waitTimeoutMillis();
     this.leakTimeMinutes = pool.leakTimeMinutes();
     this.maxAgeMillis = pool.maxAgeMillis();
+    this.validateStaleMillis = pool.validateStaleMillis();
     this.busyList = new BusyConnectionBuffer(maxSize, 20);
     this.freeList = new FreeConnectionBuffer();
     this.lock = new ReentrantLock(false);
@@ -184,11 +186,32 @@ void returnPooledConnection(PooledConnection c, boolean forceClose) {
   }
 
   private PooledConnection extractFromFreeList() {
-    PooledConnection c = freeList.remove();
+    if (freeList.isEmpty()) {
+      return null;
+    }
+    final PooledConnection c = freeList.remove();
+    if (validateStaleMillis > 0 && staleEviction(c)) {
+      c.closeConnectionFully(false);
+      return null;
+    }
     registerBusyConnection(c);
     return c;
   }
 
+  private boolean staleEviction(PooledConnection c) {
+    if (!stale(c)) {
+      return false;
+    }
+    if (Log.isLoggable(DEBUG)) {
+      Log.debug("stale connection validation millis:{0}", (System.currentTimeMillis() - c.lastUsedTime()));
+    }
+    return !pool.validateConnection(c);
+  }
+
+  private boolean stale(PooledConnection c) {
+    return c.lastUsedTime() < System.currentTimeMillis() - validateStaleMillis;
+  }
+
   PooledConnection obtainConnection() throws SQLException {
     try {
       PooledConnection pc = _obtainConnection();
@@ -224,9 +247,9 @@ private PooledConnection _obtainConnection() throws InterruptedException, SQLExc
       hitCount++;
       // are other threads already waiting? (they get priority)
       if (waitingThreads == 0) {
-        if (!freeList.isEmpty()) {
-          // we have a free connection to return
-          return extractFromFreeList();
+        PooledConnection freeConnection = extractFromFreeList();
+        if (freeConnection != null) {
+          return freeConnection;
         }
         if (busyList.size() < maxSize) {
           // grow the connection pool

ebean-datasource/src/test/java/io/ebean/datasource/test/FactoryTest.java

@@ -53,6 +53,31 @@ void readOnly() throws Exception {
     pool.shutdown();
   }
 
+  @Test
+  void staleValidate() throws Exception {
+    DataSourcePool pool = DataSourcePool.builder()
+      .name("staleValidate")
+      .url("jdbc:h2:mem:heartbeatTrimOnly")
+      .username("sa")
+      .password("")
+      .readOnly(true)
+      .autoCommit(true)
+      .heartbeatFreqSecs(1)
+      .trimPoolFreqSecs(1)
+      .maxInactiveTimeSecs(1)
+      .validateOnHeartbeat(false)
+      // .useLambdaCheck(true)
+      .build();
+
+
+    Thread.sleep(3000);
+    Connection connection1 = pool.getConnection();
+    connection1.close();
+    System.out.println("done");
+    Thread.sleep(2000);
+    pool.shutdown();
+  }
+
   // @Disabled
   @Test
   void heartbeatTrimOnly() throws Exception {