Merge pull request #146 from ebean-orm/feature/add-fastmode

Add a fastMode for the common case when there are no new migrations to run

Author: rbygrave

ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java

@@ -66,6 +66,7 @@ public class MigrationConfig {
   private String platform;
   private Properties properties;
   private boolean earlyChecksumMode;
+  private boolean fastMode;
 
   /**
    * Return the name of the migration table.
@@ -468,6 +469,7 @@ public void load(Properties props) {
     dbPassword = property("password", dbPassword);
     dbUrl = property("url", dbUrl);
     dbSchema = property("schema", dbSchema);
+    fastMode = property("fastMode", fastMode);
     skipMigrationRun = property("skipMigrationRun", skipMigrationRun);
     skipChecksum = property("skipChecksum", skipChecksum);
     earlyChecksumMode = property("earlyChecksumMode", earlyChecksumMode);
@@ -592,6 +594,22 @@ public void setEarlyChecksumMode(boolean earlyChecksumMode) {
     this.earlyChecksumMode = earlyChecksumMode;
   }
 
+  /**
+   * Return true if fastMode is turned on.
+   */
+  public boolean isFastMode() {
+    return fastMode;
+  }
+
+  /**
+   * Set true to enable fastMode. This will perform an initial check for the exact same number
+   * of migrations and matching checksums without any locking. If anything does not match
+   * then the normal migration is performed with appropriate locking etc.
+   */
+  public void setFastMode(boolean fastMode) {
+    this.fastMode = fastMode;
+  }
+
   /**
    * Default factory. Uses the migration's class loader and injects the config if necessary.
    *

ebean-migration/src/main/java/io/ebean/migration/runner/MigrationEngine.java

@@ -9,6 +9,8 @@
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 import static java.lang.System.Logger.Level.*;
 import static java.lang.System.Logger.Level.WARNING;
@@ -22,13 +24,17 @@ public class MigrationEngine {
 
   private final MigrationConfig migrationConfig;
   private final boolean checkStateOnly;
+  private final boolean fastMode;
+  private int fastModeCount;
+  private MigrationTable table;
 
   /**
    * Create with the MigrationConfig.
    */
   public MigrationEngine(MigrationConfig migrationConfig, boolean checkStateOnly) {
     this.migrationConfig = migrationConfig;
     this.checkStateOnly = checkStateOnly;
+    this.fastMode = !checkStateOnly && migrationConfig.isFastMode();
   }
 
   /**
@@ -41,11 +47,18 @@ public List<MigrationResource> run(Connection connection) {
         log.log(DEBUG, "no migrations to check");
         return Collections.emptyList();
       }
-
       long startMs = System.currentTimeMillis();
-      MigrationTable table = initialiseMigrationTable(connection);
+      setAutoCommitFalse(connection);
+      table = initMigrationTable(connection);
+      if (fastMode && fastModeCheck(resources.versions())) {
+        long checkMs = System.currentTimeMillis() - startMs;
+        log.log(INFO, "DB migrations completed in {0}ms - totalMigrations:{1}", checkMs, fastModeCount);
+        return Collections.emptyList();
+      }
+
+      initialiseMigrationTable(connection);
       try {
-        List<MigrationResource> result = runMigrations(resources.versions(), table, checkStateOnly);
+        List<MigrationResource> result = runMigrations(resources.versions());
         connection.commit();
         if (!checkStateOnly) {
           long commitMs = System.currentTimeMillis();
@@ -71,15 +84,65 @@ public List<MigrationResource> run(Connection connection) {
     }
   }
 
-  private MigrationTable initialiseMigrationTable(Connection connection) {
+  private static void setAutoCommitFalse(Connection connection) {
     try {
       connection.setAutoCommit(false);
-      MigrationPlatform platform = derivePlatformName(migrationConfig, connection);
-      new MigrationSchema(migrationConfig, connection).createAndSetIfNeeded();
+    } catch (SQLException e) {
+      throw new MigrationException("Error running DB migrations", e);
+    }
+  }
+
+  private MigrationTable initMigrationTable(Connection connection) {
+    final MigrationPlatform platform = derivePlatformName(migrationConfig, connection);
+    return new MigrationTable(migrationConfig, connection, checkStateOnly, platform);
+  }
+
+  private boolean fastModeCheck(List<LocalMigrationResource> versions) {
+    try {
+      final List<MigrationMetaRow> rows = table.fastRead();
+      if (rows.size() != versions.size() + 1) {
+        // difference in count of migrations
+        return false;
+      }
+      final Map<String, Integer> dbChecksums = dbChecksumMap(rows);
+      for (LocalMigrationResource local : versions) {
+        Integer dbChecksum = dbChecksums.get(local.key());
+        if (dbChecksum == null) {
+          // no match, unexpected missing migration
+          return false;
+        }
+        int localChecksum = checksumFor(local);
+        if (localChecksum != dbChecksum) {
+          // no match, perhaps repeatable migration change
+          return false;
+        }
+      }
+      // successful fast check
+      fastModeCount = versions.size();
+      return true;
+    } catch (SQLException e) {
+      // probably migration table does not exist
+      return false;
+    }
+  }
 
-      final MigrationTable table = new MigrationTable(migrationConfig, connection, checkStateOnly, platform);
+  private static Map<String, Integer> dbChecksumMap(List<MigrationMetaRow> rows) {
+    return rows.stream().collect(Collectors.toMap(MigrationMetaRow::version, MigrationMetaRow::checksum));
+  }
+
+  private int checksumFor(LocalMigrationResource local) {
+    if (local instanceof LocalUriMigrationResource) {
+      return ((LocalUriMigrationResource)local).checksum();
+    } else if (local instanceof LocalDdlMigrationResource) {
+      return Checksum.calculate(local.content());
+    } else {
+      return ((LocalJdbcMigrationResource) local).checksum();
+    }
+  }
+
+  private void initialiseMigrationTable(Connection connection) {
+    try {
       table.createIfNeededAndLock();
-      return table;
     } catch (Throwable e) {
       rollback(connection);
       throw new MigrationException("Error initialising db migrations table", e);
@@ -89,13 +152,13 @@ private MigrationTable initialiseMigrationTable(Connection connection) {
   /**
    * Run all the migrations as needed.
    */
-  private List<MigrationResource> runMigrations(List<LocalMigrationResource> localVersions, MigrationTable table, boolean checkStateMode) throws SQLException {
+  private List<MigrationResource> runMigrations(List<LocalMigrationResource> localVersions) throws SQLException {
     // get the migrations in version order
     if (table.isEmpty()) {
       LocalMigrationResource initVersion = lastInitVersion();
       if (initVersion != null) {
         // run using a dbinit script
-        log.log(INFO, "dbinit migration version:{0}  local migrations:{1}  checkState:{2}", initVersion, localVersions.size(), checkStateMode);
+        log.log(INFO, "dbinit migration version:{0}  local migrations:{1}  checkState:{2}", initVersion, localVersions.size(), checkStateOnly);
         return table.runInit(initVersion, localVersions);
       }
     }

ebean-migration/src/main/java/io/ebean/migration/runner/MigrationMetaRow.java

@@ -12,8 +12,8 @@
 @SuppressWarnings("SqlSourceToSinkFlow")
 final class MigrationMetaRow {
 
-  private final int id;
-  private final String type;
+  private int id;
+  private String type;
   private final String version;
   private String comment;
   private int checksum;
@@ -45,6 +45,17 @@ final class MigrationMetaRow {
     checksum = row.getInt(4);
   }
 
+  private MigrationMetaRow(int checksum, String version) {
+    this.checksum = checksum;
+    this.version = version;
+  }
+
+  static MigrationMetaRow fastRead(ResultSet row) throws SQLException {
+    final var checksum = row.getInt(1);
+    final var version = row.getString(2);
+    return new MigrationMetaRow(checksum, version);
+  }
+
   @Override
   public String toString() {
     return "id:" + id + " type:" + type + " checksum:" + checksum + " version:" + version;

ebean-migration/src/main/java/io/ebean/migration/runner/MigrationPlatform.java

@@ -19,6 +19,7 @@ class MigrationPlatform {
 
   private static final String BASE_SELECT_ID = "select id from ";
   private static final String BASE_SELECT = "select id, mtype, mversion, mchecksum from ";
+  private static final String SELECT_FAST_READ = "select mchecksum, mversion from ";
 
   /**
    * Standard row locking for db migration table.
@@ -66,9 +67,8 @@ private static void backoff(int attempt) {
 
   private int lockRows(String sqlTable, Connection connection) throws SQLException {
     int rowCount = 0;
-    final String selectSql = sqlSelectForUpdate(sqlTable);
     try (Statement query = connection.createStatement()) {
-      try (ResultSet resultSet = query.executeQuery(selectSql)) {
+      try (ResultSet resultSet = query.executeQuery(sqlSelectForUpdate(sqlTable))) {
         while (resultSet.next()) {
           resultSet.getInt(1);
           rowCount++;
@@ -78,6 +78,20 @@ private int lockRows(String sqlTable, Connection connection) throws SQLException
     return rowCount;
   }
 
+  List<MigrationMetaRow> fastReadMigrations(String sqlTable, Connection connection) throws SQLException {
+    List<MigrationMetaRow> rows = new ArrayList<>();
+    try (Statement query = connection.createStatement()) {
+      try (ResultSet resultSet = query.executeQuery(sqlSelectForFastRead(sqlTable))) {
+        while (resultSet.next()) {
+          rows.add(MigrationMetaRow.fastRead(resultSet));
+        }
+      }
+    } finally {
+      connection.rollback();
+    }
+    return rows;
+  }
+
   /**
    * Read the existing migrations from the db migration table.
    */
@@ -108,6 +122,10 @@ String sqlSelectForReading(String table) {
     return BASE_SELECT + table + forUpdateSuffix;
   }
 
+  String sqlSelectForFastRead(String table) {
+    return SELECT_FAST_READ + table;
+  }
+
   static final class LogicalLock extends MigrationPlatform {
 
     @Override

ebean-migration/src/main/java/io/ebean/migration/runner/MigrationSchema.java

@@ -26,13 +26,17 @@ final class MigrationSchema {
   /**
    * Construct with configuration and connection.
    */
-  MigrationSchema(MigrationConfig migrationConfig, Connection connection) {
+  private MigrationSchema(MigrationConfig migrationConfig, Connection connection) {
     this.dbSchema = trim(migrationConfig.getDbSchema());
     this.createSchemaIfNotExists = migrationConfig.isCreateSchemaIfNotExists();
     this.setCurrentSchema = migrationConfig.isSetCurrentSchema();
     this.connection = connection;
   }
 
+  static void createIfNeeded(MigrationConfig config, Connection connection) throws SQLException {
+    new MigrationSchema(config, connection).createAndSetIfNeeded();
+  }
+
   private String trim(String dbSchema) {
     return (dbSchema == null) ? null : dbSchema.trim();
   }
@@ -52,6 +56,7 @@ void createAndSetIfNeeded() throws SQLException {
     }
   }
 
+  @SuppressWarnings("SqlDialectInspection")
   private void createSchemaIfNeeded() throws SQLException {
     if (!schemaExists()) {
       log.log(INFO, "Creating schema: {0}", dbSchema);

ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java

@@ -25,6 +25,7 @@ final class MigrationTable {
   private static final int EARLY_MODE_CHECKSUM = 1;
   private static final int AUTO_PATCH_CHECKSUM = -1;
 
+  private final MigrationConfig config;
   private final Connection connection;
   private final boolean checkStateOnly;
   private boolean earlyChecksumMode;
@@ -71,11 +72,13 @@ final class MigrationTable {
   private int executionCount;
   private boolean patchLegacyChecksums;
   private MigrationMetaRow initMetaRow;
+  private boolean tableKnownToExist;
 
   /**
    * Construct with server, configuration and jdbc connection (DB admin user).
    */
   MigrationTable(MigrationConfig config, Connection connection, boolean checkStateOnly, MigrationPlatform platform) {
+    this.config = config;
     this.platform = platform;
     this.connection = connection;
     this.scriptRunner = new MigrationScriptRunner(connection, platform);
@@ -146,25 +149,28 @@ private ScriptTransform createScriptTransform(MigrationConfig config) {
    * </p>
    */
   void createIfNeededAndLock() throws SQLException, IOException {
-    SQLException sqlEx = null;
-    if (!tableExists()) {
-      try {
-        createTable();
-      } catch (SQLException e) {
-        if (tableExists()) {
-          sqlEx = e;
-          log.log(INFO, "Ignoring error during table creation, as an other process may have created the table", e);
-        } else {
-          throw e;
+    SQLException suppressedException = null;
+    if (!tableKnownToExist) {
+      MigrationSchema.createIfNeeded(config, connection);
+      if (!tableExists()) {
+        try {
+          createTable();
+        } catch (SQLException e) {
+          if (tableExists()) {
+            suppressedException = e;
+            log.log(INFO, "Ignoring error during table creation, as an other process may have created the table", e);
+          } else {
+            throw e;
+          }
         }
       }
     }
     try {
       obtainLockWithWait();
     } catch (RuntimeException re) {
       // catch "failed to obtain row locks"
-      if (sqlEx != null) {
-        re.addSuppressed(sqlEx);
+      if (suppressedException != null) {
+        re.addSuppressed(suppressedException);
       }
       throw re;
     }
@@ -187,6 +193,13 @@ void unlockMigrationTable() {
     platform.unlockMigrationTable(sqlTable, connection);
   }
 
+  List<MigrationMetaRow> fastRead() throws SQLException {
+    final var result = platform.fastReadMigrations(sqlTable, connection);
+    // if we know the migration table exists we can skip those checks
+    tableKnownToExist = !result.isEmpty();
+    return result;
+  }
+
   /**
    * Read the migration table with details on what migrations have run.
    * This must execute after we have completed the wait for the lock on

ebean-migration/src/test/java/io/ebean/migration/runner/MigrationEarlyModeTest.java

@@ -47,6 +47,7 @@ void testEarlyMode() {
     config.setDbPassword(pw);
     config.setMigrationPath("dbmig_postgres_early");
     config.setRunPlaceholderMap(Map.of("my_table_name", "my_table"));
+    config.setFastMode(true);
 
     // legacy mode
     new MigrationRunner(config).run(dataSource);

ebean-migration/src/test/java/io/ebean/migration/runner/MigrationSchemaTest.java

@@ -16,8 +16,7 @@ void testCreateAndSetIfNeeded() throws Exception {
 
     Connection connection = config.createConnection();
 
-    MigrationSchema migrationSchema = new MigrationSchema(config, connection);
-    migrationSchema.createAndSetIfNeeded();
+    MigrationSchema.createIfNeeded(config, connection);
   }
 
   private MigrationConfig createMigrationConfig() {
@@ -28,4 +27,4 @@ private MigrationConfig createMigrationConfig() {
     config.setDbUrl("jdbc:h2:mem:db1");
     return config;
   }
-}
+}