Skip to content

Commit 6ad439e

Browse files
rbygraverbygrave
authored
Merge pull request #142 from ebean-orm/wip/91-early-checksum-2
Add earlyChecksumMode, with automatic patching of legacy checksums
2 parents 73a1c08 + 1097773 commit 6ad439e

28 files changed

+266
-19
lines changed

‎ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ public class MigrationConfig {
6565
private String basePlatform;
6666
private String platform;
6767
private Properties properties;
68+
private boolean earlyChecksumMode;
6869

6970
/**
7071
* Return the name of the migration table.
@@ -469,6 +470,7 @@ public void load(Properties props) {
469470
dbSchema = getProperty("schema", dbSchema);
470471
skipMigrationRun = getBool("skipMigrationRun", skipMigrationRun);
471472
skipChecksum = getBool("skipChecksum", skipChecksum);
473+
earlyChecksumMode = getBool("earlyChecksumMode", earlyChecksumMode);
472474
createSchemaIfNotExists = getBool("createSchemaIfNotExists", createSchemaIfNotExists);
473475
setCurrentSchema = getBool("setCurrentSchema", setCurrentSchema);
474476
basePlatform = getProperty("basePlatform", basePlatform);
@@ -575,6 +577,21 @@ public void setPlatform(String platform) {
575577
this.platform = platform;
576578
}
577579

580+
/**
581+
* Return true if using the earlyChecksumMode which means checksums are computed
582+
* before any expressions in the scripts are translated.
583+
*/
584+
public boolean isEarlyChecksumMode() {
585+
return earlyChecksumMode;
586+
}
587+
588+
/**
589+
* Set to true to turn on earlyChecksumMode.
590+
*/
591+
public void setEarlyChecksumMode(boolean earlyChecksumMode) {
592+
this.earlyChecksumMode = earlyChecksumMode;
593+
}
594+
578595
/**
579596
* Default factory. Uses the migration's class loader and injects the config if necessary.
580597
*

‎ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResources.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ boolean readInitResources() {
4848
*/
4949
boolean readResources() {
5050
if (readFromIndex()) {
51+
// automatically enable earlyChecksumMode when using index file with pre-computed checksums
52+
migrationConfig.setEarlyChecksumMode(true);
5153
return true;
5254
}
5355
return readResourcesForPath(migrationConfig.getMigrationPath());
@@ -86,7 +88,7 @@ private boolean loadFromIndexFile(URL idx, String base) {
8688
if (pair.length == 2) {
8789
final var checksum = Integer.parseInt(pair[0]);
8890
final var location = pair[1].trim();
89-
final String substring = location.substring(0, location.length() - 4);
91+
final var substring = location.substring(0, location.length() - 4);
9092
final var version = MigrationVersion.parse(substring);
9193
final var url = resource(base + location);
9294
versions.add(new LocalUriMigrationResource(version, location, url, checksum));

‎ebean-migration/src/main/java/io/ebean/migration/runner/MigrationEngine.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public List<MigrationResource> run(Connection connection) {
4949
connection.commit();
5050
if (!checkStateOnly) {
5151
long commitMs = System.currentTimeMillis();
52-
log.log(INFO, "DB migrations completed in {0}ms - executed:{1} totalMigrations:{2}", (commitMs - startMs), table.count(), table.size());
52+
log.log(INFO, "DB migrations completed in {0}ms - executed:{1} totalMigrations:{2} mode:{3}", (commitMs - startMs), table.count(), table.size(), table.mode());
5353
int countNonTransactional = table.runNonTransactional();
5454
if (countNonTransactional > 0) {
5555
log.log(INFO, "Non-transactional DB migrations completed in {0}ms - executed:{1}", (System.currentTimeMillis() - commitMs), countNonTransactional);

‎ebean-migration/src/main/java/io/ebean/migration/runner/MigrationMetaRow.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ final class MigrationMetaRow {
5252

5353
@Override
5454
public String toString() {
55-
return "id:" + id + " type:" + type + " runVersion:" + version + " comment:" + comment + " runOn:" + runOn + " runBy:" + runBy;
55+
return "id:" + id + " type:" + type + " checksum:" + checksum + " runVersion:" + version + " comment:" + comment + " runOn:" + runOn + " runBy:" + runBy;
5656
}
5757

5858
/**

‎ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java‎

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,13 @@ final class MigrationTable {
2121
static final System.Logger log = AppLog.getLogger("io.ebean.DDL");
2222

2323
private static final String INIT_VER_0 = "0";
24+
private static final int LEGACY_MODE_CHECKSUM = 0;
25+
private static final int EARLY_MODE_CHECKSUM = 1;
26+
private static final int AUTO_PATCH_CHECKSUM = -1;
2427

2528
private final Connection connection;
2629
private final boolean checkStateOnly;
30+
private boolean earlyChecksumMode;
2731
private final MigrationPlatform platform;
2832
private final MigrationScriptRunner scriptRunner;
2933
private final String catalog;
@@ -65,6 +69,8 @@ final class MigrationTable {
6569
private MigrationVersion dbInitVersion;
6670

6771
private int executionCount;
72+
private boolean patchLegacyChecksums;
73+
private MigrationMetaRow initMetaRow;
6874

6975
/**
7076
* Construct with server, configuration and jdbc connection (DB admin user).
@@ -74,6 +80,7 @@ final class MigrationTable {
7480
this.connection = connection;
7581
this.scriptRunner = new MigrationScriptRunner(connection, platform);
7682
this.checkStateOnly = checkStateOnly;
83+
this.earlyChecksumMode = config.isEarlyChecksumMode();
7784
this.migrations = new LinkedHashMap<>();
7885
this.catalog = null;
7986
this.allowErrorInRepeatable = config.isAllowErrorInRepeatable();
@@ -296,20 +303,25 @@ private boolean shouldRun(LocalMigrationResource localVersion, LocalMigrationRes
296303
private boolean runMigration(LocalMigrationResource local, MigrationMetaRow existing) throws SQLException {
297304
String script = null;
298305
int checksum;
306+
int checksum2 = 0;
299307
if (local instanceof LocalUriMigrationResource) {
300-
script = convertScript(local.content());
301308
checksum = ((LocalUriMigrationResource)local).checksum();
302-
} else if (local instanceof LocalDdlMigrationResource) {
309+
checksum2 = patchLegacyChecksums ? AUTO_PATCH_CHECKSUM : 0;
303310
script = convertScript(local.content());
304-
checksum = Checksum.calculate(script);
311+
} else if (local instanceof LocalDdlMigrationResource) {
312+
final String content = local.content();
313+
script = convertScript(content);
314+
// checksum on original content (NEW) or converted script content (LEGACY)
315+
checksum = Checksum.calculate(earlyChecksumMode ? content : script);
316+
checksum2 = patchLegacyChecksums ? Checksum.calculate(script) : 0;
305317
} else {
306318
checksum = ((LocalJdbcMigrationResource) local).checksum();
307319
}
308320

309321
if (existing == null && patchInsertMigration(local, checksum)) {
310322
return true;
311323
}
312-
if (existing != null && skipMigration(checksum, local, existing)) {
324+
if (existing != null && skipMigration(checksum, checksum2, local, existing)) {
313325
return true;
314326
}
315327
executeMigration(local, script, checksum, existing);
@@ -333,12 +345,19 @@ private boolean patchInsertMigration(LocalMigrationResource local, int checksum)
333345
/**
334346
* Return true if the migration should be skipped.
335347
*/
336-
boolean skipMigration(int checksum, LocalMigrationResource local, MigrationMetaRow existing) throws SQLException {
348+
boolean skipMigration(int checksum, int checksum2, LocalMigrationResource local, MigrationMetaRow existing) throws SQLException {
337349
boolean matchChecksum = (existing.checksum() == checksum);
338350
if (matchChecksum) {
339351
log.log(TRACE, "skip unchanged migration {0}", local.location());
340352
return true;
341353

354+
} else if (patchLegacyChecksums && (existing.checksum() == checksum2 || checksum2 == AUTO_PATCH_CHECKSUM)) {
355+
if (!checkStateOnly) {
356+
log.log(INFO, "Patch migration, set early mode checksum on {0}", local.location());
357+
existing.resetChecksum(checksum, connection, updateChecksumSql);
358+
}
359+
return true;
360+
342361
} else if (patchResetChecksum(existing, checksum)) {
343362
log.log(INFO, "Patch migration, reset checksum on {0}", local.location());
344363
return true;
@@ -424,7 +443,8 @@ private void insertIntoHistory(LocalMigrationResource local, int checksum, long
424443
}
425444

426445
private MigrationMetaRow createInitMetaRow() {
427-
return new MigrationMetaRow(0, "I", INIT_VER_0, "<init>", 0, envUserName, runOn, 0);
446+
final int mode = earlyChecksumMode ? EARLY_MODE_CHECKSUM : LEGACY_MODE_CHECKSUM;
447+
return new MigrationMetaRow(0, "I", INIT_VER_0, "<init>", mode, envUserName, runOn, 0);
428448
}
429449

430450
/**
@@ -460,7 +480,12 @@ private String convertScript(String script) {
460480
*/
461481
private void addMigration(String key, MigrationMetaRow metaRow) {
462482
if (INIT_VER_0.equals(key)) {
463-
// ignore the version 0 <init> row
483+
if (metaRow.checksum() == EARLY_MODE_CHECKSUM && !earlyChecksumMode) {
484+
log.log(DEBUG, "automatically detected earlyChecksumMode");
485+
earlyChecksumMode = true;
486+
}
487+
initMetaRow = metaRow;
488+
patchLegacyChecksums = earlyChecksumMode && metaRow.checksum() == LEGACY_MODE_CHECKSUM;
464489
return;
465490
}
466491
lastMigration = metaRow;
@@ -501,6 +526,10 @@ List<MigrationResource> runAll(List<LocalMigrationResource> localVersions) throw
501526
break;
502527
}
503528
}
529+
if (patchLegacyChecksums && !checkStateOnly) {
530+
// only patch the legacy checksums once
531+
initMetaRow.resetChecksum(EARLY_MODE_CHECKSUM, connection, updateChecksumSql);
532+
}
504533
return checkMigrations;
505534
}
506535

@@ -561,4 +590,11 @@ int runNonTransactional() {
561590
int count() {
562591
return executionCount;
563592
}
593+
594+
/**
595+
* Return the mode being used by this migration run.
596+
*/
597+
String mode() {
598+
return !earlyChecksumMode ? "legacy" : (patchLegacyChecksums ? "earlyChecksum with patching" : "earlyChecksum");
599+
}
564600
}

‎ebean-migration/src/test/java/io/ebean/migration/MigrationRunner_FastCheckTest.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,15 @@
11
package io.ebean.migration;
22

3+
import io.ebean.datasource.DataSourceConfig;
4+
import io.ebean.datasource.DataSourceFactory;
5+
import io.ebean.datasource.DataSourcePool;
36
import org.junit.jupiter.api.Test;
47

8+
import java.util.List;
9+
import java.util.Map;
10+
11+
import static org.assertj.core.api.Assertions.assertThat;
12+
513
public class MigrationRunner_FastCheckTest {
614

715
private MigrationConfig createMigrationConfig() {
@@ -39,4 +47,54 @@ public void run_when() {
3947
runner.run();
4048
}
4149

50+
@Test
51+
public void autoEnableEarlyMode_when_indexFileAddedToExistingMigrations() {
52+
53+
String url = "jdbc:h2:mem:autoEnableEarlyMode";
54+
DataSourceConfig dataSourceConfig = new DataSourceConfig()
55+
.setUrl(url)
56+
.setUsername("sa")
57+
.setPassword("");
58+
59+
DataSourcePool dataSource = DataSourceFactory.create("test", dataSourceConfig);
60+
61+
MigrationConfig config = new MigrationConfig();
62+
config.setPlatform("h2");
63+
config.setRunPlaceholderMap(Map.of("my_table_name", "bar"));
64+
65+
// initial traditional migration
66+
config.setMigrationPath("indexB_0");
67+
MigrationRunner runner = new MigrationRunner(config);
68+
runner.run(dataSource);
69+
70+
assertThat(config.isEarlyChecksumMode()).isFalse();
71+
72+
// add an index file now, expect automatically go to early mode + patch checksums
73+
config.setMigrationPath("indexB_1");
74+
new MigrationRunner(config).run(dataSource);
75+
assertThat(config.isEarlyChecksumMode()).isTrue();
76+
77+
// early mode via <init> row + add an extra migration
78+
config.setMigrationPath("indexB_2");
79+
new MigrationRunner(config).run(dataSource);
80+
81+
dataSource.shutdown();
82+
}
83+
84+
@Test
85+
public void autoEnableEarlyMode() {
86+
87+
MigrationConfig config = createMigrationConfig();
88+
89+
config.setPlatform("h2");
90+
config.setMigrationPath("indexB_1");
91+
config.setDbUrl("jdbc:h2:mem:autoEnableEarlyMode_simple");
92+
config.setRunPlaceholderMap(Map.of("my_table_name", "bar"));
93+
94+
MigrationRunner runner = new MigrationRunner(config);
95+
runner.run();
96+
97+
assertThat(config.isEarlyChecksumMode()).isTrue();
98+
}
99+
42100
}

‎ebean-migration/src/test/java/io/ebean/migration/runner/MigrationEarlyModeTest.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
package io.ebean.migration.runner;
2+
3+
import io.avaje.applog.AppLog;
4+
import io.ebean.datasource.DataSourceConfig;
5+
import io.ebean.datasource.DataSourceFactory;
6+
import io.ebean.datasource.DataSourcePool;
7+
import io.ebean.migration.MigrationConfig;
8+
import io.ebean.migration.MigrationRunner;
9+
import io.ebean.test.containers.PostgresContainer;
10+
import org.junit.jupiter.api.Test;
11+
12+
import java.io.IOException;
13+
import java.sql.Connection;
14+
import java.sql.PreparedStatement;
15+
import java.sql.SQLException;
16+
import java.util.Map;
17+
18+
import static java.lang.System.Logger.Level.INFO;
19+
import static org.assertj.core.api.Assertions.fail;
20+
21+
class MigrationEarlyModeTest {
22+
23+
private static final System.Logger log = AppLog.getLogger(MigrationEarlyModeTest.class);
24+
25+
String un = "mig_early";
26+
String pw = "test";
27+
28+
private static PostgresContainer createPostgres() {
29+
return PostgresContainer.builder("15")
30+
.port(9823)
31+
.containerName("pg15")
32+
.user("mig_early")
33+
.dbName("mig_early")
34+
.build();
35+
}
36+
37+
@Test
38+
void testEarlyMode() {
39+
createPostgres().startWithDropCreate();
40+
41+
String url = createPostgres().jdbcUrl();
42+
DataSourcePool dataSource = dataSource(url);
43+
44+
MigrationConfig config = new MigrationConfig();
45+
config.setDbUrl(url);
46+
config.setDbUsername(un);
47+
config.setDbPassword(pw);
48+
config.setMigrationPath("dbmig_postgres_early");
49+
config.setRunPlaceholderMap(Map.of("my_table_name", "my_table"));
50+
51+
// legacy mode
52+
new MigrationRunner(config).run(dataSource);
53+
54+
// early mode
55+
log.log(INFO, "-- EARLY MODE -- ");
56+
config.setEarlyChecksumMode(true);
57+
new MigrationRunner(config).run(dataSource);
58+
59+
log.log(INFO, "-- RE-RUN EARLY MODE -- ");
60+
new MigrationRunner(config).run(dataSource);
61+
62+
log.log(INFO, "-- LEGACY MODE AGAIN (will auto detect early mode) -- ");
63+
config.setEarlyChecksumMode(false);
64+
new MigrationRunner(config).run(dataSource);
65+
66+
log.log(INFO, "-- LEGACY MODE with more migrations -- ");
67+
68+
config.setRunPlaceholderMap(Map.of("my_table_name", "my_table", "other_table_name", "other"));
69+
config.setMigrationPath("dbmig_postgres_early1");
70+
new MigrationRunner(config).run(dataSource);
71+
72+
log.log(INFO, "-- EARLY MODE again -- ");
73+
config.setEarlyChecksumMode(true);
74+
new MigrationRunner(config).run(dataSource);
75+
76+
}
77+
78+
private DataSourcePool dataSource(String url) {
79+
DataSourceConfig dataSourceConfig = new DataSourceConfig()
80+
.setUrl(url)
81+
.setUsername(un)
82+
.setPassword(pw);
83+
return DataSourceFactory.create("mig_early", dataSourceConfig);
84+
}
85+
}

‎ebean-migration/src/test/java/io/ebean/migration/runner/MigrationTable2Test.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,9 @@ void test_skipMigration_Repeatable() throws Exception {
4848

4949
MigrationTable mt = new MigrationTable(config, null, false, new MigrationPlatform());
5050
// checksum different - no skip
51-
assertFalse(mt.skipMigration(100, local, existing));
51+
assertFalse(mt.skipMigration(100, 100, local, existing));
5252
// checksum same - skip
53-
assertTrue(mt.skipMigration(42, local, existing));
53+
assertTrue(mt.skipMigration(42, 42, local, existing));
5454
}
5555

5656
@Test
@@ -66,19 +66,20 @@ void test_skipMigration_skipChecksum() throws Exception {
6666
MigrationMetaRow existing = new MigrationMetaRow(12, "R", "", "comment", 42, null, null, 0);
6767

6868
// repeatable checksum mismatch
69-
assertFalse(mt.skipMigration(44, local, existing));
69+
assertFalse(mt.skipMigration(44, 44, local, existing));
7070

7171
// repeatable match checksum
72-
assertTrue(mt.skipMigration(42, local, existing));
72+
assertTrue(mt.skipMigration(42, 42, local, existing));
73+
// assertTrue(mt.skipMigration(99, 42, local, existing));
7374

7475
LocalMigrationResource localVer = local("V1__hello");
7576
MigrationMetaRow localExisting = new MigrationMetaRow(12, "V", "1", "comment", 42, null, null, 0);
7677

7778
// re-run on checksum mismatch and skipChecksum
78-
assertFalse(mt.skipMigration(44, localVer, localExisting));
79+
assertFalse(mt.skipMigration(44, 44, localVer, localExisting));
7980

8081
// match checksum so skip
81-
assertTrue(mt.skipMigration(42, localVer, localExisting));
82+
assertTrue(mt.skipMigration(42, 42, localVer, localExisting));
8283
}
8384

8485
private LocalMigrationResource local(String raw) {

0 commit comments

Comments
 (0)