Merge pull request #138 from ebean-orm/feature/read-idx-files

rbygrave · web-flow · commit 2a1e5eb67062 · 2023-10-29T23:29:04.000+13:00
Read idx migrations (index) files and support GraalVM native image,
diff --git a/ebean-migration/pom.xml b/ebean-migration/pom.xml
@@ -135,8 +135,8 @@ mvn install:install-file -Dfile=/some/path/to/ojdbc7.jar -DgroupId=oracle \
 
     <dependency>
       <groupId>io.ebean</groupId>
-      <artifactId>ebean-test-docker</artifactId>
-      <version>5.4</version>
+      <artifactId>ebean-test-containers</artifactId>
+      <version>7.1</version>
       <scope>test</scope>
     </dependency>
 
diff --git a/ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java b/ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java
@@ -549,7 +549,7 @@ public void setName(String name) {
    * Return the base platform that was set.
    */
   public String getBasePlatform() {
-    return basePlatform;
+    return basePlatform != null ? basePlatform : platform;
   }
 
   /**
diff --git a/ebean-migration/src/main/java/io/ebean/migration/MigrationVersion.java b/ebean-migration/src/main/java/io/ebean/migration/MigrationVersion.java
@@ -190,6 +190,9 @@ public static String trim(String raw) {
    * Parse the raw version string into a MigrationVersion.
    */
   public static MigrationVersion parse(String raw) {
+    if (raw.endsWith(".sql")) {
+      raw = raw.substring(0, raw.length() - 4);
+    }
     if (raw.startsWith("V") || raw.startsWith("v")) {
       raw = raw.substring(1);
     }
diff --git a/ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResources.java b/ebean-migration/src/main/java/io/ebean/migration/runner/LocalMigrationResources.java
@@ -6,6 +6,8 @@
 import io.ebean.migration.MigrationConfig;
 import io.ebean.migration.MigrationVersion;
 
+import java.io.*;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -45,9 +47,60 @@ boolean readInitResources() {
    * Read all the migration resources (SQL scripts) returning true if there are versions.
    */
   boolean readResources() {
+    if (readFromIndex()) {
+      return true;
+    }
     return readResourcesForPath(migrationConfig.getMigrationPath());
   }
 
+  private boolean readFromIndex() {
+    final var base = "/" + migrationConfig.getMigrationPath() + "/";
+    final var basePlatform = migrationConfig.getBasePlatform();
+    final var indexName = "idx_" + basePlatform + ".migrations";
+    URL idx = resource(base + indexName);
+    if (idx != null) {
+      return loadFromIndexFile(idx, base);
+    }
+    idx = resource(base + basePlatform + '/' + indexName);
+    if (idx != null) {
+      return loadFromIndexFile(idx, base + basePlatform + '/');
+    }
+    final var platform = migrationConfig.getPlatform();
+    idx = resource(base + platform + indexName);
+    if (idx != null) {
+      return loadFromIndexFile(idx, base + platform + '/');
+    }
+    return false;
+  }
+
+  private URL resource(String base) {
+    return LocalMigrationResources.class.getResource(base);
+  }
+
+  private boolean loadFromIndexFile(URL idx, String base) {
+    try (var reader = new LineNumberReader(new InputStreamReader(idx.openStream()))) {
+      String line;
+      while ((line = reader.readLine()) != null) {
+        if (!line.isEmpty()) {
+          final String[] pair = line.split(",");
+          if (pair.length == 2) {
+            final var checksum = Integer.parseInt(pair[0]);
+            final var location = pair[1].trim();
+            final String substring = location.substring(0, location.length() - 4);
+            final var version = MigrationVersion.parse(substring);
+            final var url = resource(base + location);
+            versions.add(new LocalUriMigrationResource(version, location, url, checksum));
+          }
+        }
+      }
+
+      return !versions.isEmpty();
+
+    } catch (IOException e) {
+      throw new UncheckedIOException("Error reading idx file", e);
+    }
+  }
+
   private boolean readResourcesForPath(String path) {
     // try to load from base platform first
     final String basePlatform = migrationConfig.getBasePlatform();
diff --git a/ebean-migration/src/main/java/io/ebean/migration/runner/LocalUriMigrationResource.java b/ebean-migration/src/main/java/io/ebean/migration/runner/LocalUriMigrationResource.java
@@ -0,0 +1,42 @@
+package io.ebean.migration.runner;
+
+import io.ebean.migration.MigrationVersion;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.StringWriter;
+import java.net.URL;
+
+/**
+ * A local URL based DB migration resource.
+ */
+final class LocalUriMigrationResource extends LocalMigrationResource {
+
+  private final URL resource;
+  private final int checksum;
+
+  LocalUriMigrationResource(MigrationVersion version, String location, URL resource, int checksum) {
+    super(version, location);
+    this.resource = resource;
+    this.checksum = checksum;
+  }
+
+  public int checksum() {
+    return checksum;
+  }
+
+  @Override
+  public String content() {
+    try (var reader = new InputStreamReader(resource.openStream())) {
+      var writer = new StringWriter(1024);
+      reader.transferTo(writer);
+      return writer.toString();
+    } catch (IOException e) {
+      throw new IllegalStateException(missingOpensMessage(), e);
+    }
+  }
+
+  private String missingOpensMessage() {
+    return "NPE reading DB migration content at [" + location + "] Probably missing an 'opens dbmigration;' in module-info.java";
+  }
+}
diff --git a/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java b/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java
@@ -296,7 +296,10 @@ private boolean shouldRun(LocalMigrationResource localVersion, LocalMigrationRes
   private boolean runMigration(LocalMigrationResource local, MigrationMetaRow existing) throws SQLException {
     String script = null;
     int checksum;
-    if (local instanceof LocalDdlMigrationResource) {
+    if (local instanceof LocalUriMigrationResource) {
+      script = convertScript(local.content());
+      checksum = ((LocalUriMigrationResource)local).checksum();
+    } else if (local instanceof LocalDdlMigrationResource) {
       script = convertScript(local.content());
       checksum = Checksum.calculate(script);
     } else {
@@ -402,13 +405,13 @@ private void executeMigration(LocalMigrationResource local, String script, int c
 
   private long executeMigration(LocalMigrationResource local, String script) throws SQLException {
     long start = System.currentTimeMillis();
-    if (local instanceof LocalDdlMigrationResource) {
-      log.log(DEBUG, "run migration {0}", local.location());
-      scriptRunner.runScript(script, "run migration version: " + local.version());
-    } else {
+    if (local instanceof LocalJdbcMigrationResource) {
       JdbcMigration migration = ((LocalJdbcMigrationResource) local).migration();
       log.log(INFO, "Executing jdbc migration version: {0} - {1}", local.version(), migration);
       migration.migrate(connection);
+    } else {
+      log.log(DEBUG, "run migration {0}", local.location());
+      scriptRunner.runScript(script, "run migration version: " + local.version());
     }
     executionCount++;
     return System.currentTimeMillis() - start;
diff --git a/ebean-migration/src/main/resources/META-INF/native-image/io.ebean.migration.ebean-migration/resource-config.json b/ebean-migration/src/main/resources/META-INF/native-image/io.ebean.migration.ebean-migration/resource-config.json
@@ -0,0 +1,10 @@
+{
+ "resources": [
+  {
+   "pattern": ".*\\.sql"
+  },
+  {
+   "pattern": ".*\\.migrations"
+  }
+ ]
+}
diff --git a/ebean-migration/src/test/java/io/ebean/migration/MigrationRunner_platform_Test.java b/ebean-migration/src/test/java/io/ebean/migration/MigrationRunner_platform_Test.java
@@ -1,8 +1,7 @@
 package io.ebean.migration;
 
 import io.ebean.ddlrunner.DdlRunner;
-import io.ebean.docker.commands.*;
-import io.ebean.docker.container.ContainerBuilderDb;
+import io.ebean.test.containers.*;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
diff --git a/ebean-migration/src/test/java/io/ebean/migration/MigrationVersionTest.java b/ebean-migration/src/test/java/io/ebean/migration/MigrationVersionTest.java
@@ -108,6 +108,18 @@ void test_parse_when_v_prefix() {
     assertThat(version.type()).isEqualTo("V");
   }
 
+  @Test
+  void test_parse_when_sql_suffix() {
+
+    MigrationVersion version = MigrationVersion.parse("v1_0__Foo.sql");
+    assertThat(version.isRepeatable()).isFalse();
+    assertThat(version.comment()).isEqualTo("Foo");
+    assertThat(version.normalised()).isEqualTo("1.0");
+    assertThat(version.asString()).isEqualTo("1_0");
+    assertThat(version.raw()).isEqualTo("1_0__Foo");
+    assertThat(version.type()).isEqualTo("V");
+  }
+
   @Test
   void repeatable_compareTo() {
 
diff --git a/ebean-migration/src/test/java/io/ebean/migration/runner/MigrationTableAsyncTest.java b/ebean-migration/src/test/java/io/ebean/migration/runner/MigrationTableAsyncTest.java
@@ -3,9 +3,9 @@
 import io.ebean.datasource.DataSourceConfig;
 import io.ebean.datasource.DataSourceFactory;
 import io.ebean.datasource.DataSourcePool;
-import io.ebean.docker.commands.*;
 import io.ebean.migration.MigrationConfig;
 import io.ebean.migration.MigrationRunner;
+import io.ebean.test.containers.*;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
diff --git a/ebean-migration/src/test/java/io/ebean/migration/runner/MigrationTableCreateTableRaceTest.java b/ebean-migration/src/test/java/io/ebean/migration/runner/MigrationTableCreateTableRaceTest.java
@@ -3,8 +3,8 @@
 import io.ebean.datasource.DataSourceConfig;
 import io.ebean.datasource.DataSourceFactory;
 import io.ebean.datasource.DataSourcePool;
-import io.ebean.docker.commands.PostgresContainer;
 import io.ebean.migration.MigrationConfig;
+import io.ebean.test.containers.PostgresContainer;
 import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
diff --git a/test-native-image/pom.xml b/test-native-image/pom.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.avaje</groupId>
+    <artifactId>java11-oss</artifactId>
+    <version>3.10</version>
+  </parent>
+
+  <groupId>org.example</groupId>
+  <artifactId>test-native-image</artifactId>
+
+  <properties>
+    <maven.compiler.release>21</maven.compiler.release>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <version.plugin.nativeimage>0.9.27</version.plugin.nativeimage>
+    <mainClass>org.example.Main</mainClass>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>io.ebean</groupId>
+      <artifactId>ebean-migration</artifactId>
+      <version>13.9.1-SNAPSHOT</version>
+    </dependency>
+
+    <dependency>
+      <groupId>io.ebean</groupId>
+      <artifactId>ebean-datasource</artifactId>
+      <version>8.0</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.postgresql</groupId>
+      <artifactId>postgresql</artifactId>
+      <version>42.6.0</version>
+    </dependency>
+
+    <dependency>
+      <groupId>io.ebean</groupId>
+      <artifactId>ebean-test-containers</artifactId>
+      <version>7.1</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>io.avaje</groupId>
+      <artifactId>junit</artifactId>
+      <version>1.1</version>
+      <scope>test</scope>
+    </dependency>
+
+  </dependencies>
+
+  <profiles>
+    <profile>
+      <id>native</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.graalvm.buildtools</groupId>
+            <artifactId>native-maven-plugin</artifactId>
+            <version>${version.plugin.nativeimage}</version>
+            <executions>
+              <execution>
+                <id>build-native</id>
+                <goals>
+                  <goal>build</goal>
+                </goals>
+                <phase>package</phase>
+                <configuration>
+                  <buildArgs>
+                    <buildArg>--no-fallback</buildArg>
+                    <buildArg>-H:IncludeLocales=de,en</buildArg>
+                  </buildArgs>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+
+</project>
diff --git a/test-native-image/src/main/java/org/example/Main.java b/test-native-image/src/main/java/org/example/Main.java
@@ -0,0 +1,30 @@
+package org.example;
+
+import io.ebean.migration.*;
+
+import java.io.InputStream;
+import java.util.List;
+import java.util.Properties;
+
+public class Main {
+
+    public static void main(String[] args) {
+      // InputStream is = Main.class.getResourceAsStream("/dbmigration/postgres/idx_postgres.migrations");
+      // System.out.println("GOT is: " + is);
+
+      MigrationConfig config = new MigrationConfig();
+      config.setDbUsername("mig");
+      config.setDbPassword("test");
+      config.setDbUrl("jdbc:postgresql://localhost:6432/mig");
+      config.setBasePlatform("postgres");
+      config.setPlatform("postgres");
+      config.setMigrationPath("dbmigration");
+
+      MigrationRunner runner = new MigrationRunner(config);
+      runner.run();
+
+      //System.out.println("state: " + runner.checkState());
+      System.out.println("DONE");
+    }
+
+}
diff --git a/test-native-image/src/main/resources/dbmigration/postgres/1.0__initial.sql b/test-native-image/src/main/resources/dbmigration/postgres/1.0__initial.sql
@@ -0,0 +1,12 @@
+-- apply changes
+create table foo (
+  id                            integer generated by default as identity not null,
+  assoc_one                     varchar(255),
+  constraint pk_foo primary key (id)
+);
+
+create table bar (
+  id                            integer generated by default as identity not null,
+  something                     varchar(255),
+  constraint pk_bar primary key (id)
+);
diff --git a/test-native-image/src/main/resources/dbmigration/postgres/1.1.sql b/test-native-image/src/main/resources/dbmigration/postgres/1.1.sql
@@ -0,0 +1,5 @@
+create table baz (
+  id                            integer generated by default as identity not null,
+  something                     varchar(255),
+  constraint pk_baz primary key (id)
+);
diff --git a/test-native-image/src/main/resources/dbmigration/postgres/idx_postgres.migrations b/test-native-image/src/main/resources/dbmigration/postgres/idx_postgres.migrations
@@ -0,0 +1,2 @@
+-745768926,     1.0__initial.sql
+39858255,       1.1.sql
diff --git a/test-native-image/src/test/java/org/example/StartPostgresContainerTest.java b/test-native-image/src/test/java/org/example/StartPostgresContainerTest.java
@@ -0,0 +1,16 @@
+package org.example;
+
+import org.junit.jupiter.api.Test;
+import io.ebean.test.containers.*;
+
+class StartPostgresContainerTest {
+
+  @Test
+  void test() {
+    PostgresContainer.builder("15")
+      .port(6432)
+      .dbName("mig")
+      .build()
+      .start();
+  }
+}

Original file line number	Diff line number	Diff line change
`@@ -549,7 +549,7 @@ public void setName(String name) {`
`549`	`549`	`* Return the base platform that was set.`
`550`	`550`	`*/`
`551`	`551`	`public String getBasePlatform() {`
`552`		`- return basePlatform;`
	`552`	`+ return basePlatform != null ? basePlatform : platform;`
`553`	`553`	`}`
`554`	`554`
`555`	`555`	`/**`