Merge pull request #37 from zonkyio/docker-databases

#22 implement support for database providers

Author: tomix26

README.md

@@ -2,6 +2,7 @@ import java.util.regex.Matcher
 
 plugins {
     id 'net.researchgate.release' version '2.6.0'
+    id 'nebula.optional-base' version '3.3.0'
 }
 
 ext {
@@ -42,6 +43,7 @@ subprojects {
     apply plugin: 'java'
     apply plugin: 'maven'
     apply plugin: 'signing'
+    apply plugin: 'nebula.optional-base'
 
     sourceCompatibility = 1.8
 
@@ -149,16 +151,18 @@ project(':embedded-database-spring-test') {
         compile project(':embedded-database-spring-test-autoconfigure')
 
         compile 'io.zonky.test:embedded-postgres:1.2.1'
-        compile 'org.springframework:spring-context:4.3.21.RELEASE'
-        compile 'org.springframework:spring-test:4.3.21.RELEASE'
+        compile 'org.testcontainers:postgresql:1.10.3', optional
+        compile 'com.opentable.components:otj-pg-embedded:0.13.1', optional
+        compile 'ru.yandex.qatools.embed:postgresql-embedded:2.10', optional
+        
+        compile 'org.springframework:spring-context:4.3.22.RELEASE'
+        compile 'org.springframework:spring-test:4.3.22.RELEASE'
         compile "org.flywaydb:flyway-core:$flywayCoreVersion"
         compile "org.flywaydb.flyway-test-extensions:flyway-spring-test:$flywayTestVersion"
         compile 'com.google.guava:guava:23.0'
         compile 'org.tukaani:xz:1.8'
-
-        testCompile 'junit:junit:4.12'
-        testCompile 'org.mockito:mockito-all:1.9.5'
-        testCompile 'org.assertj:assertj-core:3.6.1'
+        
+        testCompile 'org.springframework.boot:spring-boot-starter-test:1.5.19.RELEASE'
         testCompile 'ch.qos.logback:logback-classic:1.2.3'
     }
 

build.gradle

@@ -25,10 +25,8 @@
 import java.lang.annotation.Target;
 
 /**
- * Annotation that can be applied to a test class to configure a embedded test database to use
+ * Annotation that can be applied to a test class to configure an embedded database to use
  * instead of any application defined {@link DataSource}.
- * </p>
- * This annotation is handled and processed by {@link io.zonky.test.db.postgres.EmbeddedPostgresContextCustomizerFactory}.
  *
  * @see io.zonky.test.db.postgres.EmbeddedPostgresContextCustomizerFactory
  * @see io.zonky.test.db.flyway.OptimizedFlywayTestExecutionListener
@@ -40,10 +38,11 @@
 public @interface AutoConfigureEmbeddedDatabase {
 
     /**
-     * If this attribute is set then a new data source
-     * with the specified name is created instead of overriding an existing one.
+     * The bean name to be used to identify the data source that will be replaced.
+     * It is only necessary if there is no existing DataSource
+     * or the context contains multiple DataSource beans.
      *
-     * @return the name of a new data source bean to be created
+     * @return the name to identify the DataSource bean
      */
     String beanName() default "";
 
@@ -62,6 +61,16 @@
      */
     EmbeddedDatabaseType type() default EmbeddedDatabaseType.POSTGRES;
 
+    /**
+     * Provider used to create the underlying embedded database,
+     * see the documentation for the comparision matrix.
+     * Note that the provider can also be configured
+     * through {@code embedded-database.provider} property.
+     *
+     * @return the provider of an embedded database
+     */
+    DatabaseProvider provider() default DatabaseProvider.DEFAULT;
+
     /**
      * What the test database can replace.
      */
@@ -80,14 +89,47 @@ enum Replace {
     }
 
     /**
-     * A supported embedded database type.
+     * The supported types of embedded databases.
      */
     enum EmbeddedDatabaseType {
 
         /**
-         * The Embedded PostgreSQL Database
+         * PostgreSQL Database
          */
         POSTGRES
 
     }
+
+    /**
+     * The supported providers of embedded databases.
+     */
+    enum DatabaseProvider {
+
+        /**
+         * Default typically equals to {@link #ZONKY} provider,
+         * unless a different default has been configured by externalized configuration.
+         */
+        DEFAULT,
+
+        /**
+         * Run the embedded database in a Docker container.
+         */
+        DOCKER,
+
+        /**
+         * Use Zonky's fork of OpenTable Embedded PostgreSQL Component to create the embedded database (https://github.com/zonkyio/embedded-postgres).
+         */
+        ZONKY,
+
+        /**
+         * Use OpenTable Embedded PostgreSQL Component to create the embedded database (https://github.com/opentable/otj-pg-embedded).
+         */
+        OPENTABLE,
+
+        /**
+         * Use Yandex's Embedded PostgreSQL Server to create the embedded database (https://github.com/yandex-qatools/postgresql-embedded).
+         */
+        YANDEX,
+
+    }
 }

embedded-database-spring-test/src/main/java/io/zonky/test/db/AutoConfigureEmbeddedDatabase.java

@@ -16,41 +16,33 @@
 
 package io.zonky.test.db.flyway;
 
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableList;
-import io.zonky.test.db.postgres.embedded.DatabasePreparer;
-import io.zonky.test.db.postgres.embedded.EmbeddedPostgres.Builder;
-import io.zonky.test.db.postgres.embedded.PreparedDbProvider;
+import io.zonky.test.db.provider.DatabaseDescriptor;
+import io.zonky.test.db.provider.DatabasePreparer;
+import io.zonky.test.db.provider.DatabaseType;
+import io.zonky.test.db.provider.GenericDatabaseProvider;
+import io.zonky.test.db.provider.ProviderType;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.flywaydb.core.Flyway;
-import org.postgresql.ds.PGSimpleDataSource;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
 import org.springframework.core.task.TaskExecutor;
 import org.springframework.util.concurrent.CompletableToListenableFutureAdapter;
 import org.springframework.util.concurrent.ListenableFuture;
 
 import javax.sql.DataSource;
 import java.io.IOException;
-import java.sql.SQLException;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.Executor;
-import java.util.concurrent.Semaphore;
-import java.util.function.Consumer;
 
 import static com.google.common.base.Preconditions.checkState;
 
 /**
  * Default implementation of {@link FlywayDataSourceContext} that is used for deferred initialization of the embedded database.
  * Note that this target source is dynamic and supports hot reloading while the application is running.
  * <p/>
- * For the reloading of the underlying data source is used cacheable {@link io.zonky.test.db.postgres.embedded.DatabasePreparer},
+ * For the reloading of the underlying data source is used cacheable {@link DatabasePreparer},
  * which can utilize a special template database to effective copy data into multiple independent databases.
  *
  * @see io.zonky.test.db.postgres.FlywayEmbeddedPostgresDataSourceFactoryBean
@@ -59,29 +51,17 @@
  */
 public class DefaultFlywayDataSourceContext implements FlywayDataSourceContext {
 
-    protected static final int MAX_DATABASE_CONNECTIONS = 300;
     protected static final int DEFAULT_MAX_RETRY_ATTEMPTS = 2;
 
-    protected static final Consumer<Builder> DEFAULT_DATABASE_CONFIGURATION = builder -> {
-        builder.setPGStartupWait(Duration.ofSeconds(20L));
-    };
-
-    protected static final Consumer<Builder> FORCED_DATABASE_CONFIGURATION =
-            builder -> builder.setServerConfig("max_connections", String.valueOf(MAX_DATABASE_CONNECTIONS));
-
-    protected static final LoadingCache<Integer, Semaphore> CONNECTION_SEMAPHORES = CacheBuilder.newBuilder()
-            .build(new CacheLoader<Integer, Semaphore>() {
-                public Semaphore load(Integer key) {
-                    return new Semaphore(MAX_DATABASE_CONNECTIONS);
-                }
-            });
-
     protected static final ThreadLocal<DataSource> preparerDataSourceHolder = new ThreadLocal<>();
 
     protected volatile CompletableFuture<DataSource> dataSourceFuture = CompletableFuture.completedFuture(null);
 
-    @Autowired(required = false)
-    protected List<Consumer<Builder>> databaseCustomizers = new ArrayList<>();
+    @Autowired
+    protected Environment environment;
+
+    @Autowired
+    protected GenericDatabaseProvider databaseProvider;
 
     protected int maxAttempts = DEFAULT_MAX_RETRY_ATTEMPTS;
 
@@ -114,29 +94,21 @@ public Object getTarget() throws Exception {
     }
 
     @Override
-    public void releaseTarget(Object target) throws Exception {
+    public void releaseTarget(Object target) {
         // nothing to do
     }
 
     @Override
     public synchronized ListenableFuture<DataSource> reload(Flyway flyway) {
         Executor executor = bootstrapExecutor != null ? bootstrapExecutor : Runnable::run;
 
-        List<Consumer<Builder>> customizers = ImmutableList.<Consumer<Builder>>builder()
-                .add(DEFAULT_DATABASE_CONFIGURATION)
-                .addAll(databaseCustomizers)
-                .add(FORCED_DATABASE_CONFIGURATION)
-                .build();
-
         CompletableFuture<DataSource> reloadFuture = dataSourceFuture.thenApplyAsync(x -> {
             for (int current = 1; current <= maxAttempts; current++) {
                 try {
+                    String providerName = environment.getProperty("embedded-database.provider", ProviderType.ZONKY.toString());
                     FlywayDatabasePreparer preparer = new FlywayDatabasePreparer(flyway);
-                    PreparedDbProvider provider = PreparedDbProvider.forPreparer(preparer, customizers);
-
-                    PGSimpleDataSource dataSource = provider.createDataSource().unwrap(PGSimpleDataSource.class);
-                    Semaphore semaphore = CONNECTION_SEMAPHORES.get(dataSource.getPortNumber());
-                    return new BlockingDataSourceWrapper(dataSource, semaphore);
+                    DatabaseDescriptor descriptor = new DatabaseDescriptor(DatabaseType.POSTGRES, ProviderType.valueOf(providerName));
+                    return databaseProvider.getDatabase(preparer, descriptor);
                 } catch (Exception e) {
                     if (ExceptionUtils.indexOfType(e, IOException.class) == -1 || current == maxAttempts) {
                         throw new CompletionException(e);
@@ -196,7 +168,7 @@ public FlywayDatabasePreparer(Flyway flyway) {
         }
 
         @Override
-        public void prepare(DataSource ds) throws SQLException {
+        public void prepare(DataSource ds) {
             preparerDataSourceHolder.set(ds);
             try {
                 flyway.migrate();

embedded-database-spring-test/src/main/java/io/zonky/test/db/flyway/DefaultFlywayDataSourceContext.java

@@ -16,6 +16,7 @@
 
 package io.zonky.test.db.logging;
 
+import org.apache.commons.lang3.StringUtils;
 import org.postgresql.ds.common.BaseDataSource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -41,7 +42,11 @@ public static void reportDataSource(DataSource dataSource, AnnotatedElement elem
     private static String getJdbcUrl(DataSource dataSource) {
         try {
             BaseDataSource ds = dataSource.unwrap(BaseDataSource.class);
-            return String.format(JDBC_FORMAT, ds.getPortNumber(), ds.getDatabaseName(), ds.getUser());
+            if (StringUtils.isBlank(ds.getPassword())) {
+                return String.format(JDBC_FORMAT, ds.getPortNumber(), ds.getDatabaseName(), ds.getUser());
+            } else {
+                return String.format(JDBC_FORMAT + "&password=%s", ds.getPortNumber(), ds.getDatabaseName(), ds.getUser(), ds.getPassword());
+            }
         } catch (Exception e) {
             logger.warn("Unexpected error occurred while resolving url to the embedded database", e);
             return "unknown";

embedded-database-spring-test/src/main/java/io/zonky/test/db/logging/EmbeddedDatabaseReporter.java

@@ -18,11 +18,16 @@
 
 import com.google.common.collect.ImmutableMap;
 import io.zonky.test.db.AutoConfigureEmbeddedDatabase;
-import io.zonky.test.db.AutoConfigureEmbeddedDatabase.EmbeddedDatabaseType;
+import io.zonky.test.db.AutoConfigureEmbeddedDatabase.DatabaseProvider;
 import io.zonky.test.db.AutoConfigureEmbeddedDatabase.Replace;
 import io.zonky.test.db.flyway.DefaultFlywayDataSourceContext;
 import io.zonky.test.db.flyway.FlywayClassUtils;
 import io.zonky.test.db.flyway.FlywayDataSourceContext;
+import io.zonky.test.db.provider.impl.DockerPostgresDatabaseProvider;
+import io.zonky.test.db.provider.impl.OpenTablePostgresDatabaseProvider;
+import io.zonky.test.db.provider.impl.PrefetchingDatabaseProvider;
+import io.zonky.test.db.provider.impl.YandexPostgresDatabaseProvider;
+import io.zonky.test.db.provider.impl.ZonkyPostgresDatabaseProvider;
 import org.apache.commons.lang3.StringUtils;
 import org.flywaydb.core.Flyway;
 import org.flywaydb.test.annotation.FlywayTest;
@@ -46,12 +51,15 @@
 import org.springframework.test.context.ContextCustomizer;
 import org.springframework.test.context.ContextCustomizerFactory;
 import org.springframework.test.context.MergedContextConfiguration;
+import org.springframework.util.Assert;
 import org.springframework.util.ObjectUtils;
 
 import javax.sql.DataSource;
 import java.lang.reflect.AnnotatedElement;
 import java.util.List;
 
+import static io.zonky.test.db.AutoConfigureEmbeddedDatabase.EmbeddedDatabaseType;
+
 /**
  * Implementation of the {@link org.springframework.test.context.ContextCustomizerFactory} interface,
  * which is responsible for initialization of the embedded postgres database and its registration to the application context.
@@ -89,7 +97,7 @@ public PreloadableEmbeddedPostgresContextCustomizer(AutoConfigureEmbeddedDatabas
 
         @Override
         public void customizeContext(ConfigurableApplicationContext context, MergedContextConfiguration mergedConfig) {
-            context.addBeanFactoryPostProcessor(new EnvironmentPostProcessor(context.getEnvironment()));
+            context.addBeanFactoryPostProcessor(new EnvironmentPostProcessor(context.getEnvironment(), databaseAnnotation));
 
             Class<?> testClass = mergedConfig.getTestClass();
             FlywayTest[] flywayAnnotations = findFlywayTestAnnotations(testClass);
@@ -126,16 +134,24 @@ public int hashCode() {
     protected static class EnvironmentPostProcessor implements BeanDefinitionRegistryPostProcessor {
 
         private final ConfigurableEnvironment environment;
+        private final AutoConfigureEmbeddedDatabase databaseAnnotation;
 
-        public EnvironmentPostProcessor(ConfigurableEnvironment environment) {
+        public EnvironmentPostProcessor(ConfigurableEnvironment environment, AutoConfigureEmbeddedDatabase databaseAnnotation) {
             this.environment = environment;
+            this.databaseAnnotation = databaseAnnotation;
         }
 
         @Override
         public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry registry) throws BeansException {
+            ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
+            builder.put("spring.test.database.replace", "NONE");
+
+            if (databaseAnnotation.provider() != DatabaseProvider.DEFAULT) {
+                builder.put("embedded-database.provider", databaseAnnotation.provider().toString());
+            }
+
             environment.getPropertySources().addFirst(new MapPropertySource(
-                    PreloadableEmbeddedPostgresContextCustomizer.class.getSimpleName(),
-                    ImmutableMap.of("spring.test.database.replace", "NONE")));
+                    PreloadableEmbeddedPostgresContextCustomizer.class.getSimpleName(), builder.build()));
         }
 
         @Override
@@ -156,8 +172,16 @@ public PreloadableEmbeddedPostgresRegistrar(AutoConfigureEmbeddedDatabase databa
 
         @Override
         public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry registry) throws BeansException {
+            Assert.isInstanceOf(ConfigurableListableBeanFactory.class, registry,
+                    "Embedded Database Auto-configuration can only be used with a ConfigurableListableBeanFactory");
             ConfigurableListableBeanFactory beanFactory = (ConfigurableListableBeanFactory) registry;
 
+            registerBeanIfMissing(registry, "defaultDatabaseProvider", PrefetchingDatabaseProvider.class);
+            registerBeanIfMissing(registry, "dockerPostgresProvider", DockerPostgresDatabaseProvider.class);
+            registerBeanIfMissing(registry, "zonkyPostgresProvider", ZonkyPostgresDatabaseProvider.class);
+            registerBeanIfMissing(registry, "openTablePostgresProvider", OpenTablePostgresDatabaseProvider.class);
+            registerBeanIfMissing(registry, "yandexPostgresProvider", YandexPostgresDatabaseProvider.class);
+
             BeanDefinitionHolder dataSourceInfo = getDataSourceBeanDefinition(beanFactory, databaseAnnotation);
 
             RootBeanDefinition dataSourceDefinition = new RootBeanDefinition();
@@ -198,6 +222,13 @@ public void postProcessBeanFactory(ConfigurableListableBeanFactory beanFactory)
         }
     }
 
+    protected static void registerBeanIfMissing(BeanDefinitionRegistry registry, String beanName, Class<?> beanClass) {
+        if (!registry.containsBeanDefinition(beanName)) {
+            RootBeanDefinition providerDefinition = new RootBeanDefinition(beanClass);
+            registry.registerBeanDefinition(beanName, providerDefinition);
+        }
+    }
+
     protected static BeanDefinitionRegistry getBeanDefinitionRegistry(ApplicationContext context) {
         if (context instanceof BeanDefinitionRegistry) {
             return (BeanDefinitionRegistry) context;