From 894c3253da02745f926f9f1e21eccc0cd9d5763c Mon Sep 17 00:00:00 2001 From: Jim Vella Date: Thu, 2 Nov 2017 16:26:27 +1000 Subject: [PATCH 1/9] Refactor default properties into property file --- src/main/java/net/apnic/whowas/App.java | 17 ++--------------- src/main/resources/application.yml | 9 +++++++++ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/main/java/net/apnic/whowas/App.java b/src/main/java/net/apnic/whowas/App.java index 52225ff..8d939af 100644 --- a/src/main/java/net/apnic/whowas/App.java +++ b/src/main/java/net/apnic/whowas/App.java @@ -1,24 +1,11 @@ package net.apnic.whowas; -import java.util.Properties; - import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @SpringBootApplication public class App { - - public static void main(String[] args) - { - SpringApplication app = new SpringApplication(App.class); - Properties defaultProps = new Properties(); - - defaultProps.setProperty( - "spring.mvc.throw-exception-if-no-handler-found", "true"); - defaultProps.setProperty("spring.resources.add-mappings", "false"); - defaultProps.setProperty("spring.mvc.favicon.enabled", "false"); - defaultProps.setProperty("management.add-application-context-header", "false"); - app.setDefaultProperties(defaultProps); - app.run(args); + public static void main(String[] args) { + SpringApplication.run(App.class, args); } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index a67d6a8..c45e190 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -30,8 +30,17 @@ spring: username: "${database.username}" password: "${database.password}" + mvc: + throw-exception-if-no-handler-found: true + favicon: + enabled: false + resources: + add-mappings: false + + management: port: 8081 + add-application-context-header: false info: build: From bc921684001f8a97a87bfc2a15641714f89425ab Mon Sep 17 00:00:00 2001 From: Jim Vella Date: Mon, 20 Nov 2017 11:35:48 +1000 Subject: [PATCH 2/9] Refactor RipeDbLoader to bean @Transactional will work now. --- src/main/java/net/apnic/whowas/loaders/RipeDbLoader.java | 6 ++++-- .../apnic/whowas/loaders/config/LoaderConfiguration.java | 9 +++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/net/apnic/whowas/loaders/RipeDbLoader.java b/src/main/java/net/apnic/whowas/loaders/RipeDbLoader.java index 62256c6..cf72649 100644 --- a/src/main/java/net/apnic/whowas/loaders/RipeDbLoader.java +++ b/src/main/java/net/apnic/whowas/loaders/RipeDbLoader.java @@ -20,17 +20,19 @@ import org.slf4j.LoggerFactory; import org.springframework.jdbc.core.JdbcOperations; +import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Isolation; import org.springframework.transaction.annotation.Transactional; +@Component public class RipeDbLoader implements Loader { private static final Logger LOGGER = LoggerFactory.getLogger(RipeDbLoader.class); private long lastSerial; private final transient JdbcOperations operations; - public RipeDbLoader(JdbcOperations jdbcOperations, long serial) { - this.lastSerial = serial; + public RipeDbLoader(JdbcOperations jdbcOperations) { + this.lastSerial = -1; this.operations = jdbcOperations; } diff --git a/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java b/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java index 17f9795..f73add8 100644 --- a/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java +++ b/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java @@ -48,6 +48,8 @@ public class LoaderConfiguration @Autowired private ApplicationContext context; + + @Autowired private RipeDbLoader dbLoader; @Value("${snapshot.file:#{null}}") @@ -56,9 +58,6 @@ public class LoaderConfiguration @Autowired History history; - @Autowired - private JdbcOperations jdbcOperations; - @Autowired SearchEngine searchEngine; @@ -105,9 +104,7 @@ private void buildTree() } @PostConstruct - public void initialise() - { - dbLoader = new RipeDbLoader(jdbcOperations, -1L); + public void initialise() { executorService.execute(this::buildTree); } From ad685a67d4098a250a43572ea5790d22844aeb07 Mon Sep 17 00:00:00 2001 From: Jim Vella Date: Thu, 2 Nov 2017 16:30:26 +1000 Subject: [PATCH 3/9] Test data is loaded before service is started --- .../net/apnic/whowas/InitialisationTest.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/test/java/net/apnic/whowas/InitialisationTest.java diff --git a/src/test/java/net/apnic/whowas/InitialisationTest.java b/src/test/java/net/apnic/whowas/InitialisationTest.java new file mode 100644 index 0000000..d373540 --- /dev/null +++ b/src/test/java/net/apnic/whowas/InitialisationTest.java @@ -0,0 +1,58 @@ +package net.apnic.whowas; + +import net.apnic.whowas.loaders.RipeDbLoader; +import org.junit.Test; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.context.embedded.EmbeddedServletContainerInitializedEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.jdbc.core.JdbcTemplate; + +import java.util.ArrayList; +import java.util.concurrent.ConcurrentLinkedQueue; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +public class InitialisationTest { + + @Test + public void initialDataIsLoadedBeforeServletIsStarted() { + ConcurrentLinkedQueue events = new ConcurrentLinkedQueue<>(); + + System.setProperty("server.port", "0"); + System.setProperty("management.port", "-1"); + + SpringApplication springApplication = new SpringApplication(App.class); + springApplication.addInitializers(configurableApplicationContext -> + configurableApplicationContext.addBeanFactoryPostProcessor(beanFactory -> + beanFactory.registerResolvableDependency( + RipeDbLoader.class, new RipeDbLoader(new JdbcTemplate()) { + @Override + public void loadWith(RevisionConsumer consumer) { + sleep(1000); + events.add("Data loaded"); + } + } + ) + ) + ); + springApplication.addListeners( + (ApplicationListener) applicationEvent -> + events.add("Servlet container started") + ); + + try(ConfigurableApplicationContext context = springApplication.run()) { + assertThat("Data was loaded before tomcat was initialised", + new ArrayList<>(events).get(0), is("Data loaded")); + } + } + + private static void sleep(int millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} From 4e4e7a34e826be4dbfaa38db6d471c5dfc6bae5f Mon Sep 17 00:00:00 2001 From: Jim Vella Date: Mon, 20 Nov 2017 11:39:40 +1000 Subject: [PATCH 4/9] Load data before starting service --- .../loaders/config/LoaderConfiguration.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java b/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java index f73add8..532e696 100644 --- a/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java +++ b/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java @@ -7,10 +7,7 @@ import java.time.temporal.ChronoUnit; import java.time.ZonedDateTime; import java.time.ZoneId; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; +import java.util.concurrent.*; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; import javax.annotation.PostConstruct; @@ -104,8 +101,14 @@ private void buildTree() } @PostConstruct - public void initialise() { - executorService.execute(this::buildTree); + public void initialise() throws ExecutionException, InterruptedException { + /* + Scoping the initial load of data to @PostConstruct + results in the loading completing before the servlet context is started, + preventing requests from being served from partially loaded data. + */ + Future initialDataLoaded = executorService.submit(this::buildTree); + initialDataLoaded.get(); } @Scheduled(fixedRate = 15000L) From 31acb64987de077ea83f08650dd6df98f89ae9c3 Mon Sep 17 00:00:00 2001 From: Jim Vella Date: Mon, 20 Nov 2017 11:58:33 +1000 Subject: [PATCH 5/9] Removed unused import --- .../net/apnic/whowas/loaders/config/LoaderConfiguration.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java b/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java index 532e696..1256084 100644 --- a/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java +++ b/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java @@ -30,7 +30,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.ApplicationContext; -import org.springframework.jdbc.core.JdbcOperations; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; From a5210d0bc65f480e97ff803a33656c43f309f517 Mon Sep 17 00:00:00 2001 From: Jim Vella Date: Mon, 20 Nov 2017 15:39:29 +1000 Subject: [PATCH 6/9] Allow time to start test container before marking unhealthy Now that the application does not signal it is healthy until the inital data load has completed, more time needs to be allowed for startup. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index bfcc402..e4e2e10 100644 --- a/pom.xml +++ b/pom.xml @@ -336,9 +336,9 @@ /app - 10s + 1m 15s - 3 + 7 curl -f http://localhost:8081/health || exit 1 From fb9d8d42e1fdf186c0740a14a43943db01c41476 Mon Sep 17 00:00:00 2001 From: Jim Vella Date: Tue, 21 Nov 2017 11:17:55 +1000 Subject: [PATCH 7/9] Revert "Allow time to start test container before marking unhealthy" This reverts commit a5210d0bc65f480e97ff803a33656c43f309f517. --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e4e2e10..bfcc402 100644 --- a/pom.xml +++ b/pom.xml @@ -336,9 +336,9 @@ /app - 1m + 10s 15s - 7 + 3 curl -f http://localhost:8081/health || exit 1 From 1ac9e9209967d833037cbabf55841e4d7a563ad4 Mon Sep 17 00:00:00 2001 From: Jim Vella Date: Tue, 21 Nov 2017 13:45:37 +1000 Subject: [PATCH 8/9] Fix the initial data load from hanging --- .../apnic/whowas/loaders/config/LoaderConfiguration.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java b/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java index 1256084..544ec02 100644 --- a/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java +++ b/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java @@ -32,6 +32,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.transaction.PlatformTransactionManager; @Configuration @EnableScheduling @@ -57,6 +58,13 @@ public class LoaderConfiguration @Autowired SearchEngine searchEngine; + /* + This is autowired here to trigger Spring to initialise the transaction manager bean + before @PostConstruct is invoked. Without this, the initial data load will hang. + */ + @Autowired + PlatformTransactionManager platformTransactionManager; + private void buildTree() { if (snapshotFile != null) { From 96312aece52b1643533fdb3ddb9aa4a74e100f71 Mon Sep 17 00:00:00 2001 From: Jim Vella Date: Tue, 21 Nov 2017 14:07:42 +1000 Subject: [PATCH 9/9] Fix scheduled refresh of data Logs were complaining that no task scheduler bean was available. --- .../apnic/whowas/loaders/config/LoaderConfiguration.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java b/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java index 544ec02..bf28bd9 100644 --- a/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java +++ b/src/main/java/net/apnic/whowas/loaders/config/LoaderConfiguration.java @@ -30,8 +30,10 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.ApplicationContext; +import org.springframework.scheduling.TaskScheduler; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.transaction.PlatformTransactionManager; @Configuration @@ -118,6 +120,11 @@ public void initialise() throws ExecutionException, InterruptedException { initialDataLoaded.get(); } + @Bean + TaskScheduler taskScheduler() { + return new ThreadPoolTaskScheduler(); + } + @Scheduled(fixedRate = 15000L) public void refreshData() {