feat: configure block timestamp threshold above which the scheduler will be OUT-OF-SERVICE (#768)

Author: jbern0rd

README.md

@@ -36,6 +36,7 @@ You can configure the _iExec Core Scheduler_ with the following properties:
 | `IEXEC_CORE_WALLET_PASSWORD` | Password to unlock the wallet of the server. | String | `whatever` |
 | `IEXEC_BLOCKCHAIN_NODE_ADDRESS` | Private URL to connect to the blockchain node. | URL | `http://localhost:8545` |
 | `POOL_ADDRESS` | On-chain address of the workerpool managed by the current _iExec Core Scheduler_. | String | `0x365E7BABAa85eC61Dffe5b520763062e6C29dA27` |
+| `CHAIN_OUTOFSERVICETHRESHOLD` | Threshold in seconds, the scheduler is OUT-OF-SERVICE when the last known block timestamp is older. | String | `PT5S` |
 | `IEXEC_START_BLOCK_NUMBER` | Subscribe to new deal events from a specific block number. | Positive integer | `0` |
 | `IEXEC_GAS_PRICE_MULTIPLIER` | Transactions will be sent with `networkGasPrice * gasPriceMultiplier`. | Float | `1.0` |
 | `IEXEC_GAS_PRICE_CAP` | In Wei, will be used for transactions if `networkGasPrice * gasPriceMultiplier > gasPriceCap` | Integer | `22000000000` |

docker-compose.yml

@@ -1,7 +1,7 @@
 services:
 
   chain:
-    image: docker-regis.iex.ec/poco-chain:1.0.0-poco-v5.5.0-voucher-v1.0.0-nethermind
+    image: docker-regis.iex.ec/poco-chain:1.1.0-poco-v6.1.0-contracts-voucher-v1.0.0-nethermind
     expose:
       - "8545"
 

src/main/java/com/iexec/core/chain/BlockchainConnectionHealthIndicator.java

@@ -47,6 +47,7 @@ public class BlockchainConnectionHealthIndicator implements HealthIndicator {
      * Interval between 2 requests onto the chain.
      */
     private final Duration pollingInterval;
+    private final Duration outOfServiceThreshold;
     /**
      * Number of consecutive failures before declaring this Scheduler is out-of-service.
      */
@@ -85,6 +86,7 @@ public BlockchainConnectionHealthIndicator(
             Clock clock) {
         this.applicationEventPublisher = applicationEventPublisher;
         this.pollingInterval = chainConfig.getBlockTime();
+        this.outOfServiceThreshold = chainConfig.getSyncTimeout();
         this.monitoringExecutor = monitoringExecutor;
         this.clock = clock;
     }
@@ -108,7 +110,7 @@ void scheduleMonitoring() {
     void checkConnection() {
         log.debug("Latest on-chain block number [block:{}]", latestBlockNumber);
         final Instant now = Instant.now();
-        final Instant threshold = Instant.ofEpochSecond(latestBlockTimestamp).plusSeconds(pollingInterval.toSeconds());
+        final Instant threshold = Instant.ofEpochSecond(latestBlockTimestamp).plusSeconds(outOfServiceThreshold.toSeconds());
         if (now.isAfter(threshold)) {
             connectionFailed();
         } else {

src/main/java/com/iexec/core/chain/ChainConfig.java

@@ -58,6 +58,10 @@ public class ChainConfig {
     @NotNull(message = "Block time must not be null")
     private Duration blockTime;
 
+    @Value("${chain.out-of-service-threshold}")
+    @NotNull(message = "OUT-OF-SERVICE threshold must not be null")
+    private Duration syncTimeout;
+
     @Value("${chain.node-address}")
     @URL(message = "Node address must be a valid URL")
     @NotEmpty(message = "Node address must not be empty")

src/main/resources/application.yml

@@ -52,6 +52,7 @@ wallet:
 
 chain:
   node-address: ${IEXEC_BLOCKCHAIN_NODE_ADDRESS:http://localhost:8545}
+  out-of-service-threshold: PT5S
   pool-address: ${POOL_ADDRESS:0x365E7BABAa85eC61Dffe5b520763062e6C29dA27}
   start-block-number: ${IEXEC_START_BLOCK_NUMBER:0}
   gas-price-multiplier: ${IEXEC_GAS_PRICE_MULTIPLIER:1.0} # txs will be sent with networkGasPrice*gasPriceMultiplier, 4.0 means super fast

src/test/java/com/iexec/core/chain/BlockchainConnectionHealthIndicatorTests.java

@@ -16,15 +16,13 @@
 
 package com.iexec.core.chain;
 
-import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.mockito.Mock;
-import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.springframework.boot.actuate.health.Health;
 import org.springframework.context.ApplicationEventPublisher;
@@ -35,8 +33,10 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 @ExtendWith(MockitoExtension.class)
@@ -57,6 +57,7 @@ class BlockchainConnectionHealthIndicatorTests {
     @BeforeEach
     void init() {
         when(chainConfig.getBlockTime()).thenReturn(BLOCK_TIME);
+        when(chainConfig.getSyncTimeout()).thenReturn(BLOCK_TIME);
 
         this.blockchainConnectionHealthIndicator = new BlockchainConnectionHealthIndicator(
                 applicationEventPublisher,
@@ -71,7 +72,7 @@ void init() {
     void shouldScheduleMonitoring() {
         blockchainConnectionHealthIndicator.scheduleMonitoring();
 
-        Mockito.verify(executor).scheduleAtFixedRate(
+        verify(executor).scheduleAtFixedRate(
                 any(),
                 eq(0L),
                 eq(5L),
@@ -141,8 +142,8 @@ void checkConnection(int previousConsecutiveFailures,
         final boolean outOfService = blockchainConnectionHealthIndicator.isOutOfService();
         final LocalDateTime firstFailure = blockchainConnectionHealthIndicator.getFirstFailure();
 
-        Assertions.assertThat(outOfService).isEqualTo(expectedOutOfService);
-        Assertions.assertThat(firstFailure).isEqualTo(expectedFirstFailure);
+        assertThat(outOfService).isEqualTo(expectedOutOfService);
+        assertThat(firstFailure).isEqualTo(expectedFirstFailure);
     }
     // endregion
 
@@ -161,7 +162,7 @@ void shouldReturnOutOfService() {
                 .build();
 
         final Health health = blockchainConnectionHealthIndicator.health();
-        Assertions.assertThat(health).isEqualTo(expectedHealth);
+        assertThat(health).isEqualTo(expectedHealth);
     }
 
     @Test
@@ -175,7 +176,7 @@ void shouldReturnUpAndNoFirstFailure() {
                 .build();
 
         final Health health = blockchainConnectionHealthIndicator.health();
-        Assertions.assertThat(health).isEqualTo(expectedHealth);
+        assertThat(health).isEqualTo(expectedHealth);
     }
 
     @Test
@@ -192,7 +193,7 @@ void shouldReturnUpButWithFirstFailure() {
                 .build();
 
         final Health health = blockchainConnectionHealthIndicator.health();
-        Assertions.assertThat(health).isEqualTo(expectedHealth);
+        assertThat(health).isEqualTo(expectedHealth);
     }
     // endregion
 

…java/com/iexec/core/ChainConfigTest.java …m/iexec/core/chain/ChainConfigTests.javasrc/test/java/com/iexec/core/ChainConfigTest.java renamed to src/test/java/com/iexec/core/chain/ChainConfigTests.java src/test/java/com/iexec/core/ChainConfigTest.java renamed to src/test/java/com/iexec/core/chain/ChainConfigTests.java

@@ -14,20 +14,21 @@
  * limitations under the License.
  */
 
-package com.iexec.core;
+package com.iexec.core.chain;
 
-import com.iexec.core.chain.ChainConfig;
 import jakarta.validation.ConstraintViolation;
 import jakarta.validation.Validation;
 import jakarta.validation.Validator;
 import jakarta.validation.ValidatorFactory;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+
 import java.time.Duration;
 import java.util.Set;
+
 import static org.assertj.core.api.Assertions.assertThat;
 
-class ChainConfigTest {
+class ChainConfigTests {
     private static final String IEXEC_NODE_ADDRESS = "https://bellecour.iex.ec";
     private static final String IEXEC_HUB_ADDRESS = "0x1a69b2eb604db8eba185df03ea4f5288dcbbd248";
     private static final String POOL_ADDRESS = "poolAddress";
@@ -48,6 +49,7 @@ void chainIdMustBePositive() {
                 false,
                 IEXEC_HUB_ADDRESS,
                 Duration.ofMillis(100),
+                Duration.ofSeconds(30),
                 IEXEC_NODE_ADDRESS,
                 POOL_ADDRESS,
                 0L,
@@ -67,6 +69,7 @@ void nodeAddressMustBeValidURL() {
                 false,
                 IEXEC_HUB_ADDRESS,
                 Duration.ofMillis(100),
+                Duration.ofSeconds(30),
                 "invalid-url", // invalid URL
                 POOL_ADDRESS,
                 0L,
@@ -86,6 +89,7 @@ void nodeAddressMustNotBeEmpty() {
                 false,
                 IEXEC_HUB_ADDRESS,
                 Duration.ofMillis(100),
+                Duration.ofSeconds(30),
                 "", // empty nodeAddress
                 POOL_ADDRESS,
                 0L,
@@ -105,6 +109,7 @@ void blockTimeMustBeAtLeast100ms() {
                 false,
                 IEXEC_HUB_ADDRESS,
                 Duration.ofMillis(99), // less than 100ms
+                Duration.ofSeconds(30),
                 IEXEC_NODE_ADDRESS,
                 POOL_ADDRESS,
                 0L,
@@ -124,6 +129,7 @@ void blockTimeMustBeAtMost20Seconds() {
                 false,
                 IEXEC_HUB_ADDRESS,
                 Duration.ofSeconds(21), // more than 20 seconds
+                Duration.ofSeconds(30),
                 IEXEC_NODE_ADDRESS,
                 POOL_ADDRESS,
                 0L,
@@ -143,6 +149,7 @@ void gasPriceMultiplierMustBePositive() {
                 false,
                 IEXEC_HUB_ADDRESS,
                 Duration.ofMillis(100),
+                Duration.ofSeconds(30),
                 IEXEC_NODE_ADDRESS,
                 POOL_ADDRESS,
                 0L,
@@ -162,6 +169,7 @@ void gasPriceCapMustBePositiveOrZero() {
                 false,
                 IEXEC_HUB_ADDRESS,
                 Duration.ofMillis(100),
+                Duration.ofSeconds(30),
                 IEXEC_NODE_ADDRESS,
                 POOL_ADDRESS,
                 0L,
@@ -181,6 +189,7 @@ void hubAddressMustBeValidEthereumAddress() {
                 false,
                 "0x0", // invalid address
                 Duration.ofMillis(100),
+                Duration.ofSeconds(30),
                 IEXEC_NODE_ADDRESS,
                 POOL_ADDRESS,
                 0L,

src/test/java/com/iexec/core/chain/WalletConfigurationTest.java

@@ -20,7 +20,9 @@
 import com.iexec.commons.poco.chain.SignerService;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.springframework.boot.convert.ApplicationConversionService;
 import org.springframework.boot.test.context.runner.ApplicationContextRunner;
+import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
 import org.web3j.crypto.WalletUtils;
 
 import java.io.File;
@@ -37,7 +39,11 @@ class WalletConfigurationTest {
     void shouldCreateBeans() throws Exception {
         final String tempWalletName = WalletUtils.generateFullNewWalletFile("changeit", tempWalletDir);
         final String tempWalletPath = tempWalletDir.getAbsolutePath() + File.separator + tempWalletName;
-        runner.withPropertyValues("chain.node-address=http://localhost:8545", "chain.pool-address=0x1", "chain.start-block-number=0", "chain.gas-price-multiplier=1.0", "chain.gas-price-cap=0")
+        runner.withPropertyValues("chain.out-of-service-threshold=PT30S", "chain.node-address=http://localhost:8545", "chain.pool-address=0x1", "chain.start-block-number=0", "chain.gas-price-multiplier=1.0", "chain.gas-price-cap=0")
+                .withBean(PropertySourcesPlaceholderConfigurer.class,
+                        PropertySourcesPlaceholderConfigurer::new)
+                .withInitializer(context ->
+                        context.getBeanFactory().setConversionService(ApplicationConversionService.getSharedInstance()))
                 .withBean(IexecHubService.class)
                 .withBean(PublicChainConfig.class, 65535, true, "http://localhost:8545", "0xC129e7917b7c7DeDfAa5Fff1FB18d5D7050fE8ca", Duration.ofSeconds(5))
                 .withBean(WalletConfiguration.class, tempWalletPath, "changeit")

src/test/java/com/iexec/core/chain/WebSocketBlockchainListenerTests.java

@@ -38,7 +38,7 @@
 import static org.awaitility.Awaitility.await;
 
 @Testcontainers
-@SpringBootTest
+@SpringBootTest(properties = "chain.out-of-service-threshold=PT30S")
 class WebSocketBlockchainListenerTests {
     private static final String CHAIN_SVC_NAME = "chain";
     private static final int CHAIN_SVC_PORT = 8545;
@@ -64,10 +64,14 @@ static void registerProperties(DynamicPropertyRegistry registry) {
                 environment.getServiceHost(CONFIG_SVC_NAME, CONFIG_SVC_PORT),
                 environment.getServicePort(CONFIG_SVC_NAME, CONFIG_SVC_PORT))
         );
-        registry.add("sprint.data.mongodb.host", () -> environment.getServiceHost(MONGO_SVC_NAME, MONGO_SVC_PORT));
+        registry.add("spring.data.mongodb.host", () -> environment.getServiceHost(MONGO_SVC_NAME, MONGO_SVC_PORT));
         registry.add("spring.data.mongodb.port", () -> environment.getServicePort(MONGO_SVC_NAME, MONGO_SVC_PORT));
     }
 
+    private static String getServiceUrl(final String serviceHost, final int servicePort) {
+        return String.format("http://%s:%s", serviceHost, servicePort);
+    }
+
     @Autowired
     private MeterRegistry meterRegistry;
 
@@ -77,10 +81,6 @@ static void registerProperties(DynamicPropertyRegistry registry) {
     @Autowired
     private Web3jService web3jService;
 
-    private static String getServiceUrl(String serviceHost, int servicePort) {
-        return "http://" + serviceHost + ":" + servicePort;
-    }
-
     @Test
     void shouldConnect() {
         await().atMost(10L, TimeUnit.SECONDS)

src/test/java/com/iexec/core/configuration/ConfigurationServiceTests.java

@@ -66,7 +66,7 @@ void init() {
     }
 
     private void initService(long startBlockNumber) throws Exception {
-        final ChainConfig chainConfig = new ChainConfig(65535, true, "", Duration.ofSeconds(5), "", "0x0", startBlockNumber, 1.0f, 0);
+        final ChainConfig chainConfig = new ChainConfig(65535, true, "", Duration.ofSeconds(5), Duration.ofSeconds(30), "", "0x0", startBlockNumber, 1.0f, 0);
         configurationService = new ConfigurationService(configurationRepository, replayConfigurationRepository, chainConfig);
         configurationService.run();
     }