Improve configuration collection update at startup and when no deals has been detected for the last hour (#750)

jbern0rd · web-flow · commit 3ea873d99c3d · 2025-03-31T17:22:42.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,8 @@ All notable changes to this project will be documented in this file.
 ### New Features
 
 - Add WebSocketBlockchainListener to fetch latest block without polling the blockchain network. (#747)
+- Update last seen block in configuration collection when no deal has been detected in the last hour. (#750)
+- Update configuration collection depending on chain.start-block-number at startup. (#750)
 
 ### Bug Fixes
 
diff --git a/README.md b/README.md
@@ -49,10 +49,11 @@ You can configure the _iExec Core Scheduler_ with the following properties:
 | `IEXEC_LOGS_AVAILABILITY_PERIOD_IN_DAYS` | Number of days to keep logs of past tasks. | Positive integer | `3` |
 
 If it is not the first startup of the _iExec Core Scheduler_ and if it received deals previously,
-the _MongoDB_ instance will contain a __configuration Collection__ in the __iexec Database__.
-The value stored in this document takes the precedence over the `IEXEC_START_BLOCK_NUMBER` configuration parameter.
-To enforce deal observation starting from the `IEXEC_START_BLOCK_NUMBER` value, the aforementioned document has to be deleted in the _MongoDB_.
-All deals prior to the `IEXEC_START_BLOCK_NUMBER` will then be ignored.
+the _MongoDB_ instance will contain both __configuration__ and __replayConfiguration__ collections in the __iexec Database__.
+The highest value between the value stored in this document and the `IEXEC_START_BLOCK_NUMBER` configuration parameter takes precedence.
+If the stored value takes precedence, the `IEXEC_START_BLOCK_NUMBER` is ignored.
+Otherwise, the __configuration__ and __replayConfiguration__ collections will be updated with `IEXEC_START_BLOCK_NUMBER` during startup.
+All deals between the previous configured value and the `IEXEC_START_BLOCK_NUMBER` will then be ignored.
 
 A more exhaustive documentation is available on [the official documentation of iExec](https://docs.iex.ec/).
 
diff --git a/src/main/java/com/iexec/core/chain/DealWatcherService.java b/src/main/java/com/iexec/core/chain/DealWatcherService.java
@@ -65,7 +65,7 @@ public class DealWatcherService {
     private final TaskService taskService;
     private final Web3jService web3jService;
     // internal variables
-    private boolean outOfService;
+    private boolean outOfService = true;
     private Disposable dealEventsSubscription;
     private Disposable dealEventSubscriptionReplay;
     @Getter
@@ -254,16 +254,16 @@ boolean shouldProcessDeal(ChainDeal chainDeal) {
      */
     @Scheduled(fixedRateString = "#{@cronConfiguration.getDealReplay()}")
     void replayDealEvent() {
-        BigInteger lastSeenBlockWithDeal = configurationService.getLastSeenBlockWithDeal();
-        BigInteger replayFromBlock = configurationService.getFromReplay();
-        if (replayFromBlock.compareTo(lastSeenBlockWithDeal) >= 0) {
-            return;
-        }
         disposeSubscription(dealEventSubscriptionReplay);
         if (outOfService) {
             log.info("OUT-OF-SERVICE do not create replay subscription");
             return;
         }
+        final BigInteger lastSeenBlockWithDeal = configurationService.getLastSeenBlockWithDeal();
+        final BigInteger replayFromBlock = configurationService.getFromReplay();
+        if (replayFromBlock.compareTo(lastSeenBlockWithDeal) >= 0) {
+            return;
+        }
         dealEventSubscriptionReplay = subscribeToDealEventInRange(replayFromBlock, lastSeenBlockWithDeal);
         configurationService.setFromReplay(lastSeenBlockWithDeal);
     }
diff --git a/src/main/java/com/iexec/core/configuration/Configuration.java b/src/main/java/com/iexec/core/configuration/Configuration.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2020 IEXEC BLOCKCHAIN TECH
+ * Copyright 2020-2025 IEXEC BLOCKCHAIN TECH
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,6 +24,7 @@
 import org.springframework.data.annotation.Version;
 
 import java.math.BigInteger;
+import java.time.Instant;
 
 @Data
 @AllArgsConstructor
@@ -38,6 +39,7 @@ class Configuration {
     private Long version;
 
     private BigInteger lastSeenBlockWithDeal;
+    private Instant lastUpdate;
 
 }
 
diff --git a/src/main/java/com/iexec/core/configuration/ConfigurationService.java b/src/main/java/com/iexec/core/configuration/ConfigurationService.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2020 IEXEC BLOCKCHAIN TECH
+ * Copyright 2020-2025 IEXEC BLOCKCHAIN TECH
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,17 +17,27 @@
 package com.iexec.core.configuration;
 
 import com.iexec.core.chain.ChainConfig;
+import com.iexec.core.chain.event.LatestBlockEvent;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.context.event.EventListener;
 import org.springframework.stereotype.Service;
 
 import java.math.BigInteger;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 
+@Slf4j
 @Service
-public class ConfigurationService {
+public class ConfigurationService implements CommandLineRunner {
 
     private final ConfigurationRepository configurationRepository;
     private final ReplayConfigurationRepository replayConfigurationRepository;
     private final ChainConfig chainConfig;
 
+    private Configuration configuration;
+    private ReplayConfiguration replayConfiguration;
+
     public ConfigurationService(ConfigurationRepository configurationRepository,
                                 ReplayConfigurationRepository replayConfigurationRepository,
                                 ChainConfig chainConfig) {
@@ -36,46 +46,84 @@ public ConfigurationService(ConfigurationRepository configurationRepository,
         this.chainConfig = chainConfig;
     }
 
-    private Configuration getConfiguration() {
-        if (configurationRepository.count() > 0)
-            return configurationRepository.findAll().get(0);
-
-        return configurationRepository.save(
-            Configuration
-                    .builder()
-                    .lastSeenBlockWithDeal(BigInteger.valueOf(chainConfig.getStartBlockNumber()))
-                    .build());
+    /**
+     * Update last scanned block in configuration.
+     *
+     * @param event Event containing last block number
+     */
+    @EventListener
+    void setLastScannedBlock(final LatestBlockEvent event) {
+        final Instant now = Instant.now();
+        if (now.isAfter(configuration.getLastUpdate().plus(1L, ChronoUnit.HOURS))) {
+            updateConfiguration(BigInteger.valueOf(event.getBlockNumber()));
+        }
     }
 
     public BigInteger getLastSeenBlockWithDeal() {
-        return this.getConfiguration().getLastSeenBlockWithDeal();
+        return configuration.getLastSeenBlockWithDeal();
     }
 
-    public void setLastSeenBlockWithDeal(BigInteger lastBlockNumber) {
-        Configuration configuration = this.getConfiguration();
-        configuration.setLastSeenBlockWithDeal(lastBlockNumber);
-        configurationRepository.save(configuration);
+    public void setLastSeenBlockWithDeal(final BigInteger lastBlockNumber) {
+        updateConfiguration(lastBlockNumber);
     }
 
-    private ReplayConfiguration getReplayConfiguration() {
-        if (replayConfigurationRepository.count() > 0)
-            return replayConfigurationRepository.findAll().get(0);
-
-        return replayConfigurationRepository.save(
-                ReplayConfiguration
-                        .builder()
-                        .fromBlockNumber(BigInteger.valueOf(chainConfig.getStartBlockNumber()))
-                        .build());
+    private synchronized void updateConfiguration(final BigInteger lastBlockNumber) {
+        configuration.setLastSeenBlockWithDeal(lastBlockNumber);
+        configuration.setLastUpdate(Instant.now());
+        configuration = configurationRepository.save(configuration);
     }
 
     public BigInteger getFromReplay() {
-        return this.getReplayConfiguration().getFromBlockNumber();
+        return replayConfiguration.getFromBlockNumber();
     }
 
     public void setFromReplay(BigInteger fromReplay) {
-        ReplayConfiguration replayConfiguration = this.getReplayConfiguration();
         replayConfiguration.setFromBlockNumber(fromReplay);
-        replayConfigurationRepository.save(replayConfiguration);
+        replayConfiguration = replayConfigurationRepository.save(replayConfiguration);
+    }
+
+    @Override
+    public void run(String... args) throws Exception {
+        final String messageDetails = String.format("[start:block:%s]", chainConfig.getStartBlockNumber());
+        final Instant now = Instant.now();
+        if (configurationRepository.count() == 0) {
+            log.info("Creating configuration {}", messageDetails);
+            configuration = configurationRepository.save(
+                    Configuration
+                            .builder()
+                            .lastSeenBlockWithDeal(BigInteger.valueOf(chainConfig.getStartBlockNumber()))
+                            .lastUpdate(now)
+                            .build());
+        } else {
+            configuration = configurationRepository.findAll().get(0);
+            if (chainConfig.getStartBlockNumber() > configuration.getLastSeenBlockWithDeal().longValue()) {
+                log.info("Updating configuration {}", messageDetails);
+                configuration.setLastSeenBlockWithDeal(BigInteger.valueOf(chainConfig.getStartBlockNumber()));
+                configuration.setLastUpdate(now);
+                configuration = configurationRepository.save(configuration);
+            } else {
+                log.info("Keeping current configuration [start-block:{}]", configuration.getLastSeenBlockWithDeal());
+            }
+        }
+        if (replayConfigurationRepository.count() == 0) {
+            log.info("Creating replay configuration {}", messageDetails);
+            replayConfiguration = replayConfigurationRepository.save(
+                    ReplayConfiguration
+                            .builder()
+                            .fromBlockNumber(BigInteger.valueOf(chainConfig.getStartBlockNumber()))
+                            .lastUpdate(now)
+                            .build());
+        } else {
+            replayConfiguration = replayConfigurationRepository.findAll().get(0);
+            if (chainConfig.getStartBlockNumber() > replayConfiguration.getFromBlockNumber().longValue()) {
+                log.info("Updating replay configuration {}", messageDetails);
+                replayConfiguration.setFromBlockNumber(BigInteger.valueOf(chainConfig.getStartBlockNumber()));
+                replayConfiguration.setLastUpdate(now);
+                replayConfiguration = replayConfigurationRepository.save(replayConfiguration);
+            } else {
+                log.info("Keeping current replay configuration [start-block:{}]", replayConfiguration.getFromBlockNumber());
+            }
+        }
     }
 
 }
diff --git a/src/main/java/com/iexec/core/configuration/ReplayConfiguration.java b/src/main/java/com/iexec/core/configuration/ReplayConfiguration.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2020 IEXEC BLOCKCHAIN TECH
+ * Copyright 2020-2025 IEXEC BLOCKCHAIN TECH
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,6 +24,7 @@
 import org.springframework.data.annotation.Version;
 
 import java.math.BigInteger;
+import java.time.Instant;
 
 @Data
 @AllArgsConstructor
@@ -38,6 +39,7 @@ class ReplayConfiguration {
     private Long version;
 
     private BigInteger fromBlockNumber;
+    private Instant lastUpdate;
 
 }
 
diff --git a/src/test/java/com/iexec/core/chain/DealWatcherServiceTests.java b/src/test/java/com/iexec/core/chain/DealWatcherServiceTests.java
@@ -332,6 +332,7 @@ void shouldReplayAllEventInRange() {
 
     @Test
     void shouldNotReplayIfFromReplayEqualsLastSeenBlock() {
+        ReflectionTestUtils.setField(dealWatcherService, OUT_OF_SERVICE_FIELD_NAME, false);
         when(configurationService.getLastSeenBlockWithDeal()).thenReturn(BigInteger.ZERO);
         when(configurationService.getFromReplay()).thenReturn(BigInteger.ZERO);
         dealWatcherService.replayDealEvent();
@@ -340,11 +341,8 @@ void shouldNotReplayIfFromReplayEqualsLastSeenBlock() {
 
     @Test
     void shouldNotReplayIfOutOfService() {
-        ReflectionTestUtils.setField(dealWatcherService, OUT_OF_SERVICE_FIELD_NAME, true);
-        when(configurationService.getLastSeenBlockWithDeal()).thenReturn(BigInteger.TEN);
-        when(configurationService.getFromReplay()).thenReturn(BigInteger.ZERO);
         dealWatcherService.replayDealEvent();
-        verifyNoInteractions(iexecHubService);
+        verifyNoInteractions(configurationService, iexecHubService);
     }
     // endregion
 
diff --git a/src/test/java/com/iexec/core/configuration/ConfigurationServiceTests.java b/src/test/java/com/iexec/core/configuration/ConfigurationServiceTests.java
