Merge pull request #668 from iExecBlockchainComputing/release/8.4.0

Release/8.4.0

Author: mcornaton

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 All notable changes to this project will be documented in this file.
 
+## [[8.4.0]](https://github.com/iExecBlockchainComputing/iexec-core/releases/tag/v8.4.0) 2024-02-29
+
+### New Features
+
+- Use `MongoTemplate` to enable document update without full rewrite. (#661)
+
+### Bug Fixes
+
+- Filter out `CONTRIBUTE_AND_FINALIZE` tasks when detecting missed `REVEALED` status update. (#658)
+- Fetch `results` on-chain when updating a replicate status in `CONTRIBUTE_AND_FINALIZE` workflow. (#659 #660)
+- Properly catch all runtime exceptions when an enclave challenge generation fails. (#663)
+
+### Quality
+
+- Use `@DataMongoTest` and `@Testcontainers` annotations in replicates, compute logs and tasks tests. (#662 #664 #665)
+
+### Dependency Upgrades
+
+- Upgrade to `iexec-common` 8.4.0. (#666)
+- Upgrade to `iexec-blockchain-adapter` 8.4.0. (#667)
+- Upgrade to `iexec-result-proxy` 8.4.0. (#667)
+- Upgrade to `iexec-sms` 8.5.0. (#667)
+
 ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-core/releases/tag/v8.3.0) 2024-01-11
 
 ### New Features

build.gradle

@@ -130,6 +130,7 @@ tasks.named("bootJar") {
 
 test {
     useJUnitPlatform()
+    systemProperty "mongo.image", "mongo:4.4.28-focal"
 }
 
 tasks.register('itest') {

gradle.properties

@@ -1,9 +1,9 @@
-version=8.3.0
-iexecCommonVersion=8.3.1
+version=8.4.0
+iexecCommonVersion=8.4.0
 iexecCommonsPocoVersion=3.2.0
-iexecBlockchainAdapterVersion=8.3.0
-iexecResultVersion=8.3.0
-iexecSmsVersion=8.4.0
+iexecBlockchainAdapterVersion=8.4.0
+iexecResultVersion=8.4.0
+iexecSmsVersion=8.5.0
 
 nexusUser
 nexusPassword

src/main/java/com/iexec/core/detector/replicate/ContributionAndFinalizationUnnotifiedDetector.java

@@ -20,6 +20,7 @@
 import com.iexec.commons.poco.chain.ChainContributionStatus;
 import com.iexec.core.chain.IexecHubService;
 import com.iexec.core.configuration.CronConfiguration;
+import com.iexec.core.replicate.Replicate;
 import com.iexec.core.replicate.ReplicatesService;
 import com.iexec.core.task.TaskService;
 import com.iexec.core.task.TaskStatus;
@@ -51,4 +52,14 @@ public ContributionAndFinalizationUnnotifiedDetector(TaskService taskService,
     public void detectOnChainChanges() {
         super.detectOnChainChanges();
     }
+
+    @Override
+    protected final boolean checkDetectionIsValid(Replicate replicate) {
+        return iexecHubService.getTaskDescription(replicate.getChainTaskId()).isEligibleToContributeAndFinalize();
+    }
+
+    @Override
+    protected boolean detectStatusReachedOnChain(Replicate replicate) {
+        return iexecHubService.isRevealed(replicate.getChainTaskId(), replicate.getWalletAddress());
+    }
 }

src/main/java/com/iexec/core/detector/replicate/ContributionUnnotifiedDetector.java

@@ -20,6 +20,7 @@
 import com.iexec.commons.poco.chain.ChainContributionStatus;
 import com.iexec.core.chain.IexecHubService;
 import com.iexec.core.configuration.CronConfiguration;
+import com.iexec.core.replicate.Replicate;
 import com.iexec.core.replicate.ReplicatesService;
 import com.iexec.core.task.TaskService;
 import com.iexec.core.task.TaskStatus;
@@ -51,4 +52,14 @@ public ContributionUnnotifiedDetector(TaskService taskService,
     public void detectOnChainChanges() {
         super.detectOnChainChanges();
     }
+
+    @Override
+    protected final boolean checkDetectionIsValid(Replicate replicate) {
+        return !iexecHubService.getTaskDescription(replicate.getChainTaskId()).isEligibleToContributeAndFinalize();
+    }
+
+    @Override
+    protected boolean detectStatusReachedOnChain(Replicate replicate) {
+        return iexecHubService.isContributed(replicate.getChainTaskId(), replicate.getWalletAddress());
+    }
 }

src/main/java/com/iexec/core/detector/replicate/RevealUnnotifiedDetector.java

@@ -20,6 +20,7 @@
 import com.iexec.commons.poco.chain.ChainContributionStatus;
 import com.iexec.core.chain.IexecHubService;
 import com.iexec.core.configuration.CronConfiguration;
+import com.iexec.core.replicate.Replicate;
 import com.iexec.core.replicate.ReplicatesService;
 import com.iexec.core.task.TaskService;
 import com.iexec.core.task.TaskStatus;
@@ -51,4 +52,14 @@ public RevealUnnotifiedDetector(TaskService taskService,
     public void detectOnChainChanges() {
         super.detectOnChainChanges();
     }
+
+    @Override
+    protected final boolean checkDetectionIsValid(Replicate replicate) {
+        return !iexecHubService.getTaskDescription(replicate.getChainTaskId()).isEligibleToContributeAndFinalize();
+    }
+
+    @Override
+    protected boolean detectStatusReachedOnChain(Replicate replicate) {
+        return iexecHubService.isRevealed(replicate.getChainTaskId(), replicate.getWalletAddress());
+    }
 }

src/main/java/com/iexec/core/detector/replicate/UnnotifiedAbstractDetector.java

@@ -30,15 +30,15 @@
 
 import java.util.List;
 
-import static com.iexec.common.replicate.ReplicateStatus.*;
+import static com.iexec.common.replicate.ReplicateStatus.getMissingStatuses;
 
 @Slf4j
 public abstract class UnnotifiedAbstractDetector {
     private static final int LESS_OFTEN_DETECTOR_FREQUENCY = 10;
 
     private final TaskService taskService;
     private final ReplicatesService replicatesService;
-    private final IexecHubService iexecHubService;
+    protected final IexecHubService iexecHubService;
 
     private final List<TaskStatus> detectWhenOffChainTaskStatuses;
     private final ReplicateStatus offchainOngoing;
@@ -129,32 +129,15 @@ public void detectOnchainDone() {
      * @param replicate The replicate to check
      * @return {@literal true} if the replicate is eligible, {@literal false} otherwise
      */
-    private boolean checkDetectionIsValid(Replicate replicate) {
-        final boolean isEligibleToContributeAndFinalize = iexecHubService.getTaskDescription(replicate.getChainTaskId())
-                .isEligibleToContributeAndFinalize();
-        return offchainDone == REVEALED
-                || (!isEligibleToContributeAndFinalize && offchainDone == CONTRIBUTED)
-                || (isEligibleToContributeAndFinalize && offchainDone == CONTRIBUTE_AND_FINALIZE_DONE);
-    }
+    protected abstract boolean checkDetectionIsValid(Replicate replicate);
 
     /**
      * Checks if {@code onchainDone} status has been reached on blockchain network.
      *
      * @param replicate Replicate whose on-chain status will be checked
      * @return {@literal true} if given status has been found on-chain, {@literal false} otherwise.
      */
-    private boolean detectStatusReachedOnChain(Replicate replicate) {
-        final String chainTaskId = replicate.getChainTaskId();
-        final String walletAddress = replicate.getWalletAddress();
-        switch (onchainDone) {
-            case CONTRIBUTED:
-                return iexecHubService.isContributed(chainTaskId, walletAddress);
-            case REVEALED:
-                return iexecHubService.isRevealed(chainTaskId, walletAddress);
-            default:
-                return false;
-        }
-    }
+    protected abstract boolean detectStatusReachedOnChain(Replicate replicate);
 
     /*
      * This method should stay private. We need to ensure that

src/main/java/com/iexec/core/replicate/ReplicatesService.java

@@ -19,6 +19,7 @@
 import com.iexec.common.replicate.*;
 import com.iexec.common.utils.ContextualLockRunner;
 import com.iexec.commons.poco.chain.ChainContribution;
+import com.iexec.commons.poco.chain.ChainTask;
 import com.iexec.commons.poco.notification.TaskNotificationType;
 import com.iexec.commons.poco.task.TaskDescription;
 import com.iexec.core.chain.IexecHubService;
@@ -30,10 +31,15 @@
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.dao.OptimisticLockingFailureException;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
 import org.springframework.retry.annotation.Recover;
 import org.springframework.retry.annotation.Retryable;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
+import org.web3j.utils.Numeric;
 
 import java.util.Collections;
 import java.util.List;
@@ -47,6 +53,7 @@
 @Service
 public class ReplicatesService {
 
+    private final MongoTemplate mongoTemplate;
     private final ReplicatesRepository replicatesRepository;
     private final IexecHubService iexecHubService;
     private final ApplicationEventPublisher applicationEventPublisher;
@@ -57,12 +64,14 @@ public class ReplicatesService {
     private final ContextualLockRunner<String> replicatesUpdateLockRunner =
             new ContextualLockRunner<>(10, TimeUnit.MINUTES);
 
-    public ReplicatesService(ReplicatesRepository replicatesRepository,
+    public ReplicatesService(MongoTemplate mongoTemplate,
+                             ReplicatesRepository replicatesRepository,
                              IexecHubService iexecHubService,
                              ApplicationEventPublisher applicationEventPublisher,
                              Web3jService web3jService,
                              ResultService resultService,
                              TaskLogsService taskLogsService) {
+        this.mongoTemplate = mongoTemplate;
         this.replicatesRepository = replicatesRepository;
         this.iexecHubService = iexecHubService;
         this.applicationEventPublisher = applicationEventPublisher;
@@ -101,26 +110,16 @@ public Optional<ReplicatesList> getReplicatesList(String chainTaskId) {
     }
 
     public List<Replicate> getReplicates(String chainTaskId) {
-        Optional<ReplicatesList> optionalList = getReplicatesList(chainTaskId);
-        if (optionalList.isEmpty()) {
-            return Collections.emptyList();
-        }
-        return optionalList.get().getReplicates();
+        return getReplicatesList(chainTaskId)
+                .map(ReplicatesList::getReplicates)
+                .orElse(Collections.emptyList());
     }
 
     public Optional<Replicate> getReplicate(String chainTaskId, String walletAddress) {
-        Optional<ReplicatesList> optional = getReplicatesList(chainTaskId);
-        if (optional.isEmpty()) {
-            return Optional.empty();
-        }
-
-        for (Replicate replicate : optional.get().getReplicates()) {
-            if (replicate.getWalletAddress().equals(walletAddress)) {
-                return Optional.of(replicate);
-            }
-        }
-
-        return Optional.empty();
+        return getReplicates(chainTaskId)
+                .stream()
+                .filter(replicate -> replicate.getWalletAddress().equals(walletAddress))
+                .findFirst();
     }
 
     public int getNbReplicatesWithCurrentStatus(String chainTaskId, ReplicateStatus... listStatus) {
@@ -250,6 +249,15 @@ public UpdateReplicateStatusArgs computeUpdateReplicateStatusArgs(String chainTa
             }
         }
 
+        if (statusUpdate.getStatus() == CONTRIBUTE_AND_FINALIZE_DONE) {
+            // TODO read chainCallbackData if CONTRIBUTE_AND_FINALIZE becomes applicable some day in the future and if latest ABI is used
+            resultLink = iexecHubService.getChainTask(chainTaskId)
+                    .map(ChainTask::getResults)
+                    .map(Numeric::hexStringToByteArray)
+                    .map(String::new)
+                    .orElse(null);
+        }
+
         return UpdateReplicateStatusArgs.builder()
                 .workerWeight(workerWeight)
                 .chainContribution(chainContribution)
@@ -402,31 +410,34 @@ Either<ReplicateStatusUpdateError, TaskNotificationType> updateReplicateStatusWi
         log.info("Replicate update request [status:{}, chainTaskId:{}, walletAddress:{}, details:{}]",
                 statusUpdate.getStatus(), chainTaskId, walletAddress, statusUpdate.getDetailsWithoutLogs());
 
-        final Optional<ReplicatesList> oReplicatesList = getReplicatesList(chainTaskId);
-        final Optional<Replicate> oReplicate = oReplicatesList
-                .flatMap(replicatesList -> replicatesList.getReplicateOfWorker(walletAddress));
-        if (oReplicatesList.isEmpty() || oReplicate.isEmpty()) {
+        final Replicate replicate = getReplicatesList(chainTaskId)
+                .flatMap(replicatesList -> replicatesList.getReplicateOfWorker(walletAddress))
+                .orElse(null);
+        if (replicate == null) {
             log.error("Cannot update replicate, could not get replicate [chainTaskId:{}, UpdateRequest:{}]",
                     chainTaskId, statusUpdate);
             return Either.left(ReplicateStatusUpdateError.UNKNOWN_REPLICATE);
         }
-        final ReplicatesList replicatesList = oReplicatesList.get();
-        final Replicate replicate = oReplicate.get();
         final ReplicateStatus newStatus = statusUpdate.getStatus();
 
         final ReplicateStatusUpdateError error = canUpdateReplicateStatus(replicate, statusUpdate, updateReplicateStatusArgs);
         if (ReplicateStatusUpdateError.NO_ERROR != error) {
             return Either.left(error);
         }
 
+        Update update = new Update();
         if (newStatus == CONTRIBUTED || newStatus == CONTRIBUTE_AND_FINALIZE_DONE) {
             replicate.setContributionHash(updateReplicateStatusArgs.getChainContribution().getResultHash());
             replicate.setWorkerWeight(updateReplicateStatusArgs.getWorkerWeight());
+            update.set("replicates.$.contributionHash", updateReplicateStatusArgs.getChainContribution().getResultHash());
+            update.set("replicates.$.workerWeight", updateReplicateStatusArgs.getWorkerWeight());
         }
 
         if (newStatus == RESULT_UPLOADED || newStatus == CONTRIBUTE_AND_FINALIZE_DONE) {
             replicate.setResultLink(updateReplicateStatusArgs.getResultLink());
             replicate.setChainCallbackData(updateReplicateStatusArgs.getChainCallbackData());
+            update.set("replicates.$.resultLink", updateReplicateStatusArgs.getResultLink());
+            update.set("replicates.$.chainCallbackData", updateReplicateStatusArgs.getChainCallbackData());
         }
 
         if (statusUpdate.getDetails() != null &&
@@ -436,10 +447,13 @@ Either<ReplicateStatusUpdateError, TaskNotificationType> updateReplicateStatusWi
             taskLogsService.addComputeLogs(chainTaskId, computeLogs);
             statusUpdate.getDetails().setComputeLogs(null);//using null here to keep light replicate
             replicate.setAppComputeLogsPresent(true);
+            update.set("replicates.$.appComputeLogsPresent", true);
         }
 
+        update.push("replicates.$.statusUpdateList", statusUpdate);
+        Query query = Query.query(Criteria.where("chainTaskId").is(chainTaskId).and("replicates.walletAddress").is(walletAddress));
+        mongoTemplate.updateFirst(query, update, ReplicatesList.class);
         replicate.updateStatus(statusUpdate);
-        replicatesRepository.save(replicatesList);
         applicationEventPublisher.publishEvent(new ReplicateUpdatedEvent(chainTaskId, walletAddress, statusUpdate));
         ReplicateStatusCause newStatusCause = statusUpdate.getDetails() != null ?
                 statusUpdate.getDetails().getCause() : null;