Add retry mechanism and set command status to FAILURE after all attempts failed (#134)

Author: jbern0rd

CHANGELOG.md

@@ -8,6 +8,10 @@ All notable changes to this project will be documented in this file.
 
 - Label REST API with `v1` version. (#132)
 
+### Bug Fixes
+
+- Add retry mechanism and set command status to `FAILURE` after all attempts failed. (#134)
+
 ### Quality
 
 - Remove `/tasks/{chainTaskId}` endpoint, the adapter must only call `initialize` and `finalize` **PoCo** methods. (#130)

src/main/java/com/iexec/blockchain/command/generic/CommandEngine.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020-2023 IEXEC BLOCKCHAIN TECH
+ * Copyright 2020-2024 IEXEC BLOCKCHAIN TECH
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,8 @@
 @Slf4j
 public abstract class CommandEngine<C extends Command<A>, A extends CommandArgs> {
 
+    private static final int MAX_ATTEMPTS = 5;
+
     private final CommandBlockchain<A> blockchainService;
     private final CommandStorage<C, A> updaterService;
     private final QueueService queueService;
@@ -88,24 +90,24 @@ public void triggerBlockchainCommand(A args) {
                     chainObjectId, args);
             return;
         }
+        int attempt = 0;
         log.info("Processing command [chainObjectId:{}, commandArgs:{}]",
                 chainObjectId, args);
-        TransactionReceipt receipt;
-        try {
-            receipt = blockchainService.sendBlockchainCommand(args);
-        } catch (Exception e) {
-            log.error("Something wrong happened while triggering blockchain " +
-                            "command [chainObjectId:{}, commandArgs:{}]",
-                    chainObjectId, args, e);
-            //TODO Update to proper status: PROCESSING_FAILED or FAILURE
-            return;
+        TransactionReceipt receipt = null;
+        while (attempt < MAX_ATTEMPTS && receipt == null) {
+            attempt++;
+            try {
+                receipt = blockchainService.sendBlockchainCommand(args);
+            } catch (Exception e) {
+                log.error("Something wrong happened while triggering command [chainObjectId:{}, commandArgs:{}, attempt:{}]",
+                        chainObjectId, args, attempt, e);
+            }
         }
         if (receipt == null) {
             log.error("Triggering blockchain command failed " +
                             "(received null receipt after blockchain send) " +
-                            "[chainObjectId:{}, commandArgs:{}]",
-                    chainObjectId, args);
-            return;
+                            "[chainObjectId:{}, commandArgs:{}, attempt:{}]",
+                    chainObjectId, args, attempt);
         }
         updaterService.updateToFinal(chainObjectId, receipt);
     }

src/main/java/com/iexec/blockchain/command/generic/CommandStorage.java

@@ -17,7 +17,6 @@
 package com.iexec.blockchain.command.generic;
 
 import com.iexec.blockchain.api.CommandStatus;
-import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.web3j.protocol.core.methods.response.TransactionReceipt;
 
@@ -88,26 +87,25 @@ public boolean updateToProcessing(String chainObjectId) {
      * @param receipt       blockchain receipt
      */
     public void updateToFinal(String chainObjectId,
-                              @NonNull TransactionReceipt receipt) {
-        Optional<C> localCommand = commandRepository
+                              TransactionReceipt receipt) {
+        final C command = commandRepository
                 .findByChainObjectId(chainObjectId)
-                .filter(command -> command.getStatus() == CommandStatus.PROCESSING);
-        if (localCommand.isEmpty()) {
+                .filter(cmd -> cmd.getStatus() == CommandStatus.PROCESSING)
+                .orElse(null);
+        if (command == null) {
+            log.error("No entry was found in database, could not update to final state");
             return;
         }
-        C command = localCommand.get();
 
-        CommandStatus status;
-        if (receipt.isStatusOK()) {
-            status = CommandStatus.SUCCESS;
+        if (receipt != null && receipt.isStatusOK()) {
+            command.setStatus(CommandStatus.SUCCESS);
             log.info("Success command with transaction receipt [chainObjectId:{}, command:{}, receipt:{}]",
                     chainObjectId, command.getClass().getSimpleName(), receipt);
         } else {
-            status = CommandStatus.FAILURE;
+            command.setStatus(CommandStatus.FAILURE);
             log.info("Failure after transaction sent [chainObjectId:{}, command:{}, receipt:{}]",
                     chainObjectId, command.getClass().getSimpleName(), receipt);
         }
-        command.setStatus(status);
         command.setTransactionReceipt(receipt);
         command.setFinalDate(Instant.now());
         commandRepository.save(command);

src/test/java/com/iexec/blockchain/command/task/initialize/TaskInitializeTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2021-2023 IEXEC BLOCKCHAIN TECH
+ * Copyright 2021-2024 IEXEC BLOCKCHAIN TECH
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -123,7 +123,7 @@ void shouldNotTriggerInitializeTaskSinceReceiptIsNull() throws Exception {
                 .thenReturn(null);
 
         taskInitializeService.triggerBlockchainCommand(args);
-        verify(updaterService, times(0))
+        verify(updaterService)
                 .updateToFinal(CHAIN_TASK_ID, null);
     }