feat: update exit cause handling to return a list of causes in PostCompute and PreCompute services (#658)

Author: Natchica

src/main/java/com/iexec/worker/compute/post/PostComputeService.java

@@ -220,11 +220,11 @@ public PostComputeResponse runTeePostCompute(TaskDescription taskDescription,
         }
         if (finalStatus == DockerRunFinalStatus.FAILED) {
             int exitCode = dockerResponse.getContainerExitCode();
-            ReplicateStatusCause exitCause = getExitCause(chainTaskId, exitCode);
+            final List<ReplicateStatusCause> exitCauses = getExitCauses(chainTaskId, exitCode);
             log.error("Failed to run tee post-compute [chainTaskId:{}, " +
-                    "exitCode:{}, exitCause:{}]", chainTaskId, exitCode, exitCause);
+                    "exitCode:{}, exitCauses:{}]", chainTaskId, exitCode, exitCauses);
             return PostComputeResponse.builder()
-                    .exitCauses(List.of(exitCause))
+                    .exitCauses(exitCauses)
                     .build();
         }
         return PostComputeResponse.builder()
@@ -233,25 +233,15 @@ public PostComputeResponse runTeePostCompute(TaskDescription taskDescription,
                 .build();
     }
 
-    private ReplicateStatusCause getExitCause(String chainTaskId, Integer exitCode) {
-        ReplicateStatusCause cause = null;
-        if (exitCode != null && exitCode != 0) {
-            switch (exitCode) {
-                case 1:
-                    // Use first cause from bulk processing for now
-                    cause = computeExitCauseService.getExitCausesAndPruneForGivenComputeStage(chainTaskId, ComputeStage.POST, POST_COMPUTE_FAILED_UNKNOWN_ISSUE).get(0);
-                    break;
-                case 2:
-                    cause = ReplicateStatusCause.POST_COMPUTE_EXIT_REPORTING_FAILED;
-                    break;
-                case 3:
-                    cause = ReplicateStatusCause.POST_COMPUTE_TASK_ID_MISSING;
-                    break;
-                default:
-                    break;
-            }
-        }
-        return cause;
+    private List<ReplicateStatusCause> getExitCauses(final String chainTaskId, final int exitCode) {
+        return switch (exitCode) {
+            case 0 -> List.of();
+            case 1 ->
+                    computeExitCauseService.getExitCausesAndPruneForGivenComputeStage(chainTaskId, ComputeStage.POST, POST_COMPUTE_FAILED_UNKNOWN_ISSUE);
+            case 2 -> List.of(ReplicateStatusCause.POST_COMPUTE_EXIT_REPORTING_FAILED);
+            case 3 -> List.of(ReplicateStatusCause.POST_COMPUTE_TASK_ID_MISSING);
+            default -> List.of(POST_COMPUTE_FAILED_UNKNOWN_ISSUE);
+        };
     }
 
     private String getTaskTeePostComputeContainerName(String chainTaskId) {

src/main/java/com/iexec/worker/compute/pre/PreComputeService.java

@@ -156,38 +156,28 @@ private List<ReplicateStatusCause> downloadDatasetAndFiles(
             Integer exitCode = prepareTeeInputData(taskDescription, secureSession);
             if (exitCode == null || exitCode != 0) {
                 String chainTaskId = taskDescription.getChainTaskId();
-                ReplicateStatusCause exitCause = getExitCause(chainTaskId, exitCode); // TODO: Handle list of exit causes
+                final List<ReplicateStatusCause> exitCauses = getExitCauses(chainTaskId, exitCode);
                 log.error("Failed to prepare TEE input data [chainTaskId:{}, exitCode:{}, exitCauses:{}]",
-                        chainTaskId, exitCode, exitCause);
-                return List.of(exitCause);
+                        chainTaskId, exitCode, exitCauses);
+                return exitCauses;
             }
         } catch (TimeoutException e) {
             return List.of(PRE_COMPUTE_TIMEOUT);
         }
         return List.of();
     }
 
-    private ReplicateStatusCause getExitCause(String chainTaskId, Integer exitCode) {
-        ReplicateStatusCause cause = null;
+    private List<ReplicateStatusCause> getExitCauses(final String chainTaskId, final Integer exitCode) {
         if (exitCode == null) {
-            cause = PRE_COMPUTE_IMAGE_MISSING;
-        } else {
-            switch (exitCode) {
-                case 1:
-                    // Use first cause from bulk processing for now
-                    cause = computeExitCauseService.getExitCausesAndPruneForGivenComputeStage(chainTaskId, ComputeStage.PRE, PRE_COMPUTE_FAILED_UNKNOWN_ISSUE).get(0);
-                    break;
-                case 2:
-                    cause = ReplicateStatusCause.PRE_COMPUTE_EXIT_REPORTING_FAILED;
-                    break;
-                case 3:
-                    cause = ReplicateStatusCause.PRE_COMPUTE_TASK_ID_MISSING;
-                    break;
-                default:
-                    break;
-            }
+            return List.of(PRE_COMPUTE_IMAGE_MISSING);
         }
-        return cause;
+        return switch (exitCode) {
+            case 1 -> computeExitCauseService.getExitCausesAndPruneForGivenComputeStage(
+                    chainTaskId, ComputeStage.PRE, PRE_COMPUTE_FAILED_UNKNOWN_ISSUE);
+            case 2 -> List.of(PRE_COMPUTE_EXIT_REPORTING_FAILED);
+            case 3 -> List.of(PRE_COMPUTE_TASK_ID_MISSING);
+            default -> List.of(PRE_COMPUTE_FAILED_UNKNOWN_ISSUE);
+        };
     }
 
 

src/test/java/com/iexec/worker/compute/post/PostComputeServiceTests.java

@@ -43,13 +43,15 @@
 import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.ArgumentCaptor;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
+import org.mockito.junit.jupiter.MockitoExtension;
 
 import java.io.File;
 import java.io.IOException;
@@ -67,17 +69,22 @@
 import static org.mockito.Mockito.*;
 
 @Slf4j
+@ExtendWith(MockitoExtension.class)
 class PostComputeServiceTests {
 
     private static final String CHAIN_TASK_ID = "CHAIN_TASK_ID";
     private static final String DATASET_URI = "DATASET_URI";
     private static final String WORKER_NAME = "WORKER_NAME";
     private static final String TEE_POST_COMPUTE_IMAGE = "TEE_POST_COMPUTE_IMAGE";
-    private static final long TEE_POST_COMPUTE_HEAP = 1024;
     private static final String TEE_POST_COMPUTE_ENTRYPOINT = "postComputeEntrypoint";
     private static final TeeSessionGenerationResponse SECURE_SESSION = mock(TeeSessionGenerationResponse.class);
     private static final long MAX_EXECUTION_TIME = 1000;
 
+    private final TeeAppProperties postComputeProperties = TeeAppProperties.builder()
+            .image(TEE_POST_COMPUTE_IMAGE)
+            .entrypoint(TEE_POST_COMPUTE_ENTRYPOINT)
+            .build();
+
     @TempDir
     public File jUnitTemporaryFolder;
     private TaskDescription taskDescription = TaskDescription.builder()
@@ -97,10 +104,6 @@ class PostComputeServiceTests {
     @Mock
     private TeeServicesManager teeServicesManager;
     @Mock
-    private TeeAppProperties preComputeProperties;
-    @Mock
-    private TeeAppProperties postComputeProperties;
-    @Mock
     private TeeServicesProperties properties;
     @Mock
     private DockerClientInstance dockerClientInstanceMock;
@@ -112,19 +115,11 @@ class PostComputeServiceTests {
     private TeeServicesPropertiesService teeServicesPropertiesService;
     @Mock
     private ComputeDurationsService postComputeDurationsService;
-
     @Mock
     private TeeService teeMockedService;
 
     @BeforeEach
     void beforeEach() {
-        MockitoAnnotations.openMocks(this);
-        when(dockerService.getClient()).thenReturn(dockerClientInstanceMock);
-        when(teeServicesManager.getTeeService(any())).thenReturn(teeMockedService);
-        when(properties.getPreComputeProperties()).thenReturn(preComputeProperties);
-        when(properties.getPostComputeProperties()).thenReturn(postComputeProperties);
-        when(teeServicesPropertiesService.getTeeServicesProperties(CHAIN_TASK_ID)).thenReturn(properties);
-
         output = jUnitTemporaryFolder.getAbsolutePath();
         iexecOut = output + IexecFileHelper.SLASH_IEXEC_OUT;
         computedJson = iexecOut + IexecFileHelper.SLASH_COMPUTED_JSON;
@@ -209,35 +204,36 @@ void shouldFailResultFilesNameCheckWhenFileNameTooLong() throws IOException {
     // endregion
 
     //region runTeePostCompute
+    void prepareMocksForTeePostCompute(DockerRunResponse dockerRunResponse) {
+        List<String> env = Arrays.asList("var0", "var1");
+        when(dockerService.getClient()).thenReturn(dockerClientInstanceMock);
+        when(teeServicesManager.getTeeService(any())).thenReturn(teeMockedService);
+        when(teeServicesPropertiesService.getTeeServicesProperties(CHAIN_TASK_ID)).thenReturn(properties);
+        when(properties.getPostComputeProperties()).thenReturn(postComputeProperties);
+        when(dockerClientInstanceMock.isImagePresent(TEE_POST_COMPUTE_IMAGE)).thenReturn(true);
+        when(teeMockedService.buildPostComputeDockerEnv(taskDescription, SECURE_SESSION)).thenReturn(env);
+        String iexecOutBind = iexecOut + ":" + IexecFileHelper.SLASH_IEXEC_OUT;
+        when(dockerService.getIexecOutBind(CHAIN_TASK_ID)).thenReturn(iexecOutBind);
+        when(workerConfigService.getWorkerName()).thenReturn(WORKER_NAME);
+        when(workerConfigService.getDockerNetworkName()).thenReturn("lasNetworkName");
+        when(sgxService.getSgxDriverMode()).thenReturn(SgxDriverMode.LEGACY);
+        when(dockerService.run(any())).thenReturn(dockerRunResponse);
+    }
+
     @Test
     void shouldRunTeePostComputeAndConnectToLasNetwork() {
-        String lasNetworkName = "networkName";
+        String lasNetworkName = "lasNetworkName";
         taskDescription = TaskDescription.builder()
                 .chainTaskId(CHAIN_TASK_ID)
                 .datasetUri(DATASET_URI)
                 .maxExecutionTime(MAX_EXECUTION_TIME)
                 .build();
-        List<String> env = Arrays.asList("var0", "var1");
-        when(postComputeProperties.getImage()).thenReturn(TEE_POST_COMPUTE_IMAGE);
-        when(postComputeProperties.getHeapSizeInBytes()).thenReturn(TEE_POST_COMPUTE_HEAP);
-        when(postComputeProperties.getEntrypoint()).thenReturn(TEE_POST_COMPUTE_ENTRYPOINT);
-        when(dockerClientInstanceMock.isImagePresent(TEE_POST_COMPUTE_IMAGE))
-                .thenReturn(true);
-        when(teeMockedService.buildPostComputeDockerEnv(taskDescription, SECURE_SESSION))
-                .thenReturn(env);
-        String iexecOutBind = iexecOut + ":" + IexecFileHelper.SLASH_IEXEC_OUT;
-        when(dockerService.getIexecOutBind(CHAIN_TASK_ID)).thenReturn(iexecOutBind);
-        when(workerConfigService.getTaskOutputDir(CHAIN_TASK_ID)).thenReturn(output);
-        when(workerConfigService.getTaskIexecOutDir(CHAIN_TASK_ID)).thenReturn(iexecOut);
-        when(workerConfigService.getWorkerName()).thenReturn(WORKER_NAME);
-        when(workerConfigService.getDockerNetworkName()).thenReturn(lasNetworkName);
         DockerRunResponse expectedDockerRunResponse = DockerRunResponse
                 .builder()
                 .finalStatus(DockerRunFinalStatus.SUCCESS)
                 .executionDuration(Duration.ofSeconds(10))
                 .build();
-        when(dockerService.run(any())).thenReturn(expectedDockerRunResponse);
-        when(sgxService.getSgxDriverMode()).thenReturn(SgxDriverMode.LEGACY);
+        prepareMocksForTeePostCompute(expectedDockerRunResponse);
         List<Device> devices = List.of(Device.parse("/dev/isgx"));
         when(sgxService.getSgxDevices()).thenReturn(devices);
 
@@ -251,10 +247,12 @@ void shouldRunTeePostComputeAndConnectToLasNetwork() {
         verify(dockerService).run(argumentCaptor.capture());
         DockerRunRequest dockerRunRequest =
                 argumentCaptor.getAllValues().get(0);
+        String iexecOutBind = iexecOut + ":" + IexecFileHelper.SLASH_IEXEC_OUT;
         HostConfig hostConfig = HostConfig.newHostConfig()
                 .withBinds(Bind.parse(iexecOutBind))
                 .withDevices(devices)
                 .withNetworkMode(lasNetworkName);
+        List<String> env = Arrays.asList("var0", "var1");
         assertThat(dockerRunRequest).isEqualTo(
                 DockerRunRequest.builder()
                         .hostConfig(hostConfig)
@@ -276,9 +274,9 @@ void shouldNotRunTeePostComputeSinceDockerImageNotFoundLocally() {
                 .datasetUri(DATASET_URI)
                 .maxExecutionTime(MAX_EXECUTION_TIME)
                 .build();
-        when(postComputeProperties.getImage()).thenReturn(TEE_POST_COMPUTE_IMAGE);
-        when(postComputeProperties.getHeapSizeInBytes()).thenReturn(TEE_POST_COMPUTE_HEAP);
-        when(postComputeProperties.getEntrypoint()).thenReturn(TEE_POST_COMPUTE_ENTRYPOINT);
+        when(dockerService.getClient()).thenReturn(dockerClientInstanceMock);
+        when(teeServicesPropertiesService.getTeeServicesProperties(CHAIN_TASK_ID)).thenReturn(properties);
+        when(properties.getPostComputeProperties()).thenReturn(postComputeProperties);
         when(dockerClientInstanceMock.isImagePresent(TEE_POST_COMPUTE_IMAGE))
                 .thenReturn(false);
 
@@ -297,29 +295,17 @@ void shouldRunTeePostComputeWithFailDockerResponse(Map.Entry<Integer, ReplicateS
                 .datasetUri(DATASET_URI)
                 .maxExecutionTime(MAX_EXECUTION_TIME)
                 .build();
-        List<String> env = Arrays.asList("var0", "var1");
-        when(postComputeProperties.getImage()).thenReturn(TEE_POST_COMPUTE_IMAGE);
-        when(postComputeProperties.getHeapSizeInBytes()).thenReturn(TEE_POST_COMPUTE_HEAP);
-        when(postComputeProperties.getEntrypoint()).thenReturn(TEE_POST_COMPUTE_ENTRYPOINT);
-        when(dockerClientInstanceMock.isImagePresent(TEE_POST_COMPUTE_IMAGE))
-                .thenReturn(true);
-        when(teeMockedService.buildPostComputeDockerEnv(taskDescription, SECURE_SESSION))
-                .thenReturn(env);
-        String iexecOutBind = iexecOut + ":" + IexecFileHelper.SLASH_IEXEC_OUT;
-        when(dockerService.getIexecOutBind(CHAIN_TASK_ID)).thenReturn(iexecOutBind);
-        when(workerConfigService.getTaskOutputDir(CHAIN_TASK_ID)).thenReturn(output);
-        when(workerConfigService.getTaskIexecOutDir(CHAIN_TASK_ID)).thenReturn(iexecOut);
-        when(workerConfigService.getWorkerName()).thenReturn(WORKER_NAME);
-        when(workerConfigService.getDockerNetworkName()).thenReturn("lasNetworkName");
         DockerRunResponse expectedDockerRunResponse =
                 DockerRunResponse.builder()
                         .finalStatus(DockerRunFinalStatus.FAILED)
                         .containerExitCode(exitCodeKeyToExpectedCauseValue.getKey())
                         .build();
-        when(dockerService.run(any())).thenReturn(expectedDockerRunResponse);
-        when(sgxService.getSgxDriverMode()).thenReturn(SgxDriverMode.LEGACY);
-        when(computeExitCauseService.getExitCausesAndPruneForGivenComputeStage(CHAIN_TASK_ID, ComputeStage.POST, POST_COMPUTE_FAILED_UNKNOWN_ISSUE))
-                .thenReturn(List.of(exitCodeKeyToExpectedCauseValue.getValue()));
+        prepareMocksForTeePostCompute(expectedDockerRunResponse);
+        // Only stub computeExitCauseService for exitCode == 1
+        if (exitCodeKeyToExpectedCauseValue.getKey() == 1) {
+            when(computeExitCauseService.getExitCausesAndPruneForGivenComputeStage(CHAIN_TASK_ID, ComputeStage.POST, POST_COMPUTE_FAILED_UNKNOWN_ISSUE))
+                    .thenReturn(List.of(exitCodeKeyToExpectedCauseValue.getValue()));
+        }
 
         PostComputeResponse postComputeResponse =
                 postComputeService.runTeePostCompute(taskDescription, SECURE_SESSION);
@@ -345,24 +331,11 @@ void shouldNotRunTeePostComputeSinceTimeout() {
                 .datasetUri(DATASET_URI)
                 .maxExecutionTime(MAX_EXECUTION_TIME)
                 .build();
-        List<String> env = Arrays.asList("var0", "var1");
-        when(postComputeProperties.getImage()).thenReturn(TEE_POST_COMPUTE_IMAGE);
-        when(postComputeProperties.getHeapSizeInBytes()).thenReturn(TEE_POST_COMPUTE_HEAP);
-        when(postComputeProperties.getEntrypoint()).thenReturn(TEE_POST_COMPUTE_ENTRYPOINT);
-        when(dockerClientInstanceMock.isImagePresent(TEE_POST_COMPUTE_IMAGE))
-                .thenReturn(true);
-        when(teeMockedService.buildPostComputeDockerEnv(taskDescription, SECURE_SESSION))
-                .thenReturn(env);
-        when(dockerService.getIexecOutBind(CHAIN_TASK_ID)).thenReturn("/iexec_out:/iexec_out");
-        when(workerConfigService.getTaskOutputDir(CHAIN_TASK_ID)).thenReturn(output);
-        when(workerConfigService.getWorkerName()).thenReturn(WORKER_NAME);
-        when(workerConfigService.getDockerNetworkName()).thenReturn("lasNetworkName");
         DockerRunResponse expectedDockerRunResponse =
                 DockerRunResponse.builder()
                         .finalStatus(DockerRunFinalStatus.TIMEOUT)
                         .build();
-        when(dockerService.run(any())).thenReturn(expectedDockerRunResponse);
-        when(sgxService.getSgxDriverMode()).thenReturn(SgxDriverMode.LEGACY);
+        prepareMocksForTeePostCompute(expectedDockerRunResponse);
 
         PostComputeResponse postComputeResponse =
                 postComputeService.runTeePostCompute(taskDescription, SECURE_SESSION);
@@ -372,5 +345,26 @@ void shouldNotRunTeePostComputeSinceTimeout() {
                 .containsExactly(ReplicateStatusCause.POST_COMPUTE_TIMEOUT);
         verify(dockerService).run(any());
     }
+
+    // region getExitCauses
+    @ParameterizedTest
+    @ValueSource(ints = {4, 5, 10, 42, 127, 255})
+    void shouldReturnUnknownIssueForUnmappedExitCodes(int exitCode) {
+        taskDescription = TaskDescription.builder()
+                .chainTaskId(CHAIN_TASK_ID)
+                .datasetUri(DATASET_URI)
+                .maxExecutionTime(MAX_EXECUTION_TIME)
+                .build();
+        final DockerRunResponse dockerResponse = DockerRunResponse.builder()
+                .finalStatus(DockerRunFinalStatus.FAILED)
+                .containerExitCode(exitCode)
+                .build();
+        prepareMocksForTeePostCompute(dockerResponse);
+        final PostComputeResponse response = postComputeService.runTeePostCompute(taskDescription, SECURE_SESSION);
+        assertThat(response.isSuccessful()).isFalse();
+        assertThat(response.getExitCauses())
+                .hasSize(1)
+                .containsExactly(POST_COMPUTE_FAILED_UNKNOWN_ISSUE);
+    }
     //endregion
 }