Fix K8s job fallback to not return incorrect zero exit code (#6746)

Author: jorgee

plugins/nf-k8s/src/main/nextflow/k8s/K8sTaskHandler.groovy

@@ -151,6 +151,8 @@ class K8sTaskHandler extends TaskHandler implements FusionAwareTask {
         final workDir = Escape.path(task.workDir)
 
         final result = new ArrayList(BashWrapperBuilder.BASH)
+        result.add('-o')
+        result.add('pipefail')
         result.add('-c')
         result.add("bash ${workDir}/${TaskRun.CMD_RUN} 2>&1 | tee ${workDir}/${TaskRun.CMD_LOG}")
         return result

plugins/nf-k8s/src/main/nextflow/k8s/client/K8sClient.groovy

@@ -419,7 +419,6 @@ class K8sClient {
                 log.warn1("Job $jobName already completed and Pod is gone")
                 final dummyPodStatus = [
                         terminated: [
-                                exitcode: 0,
                                 reason: "Completed",
                                 startedAt: jobStatus.startTime,
                                 finishedAt: jobStatus.completionTime,

plugins/nf-k8s/src/test/nextflow/k8s/K8sTaskHandlerTest.groovy

@@ -96,7 +96,7 @@ class K8sTaskHandlerTest extends Specification {
                 containers: [[
                     name:'nf-123',
                     image:'debian:latest',
-                    args:['/bin/bash', '-ue','-c','bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log']
+                    args:['/bin/bash', '-ue', '-o', 'pipefail', '-c', 'bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log']
                 ]]
             ]
         ]
@@ -123,7 +123,7 @@ class K8sTaskHandlerTest extends Specification {
         and:
         result.metadata.labels == [sessionId: 'xxx']
         result.metadata.annotations == [evict: 'false']
-        result.spec.containers[0].command == ['/bin/bash', '-ue', '-c','bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log']
+        result.spec.containers[0].command == ['/bin/bash', '-ue', '-o', 'pipefail', '-c', 'bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log']
         result.spec.containers[0].resources == [ requests: [cpu:1] ]
         result.spec.containers[0].env == [ [name:'NXF_OWNER', value:'501:502'] ]
 
@@ -148,7 +148,7 @@ class K8sTaskHandlerTest extends Specification {
         and:
         result.metadata.namespace == 'namespace-x'
         result.spec.containers[0].image == 'user/alpine:1.0'
-        result.spec.containers[0].command == ['/bin/bash', '-ue', '-c','bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log']
+        result.spec.containers[0].command == ['/bin/bash', '-ue', '-o', 'pipefail', '-c', 'bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log']
         result.spec.containers[0].resources == [ requests: [cpu:4, memory:'16384Mi'], limits: [memory:'16384Mi'] ]
 
     }
@@ -428,7 +428,7 @@ class K8sTaskHandlerTest extends Specification {
                         containers: [[
                             name: 'nf-123',
                             image: 'debian:latest',
-                            command: ['/bin/bash', '-ue','-c','bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log']
+                            command: ['/bin/bash', '-ue', '-o', 'pipefail', '-c', 'bash /some/work/dir/.command.run 2>&1 | tee /some/work/dir/.command.log']
                         ]]
                     ]
                 ]

plugins/nf-k8s/src/test/nextflow/k8s/client/K8sClientTest.groovy

@@ -1059,4 +1059,47 @@ class K8sClientTest extends Specification {
         def e = thrown(PodUnschedulableException)
         e.message == "K8s pod in Failed state"
     }
+
+    def 'should fallback to job status when pod is gone and not return hardcoded exit code' () {
+        given:
+        def JOB_STATUS_JSON = '''
+        {
+            "apiVersion": "batch/v1",
+            "kind": "Job",
+            "metadata": {
+                "name": "test-job"
+            },
+            "status": {
+                "succeeded": 1,
+                "startTime": "2025-01-15T10:00:00Z",
+                "completionTime": "2025-01-15T10:05:00Z",
+                "conditions": [
+                    {
+                        "type": "Complete",
+                        "status": "True",
+                        "lastProbeTime": "2025-01-15T10:05:00Z",
+                        "lastTransitionTime": "2025-01-15T10:05:00Z"
+                    }
+                ]
+            }
+        }
+        '''
+        def client = Spy(K8sClient)
+        final JOB_NAME = 'test-job'
+
+        when:
+        def result = client.jobStateFallback0(JOB_NAME)
+
+        then:
+        1 * client.jobStatus(JOB_NAME) >> new K8sResponseJson(JOB_STATUS_JSON)
+
+        and:
+        result.terminated != null
+        result.terminated.reason == 'Completed'
+        result.terminated.startedAt == '2025-01-15T10:00:00Z'
+        result.terminated.finishedAt == '2025-01-15T10:05:00Z'
+        // The key assertion: exitCode should not be present (null) so fallback to .exitcode file works
+        result.terminated.exitCode == null
+        result.terminated.exitcode == null
+    }
 }