[PER-44] NoAvailableHost exit code 143

Author: GallVp

README.md

@@ -26,6 +26,8 @@ Otherwise, the worker nodes won't be able to see the task files.
   - [Configure with environment variables](#configure-with-environment-variables)
   - [Configure with Nextflow secrets](#configure-with-nextflow-secrets)
   - [Configuration best practices](#configuration-best-practices)
+    - [S3](#s3)
+    - [errorStrategy](#errorstrategy)
   - [Configure s3 work directory](#configure-s3-work-directory)
   - [Configure s3fs work directory](#configure-s3fs-work-directory)
   - [Configure fusion FS over s3](#configure-fusion-fs-over-s3)
@@ -214,6 +216,8 @@ Unknown config secret 'MMC_USERNAME'
 
 ### Configuration best practices
 
+#### S3
+
 When you are using s3, it's recommended to update the aws client configurations
 based on your environment and the workload.  Here is an example:
 ```groovy
@@ -239,17 +243,34 @@ aws {
 }
 ```
 
+#### errorStrategy
+
 If you are sure that the workflow file is properly composed, it's recommended to
 set proper error strategy and retry limit in the process scope to make sure
 the workflow can be completed.
-Here is an example:
+
+Here is the recommended error strategy:
 ```groovy
 process {
-  errorStrategy='retry'
-  maxRetries=5  
+    errorStrategy = {
+        if ( task.exitStatus == 143 ) { // NoAvailableHost
+            sleep(Math.pow(2, task.attempt) * 2 * 60000 as long)
+            return 'retry'
+        }
+  
+        task.exitStatus in ((130..142) + 145 + 104 + 175)
+        ? 'retry'
+        : 'finish'
+    }
+    maxRetries = 5
 }
 ```
 
+This strategy increases max retries to 5 and generally adheres to the nf-core strategy.
+The addition is that for exit code 143 which represents NoAvailableHost resulting from no or 
+low availability of spot instances, Nextflow waits for 4, 8, 16, 32, 64 minutes for
+consecutive retries of a task.
+
 ### Configure s3 work directory
 
 To enable s3 as work directory, user need to set work directory to a s3 bucket.

plugins/nf-float/src/main/com/memverge/nextflow/FloatGridExecutor.groovy

@@ -643,6 +643,7 @@ fi
         STATUS_MAP.put(FloatStatus.PENDING, QueueStatus.PENDING)
         STATUS_MAP.put(FloatStatus.RUNNING, QueueStatus.RUNNING)
         STATUS_MAP.put(FloatStatus.DONE, QueueStatus.DONE)
+        STATUS_MAP.put(FloatStatus.NOAVAILABLEHOST, QueueStatus.ERROR)
         STATUS_MAP.put(FloatStatus.ERROR, QueueStatus.ERROR)
         STATUS_MAP.put(FloatStatus.UNKNOWN, QueueStatus.UNKNOWN)
     }

plugins/nf-float/src/main/com/memverge/nextflow/FloatJob.groovy

@@ -24,6 +24,7 @@ enum FloatStatus {
     PENDING,
     RUNNING,
     DONE,
+    NOAVAILABLEHOST,
     ERROR,
     UNKNOWN,
 
@@ -46,7 +47,7 @@ enum FloatStatus {
             'CheckpointFailed' : ERROR,
             'WaitingForLicense': ERROR,
             'Timedout'         : ERROR,
-            'NoAvailableHost'  : ERROR,
+            'NoAvailableHost'  : NOAVAILABLEHOST,
             'Unknown'          : UNKNOWN,
     ]
 
@@ -58,12 +59,8 @@ enum FloatStatus {
         return this == PENDING || this == RUNNING
     }
 
-    boolean isFinished() {
-        return this == ERROR || this == DONE
-    }
-
-    boolean isError() {
-        return this == ERROR
+    boolean isDoneOrFailed() {
+        return this == DONE || this == ERROR || this == NOAVAILABLEHOST
     }
 }
 
@@ -125,8 +122,8 @@ class FloatJob {
         return status ? status.isRunning() : false
     }
 
-    boolean isFinished() {
-        return status ? status.isFinished() : false
+    boolean isDoneOrFailed() {
+        return status ? status.isDoneOrFailed() : false
     }
 
     static List<FloatJob> parseJobMap(String input) {

plugins/nf-float/src/main/com/memverge/nextflow/FloatJobs.groovy

@@ -44,7 +44,7 @@ class FloatJobs {
     }
 
     TaskId getTaskID(String floatJobID) {
-        // go through the all float jobs to find the task id
+        // go through all float jobs to find the task id
         for (FloatJob job : nfJobID2FloatJob.values()) {
             if (job.floatJobID == floatJobID) {
                 // extract the task id from the nfJobID
@@ -74,7 +74,7 @@ class FloatJobs {
             log.error "[FLOAT] job nf Job ID is null or empty for job ${job.floatJobID}"
         } else {
             FloatJob existingJob = nfJobID2FloatJob.get(job.nfJobID)
-            if (existingJob != null && existingJob.finished) {
+            if (existingJob != null && existingJob.doneOrFailed) {
                 log.info "[FLOAT] job ${job.nfJobID} already finished, no need to update"
                 job = existingJob
             } else {
@@ -83,7 +83,7 @@ class FloatJobs {
                 }
                 nfJobID2FloatJob.put(job.nfJobID, job)
             }
-            if (job.finished) {
+            if (job.doneOrFailed) {
                 refreshWorkDir(job.nfJobID)
             }
         }

plugins/nf-float/src/main/com/memverge/nextflow/FloatTaskHandler.groovy

@@ -73,20 +73,25 @@ class FloatTaskHandler extends GridTaskHandler {
     boolean checkIfCompleted() {
         final FloatStatus st =  floatExecutor.getJobStatus(task)
         log.debug "got status ${st} for ${task.id} from float executor"
-        if (st.finished) {
-            status = COMPLETED
+        if (st.doneOrFailed) {
+            this.status = COMPLETED
             task.exitStatus = readExitStatus()
             if (task.exitStatus == null) {
                 log.debug "can't get ${task.id} exit status from file system"
                 task.exitStatus = floatExecutor.getJobRC(task.id)
             }
             log.debug "set ${task.id} exit status to ${task.exitStatus}"
             if (task.exitStatus == null) {
-                if (st.isError()) {
+                task.exitStatus = 0
+
+                if (st == FloatStatus.ERROR) {
                     task.exitStatus = 1
-                } else {
-                    task.exitStatus = 0
                 }
+
+                if (st == FloatStatus.NOAVAILABLEHOST) {
+                    task.exitStatus = 143
+                }
+
                 log.info "both .exitcode and rc are empty for ${task.id}," +
                         "set exit to ${task.exitStatus}"
             }

plugins/nf-float/src/test/com/memverge/nextflow/FloatJobTest.groovy

@@ -127,6 +127,69 @@ class FloatJobTest extends Specification {
         job.status == FloatStatus.PENDING
     }
 
+    def "check job failed with NoAvailableHost"() {
+        given:
+        final out = """
+            id: uqaj8l6a407u88v0106ol
+            name: subread-3nyw14-r6a.2xlarge
+            workingHost: ""
+            category: failed
+            status: NoAvailableHost
+            cpu: 8
+            duration: 1h5m49s
+            queueTime: 0s
+            submitTime: "2025-06-20T09:09:34Z"
+            execTime: "2025-06-20T09:09:34Z"
+            endTime: "2025-06-20T10:15:23Z"
+            lastUpdate: "2025-06-20T10:15:23Z"
+            memGB: 64
+            scheduler: mmcloud
+            imageID: quay.io/biocontainers/subread:2.0.6--he4a0461_2
+            stdout: stdout.autosave
+            stderr: stderr.autosave
+            memUsed: 0.00 B
+            cpuMax: 24
+            memGBMax: 144
+            instanceType: r6a.2xlarge
+            coreHours: 0.6784628924888889
+            """.stripIndent().trim()
+
+        when:
+        final job = FloatJob.parse(out)
+
+        then:
+        ! job.isRunning()
+        job.status == FloatStatus.NOAVAILABLEHOST
+    }
+
+    def "check job failed with FailToExecute"() {
+        given:
+        final out = """
+            id: 3cfmkg882qn09q9o8jfkf
+            name: rseqc-2609uc-m5zn.3xlarge
+            workingHost: ""
+            category: failed
+            status: FailToExecute
+            cpu: 6
+            duration: 0s
+            queueTime: 0s
+            submitTime: "2025-06-20T09:13:54Z"
+            execTime: "2025-06-20T09:13:54Z"
+            endTime: "2025-06-20T09:13:55Z"
+            lastUpdate: "2025-06-20T09:13:55Z"
+            memGB: 36
+            cpuMax: 24
+            memGBMax: 144
+            """.stripIndent().trim()
+
+        when:
+        final job = FloatJob.parse(out)
+
+        then:
+        ! job.isRunning()
+        job.status == FloatStatus.ERROR
+    }
+
     def "get queue status"() {
         given:
         final out = """