[cuebot] Improve nimby retry logic (AcademySoftwareFoundation#2169)

Frames that got killed with SIGTERM due to their executing host having
become nimby locked should always be retried, regardless of their layer
maxretries limit.

Also increase NO_RETRY_LIMIT to prevent long running frames from not
being retried.

Author: DiegoTavares

cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java

@@ -81,7 +81,7 @@ public interface Dispatcher {
     public static final int DOCKER_EXIT_STATUS_MEMORY_FAILURE = 137;
 
     // max retry time
-    public static final int FRAME_TIME_NO_RETRY = 3600 * 8;
+    public static final int FRAME_TIME_NO_RETRY = 3600 * 12;
 
     // The maximum amount of virtual memory a frame can be using
     // without being penalized for it.

cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java

@@ -609,13 +609,17 @@ else if (frame.state.equals(FrameState.DEAD)) {
                 report = FrameCompleteReport.newBuilder(report)
                         .setExitStatus(FrameExitStatus.SKIP_RETRY_VALUE).build();
                 newState = FrameState.WAITING;
-                // exemption code 256
             } else if ((report.getExitStatus() == FrameExitStatus.FAILED_LAUNCH_VALUE
                     || report.getExitSignal() == FrameExitStatus.FAILED_LAUNCH_VALUE)
                     && (frame.retries < job.maxRetries)) {
+                // exemption code 256
                 report = FrameCompleteReport.newBuilder(report)
                         .setExitStatus(report.getExitStatus()).build();
                 newState = FrameState.WAITING;
+            } else if (report.getHost().getNimbyLocked() && report.getExitSignal() == 15) {
+                // If frame got killed because the host was nimby locked,
+                // retry even if retry count is higher than maxretrycount
+                newState = FrameState.WAITING;
             } else if (job.autoEat) {
                 newState = FrameState.EATEN;
                 // ETC Time out and LLU timeout