Merge pull request #100 from redskyops/fix/trial-job-status

Detect unschedulable trial job pods

Author: jgustie

controllers/trial_job_controller.go

@@ -179,14 +179,7 @@ func (r *TrialJobReconciler) applyJobStatus(ctx context.Context, t *redskyv1alph
 	if matchingSelector, err := meta.MatchingSelector(job.Spec.Selector); err == nil {
 		podList := &corev1.PodList{}
 		if err := r.List(ctx, podList, client.InNamespace(job.Namespace), matchingSelector); err == nil {
-			startedAt, finishedAt = containerTime(podList)
-
-			// Check if the job has a start/completion time, but it is not yet reflected in the pod state we are seeing
-			if (startedAt == nil && job.Status.StartTime != nil) || (finishedAt == nil && job.Status.CompletionTime != nil) {
-				return false, true
-			}
-
-			// Look for pod failures (edge case where job controller doesn't update status properly, e.g. initContainer failure)
+			// Look for pod failures (edge case where job controller doesn't update status properly, e.g. initContainer failure or unschedulable)
 			for i := range podList.Items {
 				s := &podList.Items[i].Status
 				if s.Phase == corev1.PodFailed {
@@ -201,6 +194,12 @@ func (r *TrialJobReconciler) applyJobStatus(ctx context.Context, t *redskyv1alph
 					}
 				}
 			}
+
+			// Check if the job has a start/completion time, but it is not yet reflected in the pod state we are seeing
+			startedAt, finishedAt = containerTime(podList)
+			if (startedAt == nil && job.Status.StartTime != nil) || (finishedAt == nil && job.Status.CompletionTime != nil) {
+				return dirty, true
+			}
 		}
 	}