improve e2e testing of failures (#236)

1. Add test for Job with stuckInit
2. Merge two failing job scenarios into one to reduce CI time

Author: dgrove-oss

test/e2e/appwrapper_test.go

@@ -280,20 +280,7 @@ var _ = Describe("AppWrapper E2E Test", func() {
 			Eventually(AppWrapperPhase(ctx, aw), 60*time.Second).Should(Equal(workloadv1beta2.AppWrapperSucceeded))
 		})
 
-		It("A failed Batch Job yields a failed AppWrapper", func() {
-			aw := toAppWrapper(failingBatchjob(500))
-			if aw.Annotations == nil {
-				aw.Annotations = make(map[string]string)
-			}
-			aw.Annotations[workloadv1beta2.FailureGracePeriodDurationAnnotation] = "0s"
-			aw.Annotations[workloadv1beta2.RetryLimitAnnotation] = "0"
-			Expect(getClient(ctx).Create(ctx, aw)).To(Succeed())
-			appwrappers = append(appwrappers, aw)
-			Expect(waitAWPodsReady(ctx, aw)).Should(Succeed())
-			Eventually(AppWrapperPhase(ctx, aw), 90*time.Second).Should(Equal(workloadv1beta2.AppWrapperFailed))
-		})
-
-		It("Failed Jobs will be retried up to retryLimit", func() {
+		It("A failed Batch Job will be Reset up to retryLimit and then Failed", func() {
 			aw := toAppWrapper(failingBatchjob(500))
 			if aw.Annotations == nil {
 				aw.Annotations = make(map[string]string)
@@ -348,6 +335,26 @@ var _ = Describe("AppWrapper E2E Test", func() {
 		})
 	})
 
+	Describe("Detection of Startup Failures", Label("slow"), Label("Kueue", "Standalone"), func() {
+		It("Job with stuck init is detected and Failed", func() {
+			aw := toAppWrapper(stuckInitBatchjob(100))
+			if aw.Annotations == nil {
+				aw.Annotations = make(map[string]string)
+			}
+			aw.Annotations[workloadv1beta2.FailureGracePeriodDurationAnnotation] = "10s"
+			aw.Annotations[workloadv1beta2.WarmupGracePeriodDurationAnnotation] = "10s"
+			aw.Annotations[workloadv1beta2.RetryLimitAnnotation] = "1"
+			aw.Annotations[workloadv1beta2.RetryPausePeriodDurationAnnotation] = "0s"
+			Expect(getClient(ctx).Create(ctx, aw)).To(Succeed())
+			appwrappers = append(appwrappers, aw)
+			Eventually(AppWrapperPhase(ctx, aw), 30*time.Second).Should(Equal(workloadv1beta2.AppWrapperRunning))
+			Eventually(AppWrapperPhase(ctx, aw), 30*time.Second).Should(Equal(workloadv1beta2.AppWrapperResetting))
+			Eventually(AppWrapperPhase(ctx, aw), 180*time.Second).Should(Equal(workloadv1beta2.AppWrapperFailed))
+			aw = getAppWrapper(ctx, types.NamespacedName{Name: aw.Name, Namespace: aw.Namespace})
+			Expect(aw.Status.Retries).Should(Equal(int32(1)))
+		})
+	})
+
 	Describe("Load Testing", Label("slow"), Label("Kueue", "Standalone"), func() {
 		It("Create 50 AppWrappers", func() {
 			const (

test/e2e/fixtures_test.go

@@ -300,6 +300,42 @@ func succeedingBatchjob(milliCPU int64) workloadv1beta2.AppWrapperComponent {
 	}
 }
 
+const stuckInitBatchJobYAML = `
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: %v
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      terminationGracePeriodSeconds: 0
+      initContainers:
+      - name: stuck-init
+        image: quay.io/project-codeflare/busybox:1.36
+        command: ["sh", "-c", "sleep 100000; exit 1"]
+      containers:
+      - name: busybox
+        image: quay.io/project-codeflare/busybox:1.36
+        command: ["sh", "-c", "sleep 10; exit 1"]
+        resources:
+          requests:
+            cpu: %v
+`
+
+func stuckInitBatchjob(milliCPU int64) workloadv1beta2.AppWrapperComponent {
+	yamlString := fmt.Sprintf(stuckInitBatchJobYAML,
+		randName("batchjob"),
+		resource.NewMilliQuantity(milliCPU, resource.DecimalSI))
+
+	jsonBytes, err := yaml.YAMLToJSON([]byte(yamlString))
+	Expect(err).NotTo(HaveOccurred())
+	return workloadv1beta2.AppWrapperComponent{
+		DeclaredPodSets: []workloadv1beta2.AppWrapperPodSet{{Path: "template.spec.template"}},
+		Template:        runtime.RawExtension{Raw: jsonBytes},
+	}
+}
+
 // This is not a useful PyTorchJob:
 // 1. Using a dummy busybox image to avoid pulling a large & rate-limited image from dockerhub
 // 2. We avoid needing the injected sidecar (alpine:3.10 from dockerhub) by not specifying a Master