add failing job with retry test (#235)

Author: dgrove-oss

test/e2e/appwrapper_test.go

@@ -293,6 +293,23 @@ var _ = Describe("AppWrapper E2E Test", func() {
 			Eventually(AppWrapperPhase(ctx, aw), 90*time.Second).Should(Equal(workloadv1beta2.AppWrapperFailed))
 		})
 
+		It("Failed Jobs will be retried up to retryLimit", func() {
+			aw := toAppWrapper(failingBatchjob(500))
+			if aw.Annotations == nil {
+				aw.Annotations = make(map[string]string)
+			}
+			aw.Annotations[workloadv1beta2.FailureGracePeriodDurationAnnotation] = "0s"
+			aw.Annotations[workloadv1beta2.RetryLimitAnnotation] = "2"
+			aw.Annotations[workloadv1beta2.RetryPausePeriodDurationAnnotation] = "5s"
+			Expect(getClient(ctx).Create(ctx, aw)).To(Succeed())
+			appwrappers = append(appwrappers, aw)
+			Expect(waitAWPodsReady(ctx, aw)).Should(Succeed())
+			Eventually(AppWrapperPhase(ctx, aw), 90*time.Second).Should(Equal(workloadv1beta2.AppWrapperResetting))
+			Eventually(AppWrapperPhase(ctx, aw), 180*time.Second).Should(Equal(workloadv1beta2.AppWrapperFailed))
+			aw = getAppWrapper(ctx, types.NamespacedName{Name: aw.Name, Namespace: aw.Namespace})
+			Expect(aw.Status.Retries).Should(Equal(int32(2)))
+		})
+
 		It("Deleting a Running Component yields a failed AppWrapper", func() {
 			aw := createAppWrapper(ctx, pytorchjob(2, 500))
 			appwrappers = append(appwrappers, aw)