fix: recreate hypervisor pod after eviction (#173)

* fix: recreate hypervisor pod after eviction

* chore: disable the default 10-minute timeout limit for test

Author: knave

Makefile

@@ -62,7 +62,7 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -timeout 0 -coverprofile cover.out
 
 # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
 # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.

internal/controller/gpunode_controller.go

@@ -393,7 +393,8 @@ func (r *GPUNodeReconciler) reconcileHypervisorPod(ctx context.Context, node *tf
 			return key.Name, nil
 		}
 
-		if currentPod.Labels[constants.LabelKeyPodTemplateHash] != utils.GetObjectHash(pool.Spec.ComponentConfig.Hypervisor) {
+		if utils.IsPodTerminated(currentPod) ||
+			currentPod.Labels[constants.LabelKeyPodTemplateHash] != utils.GetObjectHash(pool.Spec.ComponentConfig.Hypervisor) {
 			if err := r.Delete(ctx, currentPod); err != nil {
 				return "", fmt.Errorf("failed to delete old hypervisor pod: %w", err)
 			}

internal/controller/gpunode_controller_test.go

@@ -55,8 +55,8 @@ var _ = Describe("GPUNode Controller", func() {
 			}, timeout, interval).Should(Succeed())
 
 			By("checking that the hypervisor pod is created")
+			pod := &corev1.Pod{}
 			Eventually(func(g Gomega) {
-				pod := &corev1.Pod{}
 				err := k8sClient.Get(ctx, types.NamespacedName{
 					Name:      fmt.Sprintf("hypervisor-%s", gpuNode.Name),
 					Namespace: utils.CurrentNamespace(),
@@ -71,37 +71,20 @@ var _ = Describe("GPUNode Controller", func() {
 				g.Expect(gpunode.Status.Phase).Should(Equal(tfv1.TensorFusionGPUNodePhaseRunning))
 			}, timeout, interval).Should(Succeed())
 
-			tfEnv.Cleanup()
-
-			// By("checking that it will recreate terminated hypervisor pod")
-			// Expect(k8sClient.Delete(ctx, pod)).Should(Succeed())
-			// Eventually(func() error {
-			// return k8sClient.Get(ctx, types.NamespacedName{
-			// Name:      fmt.Sprintf("hypervisor-%s", gpuNode.Name),
-			// Namespace: utils.CurrentNamespace(),
-			// }, pod)
-			// }, timeout, interval).Should(Succeed())
+			By("checking the hypervisor pod should be recreated when enters terminated status")
+			pod.Status.Phase = corev1.PodFailed
+			Expect(k8sClient.Status().Update(ctx, pod)).Should(Succeed())
+			Eventually(func(g Gomega) {
+				newPod := &corev1.Pod{}
+				err := k8sClient.Get(ctx, types.NamespacedName{
+					Name:      fmt.Sprintf("hypervisor-%s", gpuNode.Name),
+					Namespace: utils.CurrentNamespace(),
+				}, newPod)
+				g.Expect(err).ShouldNot(HaveOccurred())
+				g.Expect(newPod.UID).ShouldNot(Equal(pod.UID))
+			}, timeout, interval).Should(Succeed())
 
-			// TODO: make this test pass when implement rolling udpate
-			// By("checking that the hypervisor config changed")
-			// tfc := getMockCluster(ctx)
-			// hypervisor := tfc.Spec.GPUPools[0].SpecTemplate.ComponentConfig.Hypervisor
-			// podTmpl := &corev1.PodTemplate{}
-			// err := json.Unmarshal(hypervisor.PodTemplate.Raw, podTmpl)
-			// Expect(err).NotTo(HaveOccurred())
-			// podTmpl.Template.Spec.Containers[0].Name = "foo"
-			// hypervisor.PodTemplate.Raw = lo.Must(json.Marshal(podTmpl))
-			// Expect(k8sClient.Update(ctx, tfc)).To(Succeed())
-			// Eventually(func() string {
-			// 	pod := &corev1.Pod{}
-			// 	if err = k8sClient.Get(ctx, types.NamespacedName{
-			// 		Name:      fmt.Sprintf("hypervisor-%s", gpuNode.Name),
-			// 		Namespace: utils.CurrentNamespace(),
-			// 	}, pod); err != nil {
-			// 		return ""
-			// 	}
-			// 	return pod.Spec.Containers[0].Name
-			// }, timeout, interval).Should(Equal("foo"))
+			tfEnv.Cleanup()
 		})
 	})
 })

internal/utils/reconcile.go

@@ -175,3 +175,7 @@ func IsPodConditionTrue(conditions []corev1.PodCondition, conditionType corev1.P
 	}
 	return false
 }
+
+func IsPodTerminated(pod *corev1.Pod) bool {
+	return pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodSucceeded
+}