Skip to content

fix(e2e): wait for PhaseReady before accessing master pod #516

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Abhra303 merged 1 commit into from
Apr 9, 2026
Merged

fix(e2e): wait for PhaseReady before accessing master pod #516

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions e2e/dragonfly_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -714,8 +714,8 @@ var _ = Describe("Dragonfly tiering test with single replica", Ordered, FlakeAtt
})

It("Resources should exist", func() {
// Wait until Dragonfly object is marked initialized
waitForDragonflyPhase(ctx, k8sClient, name, namespace, controller.PhaseResourcesCreated, 2*time.Minute)
// Wait until Dragonfly object is marked initialized and master is elected
waitForDragonflyPhase(ctx, k8sClient, name, namespace, controller.PhaseReady, 2*time.Minute)
waitForStatefulSetReady(ctx, k8sClient, name, namespace, 2*time.Minute)

// Check for service and statefulset
Expand Down Expand Up @@ -833,8 +833,8 @@ var _ = Describe("Dragonfly PVC Test with single replica", Ordered, FlakeAttempt
})

It("Resources should exist", func() {
// Wait until Dragonfly object is marked initialized
waitForDragonflyPhase(ctx, k8sClient, name, namespace, controller.PhaseResourcesCreated, 2*time.Minute)
// Wait until Dragonfly object is marked initialized and master is elected
waitForDragonflyPhase(ctx, k8sClient, name, namespace, controller.PhaseReady, 2*time.Minute)
waitForStatefulSetReady(ctx, k8sClient, name, namespace, 2*time.Minute)

// Check for service and statefulset
Expand Down Expand Up @@ -888,6 +888,10 @@ var _ = Describe("Dragonfly PVC Test with single replica", Ordered, FlakeAttempt
// Wait until Dragonfly object is marked initialized
waitForDragonflyPhase(ctx, k8sClient, name, namespace, controller.PhaseReady, 2*time.Minute)
waitForStatefulSetReady(ctx, k8sClient, name, namespace, 2*time.Minute)
// Phase may already be Ready from before deletion; wait explicitly for the
// lifecycle controller to finish master election on the recreated pod.
err = waitForMasterPod(ctx, k8sClient, name, namespace, 2*time.Minute)
Expect(err).To(BeNil())
// check if the pod is created
err = k8sClient.Get(ctx, types.NamespacedName{
Name: fmt.Sprintf("%s-0", name),
Expand Down
22 changes: 22 additions & 0 deletions e2e/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,28 @@ func parseTieredEntriesFromInfo(info string) (int64, error) {
return 0, fmt.Errorf("tiered_entries not found")
}

// waitForMasterPod polls until at least one pod with role=master exists. Use this
// after waitForStatefulSetReady to guarantee the lifecycle controller has finished
// master election before the test tries to connect.
func waitForMasterPod(ctx context.Context, c client.Client, name, namespace string, maxDuration time.Duration) error {
ctx, cancel := context.WithTimeout(ctx, maxDuration)
defer cancel()
for {
select {
case <-ctx.Done():
return fmt.Errorf("timed out waiting for master pod for %s", name)
default:
var pods corev1.PodList
if err := c.List(ctx, &pods, client.InNamespace(namespace), client.MatchingLabels{
resources.DragonflyNameLabelKey: name,
resources.RoleLabelKey: resources.Master,
}); err == nil && len(pods.Items) > 0 {
return nil
}
}
}
}

func waitForStatefulSetReady(ctx context.Context, c client.Client, name, namespace string, maxDuration time.Duration) error {
ctx, cancel := context.WithTimeout(ctx, maxDuration)
defer cancel()
Expand Down