Fix Ring/Memberlist Join Test Flakiness (#5992)

The tests rely on the CPU's wall clock which can be unreliable if there is CPU starvation. If we give the tests enough time to run, they are more much more reliable and resilient to CPU starvation.

Signed-off-by: Charlie Le <[email protected]>

Author: CharlieTLe

pkg/ring/kv/memberlist/memberlist_client_test.go

@@ -827,7 +827,7 @@ func TestMemberlistJoinOnStarting(t *testing.T) {
 		return mkv2.memberlist.NumMembers()
 	}
 
-	poll(t, 5*time.Second, 2, membersFunc)
+	poll(t, 1*time.Minute, 2, membersFunc)
 }
 
 func getFreePorts(count int) ([]int, error) {
@@ -1103,20 +1103,20 @@ func TestRejoin(t *testing.T) {
 		return mkv2.memberlist.NumMembers()
 	}
 
-	poll(t, 5*time.Second, 2, membersFunc)
+	poll(t, 1*time.Minute, 2, membersFunc)
 
 	// Shutdown first KV
 	require.NoError(t, services.StopAndAwaitTerminated(context.Background(), mkv1))
 
 	// Second KV should see single member now.
-	poll(t, 5*time.Second, 1, membersFunc)
+	poll(t, 1*time.Minute, 1, membersFunc)
 
 	// Let's start first KV again. It is not configured to join the cluster, but KV2 is rejoining.
 	mkv1 = NewKV(cfg1, log.NewNopLogger(), &dnsProviderMock{}, prometheus.NewPedanticRegistry())
 	require.NoError(t, services.StartAndAwaitRunning(context.Background(), mkv1))
 	defer services.StopAndAwaitTerminated(context.Background(), mkv1) //nolint:errcheck
 
-	poll(t, 5*time.Second, 2, membersFunc)
+	poll(t, 1*time.Minute, 2, membersFunc)
 }
 
 func TestMessageBuffer(t *testing.T) {

pkg/ring/lifecycler_test.go

@@ -494,17 +494,17 @@ func TestCheckReady_CheckRingHealth(t *testing.T) {
 	}{
 		"should wait until the self instance is ACTIVE and healthy in the ring when 'check ring health' is disabled": {
 			checkRingHealthEnabled: false,
-			firstJoinAfter:         time.Second,
-			secondJoinAfter:        3 * time.Second,
-			expectedFirstMinReady:  time.Second,
-			expectedFirstMaxReady:  2 * time.Second,
+			firstJoinAfter:         1000 * time.Millisecond,
+			secondJoinAfter:        3000 * time.Millisecond,
+			expectedFirstMinReady:  900 * time.Millisecond,
+			expectedFirstMaxReady:  1900 * time.Millisecond,
 		},
 		"should wait until all instances are ACTIVE and healthy in the ring when 'check ring health' is enabled": {
 			checkRingHealthEnabled: true,
-			firstJoinAfter:         time.Second,
-			secondJoinAfter:        3 * time.Second,
-			expectedFirstMinReady:  3 * time.Second,
-			expectedFirstMaxReady:  4 * time.Second,
+			firstJoinAfter:         1000 * time.Millisecond,
+			secondJoinAfter:        3000 * time.Millisecond,
+			expectedFirstMinReady:  2900 * time.Millisecond,
+			expectedFirstMaxReady:  3900 * time.Millisecond,
 		},
 	}