Fix flaky TestNewDistributorsCanPushToOldIngestersWithReplication test due SG not ready (#5975)

* Fixing Flaky integration test

Signed-off-by: alanprot <[email protected]>

* lint

Signed-off-by: alanprot <[email protected]>

* fix TestQuerierWithBlocksStorageRunningInMicroservicesMode

Signed-off-by: alanprot <[email protected]>

---------

Signed-off-by: alanprot <[email protected]>

Author: alanprot

integration/backward_compatibility_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/model/labels"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -210,6 +211,14 @@ func checkQueries(
 
 	for name, c := range cases {
 		t.Run(name, func(t *testing.T) {
+			// Start store gateway.
+			storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), c.storeGatewayFlags, c.storeGatewayImage)
+
+			require.NoError(t, s.Start(storeGateway))
+			defer func() {
+				require.NoError(t, s.Stop(storeGateway))
+			}()
+
 			// Start query-frontend.
 			queryFrontend := e2ecortex.NewQueryFrontend("query-frontend", c.queryFrontendFlags, c.queryFrontendImage)
 			require.NoError(t, s.Start(queryFrontend))
@@ -227,18 +236,22 @@ func checkQueries(
 				require.NoError(t, s.Stop(querier))
 			}()
 
-			// Start store gateway.
-			storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), c.storeGatewayFlags, c.storeGatewayImage)
-
-			require.NoError(t, s.Start(storeGateway))
-			defer func() {
-				require.NoError(t, s.Stop(storeGateway))
-			}()
-
 			// Wait until querier and query-frontend are ready, and the querier has updated the ring.
 			require.NoError(t, s.WaitReady(querier, queryFrontend, storeGateway))
 			expectedTokens := float64((numIngesters + 1) * 512) // Ingesters and Store Gateway.
 			require.NoError(t, querier.WaitSumMetrics(e2e.Equals(expectedTokens), "cortex_ring_tokens_total"))
+			require.NoError(t, storeGateway.WaitSumMetrics(e2e.Greater(0), "cortex_storegateway_bucket_sync_total"))
+
+			// Wait store-gateways and ingesters appears on querier ring
+			require.NoError(t, querier.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"cortex_ring_members"}, e2e.WithLabelMatchers(
+				labels.MustNewMatcher(labels.MatchEqual, "name", "store-gateway-client"),
+				labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE"),
+			)))
+
+			require.NoError(t, querier.WaitSumMetricsWithOptions(e2e.Greater(0), []string{"cortex_ring_members"}, e2e.WithLabelMatchers(
+				labels.MustNewMatcher(labels.MatchEqual, "name", "ingester"),
+				labels.MustNewMatcher(labels.MatchEqual, "state", "ACTIVE"),
+			)))
 
 			// Query the series.
 			for _, endpoint := range []string{queryFrontend.HTTPEndpoint(), querier.HTTPEndpoint()} {

integration/querier_test.go

@@ -217,7 +217,7 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
 					require.NoError(t, s.StartAndWaitReady(compactor))
 				} else {
 					// Wait until the querier has discovered the uploaded blocks.
-					require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2), "cortex_blocks_meta_synced"))
+					require.NoError(t, querier.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"cortex_blocks_meta_synced"}, e2e.WaitMissingMetrics))
 				}
 
 				// Wait until the store-gateway has synched the new uploaded blocks. When sharding is enabled