pkg/monitor,pkg/util/changefeed,Makefile: fix changefeed test races

Replace AllIteratorsConsumed()-based synchronization in changefeed
tests with barriers derived from GetLastProcessed(), which advances
only after OnAllPendingProcessed() completes a full sweep. This is
the correct settled-cache barrier.

Key changes:
- pkg/util/changefeed/subscriptioncache_test.go: wait for two
  GetLastProcessed() advancements after each mutation group to
  guarantee a complete post-mutation sweep has run
- pkg/monitor/worker_test.go: replace vacuous len(docs) barrier with
  lastClusterChangefeed timestamp barrier; same two-advancement
  pattern for both cluster and subscription caches
- pkg/database/cosmosdb: add per-iterator sync.Mutex to
  fakeSubscriptionDocumentIterator and fakeOpenShiftClusterDocumentIterator;
  fix ChangeFeed() to hold a write lock (not read lock) while
  appending to changeFeedIterators
- pkg/database/cosmosdb: delete AllIteratorsConsumed() extension
  methods (no callers remain)
- pkg/monitor/test_helpers.go: remove unused fake client fields from
  TestEnvironment
- Makefile: add -race flag to unit-test-go target

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: ventifus

Makefile

@@ -334,7 +334,7 @@ validate-fips: $(BINGO)
 
 .PHONY: unit-test-go
 unit-test-go: $(GOTESTSUM)
-	$(GOTESTSUM) --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./...
+	$(GOTESTSUM) --format pkgname --junitfile report.xml -- -race -coverprofile=cover.out ./...
 
 .PHONY: unit-test-go-coverpkg
 unit-test-go-coverpkg: $(GOTESTSUM)

pkg/database/cosmosdb/openshiftcluster_ext.go

@@ -32,26 +32,24 @@ var fakeClusterVisitMonitoringAttempts = map[string]*int{}
 
 // TestEnvironment contains all the test setup components
 type TestEnvironment struct {
-	OpenShiftClusterDB     database.OpenShiftClusters
-	SubscriptionsDB        database.Subscriptions
-	MonitorsDB             database.Monitors
-	OpenShiftClusterClient *cosmosdb.FakeOpenShiftClusterDocumentClient
-	SubscriptionsClient    *cosmosdb.FakeSubscriptionDocumentClient
-	FakeMonitorsDBClient   *cosmosdb.FakeMonitorDocumentClient
-	Controller             *gomock.Controller
-	TestLogger             *logrus.Entry
-	Dialer                 *mock_proxy.MockDialer
-	MockEnv                *mock_env.MockInterface
-	NoopMetricsEmitter     noop.Noop
-	NoopClusterMetrics     noop.Noop
-	DBGroup                monitorDBs
+	OpenShiftClusterDB   database.OpenShiftClusters
+	SubscriptionsDB      database.Subscriptions
+	MonitorsDB           database.Monitors
+	FakeMonitorsDBClient *cosmosdb.FakeMonitorDocumentClient
+	Controller           *gomock.Controller
+	TestLogger           *logrus.Entry
+	Dialer               *mock_proxy.MockDialer
+	MockEnv              *mock_env.MockInterface
+	NoopMetricsEmitter   noop.Noop
+	NoopClusterMetrics   noop.Noop
+	DBGroup              monitorDBs
 }
 
 // SetupTestEnvironment creates a common test environment for monitor tests
 func SetupTestEnvironment(t *testing.T) *TestEnvironment {
 	// Create databases
-	openShiftClusterDB, openShiftClusterClient := testdatabase.NewFakeOpenShiftClusters()
-	subscriptionsDB, subscriptionsClient := testdatabase.NewFakeSubscriptions()
+	openShiftClusterDB, _ := testdatabase.NewFakeOpenShiftClusters()
+	subscriptionsDB, _ := testdatabase.NewFakeSubscriptions()
 	monitorsDB, fakeMonitorsDBClient := testdatabase.NewFakeMonitors()
 
 	// Create mocks
@@ -85,19 +83,17 @@ func SetupTestEnvironment(t *testing.T) *TestEnvironment {
 	f.Create()
 
 	return &TestEnvironment{
-		OpenShiftClusterDB:     openShiftClusterDB,
-		SubscriptionsDB:        subscriptionsDB,
-		MonitorsDB:             monitorsDB,
-		OpenShiftClusterClient: openShiftClusterClient,
-		SubscriptionsClient:    subscriptionsClient,
-		FakeMonitorsDBClient:   fakeMonitorsDBClient,
-		Controller:             ctrl,
-		TestLogger:             testlogger,
-		Dialer:                 dialer,
-		MockEnv:                mockEnv,
-		NoopMetricsEmitter:     noopMetricsEmitter,
-		NoopClusterMetrics:     noopClusterMetricsEmitter,
-		DBGroup:                dbs,
+		OpenShiftClusterDB:   openShiftClusterDB,
+		SubscriptionsDB:      subscriptionsDB,
+		MonitorsDB:           monitorsDB,
+		FakeMonitorsDBClient: fakeMonitorsDBClient,
+		Controller:           ctrl,
+		TestLogger:           testlogger,
+		Dialer:               dialer,
+		MockEnv:              mockEnv,
+		NoopMetricsEmitter:   noopMetricsEmitter,
+		NoopClusterMetrics:   noopClusterMetricsEmitter,
+		DBGroup:              dbs,
 	}
 }
 

pkg/database/cosmosdb/subscriptions_ext.go

@@ -138,13 +138,50 @@ func TestChangefeedOperations(t *testing.T) {
 				}
 			}
 
-			// Wait for changefeeds to be consumed
-			assert.Eventually(t, env.OpenShiftClusterClient.AllIteratorsConsumed, time.Second, 10*time.Millisecond)
-			assert.Eventually(t, env.SubscriptionsClient.AllIteratorsConsumed, time.Second, 10*time.Millisecond)
+			// Capture baselines after mutations so that a changefeed sweep
+			// that ran before the mutations cannot satisfy the barriers below.
+			// Require two advancements: the first sweep after the baseline may
+			// have already fetched an empty page before the mutation was
+			// appended, so its OnAllPendingProcessed() can fire without the
+			// mutation being processed. A second advancement guarantees a full
+			// sweep ran after the mutation was visible to the iterator.
+			beforeCluster, _ := mon.lastClusterChangefeed.Load().(time.Time)
+			beforeSubs, _ := mon.subs.GetLastProcessed()
+
+			seenClusterAdvance := false
+			assert.Eventually(t, func() bool {
+				ts, ok := mon.lastClusterChangefeed.Load().(time.Time)
+				if !ok || !ts.After(beforeCluster) {
+					return false
+				}
+				if !seenClusterAdvance {
+					seenClusterAdvance = true
+					beforeCluster = ts
+					return false
+				}
+				return true
+			}, time.Second, 10*time.Millisecond)
+
+			seenSubsAdvance := false
+			assert.Eventually(t, func() bool {
+				last, ok := mon.subs.GetLastProcessed()
+				if !ok || !last.After(beforeSubs) {
+					return false
+				}
+				if !seenSubsAdvance {
+					seenSubsAdvance = true
+					beforeSubs = last
+					return false
+				}
+				return true
+			}, time.Second, 10*time.Millisecond)
 
 			// Validate expected results
-			if len(mon.docs) != op.expectDocs {
-				t.Errorf("%s: expected %d documents in cache, got %d", op.name, op.expectDocs, len(mon.docs))
+			mon.mu.RLock()
+			nDocs := len(mon.docs)
+			mon.mu.RUnlock()
+			if nDocs != op.expectDocs {
+				t.Errorf("%s: expected %d documents in cache, got %d", op.name, op.expectDocs, nDocs)
 			}
 			if mon.subs.GetCacheSize() != op.expectSubs {
 				t.Errorf("%s: expected %d subscriptions in cache, got %d", op.name, op.expectSubs, mon.subs.GetCacheSize())

pkg/database/cosmosdb/zz_generated_openshiftclusterdocument_fake.go

@@ -70,7 +70,7 @@ func TestSubscriptionChangefeed(t *testing.T) {
 	for _, tC := range testCases {
 		t.Run(tC.desc, func(t *testing.T) {
 			startedTime := time.Now().UnixNano()
-			subscriptionsDB, subscriptionsClient := testdatabase.NewFakeSubscriptions()
+			subscriptionsDB, _ := testdatabase.NewFakeSubscriptions()
 			_, log := testlog.LogForTesting(t)
 
 			// need to register the changefeed before making documents
@@ -147,7 +147,6 @@ func TestSubscriptionChangefeed(t *testing.T) {
 			go RunChangefeed(t.Context(), log, subscriptionChangefeed, 100*time.Microsecond, 1, cache, stop)
 
 			cache.WaitForInitialPopulation()
-			assert.Eventually(t, subscriptionsClient.AllIteratorsConsumed, time.Second, 1*time.Millisecond)
 
 			// Create some after initially populated
 			_, err := subscriptionsDB.Create(t.Context(), &api.SubscriptionDocument{
@@ -185,7 +184,26 @@ func TestSubscriptionChangefeed(t *testing.T) {
 				},
 			})
 			require.NoError(t, err)
-			assert.Eventually(t, subscriptionsClient.AllIteratorsConsumed, time.Second, 1*time.Millisecond)
+			// Capture baseline after mutations. Require two advancements: the
+			// first sweep after the baseline may have already fetched an empty
+			// page before the mutation was appended, so its
+			// OnAllPendingProcessed() can fire without the mutation being
+			// processed. A second advancement guarantees a full sweep ran after
+			// the mutation was visible to the iterator.
+			seen := false
+			before, _ := cache.GetLastProcessed()
+			assert.Eventually(t, func() bool {
+				last, ok := cache.GetLastProcessed()
+				if !ok || !last.After(before) {
+					return false
+				}
+				if !seen {
+					seen = true
+					before = last
+					return false
+				}
+				return true
+			}, time.Second, 1*time.Millisecond)
 
 			// Switch a registered to suspended
 			old2, err := subscriptionsDB.Get(t.Context(), "8c90b62a-3783-4ea6-a8c8-cbaee4667ffd")
@@ -201,7 +219,20 @@ func TestSubscriptionChangefeed(t *testing.T) {
 				},
 			})
 			require.NoError(t, err)
-			assert.Eventually(t, subscriptionsClient.AllIteratorsConsumed, time.Second, 1*time.Millisecond)
+			seen = false
+			before, _ = cache.GetLastProcessed()
+			assert.Eventually(t, func() bool {
+				last, ok := cache.GetLastProcessed()
+				if !ok || !last.After(before) {
+					return false
+				}
+				if !seen {
+					seen = true
+					before = last
+					return false
+				}
+				return true
+			}, time.Second, 1*time.Millisecond)
 
 			// Switch a registered to deleted
 			old3, err := subscriptionsDB.Get(t.Context(), "4e07b0f5-c789-4817-9079-94012b04e1c9")
@@ -217,7 +248,20 @@ func TestSubscriptionChangefeed(t *testing.T) {
 				},
 			})
 			require.NoError(t, err)
-			assert.Eventually(t, subscriptionsClient.AllIteratorsConsumed, time.Second, 1*time.Millisecond)
+			seen = false
+			before, _ = cache.GetLastProcessed()
+			assert.Eventually(t, func() bool {
+				last, ok := cache.GetLastProcessed()
+				if !ok || !last.After(before) {
+					return false
+				}
+				if !seen {
+					seen = true
+					before = last
+					return false
+				}
+				return true
+			}, time.Second, 1*time.Millisecond)
 
 			// Validate the expected cache contents
 			assert.Equal(t, tC.expected, maps.Collect(cache.subs.All()))