From cba9e6f89de8fab32fa753f892c4407c7ffe3e79 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 00:54:07 -0500
Subject: [PATCH 001/104] =?UTF-8?q?wrong=20even=20with=20just=20this=20?=
 =?UTF-8?q?=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 internal/verifier/change_stream_test.go      |  15 +--
 internal/verifier/integration_test_suite.go  | 100 +++++++++++++++++++
 internal/verifier/migration_verifier_test.go |  22 ++++
 3 files changed, 130 insertions(+), 7 deletions(-)
 create mode 100644 internal/verifier/integration_test_suite.go

diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index 7170129b..cc0f3413 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -33,9 +33,9 @@ func TestChangeStreamFilter(t *testing.T) {
 // TestChangeStreamResumability creates a verifier, starts its change stream,
 // terminates that verifier, updates the source cluster, starts a new
 // verifier with change stream, and confirms that things look as they should.
-func (suite *MultiSourceVersionTestSuite) TestChangeStreamResumability() {
+func (suite *IntegrationTestSuite) TestChangeStreamResumability() {
 	func() {
-		verifier1 := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+		verifier1 := suite.BuildVerifier()
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 		err := verifier1.StartChangeStream(ctx)
@@ -54,7 +54,7 @@ func (suite *MultiSourceVersionTestSuite) TestChangeStreamResumability() {
 		)
 	suite.Require().NoError(err)
 
-	verifier2 := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+	verifier2 := suite.BuildVerifier()
 
 	suite.Require().Empty(
 		suite.fetchVerifierRechecks(ctx, verifier2),
@@ -99,7 +99,7 @@ func (suite *MultiSourceVersionTestSuite) TestChangeStreamResumability() {
 	)
 }
 
-func (suite *MultiSourceVersionTestSuite) getClusterTime(ctx context.Context, client *mongo.Client) primitive.Timestamp {
+func (suite *IntegrationTestSuite) getClusterTime(ctx context.Context, client *mongo.Client) primitive.Timestamp {
 	sess, err := client.StartSession()
 	suite.Require().NoError(err, "should start session")
 
@@ -112,7 +112,7 @@ func (suite *MultiSourceVersionTestSuite) getClusterTime(ctx context.Context, cl
 	return newTime
 }
 
-func (suite *MultiSourceVersionTestSuite) fetchVerifierRechecks(ctx context.Context, verifier *Verifier) []bson.M {
+func (suite *IntegrationTestSuite) fetchVerifierRechecks(ctx context.Context, verifier *Verifier) []bson.M {
 	recheckDocs := []bson.M{}
 
 	recheckColl := verifier.verificationDatabase().Collection(recheckQueue)
@@ -199,8 +199,9 @@ func (suite *MultiSourceVersionTestSuite) TestNoStartAtTime() {
 	suite.Require().LessOrEqual(origStartTs.Compare(*verifier.srcStartAtTs), 0)
 }
 
-func (suite *MultiSourceVersionTestSuite) TestWithChangeEventsBatching() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestWithChangeEventsBatching() {
+	verifier := suite.BuildVerifier()
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
new file mode 100644
index 00000000..edda4715
--- /dev/null
+++ b/internal/verifier/integration_test_suite.go
@@ -0,0 +1,100 @@
+package verifier
+
+import (
+	"context"
+	"strings"
+
+	mapset "github.com/deckarep/golang-set/v2"
+	"github.com/stretchr/testify/suite"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.mongodb.org/mongo-driver/mongo/writeconcern"
+)
+
+type IntegrationTestSuite struct {
+	suite.Suite
+	srcConnStr, dstConnStr, metaConnStr             string
+	srcMongoClient, dstMongoClient, metaMongoClient *mongo.Client
+	initialDbNames                                  mapset.Set[string]
+}
+
+var _ suite.TestingSuite = &IntegrationTestSuite{}
+
+func (suite *IntegrationTestSuite) SetupSuite() {
+	ctx := context.Background()
+	clientOpts := options.Client().ApplyURI(suite.srcConnStr).SetAppName("Verifier Test Suite").SetWriteConcern(writeconcern.Majority())
+	var err error
+
+	suite.srcMongoClient, err = mongo.Connect(ctx, clientOpts)
+	suite.Require().NoError(err)
+
+	clientOpts = options.Client().ApplyURI(suite.dstConnStr).SetAppName("Verifier Test Suite").SetWriteConcern(writeconcern.Majority())
+	suite.dstMongoClient, err = mongo.Connect(ctx, clientOpts)
+	suite.Require().NoError(err)
+
+	clientOpts = options.Client().ApplyURI(suite.metaConnStr).SetAppName("Verifier Test Suite")
+	suite.metaMongoClient, err = mongo.Connect(ctx, clientOpts)
+	suite.Require().NoError(err)
+
+	suite.initialDbNames = mapset.NewSet[string]()
+	for _, client := range []*mongo.Client{suite.srcMongoClient, suite.dstMongoClient, suite.metaMongoClient} {
+		dbNames, err := client.ListDatabaseNames(ctx, bson.D{})
+		suite.Require().NoError(err)
+		for _, dbName := range dbNames {
+			suite.initialDbNames.Add(dbName)
+		}
+	}
+}
+
+func (suite *IntegrationTestSuite) TearDownTest() {
+	suite.T().Logf("Tearing down test %#q", suite.T().Name())
+
+	ctx := context.Background()
+	for _, client := range []*mongo.Client{suite.srcMongoClient, suite.dstMongoClient, suite.metaMongoClient} {
+		dbNames, err := client.ListDatabaseNames(ctx, bson.D{})
+		suite.Require().NoError(err)
+		for _, dbName := range dbNames {
+			if !suite.initialDbNames.Contains(dbName) {
+				suite.T().Logf("Dropping database %#q, which seems to have been created during test %#q.", dbName, suite.T().Name())
+
+				err = client.Database(dbName).Drop(ctx)
+				suite.Require().NoError(err)
+			}
+		}
+	}
+}
+
+func (suite *IntegrationTestSuite) BuildVerifier() *Verifier {
+	qfilter := QueryFilter{Namespace: "keyhole.dealers"}
+	task := VerificationTask{QueryFilter: qfilter}
+
+	verifier := NewVerifier(VerifierSettings{})
+	//verifier.SetStartClean(true)
+	verifier.SetNumWorkers(3)
+	verifier.SetGenerationPauseDelayMillis(0)
+	verifier.SetWorkerSleepDelayMillis(0)
+
+	ctx := context.Background()
+
+	suite.Require().NoError(verifier.SetSrcURI(ctx, suite.srcConnStr))
+	suite.Require().NoError(verifier.SetDstURI(ctx, suite.dstConnStr))
+	suite.Require().NoError(verifier.SetMetaURI(ctx, suite.metaConnStr))
+	verifier.SetLogger("stderr")
+	verifier.SetMetaDBName("VERIFIER_META")
+
+	verifier.verificationDatabase().Drop(ctx)
+	suite.Require().NoError(verifier.srcClientCollection(&task).Drop(ctx))
+	suite.Require().NoError(verifier.dstClientCollection(&task).Drop(ctx))
+	suite.Require().NoError(verifier.AddMetaIndexes(ctx))
+	return verifier
+}
+
+func (suite *IntegrationTestSuite) DBNameForTest() string {
+	name := suite.T().Name()
+	return strings.ReplaceAll(
+		strings.ReplaceAll(name, "/", "-"),
+		".",
+		"-",
+	)
+}
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index eab99d4b..39e0eacf 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -10,6 +10,7 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"os"
 	"regexp"
 	"sort"
 	"testing"
@@ -115,6 +116,27 @@ func TestVerifierMultiMetaVersion(t *testing.T) {
 	runMultipleVersionTests(t, testSuite, srcVersions, destVersions, metaVersions)
 }
 
+func TestIntegration(t *testing.T) {
+	envVals := map[string]string{}
+
+	for _, name := range []string{"MVTEST_SRC", "MVTEST_DST", "MVTEST_META"} {
+		connStr := os.Getenv(name)
+		if connStr == "" {
+			t.Fatalf("%s requires %#q in environment.", t.Name(), name)
+		}
+
+		envVals[name] = connStr
+	}
+
+	testSuite := &IntegrationTestSuite{
+		srcConnStr:  envVals["MVTEST_SRC"],
+		dstConnStr:  envVals["MVTEST_DST"],
+		metaConnStr: envVals["MVTEST_META"],
+	}
+
+	suite.Run(t, testSuite)
+}
+
 func runMultipleVersionTests(t *testing.T, testSuite WithMongodsTestingSuite,
 	srcVersions, destVersions, metaVersions []string) {
 	for _, srcVersion := range srcVersions {

From ccaa8999fab244bacbba234cb53a7ac25ec3cfad Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 01:12:14 -0500
Subject: [PATCH 002/104] add a bit of forgotten shutdown

---
 internal/verifier/change_stream.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index e03a0c39..edf738e2 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -123,6 +123,8 @@ func (verifier *Verifier) GetChangeStreamFilter() []bson.D {
 }
 
 func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.ChangeStream) {
+	defer cs.Close(ctx)
+
 	var lastPersistedTime time.Time
 
 	persistResumeTokenIfNeeded := func() error {
@@ -179,6 +181,9 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 
 		// If the context is canceled, return immmediately.
 		case <-ctx.Done():
+			verifier.logger.Debug().
+				Err(ctx.Err()).
+				Msg("Change stream quitting.")
 			return
 
 		// If the changeStreamEnderChan has a message, the user has indicated that

From 91003f371ee0df582a55917bd7f80e3b5d9a3168 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 01:42:18 -0500
Subject: [PATCH 003/104] fix

---
 internal/verifier/change_stream.go          |  7 ++++-
 internal/verifier/change_stream_test.go     |  4 +--
 internal/verifier/integration_test_suite.go | 29 +++++++++++++++++++--
 3 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index edf738e2..1701682a 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -333,6 +333,10 @@ func (v *Verifier) getChangeStreamMetadataCollection() *mongo.Collection {
 
 func (verifier *Verifier) loadChangeStreamResumeToken(ctx context.Context) (bson.Raw, error) {
 	coll := verifier.getChangeStreamMetadataCollection()
+	verifier.logger.Debug().
+		Str("db", coll.Database().Name()).
+		Str("coll", coll.Name()).
+		Msg("Seeking persisted resume token.")
 
 	token, err := coll.FindOne(
 		ctx,
@@ -360,7 +364,8 @@ func (verifier *Verifier) persistChangeStreamResumeToken(ctx context.Context, cs
 	if err == nil {
 		ts, err := extractTimestampFromResumeToken(token)
 
-		logEvent := verifier.logger.Debug()
+		logEvent := verifier.logger.Debug().
+			Interface("token", token)
 
 		if err == nil {
 			logEvent = addUnixTimeToLogEvent(ts.T, logEvent)
diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index cc0f3413..ba165604 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -46,7 +46,7 @@ func (suite *IntegrationTestSuite) TestChangeStreamResumability() {
 	defer cancel()
 
 	_, err := suite.srcMongoClient.
-		Database("testDb").
+		Database(suite.DBNameForTest()).
 		Collection("testColl").
 		InsertOne(
 			ctx,
@@ -89,7 +89,7 @@ func (suite *IntegrationTestSuite) TestChangeStreamResumability() {
 
 	suite.Assert().Equal(
 		bson.M{
-			"db":         "testDb",
+			"db":         suite.DBNameForTest(),
 			"coll":       "testColl",
 			"generation": int32(0),
 			"docID":      "heyhey",
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index edda4715..d13c37b3 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -12,6 +12,8 @@ import (
 	"go.mongodb.org/mongo-driver/mongo/writeconcern"
 )
 
+const metaDBName = "VERIFIER_TEST_META"
+
 type IntegrationTestSuite struct {
 	suite.Suite
 	srcConnStr, dstConnStr, metaConnStr             string
@@ -47,6 +49,30 @@ func (suite *IntegrationTestSuite) SetupSuite() {
 	}
 }
 
+func (suite *IntegrationTestSuite) SetupTest() {
+	ctx := context.Background()
+
+	dbname := suite.DBNameForTest()
+
+	suite.Require().NoError(
+		suite.srcMongoClient.Database(dbname).Drop(ctx),
+		"should drop source db %#q",
+		dbname,
+	)
+
+	suite.Require().NoError(
+		suite.dstMongoClient.Database(dbname).Drop(ctx),
+		"should drop destination db %#q",
+		dbname,
+	)
+
+	suite.Require().NoError(
+		suite.metaMongoClient.Database(metaDBName).Drop(ctx),
+		"should drop destination db %#q",
+		dbname,
+	)
+}
+
 func (suite *IntegrationTestSuite) TearDownTest() {
 	suite.T().Logf("Tearing down test %#q", suite.T().Name())
 
@@ -81,9 +107,8 @@ func (suite *IntegrationTestSuite) BuildVerifier() *Verifier {
 	suite.Require().NoError(verifier.SetDstURI(ctx, suite.dstConnStr))
 	suite.Require().NoError(verifier.SetMetaURI(ctx, suite.metaConnStr))
 	verifier.SetLogger("stderr")
-	verifier.SetMetaDBName("VERIFIER_META")
+	verifier.SetMetaDBName(metaDBName)
 
-	verifier.verificationDatabase().Drop(ctx)
 	suite.Require().NoError(verifier.srcClientCollection(&task).Drop(ctx))
 	suite.Require().NoError(verifier.dstClientCollection(&task).Drop(ctx))
 	suite.Require().NoError(verifier.AddMetaIndexes(ctx))

From 769d8eb60a76345440eb94fe9e495b3d9fafe3c3 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 01:46:16 -0500
Subject: [PATCH 004/104] fixes

---
 internal/verifier/change_stream_test.go      |  12 +-
 internal/verifier/migration_verifier_test.go | 159 +++----------------
 internal/verifier/recheck_test.go            |  22 +--
 internal/verifier/reset_test.go              |   8 +-
 4 files changed, 45 insertions(+), 156 deletions(-)

diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index ba165604..c7042618 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -126,8 +126,8 @@ func (suite *IntegrationTestSuite) fetchVerifierRechecks(ctx context.Context, ve
 	return recheckDocs
 }
 
-func (suite *MultiSourceVersionTestSuite) TestStartAtTimeNoChanges() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestStartAtTimeNoChanges() {
+	verifier := suite.BuildVerifier()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	sess, err := suite.srcMongoClient.StartSession()
@@ -146,8 +146,8 @@ func (suite *MultiSourceVersionTestSuite) TestStartAtTimeNoChanges() {
 	suite.Require().Equal(verifier.srcStartAtTs, origStartTs)
 }
 
-func (suite *MultiSourceVersionTestSuite) TestStartAtTimeWithChanges() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
+	verifier := suite.BuildVerifier()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	sess, err := suite.srcMongoClient.StartSession()
@@ -181,8 +181,8 @@ func (suite *MultiSourceVersionTestSuite) TestStartAtTimeWithChanges() {
 	suite.Require().Equal(verifier.srcStartAtTs, newStartTs)
 }
 
-func (suite *MultiSourceVersionTestSuite) TestNoStartAtTime() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestNoStartAtTime() {
+	verifier := suite.BuildVerifier()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	sess, err := suite.srcMongoClient.StartSession()
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index 39e0eacf..2e84b86d 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -31,91 +31,6 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-var macArmMongoVersions []string = []string{
-	"6.2.0", "6.0.1",
-}
-
-var preMacArmMongoVersions []string = []string{
-	"5.3.2", "5.0.11",
-	"4.4.16", "4.2.22",
-}
-
-type MultiDataVersionTestSuite struct {
-	WithMongodsTestSuite
-}
-
-type MultiSourceVersionTestSuite struct {
-	WithMongodsTestSuite
-}
-
-type MultiMetaVersionTestSuite struct {
-	WithMongodsTestSuite
-}
-
-func buildVerifier(t *testing.T, srcMongoInstance MongoInstance, dstMongoInstance MongoInstance, metaMongoInstance MongoInstance) *Verifier {
-	qfilter := QueryFilter{Namespace: "keyhole.dealers"}
-	task := VerificationTask{QueryFilter: qfilter}
-
-	verifier := NewVerifier(VerifierSettings{})
-	//verifier.SetStartClean(true)
-	verifier.SetNumWorkers(3)
-	verifier.SetGenerationPauseDelayMillis(0)
-	verifier.SetWorkerSleepDelayMillis(0)
-	require.Nil(t, verifier.SetMetaURI(context.Background(), "mongodb://localhost:"+metaMongoInstance.port))
-	require.Nil(t, verifier.SetSrcURI(context.Background(), "mongodb://localhost:"+srcMongoInstance.port))
-	require.Nil(t, verifier.SetDstURI(context.Background(), "mongodb://localhost:"+dstMongoInstance.port))
-	verifier.SetLogger("stderr")
-	verifier.SetMetaDBName("VERIFIER_META")
-	require.Nil(t, verifier.verificationTaskCollection().Drop(context.Background()))
-	require.Nil(t, verifier.refetchCollection().Drop(context.Background()))
-	require.Nil(t, verifier.srcClientCollection(&task).Drop(context.Background()))
-	require.Nil(t, verifier.dstClientCollection(&task).Drop(context.Background()))
-	require.Nil(t, verifier.verificationDatabase().Collection(recheckQueue).Drop(context.Background()))
-	require.Nil(t, verifier.AddMetaIndexes(context.Background()))
-	return verifier
-}
-
-func getAllVersions(t *testing.T) []string {
-	var versions []string
-	versions = append(versions, macArmMongoVersions...)
-
-	os, arch := getOSAndArchFromEnv(t)
-
-	if os != "macos" || arch != "arm64" {
-		versions = append(versions, preMacArmMongoVersions...)
-	}
-
-	return versions
-}
-
-func getLatestVersion() string {
-	return macArmMongoVersions[0]
-}
-
-func TestVerifierMultiversion(t *testing.T) {
-	testSuite := new(MultiDataVersionTestSuite)
-	srcVersions := getAllVersions(t)
-	destVersions := getAllVersions(t)
-	metaVersions := []string{getLatestVersion()}
-	runMultipleVersionTests(t, testSuite, srcVersions, destVersions, metaVersions)
-}
-
-func TestVerifierMultiSourceversion(t *testing.T) {
-	testSuite := new(MultiSourceVersionTestSuite)
-	srcVersions := getAllVersions(t)
-	destVersions := []string{getLatestVersion()}
-	metaVersions := []string{getLatestVersion()}
-	runMultipleVersionTests(t, testSuite, srcVersions, destVersions, metaVersions)
-}
-
-func TestVerifierMultiMetaVersion(t *testing.T) {
-	srcVersions := []string{getLatestVersion()}
-	destVersions := []string{getLatestVersion()}
-	metaVersions := getAllVersions(t)
-	testSuite := new(MultiMetaVersionTestSuite)
-	runMultipleVersionTests(t, testSuite, srcVersions, destVersions, metaVersions)
-}
-
 func TestIntegration(t *testing.T) {
 	envVals := map[string]string{}
 
@@ -137,34 +52,8 @@ func TestIntegration(t *testing.T) {
 	suite.Run(t, testSuite)
 }
 
-func runMultipleVersionTests(t *testing.T, testSuite WithMongodsTestingSuite,
-	srcVersions, destVersions, metaVersions []string) {
-	for _, srcVersion := range srcVersions {
-		for _, destVersion := range destVersions {
-			for _, metaVersion := range metaVersions {
-				testName := srcVersion + "->" + destVersion + ":" + metaVersion
-				t.Run(testName, func(t *testing.T) {
-					// TODO: this should be able to be run in parallel but we run killall mongod in the start of each of these test cases
-					// For now we are going to leave killall in because the tests don't take long and adding the killall makes them very safe
-					// t.Parallel()
-					testSuite.SetMetaInstance(MongoInstance{
-						version: metaVersion,
-					})
-					testSuite.SetSrcInstance(MongoInstance{
-						version: srcVersion,
-					})
-					testSuite.SetDstInstance(MongoInstance{
-						version: destVersion,
-					})
-					suite.Run(t, testSuite)
-				})
-			}
-		}
-	}
-}
-
-func (suite *MultiDataVersionTestSuite) TestVerifierFetchDocuments() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestVerifierFetchDocuments() {
+	verifier := suite.BuildVerifier()
 	ctx := context.Background()
 	drop := func() {
 		err := verifier.srcClient.Database("keyhole").Drop(ctx)
@@ -248,9 +137,9 @@ func (suite *MultiDataVersionTestSuite) TestVerifierFetchDocuments() {
 	)
 }
 
-func (suite *MultiMetaVersionTestSuite) TestGetNamespaceStatistics_Recheck() {
+func (suite *IntegrationTestSuite) TestGetNamespaceStatistics_Recheck() {
 	ctx := context.Background()
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+	verifier := suite.BuildVerifier()
 
 	err := verifier.HandleChangeStreamEvents(
 		ctx,
@@ -308,9 +197,9 @@ func (suite *MultiMetaVersionTestSuite) TestGetNamespaceStatistics_Recheck() {
 	)
 }
 
-func (suite *MultiMetaVersionTestSuite) TestGetNamespaceStatistics_Gen0() {
+func (suite *IntegrationTestSuite) TestGetNamespaceStatistics_Gen0() {
 	ctx := context.Background()
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+	verifier := suite.BuildVerifier()
 
 	stats, err := verifier.GetNamespaceStatistics(ctx)
 	suite.Require().NoError(err)
@@ -499,9 +388,9 @@ func (suite *MultiMetaVersionTestSuite) TestGetNamespaceStatistics_Gen0() {
 	)
 }
 
-func (suite *MultiMetaVersionTestSuite) TestFailedVerificationTaskInsertions() {
+func (suite *IntegrationTestSuite) TestFailedVerificationTaskInsertions() {
 	ctx := context.Background()
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+	verifier := suite.BuildVerifier()
 	err := verifier.InsertFailedCompareRecheckDocs("foo.bar", []interface{}{42}, []int{100})
 	suite.Require().NoError(err)
 	err = verifier.InsertFailedCompareRecheckDocs("foo.bar", []interface{}{43, 44}, []int{100, 100})
@@ -766,8 +655,8 @@ func TestVerifierCompareDocs(t *testing.T) {
 	}
 }
 
-func (suite *MultiDataVersionTestSuite) TestVerifierCompareViews() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestVerifierCompareViews() {
+	verifier := suite.BuildVerifier()
 	ctx := context.Background()
 
 	err := suite.srcMongoClient.Database("testDb").CreateView(ctx, "sameView", "testColl", bson.A{bson.D{{"$project", bson.D{{"_id", 1}}}}})
@@ -879,8 +768,8 @@ func (suite *MultiDataVersionTestSuite) TestVerifierCompareViews() {
 	}
 }
 
-func (suite *MultiDataVersionTestSuite) TestVerifierCompareMetadata() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestVerifierCompareMetadata() {
+	verifier := suite.BuildVerifier()
 	ctx := context.Background()
 
 	// Collection exists only on source.
@@ -985,8 +874,8 @@ func (suite *MultiDataVersionTestSuite) TestVerifierCompareMetadata() {
 	suite.Equal(verificationTaskCompleted, task.Status)
 }
 
-func (suite *MultiDataVersionTestSuite) TestVerifierCompareIndexes() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestVerifierCompareIndexes() {
+	verifier := suite.BuildVerifier()
 	ctx := context.Background()
 
 	// Missing index on destination.
@@ -1259,8 +1148,8 @@ func TestVerifierCompareIndexSpecs(t *testing.T) {
 	}
 }
 
-func (suite *MultiDataVersionTestSuite) TestVerifierNamespaceList() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestVerifierNamespaceList() {
+	verifier := suite.BuildVerifier()
 	ctx := context.Background()
 
 	// Collections on source only
@@ -1362,8 +1251,8 @@ func (suite *MultiDataVersionTestSuite) TestVerifierNamespaceList() {
 	suite.ElementsMatch([]string{"testDb1.testColl1", "testDb1.testColl2", "testDb1.testView1"}, verifier.dstNamespaces)
 }
 
-func (suite *MultiDataVersionTestSuite) TestVerificationStatus() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestVerificationStatus() {
+	verifier := suite.BuildVerifier()
 	ctx := context.Background()
 
 	metaColl := verifier.verificationDatabase().Collection(verificationTasksCollection)
@@ -1385,9 +1274,9 @@ func (suite *MultiDataVersionTestSuite) TestVerificationStatus() {
 	suite.Equal(1, status.CompletedTasks, "completed tasks not equal")
 }
 
-func (suite *MultiDataVersionTestSuite) TestGenerationalRechecking() {
+func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+	verifier := suite.BuildVerifier()
 	verifier.SetSrcNamespaces([]string{"testDb1.testColl1"})
 	verifier.SetDstNamespaces([]string{"testDb2.testColl3"})
 	verifier.SetNamespaceMap()
@@ -1493,11 +1382,11 @@ func (suite *MultiDataVersionTestSuite) TestGenerationalRechecking() {
 	require.NoError(suite.T(), errGroup.Wait())
 }
 
-func (suite *MultiDataVersionTestSuite) TestVerifierWithFilter() {
+func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
 	filter := map[string]any{"inFilter": map[string]any{"$ne": false}}
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+	verifier := suite.BuildVerifier()
 	verifier.SetSrcNamespaces([]string{"testDb1.testColl1"})
 	verifier.SetDstNamespaces([]string{"testDb2.testColl3"})
 	verifier.SetNamespaceMap()
@@ -1605,7 +1494,7 @@ func (suite *MultiDataVersionTestSuite) TestVerifierWithFilter() {
 	<-checkDoneChan
 }
 
-func (suite *MultiDataVersionTestSuite) TestPartitionWithFilter() {
+func (suite *IntegrationTestSuite) TestPartitionWithFilter() {
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
 	ctx := context.Background()
@@ -1615,7 +1504,7 @@ func (suite *MultiDataVersionTestSuite) TestPartitionWithFilter() {
 		{"$gte": []any{"$n", 100}}, {"$lt": []any{"$n", 200}}}}}
 
 	// Set up the verifier for testing.
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+	verifier := suite.BuildVerifier()
 	verifier.SetSrcNamespaces([]string{"testDb1.testColl1"})
 	verifier.SetNamespaceMap()
 	verifier.globalFilter = filter
diff --git a/internal/verifier/recheck_test.go b/internal/verifier/recheck_test.go
index 735bcf7f..e4b14b50 100644
--- a/internal/verifier/recheck_test.go
+++ b/internal/verifier/recheck_test.go
@@ -10,8 +10,8 @@ import (
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
-func (suite *MultiMetaVersionTestSuite) TestFailedCompareThenReplace() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestFailedCompareThenReplace() {
+	verifier := suite.BuildVerifier()
 	ctx := context.Background()
 
 	suite.Require().NoError(
@@ -73,7 +73,7 @@ func (suite *MultiMetaVersionTestSuite) TestFailedCompareThenReplace() {
 	)
 }
 
-func (suite *MultiMetaVersionTestSuite) fetchRecheckDocs(ctx context.Context, verifier *Verifier) []RecheckDoc {
+func (suite *IntegrationTestSuite) fetchRecheckDocs(ctx context.Context, verifier *Verifier) []RecheckDoc {
 	metaColl := suite.metaMongoClient.Database(verifier.metaDBName).Collection(recheckQueue)
 
 	cursor, err := metaColl.Find(ctx, bson.D{})
@@ -86,8 +86,8 @@ func (suite *MultiMetaVersionTestSuite) fetchRecheckDocs(ctx context.Context, ve
 	return results
 }
 
-func (suite *MultiMetaVersionTestSuite) TestLargeIDInsertions() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestLargeIDInsertions() {
+	verifier := suite.BuildVerifier()
 	ctx := context.Background()
 
 	overlyLarge := 7 * 1024 * 1024 // Three of these exceed our 16MB limit, but two do not
@@ -147,8 +147,8 @@ func (suite *MultiMetaVersionTestSuite) TestLargeIDInsertions() {
 	suite.ElementsMatch([]VerificationTask{t1, t2}, actualTasks)
 }
 
-func (suite *MultiMetaVersionTestSuite) TestLargeDataInsertions() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestLargeDataInsertions() {
+	verifier := suite.BuildVerifier()
 	verifier.partitionSizeInBytes = 1024 * 1024
 	ctx := context.Background()
 
@@ -209,8 +209,8 @@ func (suite *MultiMetaVersionTestSuite) TestLargeDataInsertions() {
 	suite.ElementsMatch([]VerificationTask{t1, t2}, actualTasks)
 }
 
-func (suite *MultiMetaVersionTestSuite) TestMultipleNamespaces() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestMultipleNamespaces() {
+	verifier := suite.BuildVerifier()
 	ctx := context.Background()
 
 	id1 := "a"
@@ -260,8 +260,8 @@ func (suite *MultiMetaVersionTestSuite) TestMultipleNamespaces() {
 	suite.ElementsMatch([]VerificationTask{t1, t2, t3, t4}, actualTasks)
 }
 
-func (suite *MultiMetaVersionTestSuite) TestGenerationalClear() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestGenerationalClear() {
+	verifier := suite.BuildVerifier()
 	ctx := context.Background()
 
 	id1 := "a"
diff --git a/internal/verifier/reset_test.go b/internal/verifier/reset_test.go
index f18fb461..f33ce6dd 100644
--- a/internal/verifier/reset_test.go
+++ b/internal/verifier/reset_test.go
@@ -10,8 +10,8 @@ import (
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
-func (suite *MultiDataVersionTestSuite) TestResetPrimaryTask() {
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+func (suite *IntegrationTestSuite) TestResetPrimaryTask() {
+	verifier := suite.BuildVerifier()
 
 	created, err := verifier.CheckIsPrimary()
 	suite.Require().NoError(err)
@@ -39,10 +39,10 @@ func (suite *MultiDataVersionTestSuite) TestResetPrimaryTask() {
 	suite.Assert().Len(taskDocs, 1)
 }
 
-func (suite *MultiDataVersionTestSuite) TestResetNonPrimaryTasks() {
+func (suite *IntegrationTestSuite) TestResetNonPrimaryTasks() {
 	ctx := context.Background()
 
-	verifier := buildVerifier(suite.T(), suite.srcMongoInstance, suite.dstMongoInstance, suite.metaMongoInstance)
+	verifier := suite.BuildVerifier()
 
 	// Create a primary task, and set it to complete.
 	created, err := verifier.CheckIsPrimary()

From dd62b706fe1e351ab39072aa2c725241ccdccf05 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:07:18 -0500
Subject: [PATCH 005/104] migrate to mtools

---
 .github/workflows/all.yml | 47 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 41 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index e17e18a9..a32bc1c9 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -12,11 +12,26 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - runsOn: macos-latest
-            mongodb_distro: mongodb-macos-arm64
+          - name: macos-latest
+            setup: brew install pip3
 
-          - runsOn: ubuntu-latest
-            mongodb_distro: mongodb-linux-x86_64-ubuntu1804
+          - name: ubuntu-latest
+            setup: apt install -y pip3
+
+        version:
+          - 4.2
+          - 4.4
+          - 5.0
+          - 6.0
+          - 7.0
+          - 8.0
+
+        topology:
+          - name: replset
+            args: --replicaset --nodes 1
+
+          - name: sharded
+            args: --sharded 2 --nodes 1
 
         go_version:
 
@@ -25,9 +40,27 @@ jobs:
           - '1.20'
           - stable
 
-    runs-on: ${{matrix.os.runsOn}}
+    runs-on: ${{matrix.runsOn}}
 
     steps:
+      - name: Install m
+        run: npm install -g m
+
+      - name: Activate MongoDB ${{ matrix.version }}
+        run: m activate ${{ matrix.version }}
+
+      - name: Install mtools
+        run: pip3 install 'mtools[all]'
+
+      - name: Start source cluster
+        run: mlaunch init --port 27020 ${{ matrix.topology.args }}
+
+      - name: Start destination cluster
+        run: mlaunch init --port 27030 ${{ matrix.topology.args }}
+
+      - name: Start metadata cluster
+        run: mlaunch init --port 27040 ${{ matrix.topology.args }}
+
       - name: Check out repository
         uses: actions/checkout@v4
 
@@ -42,4 +75,6 @@ jobs:
       - name: Test
         run: go test -v ./...
         env:
-          MONGODB_DISTRO: ${{matrix.os.mongodb_distro}}
+          MVTEST_SRC: localhost:27020
+          MVTEST_DST: localhost:27030
+          MVTEST_META: localhost:27040

From e279dadb8fc240c44c6650cee255c120e0c14eaf Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:08:18 -0500
Subject: [PATCH 006/104] fix typo

---
 .github/workflows/all.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index a32bc1c9..956f2477 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -12,10 +12,10 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - name: macos-latest
+          - runsOn: macos-latest
             setup: brew install pip3
 
-          - name: ubuntu-latest
+          - runsOn: ubuntu-latest
             setup: apt install -y pip3
 
         version:
@@ -40,7 +40,7 @@ jobs:
           - '1.20'
           - stable
 
-    runs-on: ${{matrix.runsOn}}
+    runs-on: ${{matrix.os.runsOn}}
 
     steps:
       - name: Install m

From e75bead1d6f60d75dc9a2395616fa58e20e7c843 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:09:39 -0500
Subject: [PATCH 007/104] print statement?

---
 .github/workflows/all.yml | 76 ++++++++++++++++++++-------------------
 1 file changed, 39 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 956f2477..134b7687 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -40,41 +40,43 @@ jobs:
           - '1.20'
           - stable
 
-    runs-on: ${{matrix.os.runsOn}}
-
     steps:
-      - name: Install m
-        run: npm install -g m
-
-      - name: Activate MongoDB ${{ matrix.version }}
-        run: m activate ${{ matrix.version }}
-
-      - name: Install mtools
-        run: pip3 install 'mtools[all]'
-
-      - name: Start source cluster
-        run: mlaunch init --port 27020 ${{ matrix.topology.args }}
-
-      - name: Start destination cluster
-        run: mlaunch init --port 27030 ${{ matrix.topology.args }}
-
-      - name: Start metadata cluster
-        run: mlaunch init --port 27040 ${{ matrix.topology.args }}
-
-      - name: Check out repository
-        uses: actions/checkout@v4
-
-      - name: Fetch Go ${{ matrix.go_version }}
-        uses: actions/setup-go@v5
-        with:
-          go-version: ${{ matrix.go_version }}
-
-      - name: Build
-        run: go build main/migration_verifier.go
-
-      - name: Test
-        run: go test -v ./...
-        env:
-          MVTEST_SRC: localhost:27020
-          MVTEST_DST: localhost:27030
-          MVTEST_META: localhost:27040
+       - run: echo ${{matrix}}
+#    runs-on: ${{matrix.os.runsOn}}
+#
+#    steps:
+#      - name: Install m
+#        run: npm install -g m
+#
+#      - name: Activate MongoDB ${{ matrix.version }}
+#        run: m activate ${{ matrix.version }}
+#
+#      - name: Install mtools
+#        run: pip3 install 'mtools[all]'
+#
+#      - name: Start source cluster
+#        run: mlaunch init --port 27020 ${{ matrix.topology.args }}
+#
+#      - name: Start destination cluster
+#        run: mlaunch init --port 27030 ${{ matrix.topology.args }}
+#
+#      - name: Start metadata cluster
+#        run: mlaunch init --port 27040 ${{ matrix.topology.args }}
+#
+#      - name: Check out repository
+#        uses: actions/checkout@v4
+#
+#      - name: Fetch Go ${{ matrix.go_version }}
+#        uses: actions/setup-go@v5
+#        with:
+#          go-version: ${{ matrix.go_version }}
+#
+#      - name: Build
+#        run: go build main/migration_verifier.go
+#
+#      - name: Test
+#        run: go test -v ./...
+#        env:
+#          MVTEST_SRC: localhost:27020
+#          MVTEST_DST: localhost:27030
+#          MVTEST_META: localhost:27040

From fb2d70ffc2ad60b3bdacf14865d6e925f7ad7c20 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:10:06 -0500
Subject: [PATCH 008/104] runs on

---
 .github/workflows/all.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 134b7687..e8444ade 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -40,6 +40,8 @@ jobs:
           - '1.20'
           - stable
 
+    runs-on: ubuntu-latest
+
     steps:
        - run: echo ${{matrix}}
 #    runs-on: ${{matrix.os.runsOn}}

From 25ec3cafaada7c533284f50b336ca4d59958629c Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:12:11 -0500
Subject: [PATCH 009/104] echo more

---
 .github/workflows/all.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index e8444ade..479ac25d 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -19,12 +19,12 @@ jobs:
             setup: apt install -y pip3
 
         version:
-          - 4.2
-          - 4.4
-          - 5.0
-          - 6.0
-          - 7.0
-          - 8.0
+          - '4.2'
+          - '4.4'
+          - '5.0'
+          - '6.0'
+          - '7.0'
+          - '8.0'
 
         topology:
           - name: replset
@@ -43,7 +43,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-       - run: echo ${{matrix}}
+       - run: echo ${{matrix.os.runsOn}}
 #    runs-on: ${{matrix.os.runsOn}}
 #
 #    steps:

From ede26ef287b984d695feed2dd4db97272802adbf Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:13:12 -0500
Subject: [PATCH 010/104] =?UTF-8?q?runsOn=20again=20=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .github/workflows/all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 479ac25d..d7438707 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -40,7 +40,7 @@ jobs:
           - '1.20'
           - stable
 
-    runs-on: ubuntu-latest
+    runs-on: ${{matrix.os.runsOn}}
 
     steps:
        - run: echo ${{matrix.os.runsOn}}

From 4ee4279a29baa24c003255ccb584825d38b3e6eb Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:13:58 -0500
Subject: [PATCH 011/104] uncomment

---
 .github/workflows/all.yml | 74 ++++++++++++++++++---------------------
 1 file changed, 35 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index d7438707..dddb5b1a 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -43,42 +43,38 @@ jobs:
     runs-on: ${{matrix.os.runsOn}}
 
     steps:
-       - run: echo ${{matrix.os.runsOn}}
-#    runs-on: ${{matrix.os.runsOn}}
-#
-#    steps:
-#      - name: Install m
-#        run: npm install -g m
-#
-#      - name: Activate MongoDB ${{ matrix.version }}
-#        run: m activate ${{ matrix.version }}
-#
-#      - name: Install mtools
-#        run: pip3 install 'mtools[all]'
-#
-#      - name: Start source cluster
-#        run: mlaunch init --port 27020 ${{ matrix.topology.args }}
-#
-#      - name: Start destination cluster
-#        run: mlaunch init --port 27030 ${{ matrix.topology.args }}
-#
-#      - name: Start metadata cluster
-#        run: mlaunch init --port 27040 ${{ matrix.topology.args }}
-#
-#      - name: Check out repository
-#        uses: actions/checkout@v4
-#
-#      - name: Fetch Go ${{ matrix.go_version }}
-#        uses: actions/setup-go@v5
-#        with:
-#          go-version: ${{ matrix.go_version }}
-#
-#      - name: Build
-#        run: go build main/migration_verifier.go
-#
-#      - name: Test
-#        run: go test -v ./...
-#        env:
-#          MVTEST_SRC: localhost:27020
-#          MVTEST_DST: localhost:27030
-#          MVTEST_META: localhost:27040
+      - name: Install m
+        run: npm install -g m
+
+      - name: Activate MongoDB ${{ matrix.version }}
+        run: m activate ${{ matrix.version }}
+
+      - name: Install mtools
+        run: pip3 install 'mtools[all]'
+
+      - name: Start source cluster
+        run: mlaunch init --port 27020 ${{ matrix.topology.args }}
+
+      - name: Start destination cluster
+        run: mlaunch init --port 27030 ${{ matrix.topology.args }}
+
+      - name: Start metadata cluster
+        run: mlaunch init --port 27040 ${{ matrix.topology.args }}
+
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Fetch Go ${{ matrix.go_version }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go_version }}
+
+      - name: Build
+        run: go build main/migration_verifier.go
+
+      - name: Test
+        run: go test -v ./...
+        env:
+          MVTEST_SRC: localhost:27020
+          MVTEST_DST: localhost:27030
+          MVTEST_META: localhost:27040

From d46f9b888c16ede78a61c862c7796324b74c36de Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:15:35 -0500
Subject: [PATCH 012/104] pipx

---
 .github/workflows/all.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index dddb5b1a..b967f100 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -13,10 +13,10 @@ jobs:
       matrix:
         os:
           - runsOn: macos-latest
-            setup: brew install pip3
+            setup: brew install pipx
 
           - runsOn: ubuntu-latest
-            setup: apt install -y pip3
+            setup: apt install -y pipx
 
         version:
           - '4.2'
@@ -50,7 +50,7 @@ jobs:
         run: m activate ${{ matrix.version }}
 
       - name: Install mtools
-        run: pip3 install 'mtools[all]'
+        run: pipx install 'mtools[all]'
 
       - name: Start source cluster
         run: mlaunch init --port 27020 ${{ matrix.topology.args }}

From e09cdbe8cc52db2ab701e98e2777c8fb5c14823b Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:17:43 -0500
Subject: [PATCH 013/104] =?UTF-8?q?not=20even=20=E2=80=9Cactivate=E2=80=9D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .github/workflows/all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index b967f100..9aa0de5b 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -47,7 +47,7 @@ jobs:
         run: npm install -g m
 
       - name: Activate MongoDB ${{ matrix.version }}
-        run: m activate ${{ matrix.version }}
+        run: m ${{ matrix.version }}
 
       - name: Install mtools
         run: pipx install 'mtools[all]'

From b04fe113a91a9d6345bcaa95480e2ee3c60ad196 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:19:50 -0500
Subject: [PATCH 014/104] always replset

---
 .github/workflows/all.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 9aa0de5b..54c76694 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -28,10 +28,9 @@ jobs:
 
         topology:
           - name: replset
-            args: --replicaset --nodes 1
 
           - name: sharded
-            args: --sharded 2 --nodes 1
+            args: --sharded 2
 
         go_version:
 
@@ -53,13 +52,13 @@ jobs:
         run: pipx install 'mtools[all]'
 
       - name: Start source cluster
-        run: mlaunch init --port 27020 ${{ matrix.topology.args }}
+        run: mlaunch init --port 27020 --replicaset ${{ matrix.topology.args }}
 
       - name: Start destination cluster
-        run: mlaunch init --port 27030 ${{ matrix.topology.args }}
+        run: mlaunch init --port 27030 --replicaset ${{ matrix.topology.args }}
 
       - name: Start metadata cluster
-        run: mlaunch init --port 27040 ${{ matrix.topology.args }}
+        run: mlaunch init --port 27040 --replicaset ${{ matrix.topology.args }}
 
       - name: Check out repository
         uses: actions/checkout@v4

From 7b2e021f44141edc683a71e6d071ae87cf075dab Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:23:20 -0500
Subject: [PATCH 015/104] yes

---
 .github/workflows/all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 54c76694..4968164f 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -46,7 +46,7 @@ jobs:
         run: npm install -g m
 
       - name: Activate MongoDB ${{ matrix.version }}
-        run: m ${{ matrix.version }}
+        run: yes | m ${{ matrix.version }}
 
       - name: Install mtools
         run: pipx install 'mtools[all]'

From 35566aae90aac122191d0a10bf493f83161fdee2 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:24:31 -0500
Subject: [PATCH 016/104] separate dirs in mlaunch

---
 .github/workflows/all.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 4968164f..f5f5bdc4 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -52,13 +52,13 @@ jobs:
         run: pipx install 'mtools[all]'
 
       - name: Start source cluster
-        run: mlaunch init --port 27020 --replicaset ${{ matrix.topology.args }}
+        run: mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}
 
       - name: Start destination cluster
-        run: mlaunch init --port 27030 --replicaset ${{ matrix.topology.args }}
+        run: mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}
 
       - name: Start metadata cluster
-        run: mlaunch init --port 27040 --replicaset ${{ matrix.topology.args }}
+        run: mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}
 
       - name: Check out repository
         uses: actions/checkout@v4

From fb8b398b18e76d729a6d88d46c0469c55f4a1926 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:31:00 -0500
Subject: [PATCH 017/104] fix connstrs & parallelize cluster setup

---
 .github/workflows/all.yml | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index f5f5bdc4..605cceaf 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -13,10 +13,10 @@ jobs:
       matrix:
         os:
           - runsOn: macos-latest
-            setup: brew install pipx
+            setup: brew install pipx parallel
 
           - runsOn: ubuntu-latest
-            setup: apt install -y pipx
+            setup: apt install -y pipx parallel
 
         version:
           - '4.2'
@@ -51,14 +51,13 @@ jobs:
       - name: Install mtools
         run: pipx install 'mtools[all]'
 
-      - name: Start source cluster
-        run: mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}
-
-      - name: Start destination cluster
-        run: mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}
-
-      - name: Start metadata cluster
-        run: mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}
+      - name: Start clusters
+        run: |-
+            {
+                echo "mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}"
+                echo "mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}"
+                echo "mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}"
+            } | parallel
 
       - name: Check out repository
         uses: actions/checkout@v4
@@ -74,6 +73,6 @@ jobs:
       - name: Test
         run: go test -v ./...
         env:
-          MVTEST_SRC: localhost:27020
-          MVTEST_DST: localhost:27030
-          MVTEST_META: localhost:27040
+          MVTEST_SRC: mongodb://localhost:27020
+          MVTEST_DST: mongodb://localhost:27030
+          MVTEST_META: mongodb://localhost:27040

From 47a91fafb6f2b786c61034fd0d4fc57e2d3220aa Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:36:49 -0500
Subject: [PATCH 018/104] fix replsets

---
 .github/workflows/all.yml | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 605cceaf..0bd8dbdf 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -13,10 +13,10 @@ jobs:
       matrix:
         os:
           - runsOn: macos-latest
-            setup: brew install pipx parallel
+            setup: brew install parallel
 
           - runsOn: ubuntu-latest
-            setup: apt install -y pipx parallel
+            setup: apt install -y parallel
 
         version:
           - '4.2'
@@ -28,9 +28,15 @@ jobs:
 
         topology:
           - name: replset
+            srcConnStr: mongodb://localhost:27020,localhost:27021,localhost:27022
+            dstConnStr: mongodb://localhost:27030,localhost:27031,localhost:27032
+            metaConnStr: mongodb://localhost:27040,localhost:27041,localhost:27042
 
           - name: sharded
             args: --sharded 2
+            srcConnStr: mongodb://localhost:27020
+            dstConnStr: mongodb://localhost:27030
+            metaConnStr: mongodb://localhost:27040
 
         go_version:
 
@@ -42,6 +48,9 @@ jobs:
     runs-on: ${{matrix.os.runsOn}}
 
     steps:
+      - name: Install packages
+        run: ${{ matrix.os.setup }}
+
       - name: Install m
         run: npm install -g m
 
@@ -73,6 +82,6 @@ jobs:
       - name: Test
         run: go test -v ./...
         env:
-          MVTEST_SRC: mongodb://localhost:27020
-          MVTEST_DST: mongodb://localhost:27030
-          MVTEST_META: mongodb://localhost:27040
+          MVTEST_SRC: ${{matrix.topology.srcConnStr}}
+          MVTEST_DST: ${{matrix.topology.dstConnStr}}
+          MVTEST_META: ${{matrix.topology.metaConnStr}}

From 994acbcbbaa554d7420290b03e4252bc3713360a Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:42:56 -0500
Subject: [PATCH 019/104] await

---
 .github/workflows/all.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 0bd8dbdf..3504e9d4 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -16,7 +16,6 @@ jobs:
             setup: brew install parallel
 
           - runsOn: ubuntu-latest
-            setup: apt install -y parallel
 
         version:
           - '4.2'
@@ -52,7 +51,7 @@ jobs:
         run: ${{ matrix.os.setup }}
 
       - name: Install m
-        run: npm install -g m
+        run: npm install -g m mongosh
 
       - name: Activate MongoDB ${{ matrix.version }}
         run: yes | m ${{ matrix.version }}
@@ -68,6 +67,9 @@ jobs:
                 echo "mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}"
             } | parallel
 
+      - name: Await source readiness
+        run: while ! $( mongosh "${{matrix.topology.srcConnStr}}" --eval 'db.ping()' ); do sleep 1; done
+
       - name: Check out repository
         uses: actions/checkout@v4
 

From 80d891fd70f2a6adaddce55ca4442fac8bbc9ccc Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:45:41 -0500
Subject: [PATCH 020/104] hello

---
 .github/workflows/all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 3504e9d4..99e6be0f 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -68,7 +68,7 @@ jobs:
             } | parallel
 
       - name: Await source readiness
-        run: while ! $( mongosh "${{matrix.topology.srcConnStr}}" --eval 'db.ping()' ); do sleep 1; done
+        run: while ! $( mongosh "${{matrix.topology.srcConnStr}}" --eval 'db.hello()' ); do sleep 1; done
 
       - name: Check out repository
         uses: actions/checkout@v4

From ee151b1cf302b07bb83489c7e9dcaa2f89ea1b8e Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:50:06 -0500
Subject: [PATCH 021/104] restate

---
 .github/workflows/all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 99e6be0f..ed4a583b 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -68,7 +68,7 @@ jobs:
             } | parallel
 
       - name: Await source readiness
-        run: while ! $( mongosh "${{matrix.topology.srcConnStr}}" --eval 'db.hello()' ); do sleep 1; done
+        run: while true; do mongosh "${{ matrix.topology.srcConnStr }}" --eval 'db.hello()' && break; sleep 1; done
 
       - name: Check out repository
         uses: actions/checkout@v4

From 2027764ca179c392055bed436aaa86411230437f Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 02:56:37 -0500
Subject: [PATCH 022/104] more debug

---
 .github/workflows/all.yml                   | 18 +++++++++++-------
 internal/verifier/integration_test_suite.go | 15 ++++++++++++---
 internal/verifier/migration_verifier.go     |  2 +-
 3 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index ed4a583b..8870a3ba 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -59,13 +59,17 @@ jobs:
       - name: Install mtools
         run: pipx install 'mtools[all]'
 
-      - name: Start clusters
-        run: |-
-            {
-                echo "mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}"
-                echo "mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}"
-                echo "mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}"
-            } | parallel
+ #     - name: Start clusters
+ #       run: |-
+ #           {
+ #               echo "mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}"
+ #               echo "mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}"
+ #               echo "mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}"
+ #           } | parallel
+
+      - run: mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}
+      - run: mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}
+      - run: mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}
 
       - name: Await source readiness
         run: while true; do mongosh "${{ matrix.topology.srcConnStr }}" --eval 'db.hello()' && break; sleep 1; done
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index d13c37b3..1a70862a 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -103,9 +103,18 @@ func (suite *IntegrationTestSuite) BuildVerifier() *Verifier {
 
 	ctx := context.Background()
 
-	suite.Require().NoError(verifier.SetSrcURI(ctx, suite.srcConnStr))
-	suite.Require().NoError(verifier.SetDstURI(ctx, suite.dstConnStr))
-	suite.Require().NoError(verifier.SetMetaURI(ctx, suite.metaConnStr))
+	suite.Require().NoError(
+		verifier.SetSrcURI(ctx, suite.srcConnStr),
+		"should set source connection string",
+	)
+	suite.Require().NoError(
+		verifier.SetDstURI(ctx, suite.dstConnStr),
+		"should set destination connection string",
+	)
+	suite.Require().NoError(
+		verifier.SetMetaURI(ctx, suite.metaConnStr),
+		"should set metadata connection string",
+	)
 	verifier.SetLogger("stderr")
 	verifier.SetMetaDBName(metaDBName)
 
diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go
index 809e4fe3..b8ff382f 100644
--- a/internal/verifier/migration_verifier.go
+++ b/internal/verifier/migration_verifier.go
@@ -276,7 +276,7 @@ func (verifier *Verifier) SetSrcURI(ctx context.Context, uri string) error {
 	var err error
 	verifier.srcClient, err = mongo.Connect(ctx, opts)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "failed to connect to %#q", uri)
 	}
 	verifier.srcBuildInfo, err = getBuildInfo(ctx, verifier.srcClient)
 	return err

From db956ad28d45a592bb7941758a4a529d039ce6df Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 03:00:59 -0500
Subject: [PATCH 023/104] ipv4

---
 .github/workflows/all.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 8870a3ba..cf227647 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -27,15 +27,15 @@ jobs:
 
         topology:
           - name: replset
-            srcConnStr: mongodb://localhost:27020,localhost:27021,localhost:27022
-            dstConnStr: mongodb://localhost:27030,localhost:27031,localhost:27032
-            metaConnStr: mongodb://localhost:27040,localhost:27041,localhost:27042
+            srcConnStr: mongodb://127.0.0.1:27020,127.0.0.1:27021,127.0.0.1:27022
+            dstConnStr: mongodb://127.0.0.1:27030,127.0.0.1:27031,127.0.0.1:27032
+            metaConnStr: mongodb://127.0.0.1:27040,127.0.0.1:27041,127.0.0.1:27042
 
           - name: sharded
             args: --sharded 2
-            srcConnStr: mongodb://localhost:27020
-            dstConnStr: mongodb://localhost:27030
-            metaConnStr: mongodb://localhost:27040
+            srcConnStr: mongodb://127.0.0.1:27020
+            dstConnStr: mongodb://127.0.0.1:27030
+            metaConnStr: mongodb://127.0.0.1:27040
 
         go_version:
 

From b22fefe4861613c54519a48190b7b7e219d86597 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 08:09:01 -0500
Subject: [PATCH 024/104] try to output logs

---
 .github/workflows/all.yml                   | 13 ++++++++-----
 internal/verifier/integration_test_suite.go |  4 ++--
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index cf227647..268ef533 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -31,11 +31,11 @@ jobs:
             dstConnStr: mongodb://127.0.0.1:27030,127.0.0.1:27031,127.0.0.1:27032
             metaConnStr: mongodb://127.0.0.1:27040,127.0.0.1:27041,127.0.0.1:27042
 
-          - name: sharded
-            args: --sharded 2
-            srcConnStr: mongodb://127.0.0.1:27020
-            dstConnStr: mongodb://127.0.0.1:27030
-            metaConnStr: mongodb://127.0.0.1:27040
+#          - name: sharded
+#            args: --sharded 2
+#            srcConnStr: mongodb://127.0.0.1:27020
+#            dstConnStr: mongodb://127.0.0.1:27030
+#            metaConnStr: mongodb://127.0.0.1:27040
 
         go_version:
 
@@ -91,3 +91,6 @@ jobs:
           MVTEST_SRC: ${{matrix.topology.srcConnStr}}
           MVTEST_DST: ${{matrix.topology.dstConnStr}}
           MVTEST_META: ${{matrix.topology.metaConnStr}}
+        continue-on-error: true
+
+      - run: ls -la src
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index 1a70862a..b7754372 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -40,9 +40,9 @@ func (suite *IntegrationTestSuite) SetupSuite() {
 	suite.Require().NoError(err)
 
 	suite.initialDbNames = mapset.NewSet[string]()
-	for _, client := range []*mongo.Client{suite.srcMongoClient, suite.dstMongoClient, suite.metaMongoClient} {
+	for _, client := range []*mongo.Client{suite.srcMongoClient, suite.dstMongoClient} {
 		dbNames, err := client.ListDatabaseNames(ctx, bson.D{})
-		suite.Require().NoError(err)
+		suite.Require().NoError(err, "should list database names")
 		for _, dbName := range dbNames {
 			suite.initialDbNames.Add(dbName)
 		}

From ca18c1935d0db3743ce40cc637fbb77f0ad61146 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 08:12:43 -0500
Subject: [PATCH 025/104] back to parallelization

---
 .github/workflows/all.yml | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 268ef533..6083bf15 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -41,7 +41,7 @@ jobs:
 
           # This is hard-coded by design in order to catch inadvertent changes
           # to the minimum-required Go version to build migration-verifier.
-          - '1.20'
+#          - '1.20'
           - stable
 
     runs-on: ${{matrix.os.runsOn}}
@@ -59,21 +59,6 @@ jobs:
       - name: Install mtools
         run: pipx install 'mtools[all]'
 
- #     - name: Start clusters
- #       run: |-
- #           {
- #               echo "mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}"
- #               echo "mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}"
- #               echo "mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}"
- #           } | parallel
-
-      - run: mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}
-      - run: mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}
-      - run: mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}
-
-      - name: Await source readiness
-        run: while true; do mongosh "${{ matrix.topology.srcConnStr }}" --eval 'db.hello()' && break; sleep 1; done
-
       - name: Check out repository
         uses: actions/checkout@v4
 
@@ -85,6 +70,23 @@ jobs:
       - name: Build
         run: go build main/migration_verifier.go
 
+      - name: Start clusters
+        run: |-
+            {
+                echo "mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}"
+                echo "mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}"
+                echo "mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}"
+            } | parallel
+
+#      - run: mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}
+#      - run: mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}
+#      - run: mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}
+
+      - name: Await source readiness
+        run: while true; do mongosh "${{ matrix.topology.srcConnStr }}" --eval 'db.hello()' && break; sleep 1; done
+
+
+
       - name: Test
         run: go test -v ./...
         env:

From 24230c89e6d92926fe5cb890fc091de4b6510754 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 08:16:43 -0500
Subject: [PATCH 026/104] add sharding back in

---
 .github/workflows/all.yml | 31 +++++--------------------------
 1 file changed, 5 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 6083bf15..fbeb63d5 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -31,18 +31,11 @@ jobs:
             dstConnStr: mongodb://127.0.0.1:27030,127.0.0.1:27031,127.0.0.1:27032
             metaConnStr: mongodb://127.0.0.1:27040,127.0.0.1:27041,127.0.0.1:27042
 
-#          - name: sharded
-#            args: --sharded 2
-#            srcConnStr: mongodb://127.0.0.1:27020
-#            dstConnStr: mongodb://127.0.0.1:27030
-#            metaConnStr: mongodb://127.0.0.1:27040
-
-        go_version:
-
-          # This is hard-coded by design in order to catch inadvertent changes
-          # to the minimum-required Go version to build migration-verifier.
-#          - '1.20'
-          - stable
+          - name: sharded
+            args: --sharded 2
+            srcConnStr: mongodb://127.0.0.1:27020
+            dstConnStr: mongodb://127.0.0.1:27030
+            metaConnStr: mongodb://127.0.0.1:27040
 
     runs-on: ${{matrix.os.runsOn}}
 
@@ -64,8 +57,6 @@ jobs:
 
       - name: Fetch Go ${{ matrix.go_version }}
         uses: actions/setup-go@v5
-        with:
-          go-version: ${{ matrix.go_version }}
 
       - name: Build
         run: go build main/migration_verifier.go
@@ -78,21 +69,9 @@ jobs:
                 echo "mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}"
             } | parallel
 
-#      - run: mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}
-#      - run: mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}
-#      - run: mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}
-
-      - name: Await source readiness
-        run: while true; do mongosh "${{ matrix.topology.srcConnStr }}" --eval 'db.hello()' && break; sleep 1; done
-
-
-
       - name: Test
         run: go test -v ./...
         env:
           MVTEST_SRC: ${{matrix.topology.srcConnStr}}
           MVTEST_DST: ${{matrix.topology.dstConnStr}}
           MVTEST_META: ${{matrix.topology.metaConnStr}}
-        continue-on-error: true
-
-      - run: ls -la src

From 51e6b08909450391915e7a81a52461a73d450f28 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 08:20:35 -0500
Subject: [PATCH 027/104] stable Go

---
 .github/workflows/all.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index fbeb63d5..2fc3343d 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -57,6 +57,8 @@ jobs:
 
       - name: Fetch Go ${{ matrix.go_version }}
         uses: actions/setup-go@v5
+        with:
+          go-version: stable
 
       - name: Build
         run: go build main/migration_verifier.go

From 91a09abc44af4309f4e8f1a08c3b165629c77698 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 08:30:28 -0500
Subject: [PATCH 028/104] skip trouble test

---
 internal/verifier/integration_test_suite.go  | 4 ++--
 internal/verifier/migration_verifier_test.go | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index b7754372..e75b6cd7 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -124,11 +124,11 @@ func (suite *IntegrationTestSuite) BuildVerifier() *Verifier {
 	return verifier
 }
 
-func (suite *IntegrationTestSuite) DBNameForTest() string {
+func (suite *IntegrationTestSuite) DBNameForTest(suffixes ...string) string {
 	name := suite.T().Name()
 	return strings.ReplaceAll(
 		strings.ReplaceAll(name, "/", "-"),
 		".",
 		"-",
-	)
+	) + strings.Join(suffixes, "")
 }
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index 2e84b86d..eb2fb81d 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1275,6 +1275,7 @@ func (suite *IntegrationTestSuite) TestVerificationStatus() {
 }
 
 func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
+	suite.T().Skip("TEMPORARY")
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	verifier := suite.BuildVerifier()
 	verifier.SetSrcNamespaces([]string{"testDb1.testColl1"})

From 5f45511e5598fcce332f8098fbaa4e8e466bdb0d Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 08:45:01 -0500
Subject: [PATCH 029/104] moar skip

---
 internal/verifier/migration_verifier_test.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index eb2fb81d..3e958907 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1149,6 +1149,7 @@ func TestVerifierCompareIndexSpecs(t *testing.T) {
 }
 
 func (suite *IntegrationTestSuite) TestVerifierNamespaceList() {
+	suite.T().Skip("TEMPORARY")
 	verifier := suite.BuildVerifier()
 	ctx := context.Background()
 

From 9d18e28967cb516eddfec6412960b406560b9d91 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 08:56:34 -0500
Subject: [PATCH 030/104] skip remaining

---
 internal/verifier/migration_verifier_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index 3e958907..ddd9ec67 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1385,6 +1385,7 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 }
 
 func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
+	suite.T().Skip("TEMPORARY")
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
 	filter := map[string]any{"inFilter": map[string]any{"$ne": false}}
@@ -1497,6 +1498,7 @@ func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 }
 
 func (suite *IntegrationTestSuite) TestPartitionWithFilter() {
+	suite.T().Skip("TEMPORARY")
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
 	ctx := context.Background()

From f6ce5fdcc990ef0d65f9f9f8475e5ac340ed7953 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 09:10:35 -0500
Subject: [PATCH 031/104] try to fix

---
 internal/verifier/change_stream_test.go | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index c7042618..c8b6fad0 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -161,6 +161,7 @@ func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
 	err = verifier.StartChangeStream(ctx)
 	suite.Require().NoError(err)
 	suite.Require().Equal(verifier.srcStartAtTs, origStartTs)
+
 	_, err = suite.srcMongoClient.Database("testDb").Collection("testColl").InsertOne(
 		sctx, bson.D{{"_id", 1}})
 	suite.Require().NoError(err)
@@ -175,10 +176,19 @@ func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
 	suite.Require().NoError(err)
 	newStartTs := sess.OperationTime()
 	suite.Require().NotNil(newStartTs)
-	suite.Require().Negative(origStartTs.Compare(*newStartTs))
+	suite.Require().Negative(
+		origStartTs.Compare(*newStartTs),
+		"session time after events should exceed the original",
+	)
+
 	verifier.changeStreamEnderChan <- struct{}{}
 	<-verifier.changeStreamDoneChan
-	suite.Require().Equal(verifier.srcStartAtTs, newStartTs)
+
+	suite.Assert().GreaterOrEqual(
+		verifier.srcStartAtTs.Compare(*newStartTs),
+		0,
+		"srcStartAtTs should be updated to be >= our session timestamp",
+	)
 }
 
 func (suite *IntegrationTestSuite) TestNoStartAtTime() {

From d96bb96d9110aaa32cf24c68c57fbc42bf32b276 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 09:44:24 -0500
Subject: [PATCH 032/104] tweak sharding

---
 .github/workflows/all.yml                   | 15 ++++++-----
 internal/verifier/change_stream_test.go     | 22 ++++++++++------
 internal/verifier/integration_test_suite.go | 28 ++++++++++++++++++++-
 3 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 2fc3343d..f322796c 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -27,15 +27,13 @@ jobs:
 
         topology:
           - name: replset
-            srcConnStr: mongodb://127.0.0.1:27020,127.0.0.1:27021,127.0.0.1:27022
-            dstConnStr: mongodb://127.0.0.1:27030,127.0.0.1:27031,127.0.0.1:27032
-            metaConnStr: mongodb://127.0.0.1:27040,127.0.0.1:27041,127.0.0.1:27042
+            srcConnStr: mongodb://localhost:27020,localhost:27021,localhost:27022
+            dstConnStr: mongodb://localhost:27030,localhost:27031,localhost:27032
 
           - name: sharded
             args: --sharded 2
-            srcConnStr: mongodb://127.0.0.1:27020
-            dstConnStr: mongodb://127.0.0.1:27030
-            metaConnStr: mongodb://127.0.0.1:27040
+            srcConnStr: mongodb://localhost:27020
+            dstConnStr: mongodb://localhost:27030
 
     runs-on: ${{matrix.os.runsOn}}
 
@@ -68,7 +66,7 @@ jobs:
             {
                 echo "mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}"
                 echo "mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}"
-                echo "mlaunch init --port 27040 --dir meta --replicaset ${{ matrix.topology.args }}"
+                echo "mlaunch init --port 27040 --dir meta --replicaset --nodes 1"
             } | parallel
 
       - name: Test
@@ -76,4 +74,5 @@ jobs:
         env:
           MVTEST_SRC: ${{matrix.topology.srcConnStr}}
           MVTEST_DST: ${{matrix.topology.dstConnStr}}
-          MVTEST_META: ${{matrix.topology.metaConnStr}}
+          MVTEST_META: mongodb://localhost:27040
+          MVTEST_TOPOLOGY: ${{matrix.topology.name}}
diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index c8b6fad0..650f7fad 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -147,6 +147,10 @@ func (suite *IntegrationTestSuite) TestStartAtTimeNoChanges() {
 }
 
 func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
+	if suite.GetTopology() == TopologySharded {
+		suite.T().Skip("Skipping pending REP-5299.")
+	}
+
 	verifier := suite.BuildVerifier()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -156,11 +160,12 @@ func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
 	_, err = suite.srcMongoClient.Database("testDb").Collection("testColl").InsertOne(
 		sctx, bson.D{{"_id", 0}})
 	suite.Require().NoError(err)
-	origStartTs := sess.OperationTime()
-	suite.Require().NotNil(origStartTs)
+
+	origSessionTime := sess.OperationTime()
+	suite.Require().NotNil(origSessionTime)
 	err = verifier.StartChangeStream(ctx)
 	suite.Require().NoError(err)
-	suite.Require().Equal(verifier.srcStartAtTs, origStartTs)
+	suite.Require().Equal(verifier.srcStartAtTs, origSessionTime)
 
 	_, err = suite.srcMongoClient.Database("testDb").Collection("testColl").InsertOne(
 		sctx, bson.D{{"_id", 1}})
@@ -174,10 +179,11 @@ func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
 	_, err = suite.srcMongoClient.Database("testDb").Collection("testColl").DeleteOne(
 		sctx, bson.D{{"_id", 1}})
 	suite.Require().NoError(err)
-	newStartTs := sess.OperationTime()
-	suite.Require().NotNil(newStartTs)
+
+	postEventsSessionTime := sess.OperationTime()
+	suite.Require().NotNil(postEventsSessionTime)
 	suite.Require().Negative(
-		origStartTs.Compare(*newStartTs),
+		origSessionTime.Compare(*postEventsSessionTime),
 		"session time after events should exceed the original",
 	)
 
@@ -185,9 +191,9 @@ func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
 	<-verifier.changeStreamDoneChan
 
 	suite.Assert().GreaterOrEqual(
-		verifier.srcStartAtTs.Compare(*newStartTs),
+		verifier.srcStartAtTs.Compare(*postEventsSessionTime),
 		0,
-		"srcStartAtTs should be updated to be >= our session timestamp",
+		"verifier.srcStartAtTs should now meet or exceed our session timestamp",
 	)
 }
 
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index e75b6cd7..fcd86e6b 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -2,6 +2,7 @@ package verifier
 
 import (
 	"context"
+	"os"
 	"strings"
 
 	mapset "github.com/deckarep/golang-set/v2"
@@ -12,7 +13,16 @@ import (
 	"go.mongodb.org/mongo-driver/mongo/writeconcern"
 )
 
-const metaDBName = "VERIFIER_TEST_META"
+type TestTopology string
+
+const (
+	metaDBName                   = "VERIFIER_TEST_META"
+	topologyEnvVar               = "MVTEST_TOPOLOGY"
+	TopologyReplset TestTopology = "replset"
+	TopologySharded TestTopology = "sharded"
+)
+
+var knownTopologies = []TestTopology{TopologyReplset, TopologySharded}
 
 type IntegrationTestSuite struct {
 	suite.Suite
@@ -91,6 +101,22 @@ func (suite *IntegrationTestSuite) TearDownTest() {
 	}
 }
 
+func (suite *IntegrationTestSuite) GetTopology() TestTopology {
+	rawTopology, found := os.LookupEnv(topologyEnvVar)
+
+	suite.Require().True(found, "Environment must contain %#q.", topologyEnvVar)
+
+	topology := TestTopology(rawTopology)
+
+	suite.Require().Contains(
+		knownTopologies,
+		topology,
+		"%#q must be a known value.",
+	)
+
+	return topology
+}
+
 func (suite *IntegrationTestSuite) BuildVerifier() *Verifier {
 	qfilter := QueryFilter{Namespace: "keyhole.dealers"}
 	task := VerificationTask{QueryFilter: qfilter}

From c78e061c07978c84413faef55c6028f12ddee00b Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 09:48:00 -0500
Subject: [PATCH 033/104] =?UTF-8?q?don=E2=80=99t=20drop=20metadata=20DBs?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 internal/verifier/integration_test_suite.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index fcd86e6b..e4b9933b 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -87,7 +87,7 @@ func (suite *IntegrationTestSuite) TearDownTest() {
 	suite.T().Logf("Tearing down test %#q", suite.T().Name())
 
 	ctx := context.Background()
-	for _, client := range []*mongo.Client{suite.srcMongoClient, suite.dstMongoClient, suite.metaMongoClient} {
+	for _, client := range []*mongo.Client{suite.srcMongoClient, suite.dstMongoClient} {
 		dbNames, err := client.ListDatabaseNames(ctx, bson.D{})
 		suite.Require().NoError(err)
 		for _, dbName := range dbNames {

From d1ff936f3561793e6b3b88110dd8c202a940da4a Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 09:52:49 -0500
Subject: [PATCH 034/104] diagnose test failure

---
 .github/workflows/all.yml                    | 20 ++++++++++----------
 internal/verifier/migration_verifier_test.go |  2 +-
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index f322796c..152dadff 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -12,23 +12,23 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - runsOn: macos-latest
-            setup: brew install parallel
+#          - runsOn: macos-latest
+#            setup: brew install parallel
 
           - runsOn: ubuntu-latest
 
         version:
-          - '4.2'
-          - '4.4'
-          - '5.0'
-          - '6.0'
-          - '7.0'
+#          - '4.2'
+#          - '4.4'
+#          - '5.0'
+#          - '6.0'
+#          - '7.0'
           - '8.0'
 
         topology:
-          - name: replset
-            srcConnStr: mongodb://localhost:27020,localhost:27021,localhost:27022
-            dstConnStr: mongodb://localhost:27030,localhost:27031,localhost:27032
+#          - name: replset
+#            srcConnStr: mongodb://localhost:27020,localhost:27021,localhost:27022
+#            dstConnStr: mongodb://localhost:27030,localhost:27031,localhost:27032
 
           - name: sharded
             args: --sharded 2
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index ddd9ec67..fccfb538 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1498,7 +1498,7 @@ func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 }
 
 func (suite *IntegrationTestSuite) TestPartitionWithFilter() {
-	suite.T().Skip("TEMPORARY")
+
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
 	ctx := context.Background()

From 2025c88532bc1c7cf8ade2b5c11bb408b0c64443 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 09:53:20 -0500
Subject: [PATCH 035/104] TestPartitionWithFilter

---
 .github/workflows/all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 152dadff..bf02bf1f 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -70,7 +70,7 @@ jobs:
             } | parallel
 
       - name: Test
-        run: go test -v ./...
+        run: go test -v ./... -run TestIntegration -testify.m TestPartitionWithFilter
         env:
           MVTEST_SRC: ${{matrix.topology.srcConnStr}}
           MVTEST_DST: ${{matrix.topology.dstConnStr}}

From 84e0c94e935b6bb5820709b7873b82d04a60005f Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 09:54:11 -0500
Subject: [PATCH 036/104] run the rest

---
 .github/workflows/all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index bf02bf1f..152dadff 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -70,7 +70,7 @@ jobs:
             } | parallel
 
       - name: Test
-        run: go test -v ./... -run TestIntegration -testify.m TestPartitionWithFilter
+        run: go test -v ./...
         env:
           MVTEST_SRC: ${{matrix.topology.srcConnStr}}
           MVTEST_DST: ${{matrix.topology.dstConnStr}}

From c18ca676154e804471db0707ae920eb59751b0e8 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 09:56:59 -0500
Subject: [PATCH 037/104] put back another

---
 internal/verifier/migration_verifier_test.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index fccfb538..3e958907 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1385,7 +1385,6 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 }
 
 func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
-	suite.T().Skip("TEMPORARY")
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
 	filter := map[string]any{"inFilter": map[string]any{"$ne": false}}
@@ -1498,7 +1497,6 @@ func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 }
 
 func (suite *IntegrationTestSuite) TestPartitionWithFilter() {
-
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
 	ctx := context.Background()

From aee9de7e70fbeb39fa7b72b1ba808bb20efce879 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 10:10:51 -0500
Subject: [PATCH 038/104] cancel contexts during test teardown

---
 internal/verifier/integration_test_suite.go  | 21 ++++++++++++--
 internal/verifier/migration_verifier_test.go | 30 ++++++++++++--------
 2 files changed, 37 insertions(+), 14 deletions(-)

diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index e4b9933b..404c813e 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	mapset "github.com/deckarep/golang-set/v2"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/suite"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -28,11 +29,22 @@ type IntegrationTestSuite struct {
 	suite.Suite
 	srcConnStr, dstConnStr, metaConnStr             string
 	srcMongoClient, dstMongoClient, metaMongoClient *mongo.Client
+	testContext                                     context.Context
+	contextCanceller                                context.CancelCauseFunc
 	initialDbNames                                  mapset.Set[string]
 }
 
 var _ suite.TestingSuite = &IntegrationTestSuite{}
 
+func (suite *IntegrationTestSuite) Context() context.Context {
+	suite.Require().NotNil(
+		suite.testContext,
+		"context must exist (i.e., be fetched only within a test)",
+	)
+
+	return suite.testContext
+}
+
 func (suite *IntegrationTestSuite) SetupSuite() {
 	ctx := context.Background()
 	clientOpts := options.Client().ApplyURI(suite.srcConnStr).SetAppName("Verifier Test Suite").SetWriteConcern(writeconcern.Majority())
@@ -60,7 +72,7 @@ func (suite *IntegrationTestSuite) SetupSuite() {
 }
 
 func (suite *IntegrationTestSuite) SetupTest() {
-	ctx := context.Background()
+	ctx, canceller := context.WithCancelCause(context.Background())
 
 	dbname := suite.DBNameForTest()
 
@@ -81,11 +93,16 @@ func (suite *IntegrationTestSuite) SetupTest() {
 		"should drop destination db %#q",
 		dbname,
 	)
+
+	suite.testContext, suite.contextCanceller = ctx, canceller
 }
 
 func (suite *IntegrationTestSuite) TearDownTest() {
 	suite.T().Logf("Tearing down test %#q", suite.T().Name())
 
+	suite.contextCanceller(errors.Errorf("tearing down test %#q", suite.T().Name()))
+	suite.testContext, suite.contextCanceller = nil, nil
+
 	ctx := context.Background()
 	for _, client := range []*mongo.Client{suite.srcMongoClient, suite.dstMongoClient} {
 		dbNames, err := client.ListDatabaseNames(ctx, bson.D{})
@@ -127,7 +144,7 @@ func (suite *IntegrationTestSuite) BuildVerifier() *Verifier {
 	verifier.SetGenerationPauseDelayMillis(0)
 	verifier.SetWorkerSleepDelayMillis(0)
 
-	ctx := context.Background()
+	ctx := suite.Context()
 
 	suite.Require().NoError(
 		verifier.SetSrcURI(ctx, suite.srcConnStr),
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index 3e958907..b0ebc92e 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -54,7 +54,7 @@ func TestIntegration(t *testing.T) {
 
 func (suite *IntegrationTestSuite) TestVerifierFetchDocuments() {
 	verifier := suite.BuildVerifier()
-	ctx := context.Background()
+	ctx := suite.Context()
 	drop := func() {
 		err := verifier.srcClient.Database("keyhole").Drop(ctx)
 		suite.Require().NoError(err)
@@ -138,7 +138,7 @@ func (suite *IntegrationTestSuite) TestVerifierFetchDocuments() {
 }
 
 func (suite *IntegrationTestSuite) TestGetNamespaceStatistics_Recheck() {
-	ctx := context.Background()
+	ctx := suite.Context()
 	verifier := suite.BuildVerifier()
 
 	err := verifier.HandleChangeStreamEvents(
@@ -198,7 +198,7 @@ func (suite *IntegrationTestSuite) TestGetNamespaceStatistics_Recheck() {
 }
 
 func (suite *IntegrationTestSuite) TestGetNamespaceStatistics_Gen0() {
-	ctx := context.Background()
+	ctx := suite.Context()
 	verifier := suite.BuildVerifier()
 
 	stats, err := verifier.GetNamespaceStatistics(ctx)
@@ -389,7 +389,7 @@ func (suite *IntegrationTestSuite) TestGetNamespaceStatistics_Gen0() {
 }
 
 func (suite *IntegrationTestSuite) TestFailedVerificationTaskInsertions() {
-	ctx := context.Background()
+	ctx := suite.Context()
 	verifier := suite.BuildVerifier()
 	err := verifier.InsertFailedCompareRecheckDocs("foo.bar", []interface{}{42}, []int{100})
 	suite.Require().NoError(err)
@@ -657,7 +657,7 @@ func TestVerifierCompareDocs(t *testing.T) {
 
 func (suite *IntegrationTestSuite) TestVerifierCompareViews() {
 	verifier := suite.BuildVerifier()
-	ctx := context.Background()
+	ctx := suite.Context()
 
 	err := suite.srcMongoClient.Database("testDb").CreateView(ctx, "sameView", "testColl", bson.A{bson.D{{"$project", bson.D{{"_id", 1}}}}})
 	suite.Require().NoError(err)
@@ -770,7 +770,7 @@ func (suite *IntegrationTestSuite) TestVerifierCompareViews() {
 
 func (suite *IntegrationTestSuite) TestVerifierCompareMetadata() {
 	verifier := suite.BuildVerifier()
-	ctx := context.Background()
+	ctx := suite.Context()
 
 	// Collection exists only on source.
 	err := suite.srcMongoClient.Database("testDb").CreateCollection(ctx, "testColl")
@@ -876,7 +876,7 @@ func (suite *IntegrationTestSuite) TestVerifierCompareMetadata() {
 
 func (suite *IntegrationTestSuite) TestVerifierCompareIndexes() {
 	verifier := suite.BuildVerifier()
-	ctx := context.Background()
+	ctx := suite.Context()
 
 	// Missing index on destination.
 	err := suite.srcMongoClient.Database("testDb").CreateCollection(ctx, "testColl1")
@@ -1149,9 +1149,12 @@ func TestVerifierCompareIndexSpecs(t *testing.T) {
 }
 
 func (suite *IntegrationTestSuite) TestVerifierNamespaceList() {
-	suite.T().Skip("TEMPORARY")
+	if suite.GetTopology() == TopologySharded {
+		suite.T().Skip("Skipping pending REP-5299.")
+	}
+
 	verifier := suite.BuildVerifier()
-	ctx := context.Background()
+	ctx := suite.Context()
 
 	// Collections on source only
 	err := suite.srcMongoClient.Database("testDb1").CreateCollection(ctx, "testColl1")
@@ -1276,7 +1279,10 @@ func (suite *IntegrationTestSuite) TestVerificationStatus() {
 }
 
 func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
-	suite.T().Skip("TEMPORARY")
+	if suite.GetTopology() == TopologySharded {
+		suite.T().Skip("Skipping pending REP-5299.")
+	}
+
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	verifier := suite.BuildVerifier()
 	verifier.SetSrcNamespaces([]string{"testDb1.testColl1"})
@@ -1396,7 +1402,7 @@ func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 	// Set this value low to test the verifier with multiple partitions.
 	verifier.partitionSizeInBytes = 50
 
-	ctx := context.Background()
+	ctx := suite.Context()
 
 	srcColl := suite.srcMongoClient.Database("testDb1").Collection("testColl1")
 	dstColl := suite.dstMongoClient.Database("testDb2").Collection("testColl3")
@@ -1499,7 +1505,7 @@ func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 func (suite *IntegrationTestSuite) TestPartitionWithFilter() {
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
-	ctx := context.Background()
+	ctx := suite.Context()
 
 	// Make a filter that filters on field "n".
 	filter := map[string]any{"$expr": map[string]any{"$and": []map[string]any{

From 9ca54ba4718f5d60cb0e82d294c342ecdf3257f7 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 10:12:04 -0500
Subject: [PATCH 039/104] godoc

---
 internal/verifier/integration_test_suite.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index 404c813e..f6517255 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -36,6 +36,8 @@ type IntegrationTestSuite struct {
 
 var _ suite.TestingSuite = &IntegrationTestSuite{}
 
+// Context returns a Context that the suite will cancel after the test.
+// Always use this rather than context.Background() in tests!
 func (suite *IntegrationTestSuite) Context() context.Context {
 	suite.Require().NotNil(
 		suite.testContext,

From 26adccc44a4f7de2ee9c796e49840f2f2c0b784d Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 10:14:11 -0500
Subject: [PATCH 040/104] remove old provisioning logic

---
 internal/verifier/unit_test_util.go | 454 ----------------------------
 1 file changed, 454 deletions(-)
 delete mode 100644 internal/verifier/unit_test_util.go

diff --git a/internal/verifier/unit_test_util.go b/internal/verifier/unit_test_util.go
deleted file mode 100644
index 806bc3ff..00000000
--- a/internal/verifier/unit_test_util.go
+++ /dev/null
@@ -1,454 +0,0 @@
-package verifier
-
-import (
-	"archive/tar"
-	"compress/gzip"
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"regexp"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"syscall"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/suite"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/mongo"
-	"go.mongodb.org/mongo-driver/mongo/options"
-	"go.mongodb.org/mongo-driver/mongo/writeconcern"
-)
-
-const (
-	replSet              = "rs0"
-	distroEnv            = "MONGODB_DISTRO"
-	currentDownloadsLink = "https://downloads.mongodb.org/current.json"
-)
-
-type MongoInstance struct {
-	port    string
-	version string
-	process *os.Process
-}
-
-// SetPort sets the MongoInstance’s port so that clients can connect
-// to it.
-func (mi *MongoInstance) SetPort(port uint64) {
-	mi.port = strconv.FormatUint(port, 10)
-}
-
-// SetProcess sets the MongoInstance’s process so that we can terminate
-// the process once we’re done with it.
-func (mi *MongoInstance) SetProcess(process *os.Process) {
-	mi.process = process
-}
-
-type WithMongodsTestingSuite interface {
-	suite.TestingSuite
-	SetSrcInstance(MongoInstance)
-	SetDstInstance(MongoInstance)
-	SetMetaInstance(MongoInstance)
-}
-
-type WithMongodsTestSuite struct {
-	suite.Suite
-	srcMongoInstance, dstMongoInstance, metaMongoInstance MongoInstance
-	srcMongoClient, dstMongoClient, metaMongoClient       *mongo.Client
-	initialDbNames                                        map[string]bool
-}
-
-func (suite *WithMongodsTestSuite) SetSrcInstance(instance MongoInstance) {
-	suite.srcMongoInstance = instance
-}
-
-func (suite *WithMongodsTestSuite) SetDstInstance(instance MongoInstance) {
-	suite.dstMongoInstance = instance
-}
-
-func (suite *WithMongodsTestSuite) SetMetaInstance(instance MongoInstance) {
-	suite.metaMongoInstance = instance
-}
-
-func (suite *WithMongodsTestSuite) SetupSuite() {
-	if testing.Short() {
-		suite.T().Skip("Skipping mongod-requiring tests in short mode")
-	}
-	err := startTestMongods(suite.T(), &suite.srcMongoInstance, &suite.dstMongoInstance, &suite.metaMongoInstance)
-	suite.Require().NoError(err)
-	ctx := context.Background()
-	clientOpts := options.Client().ApplyURI("mongodb://localhost:" + suite.srcMongoInstance.port).SetAppName("Verifier Test Suite").SetWriteConcern(writeconcern.Majority())
-	suite.srcMongoClient, err = mongo.Connect(ctx, clientOpts)
-	suite.Require().NoError(err)
-	clientOpts = options.Client().ApplyURI("mongodb://localhost:" + suite.dstMongoInstance.port).SetAppName("Verifier Test Suite").SetWriteConcern(writeconcern.Majority())
-	suite.dstMongoClient, err = mongo.Connect(ctx, clientOpts)
-	suite.Require().NoError(err)
-	clientOpts = options.Client().ApplyURI("mongodb://localhost:" + suite.metaMongoInstance.port).SetAppName("Verifier Test Suite")
-	suite.metaMongoClient, err = mongo.Connect(ctx, clientOpts)
-	suite.startReplSets()
-	suite.Require().NoError(err)
-	suite.initialDbNames = map[string]bool{}
-	for _, client := range []*mongo.Client{suite.srcMongoClient, suite.dstMongoClient, suite.metaMongoClient} {
-		dbNames, err := client.ListDatabaseNames(ctx, bson.D{})
-		suite.Require().NoError(err)
-		for _, dbName := range dbNames {
-			suite.initialDbNames[dbName] = true
-		}
-	}
-}
-
-func (suite *WithMongodsTestSuite) startReplSets() {
-	ctx := context.Background()
-
-	for _, instance := range []MongoInstance{suite.srcMongoInstance, suite.metaMongoInstance} {
-		clientOpts := options.
-			Client().
-			ApplyURI("mongodb://localhost:" + instance.port).
-			SetDirect(true).
-			SetAppName("Verifier Test Suite")
-		directClient, err := mongo.Connect(ctx, clientOpts)
-		suite.Require().NoError(err)
-		command := bson.M{
-			"replSetInitiate": bson.M{
-				"_id": replSet,
-				"members": bson.A{
-					bson.M{"_id": 0, "host": "localhost:" + instance.port},
-				},
-			},
-		}
-		err = directClient.Database("admin").RunCommand(ctx, command).Err()
-		suite.Require().NoError(err, "should initiate replication")
-	}
-}
-
-func (suite *WithMongodsTestSuite) TearDownSuite() {
-	suite.T().Log("Shutting down mongod instances …")
-
-	instances := []*MongoInstance{
-		&suite.srcMongoInstance,
-		&suite.dstMongoInstance,
-		&suite.metaMongoInstance,
-	}
-
-	theSignal := syscall.SIGTERM
-
-	for _, instance := range instances {
-		proc := instance.process
-
-		if proc != nil {
-			pid := instance.process.Pid
-			suite.T().Logf("Sending SIGTERM to process %d", pid)
-			err := instance.process.Signal(theSignal)
-			if err != nil {
-				suite.T().Logf("Failed to signal process %d: %v", pid, err)
-			}
-		}
-	}
-}
-
-func (suite *WithMongodsTestSuite) TearDownTest() {
-	suite.T().Logf("Tearing down test %#q", suite.T().Name())
-
-	ctx := context.Background()
-	for _, client := range []*mongo.Client{suite.srcMongoClient, suite.dstMongoClient, suite.metaMongoClient} {
-		dbNames, err := client.ListDatabaseNames(ctx, bson.D{})
-		suite.Require().NoError(err)
-		for _, dbName := range dbNames {
-			if !suite.initialDbNames[dbName] {
-				suite.T().Logf("Dropping database %#q, which seems to have been created during test %#q.", dbName, suite.T().Name())
-
-				err = client.Database(dbName).Drop(ctx)
-				suite.Require().NoError(err)
-			}
-		}
-	}
-}
-
-var cachePath = filepath.Join("mongodb_exec")
-var mongoDownloadMutex sync.Mutex
-
-func startTestMongods(t *testing.T, srcMongoInstance *MongoInstance, dstMongoInstance *MongoInstance, metaMongoInstance *MongoInstance) error {
-
-	// Ideally we’d start the mongods in parallel, but in development that
-	// seemed to cause mongod to break on `--port 0`.
-	start := time.Now()
-	err := startOneMongod(t, srcMongoInstance, "--replSet", replSet)
-	if err != nil {
-		return err
-	}
-
-	err = startOneMongod(t, dstMongoInstance)
-	if err != nil {
-		return err
-	}
-
-	err = startOneMongod(t, metaMongoInstance, "--replSet", replSet)
-	if err != nil {
-		return err
-	}
-
-	t.Logf("Time elapsed creating mongod instances: %v", time.Since(start))
-
-	return nil
-}
-
-func logpath(target string) string {
-	return filepath.Join(target, "log")
-}
-
-// startOneMongod execs `path` with `extraArgs`. This mongod binds to
-// “port 0”, which causes the OS to pick an arbitrary free port. It also
-// creates a temporary directory for the mongod to do its work. Thus
-// we avoid potential race conditions and interference between test
-// runs & suites.
-//
-// The returns are the process, its listening TCP port, its dbpath,
-// and whatever error may have happened.
-func startOneMongod(t *testing.T, instance *MongoInstance, extraArgs ...string) error {
-
-	// MongoDB 5.0+ writes its logs in line-delimited JSON;
-	// older versions write free text.
-	portRegexpJson := regexp.MustCompile(`"port":([1-9][0-9]*)`)
-	portRegexp := regexp.MustCompile(`port\s([1-9][0-9]*)`)
-
-	mongodPath, err := getMongod(t, *instance)
-	if err != nil {
-		return err
-	}
-
-	dir, err := os.MkdirTemp("", "*")
-	if err != nil {
-		return err
-	}
-
-	lpath := logpath(dir)
-
-	cmdargs := []string{
-		"--port", "0",
-		"--dbpath", dir,
-		"--logpath", lpath,
-	}
-
-	cmdargs = append(cmdargs, extraArgs...)
-
-	t.Logf("Starting mongod: %s %v", mongodPath, cmdargs)
-
-	cmd := exec.Command(mongodPath, cmdargs...)
-
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-
-	if err := cmd.Start(); err != nil {
-		return err
-	}
-
-	instance.SetProcess(cmd.Process)
-
-	pid := cmd.Process.Pid
-
-	var finished atomic.Bool
-
-	go func() {
-		err := cmd.Wait()
-		finished.Store(true)
-
-		// Use Printf rather than t.Logf below in order to prevent
-		// the race checker from complaining.
-
-		if err == nil {
-			fmt.Printf("mongod process %d ended successfully\n", pid)
-		} else {
-			fmt.Printf("mongod process %d: %v\n", pid, err)
-		}
-
-		err = os.RemoveAll(dir)
-		if err != nil {
-			fmt.Printf("Failed to remove %s: %v\n", dir, err)
-		}
-	}()
-
-	duration := time.Minute * 5
-	endAt := time.Now().Add(duration)
-
-	for time.Now().Before(endAt) {
-		if finished.Load() {
-			return fmt.Errorf("mongod process %d ended without storing its listening port in %s", pid, lpath)
-		}
-
-		content, err := os.ReadFile(lpath)
-
-		if err == nil {
-			match := portRegexpJson.FindSubmatch(content)
-			if match == nil {
-				match = portRegexp.FindSubmatch(content)
-			}
-
-			if match != nil {
-				port, _ := strconv.ParseUint(string(match[1]), 10, 16)
-				instance.SetPort(port)
-				return nil
-			}
-		} else if os.IsNotExist(err) {
-			// The log file isn’t created (yet?); loop again.
-		} else {
-			return fmt.Errorf("Unexpected error while reading logfile %s: %v", lpath, err)
-		}
-
-		time.Sleep(time.Millisecond * 100)
-	}
-
-	return fmt.Errorf("Timed out (%v) waiting to find mongod process %d’s listening port in %s", duration, pid, lpath)
-}
-
-var osUrlDir = map[string]string{
-	"linux": "linux",
-	"macos": "osx",
-}
-
-func getMongoDBDirFromDistro(ourOs, localMongoDistro string) string {
-	mongoDBDir := localMongoDistro
-
-	switch ourOs {
-	case "linux":
-	case "macos":
-		if strings.Contains(localMongoDistro, "arm64") {
-			mongoDBDir = strings.Replace(mongoDBDir, "arm64", "aarch64", 1)
-		}
-	default:
-		panic("Unknown OS: " + ourOs)
-	}
-
-	return filepath.Join(cachePath, mongoDBDir)
-}
-
-// We could use runtime.GOOS and runtime.GOARCH, but since we need the
-// distro anyway to have a download URL we might as well derive the
-// OS and arch from that, too.
-func getOSAndArchFromEnv(t *testing.T) (string, string) {
-	localMongoDistro := os.Getenv(distroEnv)
-	if localMongoDistro == "" {
-		t.Fatalf(`Please set %s in the environment.
-
-Example values:
-- mongodb-linux-x86_64-rhel70
-- mongodb-linux-x86_64-ubuntu1804
-- mongodb-macos-arm64
-
-Links can be found at: %s`,
-			distroEnv, currentDownloadsLink)
-	}
-
-	re := regexp.MustCompile(`^mongodb-([^-]+)-([^-]+)`)
-	pieces := re.FindStringSubmatch(localMongoDistro)
-
-	if pieces == nil {
-		t.Fatalf("Unexpected %s; expected %v", localMongoDistro, re)
-	}
-
-	return pieces[1], pieces[2]
-}
-
-func getMongod(t *testing.T, mongoInstance MongoInstance) (string, error) {
-	ourOs, _ := getOSAndArchFromEnv(t)
-
-	localMongoDistro := os.Getenv(distroEnv) + "-" + mongoInstance.version
-
-	mongoDBDir := getMongoDBDirFromDistro(ourOs, localMongoDistro)
-	mongod := filepath.Join(mongoDBDir, "bin", "mongod")
-	mongoDownloadMutex.Lock()
-	defer mongoDownloadMutex.Unlock()
-	if _, err := os.Stat(mongoDBDir); os.IsNotExist(err) {
-		url := fmt.Sprintf("https://fastdl.mongodb.org/%s/%s.tgz", osUrlDir[ourOs], localMongoDistro)
-		t.Logf("Downloading %s", url)
-		r, err := curl(url)
-		if err != nil {
-			return "", err
-		}
-		defer r.Close()
-		return mongod, untar(r, cachePath)
-	}
-	return mongod, nil
-}
-
-func curl(url string) (io.ReadCloser, error) {
-	resp, err := http.Get(url)
-	if err != nil {
-		return nil, err
-	}
-
-	// Ensure that the HTTP response was a 2xx.
-	if resp.StatusCode/200 != 1 {
-		return nil, fmt.Errorf("HTTP %s (%s)", resp.Status, url)
-	}
-
-	return resp.Body, nil
-}
-
-func untar(r io.Reader, dst string) error {
-	gzr, err := gzip.NewReader(r)
-	if err != nil {
-		return err
-	}
-	defer gzr.Close()
-
-	tr := tar.NewReader(gzr)
-
-	for {
-		header, err := tr.Next()
-
-		switch {
-
-		// if no more files are found return
-		case err == io.EOF:
-			return nil
-
-		// return any other error
-		case err != nil:
-			return err
-
-		// if the header is nil, just skip it (not sure how this happens)
-		case header == nil:
-			continue
-		}
-
-		// the target location where the dir/file should be created
-		target := filepath.Join(dst, header.Name)
-
-		// check the file type
-		switch header.Typeflag {
-
-		// if its a dir and it doesn't exist create it
-		case tar.TypeDir:
-			if _, err := os.Stat(target); err != nil {
-				if err := os.MkdirAll(target, 0755); err != nil {
-					return err
-				}
-			}
-
-		// if it's a file create it, but make sure to create the parent
-		case tar.TypeReg:
-			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
-				return err
-			}
-			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
-			if err != nil {
-				return err
-			}
-
-			// copy over contents
-			if _, err := io.Copy(f, tr); err != nil {
-				return err
-			}
-
-			// manually close here after each file operation; defering would cause each file close
-			// to wait until all operations have completed.
-			f.Close()
-		}
-	}
-}

From 311fde724326b58f5bae49b49ccb882a29d8d454 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 10:15:40 -0500
Subject: [PATCH 041/104] skipit

---
 internal/verifier/migration_verifier_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index b0ebc92e..d766f098 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1391,6 +1391,10 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 }
 
 func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
+	if suite.GetTopology() == TopologySharded {
+		suite.T().Skip("Skipping pending REP-5299.")
+	}
+
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
 	filter := map[string]any{"inFilter": map[string]any{"$ne": false}}

From ad44fc1fca6e9667c23ab77b72ca4a3a3792c572 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 10:16:56 -0500
Subject: [PATCH 042/104] restore tests

---
 .github/workflows/all.yml | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 152dadff..138d6620 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -11,31 +11,26 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os:
-#          - runsOn: macos-latest
-#            setup: brew install parallel
-
-          - runsOn: ubuntu-latest
-
         version:
-#          - '4.2'
-#          - '4.4'
-#          - '5.0'
-#          - '6.0'
-#          - '7.0'
+          - '4.2'
+          - '4.4'
+          - '5.0'
+          - '6.0'
+          - '7.0'
           - '8.0'
 
         topology:
-#          - name: replset
-#            srcConnStr: mongodb://localhost:27020,localhost:27021,localhost:27022
-#            dstConnStr: mongodb://localhost:27030,localhost:27031,localhost:27032
+          - name: replset
+            srcConnStr: mongodb://localhost:27020,localhost:27021,localhost:27022
+            dstConnStr: mongodb://localhost:27030,localhost:27031,localhost:27032
 
           - name: sharded
             args: --sharded 2
             srcConnStr: mongodb://localhost:27020
             dstConnStr: mongodb://localhost:27030
 
-    runs-on: ${{matrix.os.runsOn}}
+    # There seems no good reason to test on other OSes … ?
+    runs-on: ubuntu-latest
 
     steps:
       - name: Install packages

From 35c34157af4a5d9dc7274166fdaea23013e7ae1e Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 10:19:54 -0500
Subject: [PATCH 043/104] change variable name

---
 .github/workflows/all.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 138d6620..40dfa8c4 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version:
+        mongodb_version:
           - '4.2'
           - '4.4'
           - '5.0'
@@ -39,8 +39,8 @@ jobs:
       - name: Install m
         run: npm install -g m mongosh
 
-      - name: Activate MongoDB ${{ matrix.version }}
-        run: yes | m ${{ matrix.version }}
+      - name: Activate MongoDB ${{ matrix.mongodb_version }}
+        run: yes | m ${{ matrix.mongodb_version }}
 
       - name: Install mtools
         run: pipx install 'mtools[all]'

From 4a1945e88a28c757d2d2e163816754c2b0734e85 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 10:30:14 -0500
Subject: [PATCH 044/104] revert & tweak

---
 internal/verifier/change_stream.go | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 1701682a..4ca5d67a 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -324,7 +324,7 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 }
 
 func addUnixTimeToLogEvent[T constraints.Integer](unixTime T, event *zerolog.Event) *zerolog.Event {
-	return event.Time("clockTime", time.Unix(int64(unixTime), int64(0)))
+	return event.Time("timestamp", time.Unix(int64(unixTime), int64(0)))
 }
 
 func (v *Verifier) getChangeStreamMetadataCollection() *mongo.Collection {
@@ -333,10 +333,6 @@ func (v *Verifier) getChangeStreamMetadataCollection() *mongo.Collection {
 
 func (verifier *Verifier) loadChangeStreamResumeToken(ctx context.Context) (bson.Raw, error) {
 	coll := verifier.getChangeStreamMetadataCollection()
-	verifier.logger.Debug().
-		Str("db", coll.Database().Name()).
-		Str("coll", coll.Name()).
-		Msg("Seeking persisted resume token.")
 
 	token, err := coll.FindOne(
 		ctx,
@@ -364,8 +360,7 @@ func (verifier *Verifier) persistChangeStreamResumeToken(ctx context.Context, cs
 	if err == nil {
 		ts, err := extractTimestampFromResumeToken(token)
 
-		logEvent := verifier.logger.Debug().
-			Interface("token", token)
+		logEvent := verifier.logger.Debug()
 
 		if err == nil {
 			logEvent = addUnixTimeToLogEvent(ts.T, logEvent)

From b12a8d2477db9ad4f0dc799f24427b6e3a970e98 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 10:30:55 -0500
Subject: [PATCH 045/104] timestampTime

---
 internal/verifier/change_stream.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 4ca5d67a..ddf8f5ba 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -324,7 +324,7 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 }
 
 func addUnixTimeToLogEvent[T constraints.Integer](unixTime T, event *zerolog.Event) *zerolog.Event {
-	return event.Time("timestamp", time.Unix(int64(unixTime), int64(0)))
+	return event.Time("timestampTime", time.Unix(int64(unixTime), int64(0)))
 }
 
 func (v *Verifier) getChangeStreamMetadataCollection() *mongo.Collection {

From 2304fca6e3df099edc4e7de416d70d249385ef4e Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 11:08:13 -0500
Subject: [PATCH 046/104] refactor build infio

---
 .github/workflows/all.yml                   |  1 -
 internal/partitions/partition.go            | 15 +---
 internal/partitions/partition_test.go       | 19 ++---
 internal/util/buildinfo.go                  | 36 +++++++++
 internal/verifier/integration_test_suite.go | 18 ++---
 internal/verifier/migration_verifier.go     | 44 +++++------
 mbson/bson_raw.go                           | 29 +++++++
 mbson/unit_test.go                          | 88 +++++++++++++++++++++
 8 files changed, 191 insertions(+), 59 deletions(-)
 create mode 100644 internal/util/buildinfo.go
 create mode 100644 mbson/bson_raw.go
 create mode 100644 mbson/unit_test.go

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 40dfa8c4..a5a49933 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -70,4 +70,3 @@ jobs:
           MVTEST_SRC: ${{matrix.topology.srcConnStr}}
           MVTEST_DST: ${{matrix.topology.dstConnStr}}
           MVTEST_META: mongodb://localhost:27040
-          MVTEST_TOPOLOGY: ${{matrix.topology.name}}
diff --git a/internal/partitions/partition.go b/internal/partitions/partition.go
index 5c9d507b..024458e8 100644
--- a/internal/partitions/partition.go
+++ b/internal/partitions/partition.go
@@ -137,7 +137,7 @@ func (p *Partition) FindCmd(
 // (e.g. use the partitions on the source to read the destination for verification)
 // If the passed-in buildinfo indicates a mongodb version < 5.0, type bracketing is not used.
 // filterAndPredicates is a slice of filter criteria that's used to construct the "filter" field in the find option.
-func (p *Partition) GetFindOptions(buildInfo *bson.M, filterAndPredicates bson.A) bson.D {
+func (p *Partition) GetFindOptions(buildInfo *util.BuildInfo, filterAndPredicates bson.A) bson.D {
 	if p == nil {
 		if len(filterAndPredicates) > 0 {
 			return bson.D{{"filter", bson.D{{"$and", filterAndPredicates}}}}
@@ -160,16 +160,9 @@ func (p *Partition) GetFindOptions(buildInfo *bson.M, filterAndPredicates bson.A
 		allowTypeBracketing := false
 		if buildInfo != nil {
 			allowTypeBracketing = true
-			versionArray, ok := (*buildInfo)["versionArray"].(bson.A)
-			//bson values are int32 or int64, never int.
-			if ok {
-				majorVersion, ok := versionArray[0].(int32)
-				if ok {
-					allowTypeBracketing = majorVersion < 5
-				} else {
-					majorVersion64, _ := versionArray[0].(int64)
-					allowTypeBracketing = majorVersion64 < 5
-				}
+
+			if buildInfo.VersionArray != nil {
+				allowTypeBracketing = buildInfo.VersionArray[0] < 5
 			}
 		}
 		if !allowTypeBracketing {
diff --git a/internal/partitions/partition_test.go b/internal/partitions/partition_test.go
index e0cae05c..53d41a0a 100644
--- a/internal/partitions/partition_test.go
+++ b/internal/partitions/partition_test.go
@@ -77,43 +77,38 @@ func (suite *UnitTestSuite) TestVersioning() {
 	filter := getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilter, filter)
 
-	// 6.0 (int64)
-	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int64(6), int64(0), int64(0), int64(0)}}, nil)
-	filter = getFilterFromFindOptions(findOptions)
-	suite.Require().Equal(expectedFilter, filter)
-
 	// 6.0
-	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int32(6), int32(0), int32(0), int32(0)}}, nil)
+	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{6, 0, 0}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilter, filter)
 
 	// 5.3.0.9
-	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int32(5), int32(3), int32(0), int32(9)}}, nil)
+	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{5, 3, 0, 9}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilter, filter)
 
 	// 7.1.3.5
-	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int32(7), int32(1), int32(3), int32(5)}}, nil)
+	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{7, 1, 3, 5}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilter, filter)
 
 	// 4.4 (int64)
-	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int64(4), int64(4), int64(0), int64(0)}}, nil)
+	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{4, 4, 0, 0}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilterWithTypeBracketing, filter)
 
 	// 4.4
-	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int32(4), int32(4), int32(0), int32(0)}}, nil)
+	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{4, 4, 0, 0}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilterWithTypeBracketing, filter)
 
 	// 4.2
-	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int32(4), int32(2), int32(0), int32(0)}}, nil)
+	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{4, 2, 0, 0}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilterWithTypeBracketing, filter)
 
 	// No version array -- assume old, require type bracketing.
-	findOptions = partition.GetFindOptions(&bson.M{"notVersionArray": bson.A{6, int32(0), int32(0), int32(0)}}, nil)
+	findOptions = partition.GetFindOptions(&util.BuildInfo{}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilterWithTypeBracketing, filter)
 }
diff --git a/internal/util/buildinfo.go b/internal/util/buildinfo.go
new file mode 100644
index 00000000..ce441961
--- /dev/null
+++ b/internal/util/buildinfo.go
@@ -0,0 +1,36 @@
+package util
+
+import (
+	"context"
+
+	"github.com/10gen/migration-verifier/mbson"
+	"github.com/pkg/errors"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type BuildInfo struct {
+	VersionArray []int
+	IsSharded    bool
+}
+
+func GetBuildInfo(ctx context.Context, client *mongo.Client) (BuildInfo, error) {
+	commandResult := client.Database("admin").RunCommand(ctx, bson.D{{"buildinfo", 1}})
+
+	rawResp, err := commandResult.Raw()
+	if err != nil {
+		return BuildInfo{}, errors.Wrap(err, "failed to fetch build info")
+	}
+
+	bi := BuildInfo{}
+	_, err = mbson.RawLookup(rawResp, &bi.VersionArray, "versionArray")
+	if err != nil {
+		return BuildInfo{}, errors.Wrap(err, "failed to decode build info version array")
+	}
+
+	var msg string
+	_, err = mbson.RawLookup(rawResp, &msg, "msg")
+	bi.IsSharded = msg == "isdbgrid"
+
+	return bi, nil
+}
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index f6517255..ea82c025 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -2,11 +2,12 @@ package verifier
 
 import (
 	"context"
-	"os"
 	"strings"
 
+	"github.com/10gen/migration-verifier/internal/util"
 	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/pkg/errors"
+	"github.com/samber/lo"
 	"github.com/stretchr/testify/suite"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -121,19 +122,10 @@ func (suite *IntegrationTestSuite) TearDownTest() {
 }
 
 func (suite *IntegrationTestSuite) GetTopology() TestTopology {
-	rawTopology, found := os.LookupEnv(topologyEnvVar)
+	buildInfo, err := util.GetBuildInfo(suite.Context(), suite.srcMongoClient)
+	suite.Require().NoError(err, "should read source's build info")
 
-	suite.Require().True(found, "Environment must contain %#q.", topologyEnvVar)
-
-	topology := TestTopology(rawTopology)
-
-	suite.Require().Contains(
-		knownTopologies,
-		topology,
-		"%#q must be a known value.",
-	)
-
-	return topology
+	return lo.Ternary(buildInfo.IsSharded, TopologySharded, "")
 }
 
 func (suite *IntegrationTestSuite) BuildVerifier() *Verifier {
diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go
index b8ff382f..2fc93753 100644
--- a/internal/verifier/migration_verifier.go
+++ b/internal/verifier/migration_verifier.go
@@ -21,6 +21,7 @@ import (
 	"github.com/10gen/migration-verifier/internal/reportutils"
 	"github.com/10gen/migration-verifier/internal/retry"
 	"github.com/10gen/migration-verifier/internal/types"
+	"github.com/10gen/migration-verifier/internal/util"
 	"github.com/10gen/migration-verifier/internal/uuidutil"
 	"github.com/olekukonko/tablewriter"
 	"github.com/pkg/errors"
@@ -88,8 +89,8 @@ type Verifier struct {
 	metaClient         *mongo.Client
 	srcClient          *mongo.Client
 	dstClient          *mongo.Client
-	srcBuildInfo       *bson.M
-	dstBuildInfo       *bson.M
+	srcBuildInfo       *util.BuildInfo
+	dstBuildInfo       *util.BuildInfo
 	numWorkers         int
 	failureDisplaySize int64
 
@@ -276,10 +277,16 @@ func (verifier *Verifier) SetSrcURI(ctx context.Context, uri string) error {
 	var err error
 	verifier.srcClient, err = mongo.Connect(ctx, opts)
 	if err != nil {
-		return errors.Wrapf(err, "failed to connect to %#q", uri)
+		return errors.Wrapf(err, "failed to connect to source %#q", uri)
 	}
-	verifier.srcBuildInfo, err = getBuildInfo(ctx, verifier.srcClient)
-	return err
+
+	buildInfo, err := util.GetBuildInfo(ctx, verifier.srcClient)
+	if err != nil {
+		return errors.Wrap(err, "failed to read source build info")
+	}
+
+	verifier.srcBuildInfo = &buildInfo
+	return nil
 }
 
 func (verifier *Verifier) SetDstURI(ctx context.Context, uri string) error {
@@ -287,10 +294,16 @@ func (verifier *Verifier) SetDstURI(ctx context.Context, uri string) error {
 	var err error
 	verifier.dstClient, err = mongo.Connect(ctx, opts)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "failed to connect to destination %#q", uri)
 	}
-	verifier.dstBuildInfo, err = getBuildInfo(ctx, verifier.dstClient)
-	return err
+
+	buildInfo, err := util.GetBuildInfo(ctx, verifier.dstClient)
+	if err != nil {
+		return errors.Wrap(err, "failed to read destination build info")
+	}
+
+	verifier.dstBuildInfo = &buildInfo
+	return nil
 }
 
 func (verifier *Verifier) SetServerPort(port int) {
@@ -429,7 +442,7 @@ func (verifier *Verifier) maybeAppendGlobalFilterToPredicates(predicates bson.A)
 	return append(predicates, verifier.globalFilter)
 }
 
-func (verifier *Verifier) getDocumentsCursor(ctx context.Context, collection *mongo.Collection, buildInfo *bson.M,
+func (verifier *Verifier) getDocumentsCursor(ctx context.Context, collection *mongo.Collection, buildInfo *util.BuildInfo,
 	startAtTs *primitive.Timestamp, task *VerificationTask) (*mongo.Cursor, error) {
 	var findOptions bson.D
 	runCommandOptions := options.RunCmd()
@@ -1402,16 +1415,3 @@ func (verifier *Verifier) getNamespaces(ctx context.Context, fieldName string) (
 	}
 	return namespaces, nil
 }
-
-func getBuildInfo(ctx context.Context, client *mongo.Client) (*bson.M, error) {
-	commandResult := client.Database("admin").RunCommand(ctx, bson.D{{"buildinfo", 1}})
-	if commandResult.Err() != nil {
-		return nil, commandResult.Err()
-	}
-	var buildInfoMap bson.M
-	err := commandResult.Decode(&buildInfoMap)
-	if err != nil {
-		return nil, err
-	}
-	return &buildInfoMap, nil
-}
diff --git a/mbson/bson_raw.go b/mbson/bson_raw.go
new file mode 100644
index 00000000..0759ffca
--- /dev/null
+++ b/mbson/bson_raw.go
@@ -0,0 +1,29 @@
+package mbson
+
+import (
+	"github.com/pkg/errors"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/x/bsonx/bsoncore"
+)
+
+// RawLookup combines bson.Raw’s LookupErr method with an additional
+// unmarshal step. The result is a convenient way to extract values from
+// bson.Raw. The returned boolean indicates whether the value was found.
+func RawLookup[T any](doc bson.Raw, dest *T, keys ...string) (bool, error) {
+	val, err := doc.LookupErr(keys...)
+
+	if err == nil {
+		return true, val.Unmarshal(dest)
+	} else if errors.Is(err, bsoncore.ErrElementNotFound) {
+		return false, nil
+	}
+
+	return false, errors.Wrapf(err, "failed to look up %+v in BSON doc", keys)
+}
+
+// RawContains is like RawLookup but makes no effort to unmarshal
+// the value.
+func RawContains(doc bson.Raw, keys ...string) (bool, error) {
+	val := any(nil)
+	return RawLookup(doc, &val, keys...)
+}
diff --git a/mbson/unit_test.go b/mbson/unit_test.go
new file mode 100644
index 00000000..0a66e92d
--- /dev/null
+++ b/mbson/unit_test.go
@@ -0,0 +1,88 @@
+package mbson
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/x/bsonx/bsoncore"
+)
+
+type UnitTestSuite struct {
+	suite.Suite
+}
+
+func TestUnitTestSuite(t *testing.T) {
+	ts := new(UnitTestSuite)
+	suite.Run(t, ts)
+}
+
+func (s *UnitTestSuite) Test_RawLookup() {
+	myDoc := bson.D{
+		{"foo", 1},
+		{"bar", "baz"},
+		{"quux", bson.A{
+			123,
+			bson.D{{"hey", "ho"}},
+		}},
+	}
+
+	myRaw, err := bson.Marshal(myDoc)
+	s.Require().NoError(err)
+
+	var myInt int
+	var myStr string
+
+	found, err := RawLookup(myRaw, &myInt, "foo")
+	s.Require().True(found)
+	s.Require().NoError(err)
+	s.Assert().EqualValues(1, myInt)
+
+	found, err = RawLookup(myRaw, &myInt, "quux", "0")
+	s.Require().True(found)
+	s.Require().NoError(err)
+	s.Assert().EqualValues(123, myInt)
+
+	found, err = RawLookup(myRaw, &myStr, "quux", "1", "hey")
+	s.Require().True(found)
+	s.Require().NoError(err)
+	s.Assert().EqualValues("ho", myStr)
+
+	found, err = RawLookup(myRaw, &myStr, "not there")
+	s.Require().NoError(err)
+	s.Assert().False(found)
+
+	myRaw = myRaw[:len(myRaw)-2]
+	_, err = RawLookup(myRaw, &myStr, "not there")
+	s.Assert().ErrorAs(err, &bsoncore.InsufficientBytesError{})
+}
+
+func (s *UnitTestSuite) Test_RawContains() {
+	myDoc := bson.D{
+		{"foo", 1},
+		{"bar", "baz"},
+		{"quux", bson.A{
+			123,
+			bson.D{{"hey", "ho"}},
+		}},
+	}
+
+	myRaw, err := bson.Marshal(myDoc)
+	s.Require().NoError(err)
+
+	has, err := RawContains(myRaw, "foo")
+	s.Require().NoError(err)
+	s.Assert().True(has, "`foo` should exist")
+
+	has, err = RawContains(myRaw, "quux", "1", "hey")
+	s.Require().NoError(err)
+	s.Assert().True(has, "deep lookup should work")
+
+	has, err = RawContains(myRaw, "not there")
+	s.Require().NoError(err)
+	s.Assert().False(has, "missing element should not exist")
+
+	myRaw = myRaw[:len(myRaw)-2]
+	_, err = RawContains(myRaw, "not there")
+	s.Assert().ErrorAs(err, &bsoncore.InsufficientBytesError{})
+}

From 7297c3cb75723c4d55bc4d4417042cd4f732e284 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 11:09:52 -0500
Subject: [PATCH 047/104] more refactor

---
 internal/util/buildinfo.go                  | 3 +++
 internal/verifier/integration_test_suite.go | 2 --
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/internal/util/buildinfo.go b/internal/util/buildinfo.go
index ce441961..1afad3fa 100644
--- a/internal/util/buildinfo.go
+++ b/internal/util/buildinfo.go
@@ -30,6 +30,9 @@ func GetBuildInfo(ctx context.Context, client *mongo.Client) (BuildInfo, error)
 
 	var msg string
 	_, err = mbson.RawLookup(rawResp, &msg, "msg")
+	if err != nil {
+		return BuildInfo{}, errors.Wrap(err, "failed to determine topology from build info")
+	}
 	bi.IsSharded = msg == "isdbgrid"
 
 	return bi, nil
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index ea82c025..a0ebdd00 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -24,8 +24,6 @@ const (
 	TopologySharded TestTopology = "sharded"
 )
 
-var knownTopologies = []TestTopology{TopologyReplset, TopologySharded}
-
 type IntegrationTestSuite struct {
 	suite.Suite
 	srcConnStr, dstConnStr, metaConnStr             string

From 524c7f1211bbaf5c6bc68edee72711e69917fbfa Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 11:31:49 -0500
Subject: [PATCH 048/104] Revert "more refactor"

This reverts commit 7297c3cb75723c4d55bc4d4417042cd4f732e284.
---
 internal/util/buildinfo.go                  | 3 ---
 internal/verifier/integration_test_suite.go | 2 ++
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/internal/util/buildinfo.go b/internal/util/buildinfo.go
index 1afad3fa..ce441961 100644
--- a/internal/util/buildinfo.go
+++ b/internal/util/buildinfo.go
@@ -30,9 +30,6 @@ func GetBuildInfo(ctx context.Context, client *mongo.Client) (BuildInfo, error)
 
 	var msg string
 	_, err = mbson.RawLookup(rawResp, &msg, "msg")
-	if err != nil {
-		return BuildInfo{}, errors.Wrap(err, "failed to determine topology from build info")
-	}
 	bi.IsSharded = msg == "isdbgrid"
 
 	return bi, nil
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index a0ebdd00..ea82c025 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -24,6 +24,8 @@ const (
 	TopologySharded TestTopology = "sharded"
 )
 
+var knownTopologies = []TestTopology{TopologyReplset, TopologySharded}
+
 type IntegrationTestSuite struct {
 	suite.Suite
 	srcConnStr, dstConnStr, metaConnStr             string

From f9cc0b8625617e4e05b5462e195f21aaf301d3fc Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 11:31:52 -0500
Subject: [PATCH 049/104] Revert "refactor build infio"

This reverts commit 2304fca6e3df099edc4e7de416d70d249385ef4e.
---
 .github/workflows/all.yml                   |  1 +
 internal/partitions/partition.go            | 15 +++-
 internal/partitions/partition_test.go       | 19 +++--
 internal/util/buildinfo.go                  | 36 ---------
 internal/verifier/integration_test_suite.go | 18 +++--
 internal/verifier/migration_verifier.go     | 44 +++++------
 mbson/bson_raw.go                           | 29 -------
 mbson/unit_test.go                          | 88 ---------------------
 8 files changed, 59 insertions(+), 191 deletions(-)
 delete mode 100644 internal/util/buildinfo.go
 delete mode 100644 mbson/bson_raw.go
 delete mode 100644 mbson/unit_test.go

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index a5a49933..40dfa8c4 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -70,3 +70,4 @@ jobs:
           MVTEST_SRC: ${{matrix.topology.srcConnStr}}
           MVTEST_DST: ${{matrix.topology.dstConnStr}}
           MVTEST_META: mongodb://localhost:27040
+          MVTEST_TOPOLOGY: ${{matrix.topology.name}}
diff --git a/internal/partitions/partition.go b/internal/partitions/partition.go
index 024458e8..5c9d507b 100644
--- a/internal/partitions/partition.go
+++ b/internal/partitions/partition.go
@@ -137,7 +137,7 @@ func (p *Partition) FindCmd(
 // (e.g. use the partitions on the source to read the destination for verification)
 // If the passed-in buildinfo indicates a mongodb version < 5.0, type bracketing is not used.
 // filterAndPredicates is a slice of filter criteria that's used to construct the "filter" field in the find option.
-func (p *Partition) GetFindOptions(buildInfo *util.BuildInfo, filterAndPredicates bson.A) bson.D {
+func (p *Partition) GetFindOptions(buildInfo *bson.M, filterAndPredicates bson.A) bson.D {
 	if p == nil {
 		if len(filterAndPredicates) > 0 {
 			return bson.D{{"filter", bson.D{{"$and", filterAndPredicates}}}}
@@ -160,9 +160,16 @@ func (p *Partition) GetFindOptions(buildInfo *util.BuildInfo, filterAndPredicate
 		allowTypeBracketing := false
 		if buildInfo != nil {
 			allowTypeBracketing = true
-
-			if buildInfo.VersionArray != nil {
-				allowTypeBracketing = buildInfo.VersionArray[0] < 5
+			versionArray, ok := (*buildInfo)["versionArray"].(bson.A)
+			//bson values are int32 or int64, never int.
+			if ok {
+				majorVersion, ok := versionArray[0].(int32)
+				if ok {
+					allowTypeBracketing = majorVersion < 5
+				} else {
+					majorVersion64, _ := versionArray[0].(int64)
+					allowTypeBracketing = majorVersion64 < 5
+				}
 			}
 		}
 		if !allowTypeBracketing {
diff --git a/internal/partitions/partition_test.go b/internal/partitions/partition_test.go
index 53d41a0a..e0cae05c 100644
--- a/internal/partitions/partition_test.go
+++ b/internal/partitions/partition_test.go
@@ -77,38 +77,43 @@ func (suite *UnitTestSuite) TestVersioning() {
 	filter := getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilter, filter)
 
+	// 6.0 (int64)
+	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int64(6), int64(0), int64(0), int64(0)}}, nil)
+	filter = getFilterFromFindOptions(findOptions)
+	suite.Require().Equal(expectedFilter, filter)
+
 	// 6.0
-	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{6, 0, 0}}, nil)
+	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int32(6), int32(0), int32(0), int32(0)}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilter, filter)
 
 	// 5.3.0.9
-	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{5, 3, 0, 9}}, nil)
+	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int32(5), int32(3), int32(0), int32(9)}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilter, filter)
 
 	// 7.1.3.5
-	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{7, 1, 3, 5}}, nil)
+	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int32(7), int32(1), int32(3), int32(5)}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilter, filter)
 
 	// 4.4 (int64)
-	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{4, 4, 0, 0}}, nil)
+	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int64(4), int64(4), int64(0), int64(0)}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilterWithTypeBracketing, filter)
 
 	// 4.4
-	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{4, 4, 0, 0}}, nil)
+	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int32(4), int32(4), int32(0), int32(0)}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilterWithTypeBracketing, filter)
 
 	// 4.2
-	findOptions = partition.GetFindOptions(&util.BuildInfo{VersionArray: []int{4, 2, 0, 0}}, nil)
+	findOptions = partition.GetFindOptions(&bson.M{"versionArray": bson.A{int32(4), int32(2), int32(0), int32(0)}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilterWithTypeBracketing, filter)
 
 	// No version array -- assume old, require type bracketing.
-	findOptions = partition.GetFindOptions(&util.BuildInfo{}, nil)
+	findOptions = partition.GetFindOptions(&bson.M{"notVersionArray": bson.A{6, int32(0), int32(0), int32(0)}}, nil)
 	filter = getFilterFromFindOptions(findOptions)
 	suite.Require().Equal(expectedFilterWithTypeBracketing, filter)
 }
diff --git a/internal/util/buildinfo.go b/internal/util/buildinfo.go
deleted file mode 100644
index ce441961..00000000
--- a/internal/util/buildinfo.go
+++ /dev/null
@@ -1,36 +0,0 @@
-package util
-
-import (
-	"context"
-
-	"github.com/10gen/migration-verifier/mbson"
-	"github.com/pkg/errors"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/mongo"
-)
-
-type BuildInfo struct {
-	VersionArray []int
-	IsSharded    bool
-}
-
-func GetBuildInfo(ctx context.Context, client *mongo.Client) (BuildInfo, error) {
-	commandResult := client.Database("admin").RunCommand(ctx, bson.D{{"buildinfo", 1}})
-
-	rawResp, err := commandResult.Raw()
-	if err != nil {
-		return BuildInfo{}, errors.Wrap(err, "failed to fetch build info")
-	}
-
-	bi := BuildInfo{}
-	_, err = mbson.RawLookup(rawResp, &bi.VersionArray, "versionArray")
-	if err != nil {
-		return BuildInfo{}, errors.Wrap(err, "failed to decode build info version array")
-	}
-
-	var msg string
-	_, err = mbson.RawLookup(rawResp, &msg, "msg")
-	bi.IsSharded = msg == "isdbgrid"
-
-	return bi, nil
-}
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index ea82c025..f6517255 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -2,12 +2,11 @@ package verifier
 
 import (
 	"context"
+	"os"
 	"strings"
 
-	"github.com/10gen/migration-verifier/internal/util"
 	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/pkg/errors"
-	"github.com/samber/lo"
 	"github.com/stretchr/testify/suite"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -122,10 +121,19 @@ func (suite *IntegrationTestSuite) TearDownTest() {
 }
 
 func (suite *IntegrationTestSuite) GetTopology() TestTopology {
-	buildInfo, err := util.GetBuildInfo(suite.Context(), suite.srcMongoClient)
-	suite.Require().NoError(err, "should read source's build info")
+	rawTopology, found := os.LookupEnv(topologyEnvVar)
 
-	return lo.Ternary(buildInfo.IsSharded, TopologySharded, "")
+	suite.Require().True(found, "Environment must contain %#q.", topologyEnvVar)
+
+	topology := TestTopology(rawTopology)
+
+	suite.Require().Contains(
+		knownTopologies,
+		topology,
+		"%#q must be a known value.",
+	)
+
+	return topology
 }
 
 func (suite *IntegrationTestSuite) BuildVerifier() *Verifier {
diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go
index 2fc93753..b8ff382f 100644
--- a/internal/verifier/migration_verifier.go
+++ b/internal/verifier/migration_verifier.go
@@ -21,7 +21,6 @@ import (
 	"github.com/10gen/migration-verifier/internal/reportutils"
 	"github.com/10gen/migration-verifier/internal/retry"
 	"github.com/10gen/migration-verifier/internal/types"
-	"github.com/10gen/migration-verifier/internal/util"
 	"github.com/10gen/migration-verifier/internal/uuidutil"
 	"github.com/olekukonko/tablewriter"
 	"github.com/pkg/errors"
@@ -89,8 +88,8 @@ type Verifier struct {
 	metaClient         *mongo.Client
 	srcClient          *mongo.Client
 	dstClient          *mongo.Client
-	srcBuildInfo       *util.BuildInfo
-	dstBuildInfo       *util.BuildInfo
+	srcBuildInfo       *bson.M
+	dstBuildInfo       *bson.M
 	numWorkers         int
 	failureDisplaySize int64
 
@@ -277,16 +276,10 @@ func (verifier *Verifier) SetSrcURI(ctx context.Context, uri string) error {
 	var err error
 	verifier.srcClient, err = mongo.Connect(ctx, opts)
 	if err != nil {
-		return errors.Wrapf(err, "failed to connect to source %#q", uri)
+		return errors.Wrapf(err, "failed to connect to %#q", uri)
 	}
-
-	buildInfo, err := util.GetBuildInfo(ctx, verifier.srcClient)
-	if err != nil {
-		return errors.Wrap(err, "failed to read source build info")
-	}
-
-	verifier.srcBuildInfo = &buildInfo
-	return nil
+	verifier.srcBuildInfo, err = getBuildInfo(ctx, verifier.srcClient)
+	return err
 }
 
 func (verifier *Verifier) SetDstURI(ctx context.Context, uri string) error {
@@ -294,16 +287,10 @@ func (verifier *Verifier) SetDstURI(ctx context.Context, uri string) error {
 	var err error
 	verifier.dstClient, err = mongo.Connect(ctx, opts)
 	if err != nil {
-		return errors.Wrapf(err, "failed to connect to destination %#q", uri)
-	}
-
-	buildInfo, err := util.GetBuildInfo(ctx, verifier.dstClient)
-	if err != nil {
-		return errors.Wrap(err, "failed to read destination build info")
+		return err
 	}
-
-	verifier.dstBuildInfo = &buildInfo
-	return nil
+	verifier.dstBuildInfo, err = getBuildInfo(ctx, verifier.dstClient)
+	return err
 }
 
 func (verifier *Verifier) SetServerPort(port int) {
@@ -442,7 +429,7 @@ func (verifier *Verifier) maybeAppendGlobalFilterToPredicates(predicates bson.A)
 	return append(predicates, verifier.globalFilter)
 }
 
-func (verifier *Verifier) getDocumentsCursor(ctx context.Context, collection *mongo.Collection, buildInfo *util.BuildInfo,
+func (verifier *Verifier) getDocumentsCursor(ctx context.Context, collection *mongo.Collection, buildInfo *bson.M,
 	startAtTs *primitive.Timestamp, task *VerificationTask) (*mongo.Cursor, error) {
 	var findOptions bson.D
 	runCommandOptions := options.RunCmd()
@@ -1415,3 +1402,16 @@ func (verifier *Verifier) getNamespaces(ctx context.Context, fieldName string) (
 	}
 	return namespaces, nil
 }
+
+func getBuildInfo(ctx context.Context, client *mongo.Client) (*bson.M, error) {
+	commandResult := client.Database("admin").RunCommand(ctx, bson.D{{"buildinfo", 1}})
+	if commandResult.Err() != nil {
+		return nil, commandResult.Err()
+	}
+	var buildInfoMap bson.M
+	err := commandResult.Decode(&buildInfoMap)
+	if err != nil {
+		return nil, err
+	}
+	return &buildInfoMap, nil
+}
diff --git a/mbson/bson_raw.go b/mbson/bson_raw.go
deleted file mode 100644
index 0759ffca..00000000
--- a/mbson/bson_raw.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package mbson
-
-import (
-	"github.com/pkg/errors"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/x/bsonx/bsoncore"
-)
-
-// RawLookup combines bson.Raw’s LookupErr method with an additional
-// unmarshal step. The result is a convenient way to extract values from
-// bson.Raw. The returned boolean indicates whether the value was found.
-func RawLookup[T any](doc bson.Raw, dest *T, keys ...string) (bool, error) {
-	val, err := doc.LookupErr(keys...)
-
-	if err == nil {
-		return true, val.Unmarshal(dest)
-	} else if errors.Is(err, bsoncore.ErrElementNotFound) {
-		return false, nil
-	}
-
-	return false, errors.Wrapf(err, "failed to look up %+v in BSON doc", keys)
-}
-
-// RawContains is like RawLookup but makes no effort to unmarshal
-// the value.
-func RawContains(doc bson.Raw, keys ...string) (bool, error) {
-	val := any(nil)
-	return RawLookup(doc, &val, keys...)
-}
diff --git a/mbson/unit_test.go b/mbson/unit_test.go
deleted file mode 100644
index 0a66e92d..00000000
--- a/mbson/unit_test.go
+++ /dev/null
@@ -1,88 +0,0 @@
-package mbson
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/suite"
-	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/x/bsonx/bsoncore"
-)
-
-type UnitTestSuite struct {
-	suite.Suite
-}
-
-func TestUnitTestSuite(t *testing.T) {
-	ts := new(UnitTestSuite)
-	suite.Run(t, ts)
-}
-
-func (s *UnitTestSuite) Test_RawLookup() {
-	myDoc := bson.D{
-		{"foo", 1},
-		{"bar", "baz"},
-		{"quux", bson.A{
-			123,
-			bson.D{{"hey", "ho"}},
-		}},
-	}
-
-	myRaw, err := bson.Marshal(myDoc)
-	s.Require().NoError(err)
-
-	var myInt int
-	var myStr string
-
-	found, err := RawLookup(myRaw, &myInt, "foo")
-	s.Require().True(found)
-	s.Require().NoError(err)
-	s.Assert().EqualValues(1, myInt)
-
-	found, err = RawLookup(myRaw, &myInt, "quux", "0")
-	s.Require().True(found)
-	s.Require().NoError(err)
-	s.Assert().EqualValues(123, myInt)
-
-	found, err = RawLookup(myRaw, &myStr, "quux", "1", "hey")
-	s.Require().True(found)
-	s.Require().NoError(err)
-	s.Assert().EqualValues("ho", myStr)
-
-	found, err = RawLookup(myRaw, &myStr, "not there")
-	s.Require().NoError(err)
-	s.Assert().False(found)
-
-	myRaw = myRaw[:len(myRaw)-2]
-	_, err = RawLookup(myRaw, &myStr, "not there")
-	s.Assert().ErrorAs(err, &bsoncore.InsufficientBytesError{})
-}
-
-func (s *UnitTestSuite) Test_RawContains() {
-	myDoc := bson.D{
-		{"foo", 1},
-		{"bar", "baz"},
-		{"quux", bson.A{
-			123,
-			bson.D{{"hey", "ho"}},
-		}},
-	}
-
-	myRaw, err := bson.Marshal(myDoc)
-	s.Require().NoError(err)
-
-	has, err := RawContains(myRaw, "foo")
-	s.Require().NoError(err)
-	s.Assert().True(has, "`foo` should exist")
-
-	has, err = RawContains(myRaw, "quux", "1", "hey")
-	s.Require().NoError(err)
-	s.Assert().True(has, "deep lookup should work")
-
-	has, err = RawContains(myRaw, "not there")
-	s.Require().NoError(err)
-	s.Assert().False(has, "missing element should not exist")
-
-	myRaw = myRaw[:len(myRaw)-2]
-	_, err = RawContains(myRaw, "not there")
-	s.Assert().ErrorAs(err, &bsoncore.InsufficientBytesError{})
-}

From fa05206878576dffcaec66eb5eef0df9aa3cd6d9 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 11:36:10 -0500
Subject: [PATCH 050/104] tweak not to require topology in env

---
 .github/workflows/all.yml                    |  1 -
 internal/verifier/change_stream_test.go      |  2 +-
 internal/verifier/integration_test_suite.go  | 26 ++++++++++----------
 internal/verifier/migration_verifier_test.go |  6 ++---
 4 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 40dfa8c4..a5a49933 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -70,4 +70,3 @@ jobs:
           MVTEST_SRC: ${{matrix.topology.srcConnStr}}
           MVTEST_DST: ${{matrix.topology.dstConnStr}}
           MVTEST_META: mongodb://localhost:27040
-          MVTEST_TOPOLOGY: ${{matrix.topology.name}}
diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index 650f7fad..89bc7877 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -147,7 +147,7 @@ func (suite *IntegrationTestSuite) TestStartAtTimeNoChanges() {
 }
 
 func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
-	if suite.GetTopology() == TopologySharded {
+	if suite.GetSrcTopology() == TopologySharded {
 		suite.T().Skip("Skipping pending REP-5299.")
 	}
 
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index f6517255..d3a112c8 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -2,11 +2,11 @@ package verifier
 
 import (
 	"context"
-	"os"
 	"strings"
 
 	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/pkg/errors"
+	"github.com/samber/lo"
 	"github.com/stretchr/testify/suite"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -23,8 +23,6 @@ const (
 	TopologySharded TestTopology = "sharded"
 )
 
-var knownTopologies = []TestTopology{TopologyReplset, TopologySharded}
-
 type IntegrationTestSuite struct {
 	suite.Suite
 	srcConnStr, dstConnStr, metaConnStr             string
@@ -120,20 +118,22 @@ func (suite *IntegrationTestSuite) TearDownTest() {
 	}
 }
 
-func (suite *IntegrationTestSuite) GetTopology() TestTopology {
-	rawTopology, found := os.LookupEnv(topologyEnvVar)
-
-	suite.Require().True(found, "Environment must contain %#q.", topologyEnvVar)
+func (suite *IntegrationTestSuite) GetSrcTopology() TestTopology {
+	hello := struct {
+		Msg string
+	}{}
 
-	topology := TestTopology(rawTopology)
+	resp := suite.srcMongoClient.Database("admin").RunCommand(
+		suite.Context(),
+		bson.D{{"hello", 1}},
+	)
 
-	suite.Require().Contains(
-		knownTopologies,
-		topology,
-		"%#q must be a known value.",
+	suite.Require().NoError(
+		resp.Decode(&hello),
+		"should fetch & decode hello",
 	)
 
-	return topology
+	return lo.Ternary(hello.Msg == "isdbgrid", TopologySharded, "")
 }
 
 func (suite *IntegrationTestSuite) BuildVerifier() *Verifier {
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index d766f098..90ade21a 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1149,7 +1149,7 @@ func TestVerifierCompareIndexSpecs(t *testing.T) {
 }
 
 func (suite *IntegrationTestSuite) TestVerifierNamespaceList() {
-	if suite.GetTopology() == TopologySharded {
+	if suite.GetSrcTopology() == TopologySharded {
 		suite.T().Skip("Skipping pending REP-5299.")
 	}
 
@@ -1279,7 +1279,7 @@ func (suite *IntegrationTestSuite) TestVerificationStatus() {
 }
 
 func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
-	if suite.GetTopology() == TopologySharded {
+	if suite.GetSrcTopology() == TopologySharded {
 		suite.T().Skip("Skipping pending REP-5299.")
 	}
 
@@ -1391,7 +1391,7 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 }
 
 func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
-	if suite.GetTopology() == TopologySharded {
+	if suite.GetSrcTopology() == TopologySharded {
 		suite.T().Skip("Skipping pending REP-5299.")
 	}
 

From 0392c20bff3e994d188c5977f38a0755823913b7 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 11:56:38 -0500
Subject: [PATCH 051/104] restore cross-version testing

---
 .github/workflows/all.yml | 60 +++++++++++++++++++++++++++++++--------
 1 file changed, 48 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index a5a49933..dd5ff01e 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -11,13 +11,40 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        mongodb_version:
-          - '4.2'
-          - '4.4'
-          - '5.0'
-          - '6.0'
-          - '7.0'
-          - '8.0'
+        mongodb_versions:
+          - src: '4.2'
+            dst:
+              - '4.2'
+              - '4.4'
+              - '5.0'
+              - '6.0'
+
+          - src: '4.4'
+            dst:
+              - '4.4'
+              - '5.0'
+              - '6.0'
+
+          - src: '5.0'
+            dst:
+              - '5.0'
+              - '6.0'
+              - '7.0'
+
+          - src: '6.0'
+            dst:
+              - '6.0'
+              - '7.0'
+              - '8.0'
+
+          - src: '7.0'
+            dst:
+              - '7.0'
+              - '8.0'
+
+          - src: '8.0'
+            dst:
+              - '8.0'
 
         topology:
           - name: replset
@@ -39,8 +66,17 @@ jobs:
       - name: Install m
         run: npm install -g m mongosh
 
-      - name: Activate MongoDB ${{ matrix.mongodb_version }}
-        run: yes | m ${{ matrix.mongodb_version }}
+      - name: Install MongoDB versions
+        run: yes | m ${{ matrix.mongodb_versions.src }} ${{ matrix.mongodb_versions.dst }} stable
+
+      - name: Set & save source version
+        run: m ${{ matrix.mongodb_versions.src }} && dirname $(readlink $(which mongod)) > .srcpath
+
+      - name: Set & save destination version
+        run: m ${{ matrix.mongodb_versions.dst }} && dirname $(readlink $(which mongod)) > .dstpath
+
+      - name: Set & save metadata version
+        run: m stable && dirname $(readlink $(which mongod)) > .metapath
 
       - name: Install mtools
         run: pipx install 'mtools[all]'
@@ -59,9 +95,9 @@ jobs:
       - name: Start clusters
         run: |-
             {
-                echo "mlaunch init --port 27020 --dir src --replicaset ${{ matrix.topology.args }}"
-                echo "mlaunch init --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}"
-                echo "mlaunch init --port 27040 --dir meta --replicaset --nodes 1"
+                echo "mlaunch init --binarypath $(cat .srcpath) --port 27020 --dir src --replicaset ${{ matrix.topology.args }}"
+                echo "mlaunch init --binarypath $(cat .dstpath) --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}"
+                echo "mlaunch init --binarypath $(cat .metapath) --port 27040 --dir meta --replicaset --nodes 1"
             } | parallel
 
       - name: Test

From 934620772c590a6db60970244ffa64fec0f56674 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 12:07:21 -0500
Subject: [PATCH 052/104] rework versions

---
 .github/workflows/all.yml | 71 +++++++++++++++------------------------
 1 file changed, 28 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index dd5ff01e..e7136188 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -11,40 +11,28 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        mongodb_versions:
-          - src: '4.2'
-            dst:
-              - '4.2'
-              - '4.4'
-              - '5.0'
-              - '6.0'
-
-          - src: '4.4'
-            dst:
-              - '4.4'
-              - '5.0'
-              - '6.0'
-
-          - src: '5.0'
-            dst:
-              - '5.0'
-              - '6.0'
-              - '7.0'
-
-          - src: '6.0'
-            dst:
-              - '6.0'
-              - '7.0'
-              - '8.0'
-
-          - src: '7.0'
-            dst:
-              - '7.0'
-              - '8.0'
-
-          - src: '8.0'
-            dst:
-              - '8.0'
+        mongodb_versions: &mongodb_versions
+          - [ '4.2', '4.2' ]
+          - [ '4.2', '4.4' ]
+          - [ '4.2', '5.0' ]
+          - [ '4.2', '6.0' ]
+
+          - [ '4.4', '4.4' ]
+          - [ '4.4', '5.0' ]
+          - [ '4.4', '6.0' ]
+
+          - [ '5.0', '5.0' ]
+          - [ '5.0', '6.0' ]
+          - [ '5.0', '7.0' ]
+
+          - [ '6.0', '6.0' ]
+          - [ '6.0', '7.0' ]
+          - [ '6.0', '8.0' ]
+
+          - [ '7.0', '7.0' ]
+          - [ '7.0', '8.0' ]
+
+          - [ '8.0', '8.0' ]
 
         topology:
           - name: replset
@@ -66,17 +54,14 @@ jobs:
       - name: Install m
         run: npm install -g m mongosh
 
-      - name: Install MongoDB versions
-        run: yes | m ${{ matrix.mongodb_versions.src }} ${{ matrix.mongodb_versions.dst }} stable
-
-      - name: Set & save source version
-        run: m ${{ matrix.mongodb_versions.src }} && dirname $(readlink $(which mongod)) > .srcpath
+      - name: Install MongoDB ${{ matrix.mongodb_versions.0 }} (source)
+        run: yes | m ${{ matrix.mongodb_versions.0 }} && dirname $(readlink $(which mongod)) > .srcpath
 
-      - name: Set & save destination version
-        run: m ${{ matrix.mongodb_versions.dst }} && dirname $(readlink $(which mongod)) > .dstpath
+      - name: Install MongoDB ${{ matrix.mongodb_versions.1 }} (destination)
+        run: yes | m ${{ matrix.mongodb_versions.1 }} && dirname $(readlink $(which mongod)) > .dstpath
 
-      - name: Set & save metadata version
-        run: m stable && dirname $(readlink $(which mongod)) > .metapath
+      - name: Install latest stable MongoDB (metadata)
+        run: yes | m stable && dirname $(readlink $(which mongod)) > .metapath
 
       - name: Install mtools
         run: pipx install 'mtools[all]'

From 5e0358781f219697140ec5e7454128f8d5fb47ff Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 12:07:44 -0500
Subject: [PATCH 053/104] no anchor

---
 .github/workflows/all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index e7136188..ab76f405 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        mongodb_versions: &mongodb_versions
+        mongodb_versions:
           - [ '4.2', '4.2' ]
           - [ '4.2', '4.4' ]
           - [ '4.2', '5.0' ]

From 12c977528535f9798353ad9131f3925cd4adbd42 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 12:08:27 -0500
Subject: [PATCH 054/104] array

---
 .github/workflows/all.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index ab76f405..39d86e7d 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -54,11 +54,11 @@ jobs:
       - name: Install m
         run: npm install -g m mongosh
 
-      - name: Install MongoDB ${{ matrix.mongodb_versions.0 }} (source)
-        run: yes | m ${{ matrix.mongodb_versions.0 }} && dirname $(readlink $(which mongod)) > .srcpath
+      - name: Install MongoDB ${{ matrix.mongodb_versions[0] }} (source)
+        run: yes | m ${{ matrix.mongodb_versions[0] }} && dirname $(readlink $(which mongod)) > .srcpath
 
-      - name: Install MongoDB ${{ matrix.mongodb_versions.1 }} (destination)
-        run: yes | m ${{ matrix.mongodb_versions.1 }} && dirname $(readlink $(which mongod)) > .dstpath
+      - name: Install MongoDB ${{ matrix.mongodb_versions[1] }} (destination)
+        run: yes | m ${{ matrix.mongodb_versions[1] }} && dirname $(readlink $(which mongod)) > .dstpath
 
       - name: Install latest stable MongoDB (metadata)
         run: yes | m stable && dirname $(readlink $(which mongod)) > .metapath

From e541382185e9a4ad7d8ac15df03bc77bb680a642 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 12:10:23 -0500
Subject: [PATCH 055/104] reorder

---
 .github/workflows/all.yml | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 39d86e7d..9dced7fe 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -47,7 +47,17 @@ jobs:
     # There seems no good reason to test on other OSes … ?
     runs-on: ubuntu-latest
 
+    name: ${{ matrix.mongodb_versions[0] }} to ${{ matrix.mongodb_versions[1] }}, ${{ matrix.topology.name }}
+
     steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Fetch Go ${{ matrix.go_version }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+
       - name: Install packages
         run: ${{ matrix.os.setup }}
 
@@ -66,14 +76,6 @@ jobs:
       - name: Install mtools
         run: pipx install 'mtools[all]'
 
-      - name: Check out repository
-        uses: actions/checkout@v4
-
-      - name: Fetch Go ${{ matrix.go_version }}
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-
       - name: Build
         run: go build main/migration_verifier.go
 

From 30db8084dce264395bad4838aed5e981e89c5ab4 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 12:15:26 -0500
Subject: [PATCH 056/104] no packages

---
 .github/workflows/all.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 9dced7fe..5cd6e1eb 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -58,9 +58,6 @@ jobs:
         with:
           go-version: stable
 
-      - name: Install packages
-        run: ${{ matrix.os.setup }}
-
       - name: Install m
         run: npm install -g m mongosh
 

From de8e127812a09cfe25bc786b7de41cd6e56f728f Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 12:20:44 -0500
Subject: [PATCH 057/104] check race

---
 .github/workflows/all.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 5cd6e1eb..7a082756 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -85,7 +85,7 @@ jobs:
             } | parallel
 
       - name: Test
-        run: go test -v ./...
+        run: go test -v ./... -race
         env:
           MVTEST_SRC: ${{matrix.topology.srcConnStr}}
           MVTEST_DST: ${{matrix.topology.dstConnStr}}

From 1d2e25fe587ba97f1387e550a614151466c1886f Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 12:32:06 -0500
Subject: [PATCH 058/104] revert needless change

---
 internal/verifier/migration_verifier.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go
index b8ff382f..809e4fe3 100644
--- a/internal/verifier/migration_verifier.go
+++ b/internal/verifier/migration_verifier.go
@@ -276,7 +276,7 @@ func (verifier *Verifier) SetSrcURI(ctx context.Context, uri string) error {
 	var err error
 	verifier.srcClient, err = mongo.Connect(ctx, opts)
 	if err != nil {
-		return errors.Wrapf(err, "failed to connect to %#q", uri)
+		return err
 	}
 	verifier.srcBuildInfo, err = getBuildInfo(ctx, verifier.srcClient)
 	return err

From 21e0666652384e549807accee86e03ebe21b1f5d Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 12:35:16 -0500
Subject: [PATCH 059/104] short mode

---
 README.md                                    | 11 +++++++++++
 internal/verifier/migration_verifier_test.go |  3 +++
 2 files changed, 14 insertions(+)

diff --git a/README.md b/README.md
index 7100ef2c..0564af17 100644
--- a/README.md
+++ b/README.md
@@ -147,6 +147,17 @@ On default settings it used about **200GB of RAM on m6id.metal machine when usin
 
 **This means it does about 1TB/20min but it is HIGHLY dependent on the source and dest machines**
 
+# Tests
+
+This project’s tests run as normal Go tests, to, with `go test`.
+
+`IntegrationTestSuite`'s tests require external clusters. You must provision these yourself.
+(See the project’s GitHub CI setup for one way to simplify it.) Once provisioned, set the relevant
+connection strings in the following environment variables:
+
+- MVTEST_SRC
+- MVTEST_DST
+- MVTEST_META
 
 # How the Verifier Works
 
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index 90ade21a..ff125fea 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -32,6 +32,9 @@ import (
 )
 
 func TestIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration tests in short-test mode.")
+	}
 	envVals := map[string]string{}
 
 	for _, name := range []string{"MVTEST_SRC", "MVTEST_DST", "MVTEST_META"} {

From e1507bd4ba12b030fd73115bb9e07bb1dfe5fd1c Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 12:45:03 -0500
Subject: [PATCH 060/104] squashed

---
 go.mod                                        |   2 +-
 internal/retry/retry.go                       |  44 ++++--
 internal/verifier/change_stream.go            |  78 ++++++----
 internal/verifier/check.go                    |  17 ++-
 internal/verifier/clustertime.go              | 141 ++++++++++++++++++
 internal/verifier/clustertime_test.go         |  13 ++
 internal/verifier/logging_setup.go            |  12 ++
 internal/verifier/migration_verifier.go       |  46 ++----
 .../verifier/migration_verifier_bench_test.go |   4 -
 mbson/bson_raw.go                             |  29 ++++
 mbson/unit_test.go                            |  88 +++++++++++
 11 files changed, 399 insertions(+), 75 deletions(-)
 create mode 100644 internal/verifier/clustertime.go
 create mode 100644 internal/verifier/clustertime_test.go
 create mode 100644 mbson/bson_raw.go
 create mode 100644 mbson/unit_test.go

diff --git a/go.mod b/go.mod
index 0f8ca4b7..fa9f2646 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module github.com/10gen/migration-verifier
 
-go 1.20
+go 1.22
 
 require (
 	github.com/cespare/permute/v2 v2.0.0-beta2
diff --git a/internal/retry/retry.go b/internal/retry/retry.go
index 4f9358fb..59bf0746 100644
--- a/internal/retry/retry.go
+++ b/internal/retry/retry.go
@@ -3,6 +3,7 @@ package retry
 import (
 	"context"
 	"math/rand"
+	"slices"
 	"time"
 
 	"github.com/10gen/migration-verifier/internal/logger"
@@ -11,6 +12,8 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 )
 
+var CustomTransientErr = errors.New("possibly-transient failure detected")
+
 // RunForUUIDAndTransientErrors retries f() for the CollectionUUIDMismatch error and for transient errors.
 // This should be used to run a driver operation that optionally specifies the `collectionUUID` parameter
 // for a collection that may have been:
@@ -273,25 +276,42 @@ func (r *Retryer) shouldRetryWithSleep(
 		return false
 	}
 
+	var whyTransient string
+
 	errCode := util.GetErrorCode(err)
-	if util.IsTransientError(err) {
-		logger.Warn().Int("error code", errCode).Err(err).Msgf(
-			"Waiting %s seconds to retry operation after transient error.", sleepTime)
-		return true
+
+	if util.IsTransientError(err) || slices.Contains(r.additionalErrorCodes, errCode) {
+		whyTransient = "Error code suggests a transient error."
+	} else if errors.Is(err, CustomTransientErr) {
+		whyTransient = "Error may be transient."
 	}
 
-	for _, code := range r.additionalErrorCodes {
-		if code == errCode {
-			logger.Warn().Int("error code", errCode).Err(err).Msgf(
-				"Waiting %s seconds to retry operation after an error because it is in our additional codes list.", sleepTime)
-			return true
+	if whyTransient == "" {
+		event := logger.Debug().Err(err)
+
+		if errCode != 0 {
+			event = event.Int("error code", errCode)
 		}
+
+		event.Msg("Not retrying on error because it appears not to be transient.")
+
+		return false
+	}
+
+	retryLogEvent := logger.Warn().
+		Err(err).
+		Stringer("duration", sleepTime)
+
+	if errCode != 0 {
+		retryLogEvent = retryLogEvent.
+			Int("error code", errCode)
 	}
 
-	logger.Debug().Err(err).Int("error code", errCode).
-		Msg("Not retrying on error because it is not transient nor is it in our additional codes list.")
+	retryLogEvent.
+		Str("reason", whyTransient).
+		Msg("Pausing then will retry operation.")
 
-	return false
+	return true
 }
 
 // Use this method for aggregates which should take a UUID in new versions but not old ones.
diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index e03a0c39..804d25e2 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -12,7 +12,6 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
-	"golang.org/x/exp/constraints"
 )
 
 const fauxDocSizeForDeleteEvents = 1024
@@ -138,12 +137,14 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 		return err
 	}
 
-	readAndHandleOneChangeEventBatch := func() (bool, error) {
+	readAndHandleOneChangeEventBatch := func() error {
 		eventsRead := 0
 		var changeEventBatch []ParsedEvent
 
 		for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
+			verifier.logger.Info().Msg("reading event")
 			gotEvent := cs.TryNext(ctx)
+			verifier.logger.Info().Err(cs.Err()).Msgf("got event? %v", gotEvent)
 
 			if !gotEvent || cs.Err() != nil {
 				break
@@ -154,21 +155,29 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 			}
 
 			if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
-				return false, errors.Wrap(err, "failed to decode change event")
+				return errors.Wrap(err, "failed to decode change event")
 			}
 
+			verifier.logger.Debug().Interface("event", changeEventBatch[eventsRead]).Msg("Got event.")
+
 			eventsRead++
 		}
 
-		if eventsRead > 0 {
-			verifier.logger.Debug().Int("eventsCount", eventsRead).Msgf("Received a batch of events.")
-			err := verifier.HandleChangeStreamEvents(ctx, changeEventBatch)
-			if err != nil {
-				return false, errors.Wrap(err, "failed to handle change events")
-			}
+		if cs.Err() != nil {
+			return errors.Wrap(cs.Err(), "change stream iteration failed")
+		}
+
+		if eventsRead == 0 {
+			return nil
 		}
 
-		return eventsRead > 0, errors.Wrap(cs.Err(), "change stream iteration failed")
+		verifier.logger.Debug().Int("eventsCount", eventsRead).Msgf("Received a batch of events.")
+		err := verifier.HandleChangeStreamEvents(ctx, changeEventBatch)
+		if err != nil {
+			return errors.Wrap(err, "failed to handle change events")
+		}
+
+		return nil
 	}
 
 	for {
@@ -184,29 +193,45 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 		// If the changeStreamEnderChan has a message, the user has indicated that
 		// source writes are ended. This means we should exit rather than continue
 		// reading the change stream since there should be no more events.
-		case <-verifier.changeStreamEnderChan:
+		case finalTs := <-verifier.changeStreamFinalTsChan:
 			verifier.logger.Debug().
-				Msg("Change stream thread received shutdown request.")
+				Interface("finalTimestamp", finalTs).
+				Msg("Change stream thread received final timestamp. Finalizing change stream.")
 
 			changeStreamEnded = true
 
 			// Read all change events until the source reports no events.
 			// (i.e., the `getMore` call returns empty)
 			for {
-				var gotEvent bool
-				gotEvent, err = readAndHandleOneChangeEventBatch()
+				var curTs primitive.Timestamp
+				curTs, err = extractTimestampFromResumeToken(cs.ResumeToken())
+				if err != nil {
+					err = errors.Wrap(err, "failed to extract timestamp from change stream's resume token")
+					break
+				}
+
+				if curTs.Compare(finalTs) >= 0 {
+					verifier.logger.Debug().
+						Interface("currentTimestamp", curTs).
+						Interface("finalTimestamp", finalTs).
+						Msg("Change stream has reached the final timestamp. Shutting down.")
 
-				if !gotEvent || err != nil {
+					break
+				}
+
+				err = readAndHandleOneChangeEventBatch()
+
+				if err != nil {
 					break
 				}
 			}
 
 		default:
-			_, err = readAndHandleOneChangeEventBatch()
-		}
+			err = readAndHandleOneChangeEventBatch()
 
-		if err == nil {
-			err = persistResumeTokenIfNeeded()
+			if err == nil {
+				err = persistResumeTokenIfNeeded()
+			}
 		}
 
 		if err != nil && !errors.Is(err, context.Canceled) {
@@ -237,9 +262,9 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 
 	infoLog := verifier.logger.Info()
 	if verifier.lastChangeEventTime == nil {
-		infoLog = infoLog.Str("changeStreamStopTime", "none")
+		infoLog = infoLog.Str("lastEventTime", "none")
 	} else {
-		infoLog = infoLog.Interface("changeStreamStopTime", *verifier.lastChangeEventTime)
+		infoLog = infoLog.Interface("lastEventTime", *verifier.lastChangeEventTime)
 	}
 
 	infoLog.Msg("Change stream is done.")
@@ -265,7 +290,7 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 
 		ts, err := extractTimestampFromResumeToken(savedResumeToken)
 		if err == nil {
-			logEvent = addUnixTimeToLogEvent(ts.T, logEvent)
+			logEvent = addTimestampToLogEvent(ts, logEvent)
 		} else {
 			verifier.logger.Warn().
 				Err(err).
@@ -284,6 +309,7 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 		return errors.Wrap(err, "failed to start session")
 	}
 	sctx := mongo.NewSessionContext(ctx, sess)
+	verifier.logger.Debug().Interface("pipeline", pipeline).Msg("Opened change stream.")
 	srcChangeStream, err := verifier.srcClient.Watch(sctx, pipeline, opts)
 	if err != nil {
 		return errors.Wrap(err, "failed to open change stream")
@@ -318,8 +344,10 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 	return nil
 }
 
-func addUnixTimeToLogEvent[T constraints.Integer](unixTime T, event *zerolog.Event) *zerolog.Event {
-	return event.Time("clockTime", time.Unix(int64(unixTime), int64(0)))
+func addTimestampToLogEvent(ts primitive.Timestamp, event *zerolog.Event) *zerolog.Event {
+	return event.
+		Interface("timestamp", ts).
+		Time("time", time.Unix(int64(ts.T), int64(0)))
 }
 
 func (v *Verifier) getChangeStreamMetadataCollection() *mongo.Collection {
@@ -358,7 +386,7 @@ func (verifier *Verifier) persistChangeStreamResumeToken(ctx context.Context, cs
 		logEvent := verifier.logger.Debug()
 
 		if err == nil {
-			logEvent = addUnixTimeToLogEvent(ts.T, logEvent)
+			logEvent = addTimestampToLogEvent(ts, logEvent)
 		} else {
 			verifier.logger.Warn().Err(err).
 				Msg("failed to extract resume token timestamp")
diff --git a/internal/verifier/check.go b/internal/verifier/check.go
index 5a60bfb5..e3a143c3 100644
--- a/internal/verifier/check.go
+++ b/internal/verifier/check.go
@@ -42,13 +42,24 @@ func (verifier *Verifier) Check(ctx context.Context, filter map[string]any) {
 	verifier.MaybeStartPeriodicHeapProfileCollection(ctx)
 }
 
-func (verifier *Verifier) waitForChangeStream() error {
+func (verifier *Verifier) waitForChangeStream(ctx context.Context) error {
 	verifier.mux.RLock()
 	csRunning := verifier.changeStreamRunning
 	verifier.mux.RUnlock()
 	if csRunning {
 		verifier.logger.Debug().Msg("Changestream still running, signalling that writes are done and waiting for change stream to exit")
-		verifier.changeStreamEnderChan <- struct{}{}
+
+		finalTs, err := GetClusterTime(
+			ctx,
+			verifier.logger,
+			verifier.srcClient,
+		)
+
+		if err != nil {
+			return errors.Wrapf(err, "failed to fetch source's cluster time")
+		}
+
+		verifier.changeStreamFinalTsChan <- finalTs
 		select {
 		case err := <-verifier.changeStreamErrChan:
 			verifier.logger.Warn().Err(err).
@@ -242,7 +253,7 @@ func (verifier *Verifier) CheckDriver(ctx context.Context, filter map[string]any
 			// It's necessary to wait for the change stream to finish before incrementing the
 			// generation number, or the last changes will not be checked.
 			verifier.mux.Unlock()
-			err := verifier.waitForChangeStream()
+			err := verifier.waitForChangeStream(ctx)
 			if err != nil {
 				return err
 			}
diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
new file mode 100644
index 00000000..7be92ea0
--- /dev/null
+++ b/internal/verifier/clustertime.go
@@ -0,0 +1,141 @@
+package verifier
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/10gen/migration-verifier/internal/logger"
+	"github.com/10gen/migration-verifier/internal/retry"
+	"github.com/10gen/migration-verifier/internal/util"
+	"github.com/10gen/migration-verifier/mbson"
+	"github.com/pkg/errors"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+const opTimeKeyInServerResponse = "operationTime"
+
+// GetClusterTime returns the remote cluster’s cluster time.
+// In so doing it creates a new oplog entry across all shards.
+// The “note” will go into the cluster’s oplog, so keep it
+// short but meaningful.
+func GetClusterTime(
+	ctx context.Context,
+	logger *logger.Logger,
+	client *mongo.Client, // could change to type=any if we need
+) (primitive.Timestamp, error) {
+	retryer := retry.New(retry.DefaultDurationLimit)
+
+	var optime primitive.Timestamp
+
+	// To get the cluster time, we submit a request to append an oplog note
+	// but set an unreasonably-early maxClusterTime. All the shards will fail
+	// the request but still send their current clusterTime to the mongos,
+	// which will return the most recent of those. Thus we can fetch the
+	// most recent cluster time without altering the oplog.
+	err := retryer.RunForTransientErrorsOnly(
+		ctx,
+		logger,
+		func(_ *retry.Info) error {
+			var err error
+			optime, err = runAppendOplogNote(ctx, client, true)
+			if err != nil {
+				err = fmt.Errorf("%w: %w", retry.CustomTransientErr, err)
+			}
+			return err
+		},
+	)
+
+	if err != nil {
+		return primitive.Timestamp{}, err
+	}
+
+	// OPTIMIZATION: We now append an oplog entry--this time for real!--to
+	// cause any lagging shards to output their events.
+	//
+	// Since this is just an optimization, failures here are nonfatal.
+	err = retryer.RunForTransientErrorsOnly(
+		ctx,
+		logger,
+		func(_ *retry.Info) error {
+			var err error
+			optime, err = runAppendOplogNote(ctx, client, false)
+			if err != nil {
+				err = fmt.Errorf("%w: %w", retry.CustomTransientErr, err)
+			}
+			return err
+		},
+	)
+	if err != nil {
+		// This isn't serious enough even to warn on, so leave it at info-level.
+		logger.Info().Err(err).
+			Msg("Failed to append oplog note; change stream may need extra time to finish.")
+	}
+
+	return optime, nil
+}
+
+func runAppendOplogNote(
+	ctx context.Context,
+	client *mongo.Client,
+	doomed bool,
+) (primitive.Timestamp, error) {
+	// We don’t need to write to any shards’ oplogs; we just
+	// need to fetch
+	cmd := bson.D{
+		{"appendOplogNote", 1},
+		{"data", bson.D{
+			{"migration-verifier", "expect StaleClusterTime error"},
+		}},
+	}
+
+	if doomed {
+		cmd = append(cmd, bson.E{"maxClusterTime", primitive.Timestamp{1, 0}})
+	}
+
+	resp := client.Database("admin").RunCommand(ctx, cmd)
+
+	err := resp.Err()
+
+	if doomed {
+		// We expect an error here; if we didn't get one then something is
+		// amiss on the server.
+		if err == nil {
+			return primitive.Timestamp{}, errors.Errorf("server request unexpectedly succeeded: %v", cmd)
+		}
+
+		if !util.IsStaleClusterTimeError(err) {
+			return primitive.Timestamp{}, errors.Errorf(
+				"unexpected error (expected StaleClusterTime) from request (%v): %v",
+				err,
+				cmd,
+			)
+		}
+	} else if err != nil {
+		return primitive.Timestamp{}, errors.Wrapf(
+			err,
+			"command (%v) failed unexpectedly",
+			cmd,
+		)
+	}
+
+	rawResponse, _ := resp.Raw()
+
+	return getOpTimeFromRawResponse(rawResponse)
+}
+
+func getOpTimeFromRawResponse(rawResponse bson.Raw) (primitive.Timestamp, error) {
+	// Get the `operationTime` from the response and return it.
+	var optime primitive.Timestamp
+
+	found, err := mbson.RawLookup(rawResponse, &optime, opTimeKeyInServerResponse)
+	if err != nil {
+		return primitive.Timestamp{}, errors.Errorf("failed to read server response (%s)", rawResponse)
+	}
+	if !found {
+		return primitive.Timestamp{}, errors.Errorf("server response (%s) lacks %#q", rawResponse, opTimeKeyInServerResponse)
+	}
+
+	return optime, nil
+}
diff --git a/internal/verifier/clustertime_test.go b/internal/verifier/clustertime_test.go
new file mode 100644
index 00000000..eab0f61e
--- /dev/null
+++ b/internal/verifier/clustertime_test.go
@@ -0,0 +1,13 @@
+package verifier
+
+import "context"
+
+func (suite *IntegrationTestSuite) TestGetClusterTime() {
+	ctx := context.Background()
+	logger, _ := getLoggerAndWriter("stdout")
+
+	ts, err := GetClusterTime(ctx, logger, suite.srcMongoClient)
+	suite.Require().NoError(err)
+
+	suite.Assert().NotZero(ts, "timestamp should be nonzero")
+}
diff --git a/internal/verifier/logging_setup.go b/internal/verifier/logging_setup.go
index 051840c0..7f4a3ac7 100644
--- a/internal/verifier/logging_setup.go
+++ b/internal/verifier/logging_setup.go
@@ -29,3 +29,15 @@ func getLogWriter(logPath string) io.Writer {
 
 	return zerolog.SyncWriter(writer)
 }
+
+func getLoggerAndWriter(logPath string) (*logger.Logger, io.Writer) {
+	writer := getLogWriter(logPath)
+
+	consoleWriter := zerolog.ConsoleWriter{
+		Out:        writer,
+		TimeFormat: timeFormat,
+	}
+
+	l := zerolog.New(consoleWriter).With().Timestamp().Logger()
+	return logger.NewLogger(&l, writer), writer
+}
diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go
index 809e4fe3..39c4fbf8 100644
--- a/internal/verifier/migration_verifier.go
+++ b/internal/verifier/migration_verifier.go
@@ -24,7 +24,6 @@ import (
 	"github.com/10gen/migration-verifier/internal/uuidutil"
 	"github.com/olekukonko/tablewriter"
 	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -49,7 +48,6 @@ const (
 	SrcNamespaceField = "query_filter.namespace"
 	DstNamespaceField = "query_filter.to"
 	NumWorkers        = 10
-	refetch           = "TODO_CHANGE_ME_REFETCH"
 	Idle              = "idle"
 	Check             = "check"
 	Recheck           = "recheck"
@@ -123,12 +121,12 @@ type Verifier struct {
 	metaDBName    string
 	srcStartAtTs  *primitive.Timestamp
 
-	mux                   sync.RWMutex
-	changeStreamRunning   bool
-	changeStreamEnderChan chan struct{}
-	changeStreamErrChan   chan error
-	changeStreamDoneChan  chan struct{}
-	lastChangeEventTime   *primitive.Timestamp
+	mux                     sync.RWMutex
+	changeStreamRunning     bool
+	changeStreamFinalTsChan chan primitive.Timestamp
+	changeStreamErrChan     chan error
+	changeStreamDoneChan    chan struct{}
+	lastChangeEventTime     *primitive.Timestamp
 
 	readConcernSetting ReadConcernSetting
 
@@ -188,15 +186,15 @@ func NewVerifier(settings VerifierSettings) *Verifier {
 	}
 
 	return &Verifier{
-		phase:                 Idle,
-		numWorkers:            NumWorkers,
-		readPreference:        readpref.Primary(),
-		partitionSizeInBytes:  400 * 1024 * 1024,
-		failureDisplaySize:    DefaultFailureDisplaySize,
-		changeStreamEnderChan: make(chan struct{}),
-		changeStreamErrChan:   make(chan error),
-		changeStreamDoneChan:  make(chan struct{}),
-		readConcernSetting:    readConcern,
+		phase:                   Idle,
+		numWorkers:              NumWorkers,
+		readPreference:          readpref.Primary(),
+		partitionSizeInBytes:    400 * 1024 * 1024,
+		failureDisplaySize:      DefaultFailureDisplaySize,
+		changeStreamFinalTsChan: make(chan primitive.Timestamp),
+		changeStreamErrChan:     make(chan error),
+		changeStreamDoneChan:    make(chan struct{}),
+		readConcernSetting:      readConcern,
 
 		// This will get recreated once gen0 starts, but we want it
 		// here in case the change streams gets an event before then.
@@ -315,15 +313,7 @@ func (verifier *Verifier) SetPartitionSizeMB(partitionSizeMB uint32) {
 }
 
 func (verifier *Verifier) SetLogger(logPath string) {
-	writer := getLogWriter(logPath)
-	verifier.writer = writer
-
-	consoleWriter := zerolog.ConsoleWriter{
-		Out:        writer,
-		TimeFormat: timeFormat,
-	}
-	l := zerolog.New(consoleWriter).With().Timestamp().Logger()
-	verifier.logger = logger.NewLogger(&l, writer)
+	verifier.logger, verifier.writer = getLoggerAndWriter(logPath)
 }
 
 func (verifier *Verifier) SetSrcNamespaces(arg []string) {
@@ -1172,10 +1162,6 @@ func (verifier *Verifier) verificationTaskCollection() *mongo.Collection {
 	return verifier.verificationDatabase().Collection(verificationTasksCollection)
 }
 
-func (verifier *Verifier) refetchCollection() *mongo.Collection {
-	return verifier.verificationDatabase().Collection(refetch)
-}
-
 func (verifier *Verifier) srcClientDatabase(dbName string) *mongo.Database {
 	db := verifier.srcClient.Database(dbName)
 	// No need to check the write concern because we do not write to the source database.
diff --git a/internal/verifier/migration_verifier_bench_test.go b/internal/verifier/migration_verifier_bench_test.go
index 900c260f..9732ded4 100644
--- a/internal/verifier/migration_verifier_bench_test.go
+++ b/internal/verifier/migration_verifier_bench_test.go
@@ -81,10 +81,6 @@ func BenchmarkGeneric(t *testing.B) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = verifier.refetchCollection().Drop(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	println("Starting tasks")
 	for _, namespace := range namespaces {
diff --git a/mbson/bson_raw.go b/mbson/bson_raw.go
new file mode 100644
index 00000000..0759ffca
--- /dev/null
+++ b/mbson/bson_raw.go
@@ -0,0 +1,29 @@
+package mbson
+
+import (
+	"github.com/pkg/errors"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/x/bsonx/bsoncore"
+)
+
+// RawLookup combines bson.Raw’s LookupErr method with an additional
+// unmarshal step. The result is a convenient way to extract values from
+// bson.Raw. The returned boolean indicates whether the value was found.
+func RawLookup[T any](doc bson.Raw, dest *T, keys ...string) (bool, error) {
+	val, err := doc.LookupErr(keys...)
+
+	if err == nil {
+		return true, val.Unmarshal(dest)
+	} else if errors.Is(err, bsoncore.ErrElementNotFound) {
+		return false, nil
+	}
+
+	return false, errors.Wrapf(err, "failed to look up %+v in BSON doc", keys)
+}
+
+// RawContains is like RawLookup but makes no effort to unmarshal
+// the value.
+func RawContains(doc bson.Raw, keys ...string) (bool, error) {
+	val := any(nil)
+	return RawLookup(doc, &val, keys...)
+}
diff --git a/mbson/unit_test.go b/mbson/unit_test.go
new file mode 100644
index 00000000..0a66e92d
--- /dev/null
+++ b/mbson/unit_test.go
@@ -0,0 +1,88 @@
+package mbson
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/x/bsonx/bsoncore"
+)
+
+type UnitTestSuite struct {
+	suite.Suite
+}
+
+func TestUnitTestSuite(t *testing.T) {
+	ts := new(UnitTestSuite)
+	suite.Run(t, ts)
+}
+
+func (s *UnitTestSuite) Test_RawLookup() {
+	myDoc := bson.D{
+		{"foo", 1},
+		{"bar", "baz"},
+		{"quux", bson.A{
+			123,
+			bson.D{{"hey", "ho"}},
+		}},
+	}
+
+	myRaw, err := bson.Marshal(myDoc)
+	s.Require().NoError(err)
+
+	var myInt int
+	var myStr string
+
+	found, err := RawLookup(myRaw, &myInt, "foo")
+	s.Require().True(found)
+	s.Require().NoError(err)
+	s.Assert().EqualValues(1, myInt)
+
+	found, err = RawLookup(myRaw, &myInt, "quux", "0")
+	s.Require().True(found)
+	s.Require().NoError(err)
+	s.Assert().EqualValues(123, myInt)
+
+	found, err = RawLookup(myRaw, &myStr, "quux", "1", "hey")
+	s.Require().True(found)
+	s.Require().NoError(err)
+	s.Assert().EqualValues("ho", myStr)
+
+	found, err = RawLookup(myRaw, &myStr, "not there")
+	s.Require().NoError(err)
+	s.Assert().False(found)
+
+	myRaw = myRaw[:len(myRaw)-2]
+	_, err = RawLookup(myRaw, &myStr, "not there")
+	s.Assert().ErrorAs(err, &bsoncore.InsufficientBytesError{})
+}
+
+func (s *UnitTestSuite) Test_RawContains() {
+	myDoc := bson.D{
+		{"foo", 1},
+		{"bar", "baz"},
+		{"quux", bson.A{
+			123,
+			bson.D{{"hey", "ho"}},
+		}},
+	}
+
+	myRaw, err := bson.Marshal(myDoc)
+	s.Require().NoError(err)
+
+	has, err := RawContains(myRaw, "foo")
+	s.Require().NoError(err)
+	s.Assert().True(has, "`foo` should exist")
+
+	has, err = RawContains(myRaw, "quux", "1", "hey")
+	s.Require().NoError(err)
+	s.Assert().True(has, "deep lookup should work")
+
+	has, err = RawContains(myRaw, "not there")
+	s.Require().NoError(err)
+	s.Assert().False(has, "missing element should not exist")
+
+	myRaw = myRaw[:len(myRaw)-2]
+	_, err = RawContains(myRaw, "not there")
+	s.Assert().ErrorAs(err, &bsoncore.InsufficientBytesError{})
+}

From 8dad68bb61d855d4f730af2359e2fbf1f90d74e9 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 12:52:20 -0500
Subject: [PATCH 061/104] =?UTF-8?q?with=20REP-5306=E2=80=99s=20changes?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 internal/verifier/change_stream_test.go      | 16 ++++++----------
 internal/verifier/migration_verifier_test.go | 12 ------------
 2 files changed, 6 insertions(+), 22 deletions(-)

diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index 89bc7877..b2678272 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -141,16 +141,12 @@ func (suite *IntegrationTestSuite) TestStartAtTimeNoChanges() {
 	err = verifier.StartChangeStream(ctx)
 	suite.Require().NoError(err)
 	suite.Require().Equal(verifier.srcStartAtTs, origStartTs)
-	verifier.changeStreamEnderChan <- struct{}{}
+	verifier.changeStreamFinalTsChan <- *origStartTs
 	<-verifier.changeStreamDoneChan
 	suite.Require().Equal(verifier.srcStartAtTs, origStartTs)
 }
 
 func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
-	if suite.GetSrcTopology() == TopologySharded {
-		suite.T().Skip("Skipping pending REP-5299.")
-	}
-
 	verifier := suite.BuildVerifier()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -187,13 +183,13 @@ func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
 		"session time after events should exceed the original",
 	)
 
-	verifier.changeStreamEnderChan <- struct{}{}
+	verifier.changeStreamFinalTsChan <- *postEventsSessionTime
 	<-verifier.changeStreamDoneChan
 
-	suite.Assert().GreaterOrEqual(
-		verifier.srcStartAtTs.Compare(*postEventsSessionTime),
-		0,
-		"verifier.srcStartAtTs should now meet or exceed our session timestamp",
+	suite.Assert().Equal(
+		*postEventsSessionTime,
+		verifier.srcStartAtTs,
+		"verifier.srcStartAtTs should now be our session timestamp",
 	)
 }
 
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index ff125fea..a4ce6c72 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1152,10 +1152,6 @@ func TestVerifierCompareIndexSpecs(t *testing.T) {
 }
 
 func (suite *IntegrationTestSuite) TestVerifierNamespaceList() {
-	if suite.GetSrcTopology() == TopologySharded {
-		suite.T().Skip("Skipping pending REP-5299.")
-	}
-
 	verifier := suite.BuildVerifier()
 	ctx := suite.Context()
 
@@ -1282,10 +1278,6 @@ func (suite *IntegrationTestSuite) TestVerificationStatus() {
 }
 
 func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
-	if suite.GetSrcTopology() == TopologySharded {
-		suite.T().Skip("Skipping pending REP-5299.")
-	}
-
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	verifier := suite.BuildVerifier()
 	verifier.SetSrcNamespaces([]string{"testDb1.testColl1"})
@@ -1394,10 +1386,6 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 }
 
 func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
-	if suite.GetSrcTopology() == TopologySharded {
-		suite.T().Skip("Skipping pending REP-5299.")
-	}
-
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
 	filter := map[string]any{"inFilter": map[string]any{"$ne": false}}

From 6742a59d4dd57631f197c83d194512c6433dcaa4 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 12:56:20 -0500
Subject: [PATCH 062/104] fix types

---
 internal/verifier/change_stream_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index b2678272..597fed8a 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -188,7 +188,7 @@ func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
 
 	suite.Assert().Equal(
 		*postEventsSessionTime,
-		verifier.srcStartAtTs,
+		*verifier.srcStartAtTs,
 		"verifier.srcStartAtTs should now be our session timestamp",
 	)
 }

From 1499c2e3a838c056cc1f65a9cabbc4709db940af Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 15:47:56 -0500
Subject: [PATCH 063/104] fix test

---
 internal/verifier/change_stream.go           | 28 +++++++++------
 internal/verifier/check.go                   | 23 +++++++++----
 internal/verifier/integration_test_suite.go  | 13 +++++++
 internal/verifier/migration_verifier_test.go | 36 ++++++++++++++------
 4 files changed, 72 insertions(+), 28 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 86886ca9..91152248 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -104,6 +104,11 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 // GetChangeStreamFilter returns an aggregation pipeline that filters
 // namespaces as per configuration.
 //
+// Note that this omits verifier.globalFilter because we still need to
+// recheck any out-filter documents that may have changed in order to
+// account for filter traversals (i.e., updates that change whether a
+// document matches the filter).
+//
 // NB: Ideally we could make the change stream give $bsonSize(fullDocument)
 // and omit fullDocument, but $bsonSize was new in MongoDB 4.4, and we still
 // want to verify migrations from 4.2. fullDocument is unlikely to be a
@@ -144,14 +149,20 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 		var changeEventBatch []ParsedEvent
 
 		for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
-			verifier.logger.Info().Msg("reading event")
 			gotEvent := cs.TryNext(ctx)
-			verifier.logger.Info().Err(cs.Err()).Msgf("got event? %v", gotEvent)
 
-			if !gotEvent || cs.Err() != nil {
+			if cs.Err() != nil {
+				return errors.Wrap(cs.Err(), "change stream iteration failed")
+			}
+
+			if !gotEvent {
 				break
 			}
 
+			verifier.logger.Debug().
+				Stringer("resumeToken", cs.ResumeToken()).
+				Msg("TryNext got an event!")
+
 			if changeEventBatch == nil {
 				changeEventBatch = make([]ParsedEvent, cs.RemainingBatchLength()+1)
 			}
@@ -160,20 +171,13 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 				return errors.Wrap(err, "failed to decode change event")
 			}
 
-			verifier.logger.Debug().Interface("event", changeEventBatch[eventsRead]).Msg("Got event.")
-
 			eventsRead++
 		}
 
-		if cs.Err() != nil {
-			return errors.Wrap(cs.Err(), "change stream iteration failed")
-		}
-
 		if eventsRead == 0 {
 			return nil
 		}
 
-		verifier.logger.Debug().Int("eventsCount", eventsRead).Msgf("Received a batch of events.")
 		err := verifier.HandleChangeStreamEvents(ctx, changeEventBatch)
 		if err != nil {
 			return errors.Wrap(err, "failed to handle change events")
@@ -302,7 +306,9 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 				Msg("Failed to extract timestamp from persisted resume token.")
 		}
 
-		logEvent.Msg("Starting change stream from persisted resume token.")
+		logEvent.
+			Interface("pipeline", pipeline).
+			Msg("Starting change stream from persisted resume token.")
 
 		opts = opts.SetStartAfter(savedResumeToken)
 	} else {
diff --git a/internal/verifier/check.go b/internal/verifier/check.go
index e3a143c3..c88ad840 100644
--- a/internal/verifier/check.go
+++ b/internal/verifier/check.go
@@ -77,6 +77,12 @@ func (verifier *Verifier) waitForChangeStream(ctx context.Context) error {
 func (verifier *Verifier) CheckWorker(ctx context.Context) error {
 	verifier.logger.Debug().Msgf("Starting %d verification workers", verifier.numWorkers)
 	ctx, cancel := context.WithCancel(ctx)
+
+	if verifier.workerSleepDelayMillis == 0 {
+		verifier.logger.Info().
+			Msg("Worker sleep delay is zero. Only tests should do this.")
+	}
+
 	wg := sync.WaitGroup{}
 	for i := 0; i < verifier.numWorkers; i++ {
 		wg.Add(1)
@@ -185,7 +191,9 @@ func (verifier *Verifier) CheckDriver(ctx context.Context, filter map[string]any
 	verifier.mux.RLock()
 	csRunning := verifier.changeStreamRunning
 	verifier.mux.RUnlock()
-	if !csRunning {
+	if csRunning {
+		verifier.logger.Debug().Msg("Check: Change stream already running.")
+	} else {
 		verifier.logger.Debug().Msg("Change stream not running; starting change stream")
 
 		err = verifier.StartChangeStream(ctx)
@@ -399,12 +407,15 @@ func (verifier *Verifier) Work(ctx context.Context, workerNum int, wg *sync.Wait
 			if errors.Is(err, mongo.ErrNoDocuments) {
 				duration := verifier.workerSleepDelayMillis * time.Millisecond
 
-				verifier.logger.Debug().
-					Int("workerNum", workerNum).
-					Stringer("duration", duration).
-					Msg("No tasks found. Sleeping.")
+				if duration > 0 {
+					verifier.logger.Debug().
+						Int("workerNum", workerNum).
+						Stringer("duration", duration).
+						Msg("No tasks found. Sleeping.")
+
+					time.Sleep(duration)
+				}
 
-				time.Sleep(duration)
 				continue
 			} else if err != nil {
 				panic(err)
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index d3a112c8..41f5d018 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -94,6 +94,19 @@ func (suite *IntegrationTestSuite) SetupTest() {
 		dbname,
 	)
 
+	for _, client := range []*mongo.Client{suite.srcMongoClient, suite.dstMongoClient} {
+		dbNames, err := client.ListDatabaseNames(ctx, bson.D{})
+		suite.Require().NoError(err, "should list database names")
+		for _, dbName := range dbNames {
+			if strings.Index(dbName, suite.DBNameForTest()) == 0 {
+				suite.T().Logf("Dropping database %#q because it seems to be left over from an earlier run of this test.", dbName)
+				suite.Require().NoError(client.Database(dbName).Drop(ctx))
+			}
+
+			suite.initialDbNames.Add(dbName)
+		}
+	}
+
 	suite.testContext, suite.contextCanceller = ctx, canceller
 }
 
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index a4ce6c72..f3b00c54 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -14,6 +14,7 @@ import (
 	"regexp"
 	"sort"
 	"testing"
+	"time"
 
 	"github.com/10gen/migration-verifier/internal/partitions"
 	"github.com/10gen/migration-verifier/internal/testutil"
@@ -1236,10 +1237,13 @@ func (suite *IntegrationTestSuite) TestVerifierNamespaceList() {
 	suite.ElementsMatch([]string{"testDb1.testColl1", "testDb1.testColl2", "testDb1.testView1"}, verifier.dstNamespaces)
 
 	// Collections in admin, config, and local should not be found
-	err = suite.srcMongoClient.Database("local").CreateCollection(ctx, "islocalSrc")
-	suite.Require().NoError(err)
-	err = suite.dstMongoClient.Database("local").CreateCollection(ctx, "islocalDest")
-	suite.Require().NoError(err)
+	if suite.GetSrcTopology() != TopologySharded {
+		err = suite.srcMongoClient.Database("local").CreateCollection(ctx, "islocalSrc")
+		suite.Require().NoError(err)
+		err = suite.dstMongoClient.Database("local").CreateCollection(ctx, "islocalDest")
+		suite.Require().NoError(err)
+	}
+
 	err = suite.srcMongoClient.Database("admin").CreateCollection(ctx, "isAdminSrc")
 	suite.Require().NoError(err)
 	err = suite.dstMongoClient.Database("admin").CreateCollection(ctx, "isAdminDest")
@@ -1388,10 +1392,13 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 	zerolog.SetGlobalLevel(zerolog.DebugLevel)
 
-	filter := map[string]any{"inFilter": map[string]any{"$ne": false}}
+	dbname1 := suite.DBNameForTest("1")
+	dbname2 := suite.DBNameForTest("2")
+
+	filter := bson.M{"inFilter": bson.M{"$ne": false}}
 	verifier := suite.BuildVerifier()
-	verifier.SetSrcNamespaces([]string{"testDb1.testColl1"})
-	verifier.SetDstNamespaces([]string{"testDb2.testColl3"})
+	verifier.SetSrcNamespaces([]string{dbname1 + ".testColl1"})
+	verifier.SetDstNamespaces([]string{dbname2 + ".testColl3"})
 	verifier.SetNamespaceMap()
 	verifier.SetIgnoreBSONFieldOrder(true)
 	// Set this value low to test the verifier with multiple partitions.
@@ -1399,8 +1406,8 @@ func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 
 	ctx := suite.Context()
 
-	srcColl := suite.srcMongoClient.Database("testDb1").Collection("testColl1")
-	dstColl := suite.dstMongoClient.Database("testDb2").Collection("testColl3")
+	srcColl := suite.srcMongoClient.Database(dbname1).Collection("testColl1")
+	dstColl := suite.dstMongoClient.Database(dbname2).Collection("testColl3")
 
 	// Documents with _id in [0, 100) should match.
 	var docs []interface{}
@@ -1432,7 +1439,11 @@ func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 		suite.Require().NoError(err)
 
 		for status.TotalTasks == 0 && verifier.generation < 10 {
-			suite.T().Logf("TotalTasks is 0 (generation=%d); waiting another generation …", verifier.generation)
+			delay := time.Second
+
+			suite.T().Logf("TotalTasks is 0 (generation=%d); waiting %s then will run another generation …", verifier.generation, delay)
+
+			time.Sleep(delay)
 			checkContinueChan <- struct{}{}
 			<-checkDoneChan
 			status, err = verifier.GetVerificationStatus()
@@ -1449,6 +1460,7 @@ func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 	suite.Require().Equal(status.FailedTasks, 0)
 
 	// Insert another document that is not in the filter.
+	// This should trigger a recheck despite being outside the filter.
 	_, err = srcColl.InsertOne(ctx, bson.M{"_id": 200, "x": 200, "inFilter": false})
 	suite.Require().NoError(err)
 
@@ -1457,11 +1469,13 @@ func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 
 	// Wait for generation to finish.
 	<-checkDoneChan
+
 	status = waitForTasks()
+
 	// There should be no failures, since the inserted document is not in the filter.
 	suite.Require().Equal(VerificationStatus{TotalTasks: 1, CompletedTasks: 1}, *status)
 
-	// Now insert in the source, this should come up next generation.
+	// Now insert in the source. This should come up next generation.
 	_, err = srcColl.InsertOne(ctx, bson.M{"_id": 201, "x": 201, "inFilter": true})
 	suite.Require().NoError(err)
 

From 0bee1e2e243e23ba58cc31efbf9a24fc8f34a170 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 15:59:53 -0500
Subject: [PATCH 064/104] more delay

---
 internal/verifier/migration_verifier_test.go | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index f3b00c54..9504ceae 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1199,8 +1199,12 @@ func (suite *IntegrationTestSuite) TestVerifierNamespaceList() {
 	suite.Require().NoError(err)
 	err = suite.dstMongoClient.Database("testDb4").CreateCollection(ctx, "testColl6")
 	suite.Require().NoError(err)
-	err = suite.dstMongoClient.Database("local").CreateCollection(ctx, "testColl7")
-	suite.Require().NoError(err)
+
+	if suite.GetSrcTopology() != TopologySharded {
+		err = suite.dstMongoClient.Database("local").CreateCollection(ctx, "testColl7")
+		suite.Require().NoError(err)
+	}
+
 	err = suite.dstMongoClient.Database("mongosync_reserved_for_internal_use").CreateCollection(ctx, "globalState")
 	suite.Require().NoError(err)
 	err = suite.dstMongoClient.Database("mongosync_reserved_for_verification_src_metadata").CreateCollection(ctx, "auditor")
@@ -1317,7 +1321,11 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 		suite.Require().NoError(err)
 
 		for status.TotalTasks == 0 && verifier.generation < 10 {
-			suite.T().Logf("TotalTasks is 0 (generation=%d); waiting another generation …", verifier.generation)
+			delay := time.Second
+
+			suite.T().Logf("TotalTasks is 0 (generation=%d); waiting %s then will run another generation …", verifier.generation, delay)
+
+			time.Sleep(delay)
 			checkContinueChan <- struct{}{}
 			<-checkDoneChan
 			status, err = verifier.GetVerificationStatus()

From dfba63d07dda428e1d6e05488c248e697d2403d2 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 16:06:11 -0500
Subject: [PATCH 065/104] go.sum

---
 .gitignore |   1 -
 go.sum     | 169 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 169 insertions(+), 1 deletion(-)
 create mode 100644 go.sum

diff --git a/.gitignore b/.gitignore
index b7c3c335..e5fbc730 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,4 @@
 dev-bin/
-go.sum
 golangci-lint
 migration_verifier
 internal/verifier/mongodb_exec/
diff --git a/go.sum b/go.sum
new file mode 100644
index 00000000..d5301ade
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,169 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/cespare/permute/v2 v2.0.0-beta2 h1:iiJWgDsInCbnwfDZiLJX7cVlJevgIM1pUiArcUylUMc=
+github.com/cespare/permute/v2 v2.0.0-beta2/go.mod h1:7e2Tqe0fjn1T4rTjDLJjQ+kNtt+UW4B9lg10asqWw0A=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g=
+github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8=
+github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
+github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
+github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
+github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
+github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0=
+github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
+github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM=
+github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
+github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU=
+github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
+github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY=
+github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc=
+github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
+github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
+github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
+github.com/urfave/cli v1.22.9 h1:cv3/KhXGBGjEXLC4bH0sLuJ9BewaAbpk5oyMOveu4pw=
+github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM=
+go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
+golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
+golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

From 6520600b79a57391f4fa50450fca1ca92f67147e Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 16:06:52 -0500
Subject: [PATCH 066/104] try 20 generations

---
 internal/verifier/migration_verifier_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index 9504ceae..564d4ac2 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1320,7 +1320,7 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 		status, err := verifier.GetVerificationStatus()
 		suite.Require().NoError(err)
 
-		for status.TotalTasks == 0 && verifier.generation < 10 {
+		for status.TotalTasks == 0 && verifier.generation < 20 {
 			delay := time.Second
 
 			suite.T().Logf("TotalTasks is 0 (generation=%d); waiting %s then will run another generation …", verifier.generation, delay)

From 302daded13af8cf79814e6bba98ac9d234a8737f Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Wed, 20 Nov 2024 16:11:55 -0500
Subject: [PATCH 067/104] more generations

---
 .github/workflows/all.yml                    | 2 ++
 internal/verifier/migration_verifier_test.go | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 7a082756..a452eb1c 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -11,6 +11,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+
+        # versions are: source, destination
         mongodb_versions:
           - [ '4.2', '4.2' ]
           - [ '4.2', '4.4' ]
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index 564d4ac2..3cdd55b2 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1320,7 +1320,7 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 		status, err := verifier.GetVerificationStatus()
 		suite.Require().NoError(err)
 
-		for status.TotalTasks == 0 && verifier.generation < 20 {
+		for status.TotalTasks == 0 && verifier.generation < 50 {
 			delay := time.Second
 
 			suite.T().Logf("TotalTasks is 0 (generation=%d); waiting %s then will run another generation …", verifier.generation, delay)
@@ -1446,7 +1446,7 @@ func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 		status, err := verifier.GetVerificationStatus()
 		suite.Require().NoError(err)
 
-		for status.TotalTasks == 0 && verifier.generation < 10 {
+		for status.TotalTasks == 0 && verifier.generation < 50 {
 			delay := time.Second
 
 			suite.T().Logf("TotalTasks is 0 (generation=%d); waiting %s then will run another generation …", verifier.generation, delay)

From 75e49b01e56d8eb7b6965e1e525cda614748533a Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 09:47:06 -0500
Subject: [PATCH 068/104] remove cruft

---
 internal/verifier/change_stream.go | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 91152248..20ff3f30 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -159,10 +159,6 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 				break
 			}
 
-			verifier.logger.Debug().
-				Stringer("resumeToken", cs.ResumeToken()).
-				Msg("TryNext got an event!")
-
 			if changeEventBatch == nil {
 				changeEventBatch = make([]ParsedEvent, cs.RemainingBatchLength()+1)
 			}
@@ -306,9 +302,7 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 				Msg("Failed to extract timestamp from persisted resume token.")
 		}
 
-		logEvent.
-			Interface("pipeline", pipeline).
-			Msg("Starting change stream from persisted resume token.")
+		logEvent.Msg("Starting change stream from persisted resume token.")
 
 		opts = opts.SetStartAfter(savedResumeToken)
 	} else {
@@ -320,7 +314,6 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 		return errors.Wrap(err, "failed to start session")
 	}
 	sctx := mongo.NewSessionContext(ctx, sess)
-	verifier.logger.Debug().Interface("pipeline", pipeline).Msg("Opened change stream.")
 	srcChangeStream, err := verifier.srcClient.Watch(sctx, pipeline, opts)
 	if err != nil {
 		return errors.Wrap(err, "failed to open change stream")

From 59de79c0b9875574c539196203e0bec66a44b6c9 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <FGasper@users.noreply.github.com>
Date: Thu, 21 Nov 2024 09:52:22 -0500
Subject: [PATCH 069/104] Update internal/verifier/change_stream.go

Co-authored-by: Michael McClimon <michael@mcclimon.org>
---
 internal/verifier/change_stream.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 20ff3f30..ec3e7888 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -215,7 +215,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 					break
 				}
 
-				if curTs.Compare(finalTs) >= 0 {
+				if curTs == finalTs || curTs.After(finalTs) {
 					verifier.logger.Debug().
 						Interface("currentTimestamp", curTs).
 						Interface("finalTimestamp", finalTs).

From 8c1af46927fc495079bcdfeacb9fc2ec493d8891 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 10:54:44 -0500
Subject: [PATCH 070/104] refactored appendOplogNote (& al.)

---
 internal/verifier/change_stream.go |  74 ++++++++++-----------
 internal/verifier/clustertime.go   | 101 +++++++++++++++++------------
 2 files changed, 98 insertions(+), 77 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 20ff3f30..2d42dd20 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -126,60 +126,60 @@ func (verifier *Verifier) GetChangeStreamFilter() []bson.D {
 	return []bson.D{stage}
 }
 
-func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.ChangeStream) {
-	defer cs.Close(ctx)
+func (verifier *Verifier) readAndHandleOneChangeEventBatch(ctx context.Context, cs *mongo.ChangeStream) error {
+	eventsRead := 0
+	var changeEventBatch []ParsedEvent
 
-	var lastPersistedTime time.Time
+	for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
+		gotEvent := cs.TryNext(ctx)
 
-	persistResumeTokenIfNeeded := func() error {
-		if time.Since(lastPersistedTime) <= minChangeStreamPersistInterval {
-			return nil
+		if cs.Err() != nil {
+			return errors.Wrap(cs.Err(), "change stream iteration failed")
 		}
 
-		err := verifier.persistChangeStreamResumeToken(ctx, cs)
-		if err == nil {
-			lastPersistedTime = time.Now()
+		if !gotEvent {
+			break
 		}
 
-		return err
-	}
+		if changeEventBatch == nil {
+			changeEventBatch = make([]ParsedEvent, cs.RemainingBatchLength()+1)
+		}
 
-	readAndHandleOneChangeEventBatch := func() error {
-		eventsRead := 0
-		var changeEventBatch []ParsedEvent
+		if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
+			return errors.Wrap(err, "failed to decode change event")
+		}
 
-		for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
-			gotEvent := cs.TryNext(ctx)
+		eventsRead++
+	}
 
-			if cs.Err() != nil {
-				return errors.Wrap(cs.Err(), "change stream iteration failed")
-			}
+	if eventsRead == 0 {
+		return nil
+	}
 
-			if !gotEvent {
-				break
-			}
+	err := verifier.HandleChangeStreamEvents(ctx, changeEventBatch)
+	if err != nil {
+		return errors.Wrap(err, "failed to handle change events")
+	}
 
-			if changeEventBatch == nil {
-				changeEventBatch = make([]ParsedEvent, cs.RemainingBatchLength()+1)
-			}
+	return nil
+}
 
-			if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
-				return errors.Wrap(err, "failed to decode change event")
-			}
+func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.ChangeStream) {
+	defer cs.Close(ctx)
 
-			eventsRead++
-		}
+	var lastPersistedTime time.Time
 
-		if eventsRead == 0 {
+	persistResumeTokenIfNeeded := func() error {
+		if time.Since(lastPersistedTime) <= minChangeStreamPersistInterval {
 			return nil
 		}
 
-		err := verifier.HandleChangeStreamEvents(ctx, changeEventBatch)
-		if err != nil {
-			return errors.Wrap(err, "failed to handle change events")
+		err := verifier.persistChangeStreamResumeToken(ctx, cs)
+		if err == nil {
+			lastPersistedTime = time.Now()
 		}
 
-		return nil
+		return err
 	}
 
 	for {
@@ -224,7 +224,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 					break
 				}
 
-				err = readAndHandleOneChangeEventBatch()
+				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
 
 				if err != nil {
 					break
@@ -232,7 +232,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 			}
 
 		default:
-			err = readAndHandleOneChangeEventBatch()
+			err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
 
 			if err == nil {
 				err = persistResumeTokenIfNeeded()
diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index 7be92ea0..942d9adc 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -23,7 +23,7 @@ const opTimeKeyInServerResponse = "operationTime"
 func GetClusterTime(
 	ctx context.Context,
 	logger *logger.Logger,
-	client *mongo.Client, // could change to type=any if we need
+	client *mongo.Client,
 ) (primitive.Timestamp, error) {
 	retryer := retry.New(retry.DefaultDurationLimit)
 
@@ -39,7 +39,7 @@ func GetClusterTime(
 		logger,
 		func(_ *retry.Info) error {
 			var err error
-			optime, err = runAppendOplogNote(ctx, client, true)
+			optime, err = syncClusterTimeAcrossShards(ctx, client)
 			if err != nil {
 				err = fmt.Errorf("%w: %w", retry.CustomTransientErr, err)
 			}
@@ -51,8 +51,9 @@ func GetClusterTime(
 		return primitive.Timestamp{}, err
 	}
 
-	// OPTIMIZATION: We now append an oplog entry--this time for real!--to
-	// cause any lagging shards to output their events.
+	// OPTIMIZATION FOR SHARDED CLUSTERS: We append another oplog entry to
+	// bring all shards to a cluster time that’s *after* the optime that we’ll
+	// return. That way any events *at*
 	//
 	// Since this is just an optimization, failures here are nonfatal.
 	err = retryer.RunForTransientErrorsOnly(
@@ -60,7 +61,7 @@ func GetClusterTime(
 		logger,
 		func(_ *retry.Info) error {
 			var err error
-			optime, err = runAppendOplogNote(ctx, client, false)
+			optime, err = syncClusterTimeAcrossShards(ctx, client)
 			if err != nil {
 				err = fmt.Errorf("%w: %w", retry.CustomTransientErr, err)
 			}
@@ -76,55 +77,75 @@ func GetClusterTime(
 	return optime, nil
 }
 
-func runAppendOplogNote(
+// Use this when we just need the correct cluster time without
+// actually changing any shards’ oplogs.
+func fetchClusterTime(
 	ctx context.Context,
 	client *mongo.Client,
-	doomed bool,
 ) (primitive.Timestamp, error) {
-	// We don’t need to write to any shards’ oplogs; we just
-	// need to fetch
-	cmd := bson.D{
-		{"appendOplogNote", 1},
-		{"data", bson.D{
-			{"migration-verifier", "expect StaleClusterTime error"},
-		}},
-	}
+	cmd, rawResponse, err := runAppendOplogNote(
+		ctx,
+		client,
+		"expect StaleClusterTime error",
+		bson.E{"maxClusterTime", primitive.Timestamp{1, 0}},
+	)
 
-	if doomed {
-		cmd = append(cmd, bson.E{"maxClusterTime", primitive.Timestamp{1, 0}})
+	// We expect an error here; if we didn't get one then something is
+	// amiss on the server.
+	if err == nil {
+		return primitive.Timestamp{}, errors.Errorf("server request unexpectedly succeeded: %v", cmd)
 	}
 
-	resp := client.Database("admin").RunCommand(ctx, cmd)
-
-	err := resp.Err()
-
-	if doomed {
-		// We expect an error here; if we didn't get one then something is
-		// amiss on the server.
-		if err == nil {
-			return primitive.Timestamp{}, errors.Errorf("server request unexpectedly succeeded: %v", cmd)
-		}
-
-		if !util.IsStaleClusterTimeError(err) {
-			return primitive.Timestamp{}, errors.Errorf(
-				"unexpected error (expected StaleClusterTime) from request (%v): %v",
-				err,
-				cmd,
-			)
-		}
-	} else if err != nil {
-		return primitive.Timestamp{}, errors.Wrapf(
+	if !util.IsStaleClusterTimeError(err) {
+		return primitive.Timestamp{}, errors.Wrap(
 			err,
-			"command (%v) failed unexpectedly",
-			cmd,
+			"unexpected error (expected StaleClusterTime) from request",
 		)
 	}
 
-	rawResponse, _ := resp.Raw()
+	return getOpTimeFromRawResponse(rawResponse)
+}
+
+func syncClusterTimeAcrossShards(
+	ctx context.Context,
+	client *mongo.Client,
+) (primitive.Timestamp, error) {
+	_, rawResponse, err := runAppendOplogNote(ctx, client, "syncing cluster time")
+
+	if err != nil {
+		return primitive.Timestamp{}, err
+	}
 
 	return getOpTimeFromRawResponse(rawResponse)
 }
 
+func runAppendOplogNote(
+	ctx context.Context,
+	client *mongo.Client,
+	note string,
+	extraPieces ...bson.E,
+) (bson.D, bson.Raw, error) {
+	cmd := append(
+		bson.D{
+			{"appendOplogNote", 1},
+			{"data", bson.D{
+				{"migration-verifier", note},
+			}},
+		},
+		extraPieces...,
+	)
+
+	resp := client.Database("admin").RunCommand(ctx, cmd)
+
+	raw, err := resp.Raw()
+
+	return cmd, raw, errors.Wrapf(
+		err,
+		"command (%v) failed unexpectedly",
+		cmd,
+	)
+}
+
 func getOpTimeFromRawResponse(rawResponse bson.Raw) (primitive.Timestamp, error) {
 	// Get the `operationTime` from the response and return it.
 	var optime primitive.Timestamp

From 0dea2b9e5240e4ba63d29e668ffb1b45175a052d Mon Sep 17 00:00:00 2001
From: Felipe Gasper <FGasper@users.noreply.github.com>
Date: Thu, 21 Nov 2024 11:41:06 -0500
Subject: [PATCH 071/104] Update internal/verifier/integration_test_suite.go

Co-authored-by: Michael McClimon <michael@mcclimon.org>
---
 internal/verifier/integration_test_suite.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index 41f5d018..ecb44783 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -98,7 +98,7 @@ func (suite *IntegrationTestSuite) SetupTest() {
 		dbNames, err := client.ListDatabaseNames(ctx, bson.D{})
 		suite.Require().NoError(err, "should list database names")
 		for _, dbName := range dbNames {
-			if strings.Index(dbName, suite.DBNameForTest()) == 0 {
+			if strings.HasPrefix(dbName, suite.DBNameForTest()) {
 				suite.T().Logf("Dropping database %#q because it seems to be left over from an earlier run of this test.", dbName)
 				suite.Require().NoError(client.Database(dbName).Drop(ctx))
 			}

From 45090916464505d398f0a701b4d37757b9507b59 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 11:43:49 -0500
Subject: [PATCH 072/104] rework cluster-time-getting

---
 internal/verifier/check.go            |   2 +-
 internal/verifier/clustertime.go      | 120 +++++---------------------
 internal/verifier/clustertime_test.go |   2 +-
 3 files changed, 22 insertions(+), 102 deletions(-)

diff --git a/internal/verifier/check.go b/internal/verifier/check.go
index c88ad840..c6e0d31e 100644
--- a/internal/verifier/check.go
+++ b/internal/verifier/check.go
@@ -49,7 +49,7 @@ func (verifier *Verifier) waitForChangeStream(ctx context.Context) error {
 	if csRunning {
 		verifier.logger.Debug().Msg("Changestream still running, signalling that writes are done and waiting for change stream to exit")
 
-		finalTs, err := GetClusterTime(
+		finalTs, err := GetNewClusterTime(
 			ctx,
 			verifier.logger,
 			verifier.srcClient,
diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index 942d9adc..a1db4daf 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -2,11 +2,9 @@ package verifier
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/10gen/migration-verifier/internal/logger"
 	"github.com/10gen/migration-verifier/internal/retry"
-	"github.com/10gen/migration-verifier/internal/util"
 	"github.com/10gen/migration-verifier/mbson"
 	"github.com/pkg/errors"
 	"go.mongodb.org/mongo-driver/bson"
@@ -16,11 +14,11 @@ import (
 
 const opTimeKeyInServerResponse = "operationTime"
 
-// GetClusterTime returns the remote cluster’s cluster time.
-// In so doing it creates a new oplog entry across all shards.
-// The “note” will go into the cluster’s oplog, so keep it
-// short but meaningful.
-func GetClusterTime(
+// GetNewClusterTime advances the remote cluster’s cluster time an returns
+// that time. In sharded clusters this advancement happens across all shards,
+// which (usefully!) equalizes the shards’ cluster time and triggers them to
+// output all events before then.
+func GetNewClusterTime(
 	ctx context.Context,
 	logger *logger.Logger,
 	client *mongo.Client,
@@ -29,125 +27,47 @@ func GetClusterTime(
 
 	var optime primitive.Timestamp
 
-	// To get the cluster time, we submit a request to append an oplog note
-	// but set an unreasonably-early maxClusterTime. All the shards will fail
-	// the request but still send their current clusterTime to the mongos,
-	// which will return the most recent of those. Thus we can fetch the
-	// most recent cluster time without altering the oplog.
 	err := retryer.RunForTransientErrorsOnly(
 		ctx,
 		logger,
 		func(_ *retry.Info) error {
 			var err error
 			optime, err = syncClusterTimeAcrossShards(ctx, client)
-			if err != nil {
-				err = fmt.Errorf("%w: %w", retry.CustomTransientErr, err)
-			}
 			return err
 		},
 	)
 
-	if err != nil {
-		return primitive.Timestamp{}, err
-	}
-
-	// OPTIMIZATION FOR SHARDED CLUSTERS: We append another oplog entry to
-	// bring all shards to a cluster time that’s *after* the optime that we’ll
-	// return. That way any events *at*
-	//
-	// Since this is just an optimization, failures here are nonfatal.
-	err = retryer.RunForTransientErrorsOnly(
-		ctx,
-		logger,
-		func(_ *retry.Info) error {
-			var err error
-			optime, err = syncClusterTimeAcrossShards(ctx, client)
-			if err != nil {
-				err = fmt.Errorf("%w: %w", retry.CustomTransientErr, err)
-			}
-			return err
-		},
-	)
-	if err != nil {
-		// This isn't serious enough even to warn on, so leave it at info-level.
-		logger.Info().Err(err).
-			Msg("Failed to append oplog note; change stream may need extra time to finish.")
-	}
-
-	return optime, nil
+	return optime, err
 }
 
-// Use this when we just need the correct cluster time without
-// actually changing any shards’ oplogs.
-func fetchClusterTime(
+func syncClusterTimeAcrossShards(
 	ctx context.Context,
 	client *mongo.Client,
 ) (primitive.Timestamp, error) {
-	cmd, rawResponse, err := runAppendOplogNote(
-		ctx,
-		client,
-		"expect StaleClusterTime error",
-		bson.E{"maxClusterTime", primitive.Timestamp{1, 0}},
-	)
-
-	// We expect an error here; if we didn't get one then something is
-	// amiss on the server.
-	if err == nil {
-		return primitive.Timestamp{}, errors.Errorf("server request unexpectedly succeeded: %v", cmd)
+	cmd := bson.D{
+		{"appendOplogNote", 1},
+		{"data", bson.D{
+			{"migration-verifier", "syncing cluster time"},
+		}},
 	}
 
-	if !util.IsStaleClusterTimeError(err) {
-		return primitive.Timestamp{}, errors.Wrap(
-			err,
-			"unexpected error (expected StaleClusterTime) from request",
-		)
-	}
+	resp := client.Database("admin").RunCommand(ctx, cmd)
 
-	return getOpTimeFromRawResponse(rawResponse)
-}
-
-func syncClusterTimeAcrossShards(
-	ctx context.Context,
-	client *mongo.Client,
-) (primitive.Timestamp, error) {
-	_, rawResponse, err := runAppendOplogNote(ctx, client, "syncing cluster time")
+	rawResponse, err := resp.Raw()
 
 	if err != nil {
-		return primitive.Timestamp{}, err
+		return primitive.Timestamp{}, errors.Wrapf(
+			err,
+			"command (%v) failed unexpectedly",
+			cmd,
+		)
 	}
 
 	return getOpTimeFromRawResponse(rawResponse)
 }
 
-func runAppendOplogNote(
-	ctx context.Context,
-	client *mongo.Client,
-	note string,
-	extraPieces ...bson.E,
-) (bson.D, bson.Raw, error) {
-	cmd := append(
-		bson.D{
-			{"appendOplogNote", 1},
-			{"data", bson.D{
-				{"migration-verifier", note},
-			}},
-		},
-		extraPieces...,
-	)
-
-	resp := client.Database("admin").RunCommand(ctx, cmd)
-
-	raw, err := resp.Raw()
-
-	return cmd, raw, errors.Wrapf(
-		err,
-		"command (%v) failed unexpectedly",
-		cmd,
-	)
-}
-
 func getOpTimeFromRawResponse(rawResponse bson.Raw) (primitive.Timestamp, error) {
-	// Get the `operationTime` from the response and return it.
+	// Return the response’s `operationTime`.
 	var optime primitive.Timestamp
 
 	found, err := mbson.RawLookup(rawResponse, &optime, opTimeKeyInServerResponse)
diff --git a/internal/verifier/clustertime_test.go b/internal/verifier/clustertime_test.go
index eab0f61e..6d2e5acb 100644
--- a/internal/verifier/clustertime_test.go
+++ b/internal/verifier/clustertime_test.go
@@ -6,7 +6,7 @@ func (suite *IntegrationTestSuite) TestGetClusterTime() {
 	ctx := context.Background()
 	logger, _ := getLoggerAndWriter("stdout")
 
-	ts, err := GetClusterTime(ctx, logger, suite.srcMongoClient)
+	ts, err := GetNewClusterTime(ctx, logger, suite.srcMongoClient)
 	suite.Require().NoError(err)
 
 	suite.Assert().NotZero(ts, "timestamp should be nonzero")

From 713dbca652c31da3fc7173c3dab1270fcbfe5f92 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 11:56:13 -0500
Subject: [PATCH 073/104] remove log

---
 internal/verifier/check.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/internal/verifier/check.go b/internal/verifier/check.go
index c6e0d31e..3e4c238f 100644
--- a/internal/verifier/check.go
+++ b/internal/verifier/check.go
@@ -78,11 +78,6 @@ func (verifier *Verifier) CheckWorker(ctx context.Context) error {
 	verifier.logger.Debug().Msgf("Starting %d verification workers", verifier.numWorkers)
 	ctx, cancel := context.WithCancel(ctx)
 
-	if verifier.workerSleepDelayMillis == 0 {
-		verifier.logger.Info().
-			Msg("Worker sleep delay is zero. Only tests should do this.")
-	}
-
 	wg := sync.WaitGroup{}
 	for i := 0; i < verifier.numWorkers; i++ {
 		wg.Add(1)

From 243cfb1e8af2f6161336f444f3590e0e50421269 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 12:20:55 -0500
Subject: [PATCH 074/104] write concern

---
 internal/verifier/clustertime.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index a1db4daf..2fa75189 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -10,6 +10,8 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.mongodb.org/mongo-driver/mongo/writeconcern"
 )
 
 const opTimeKeyInServerResponse = "operationTime"
@@ -51,7 +53,12 @@ func syncClusterTimeAcrossShards(
 		}},
 	}
 
-	resp := client.Database("admin").RunCommand(ctx, cmd)
+	resp := client.
+		Database(
+			"admin",
+			options.Database().SetWriteConcern(writeconcern.Majority()),
+		).
+		RunCommand(ctx, cmd)
 
 	rawResponse, err := resp.Raw()
 

From 79d5675e06b437572cf836ca164a6c151e7cf7ea Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 14:25:41 -0500
Subject: [PATCH 075/104] revert retryer fix

---
 internal/retry/retry.go | 44 +++++++++++------------------------------
 1 file changed, 12 insertions(+), 32 deletions(-)

diff --git a/internal/retry/retry.go b/internal/retry/retry.go
index 59bf0746..4f9358fb 100644
--- a/internal/retry/retry.go
+++ b/internal/retry/retry.go
@@ -3,7 +3,6 @@ package retry
 import (
 	"context"
 	"math/rand"
-	"slices"
 	"time"
 
 	"github.com/10gen/migration-verifier/internal/logger"
@@ -12,8 +11,6 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 )
 
-var CustomTransientErr = errors.New("possibly-transient failure detected")
-
 // RunForUUIDAndTransientErrors retries f() for the CollectionUUIDMismatch error and for transient errors.
 // This should be used to run a driver operation that optionally specifies the `collectionUUID` parameter
 // for a collection that may have been:
@@ -276,42 +273,25 @@ func (r *Retryer) shouldRetryWithSleep(
 		return false
 	}
 
-	var whyTransient string
-
 	errCode := util.GetErrorCode(err)
-
-	if util.IsTransientError(err) || slices.Contains(r.additionalErrorCodes, errCode) {
-		whyTransient = "Error code suggests a transient error."
-	} else if errors.Is(err, CustomTransientErr) {
-		whyTransient = "Error may be transient."
+	if util.IsTransientError(err) {
+		logger.Warn().Int("error code", errCode).Err(err).Msgf(
+			"Waiting %s seconds to retry operation after transient error.", sleepTime)
+		return true
 	}
 
-	if whyTransient == "" {
-		event := logger.Debug().Err(err)
-
-		if errCode != 0 {
-			event = event.Int("error code", errCode)
+	for _, code := range r.additionalErrorCodes {
+		if code == errCode {
+			logger.Warn().Int("error code", errCode).Err(err).Msgf(
+				"Waiting %s seconds to retry operation after an error because it is in our additional codes list.", sleepTime)
+			return true
 		}
-
-		event.Msg("Not retrying on error because it appears not to be transient.")
-
-		return false
-	}
-
-	retryLogEvent := logger.Warn().
-		Err(err).
-		Stringer("duration", sleepTime)
-
-	if errCode != 0 {
-		retryLogEvent = retryLogEvent.
-			Int("error code", errCode)
 	}
 
-	retryLogEvent.
-		Str("reason", whyTransient).
-		Msg("Pausing then will retry operation.")
+	logger.Debug().Err(err).Int("error code", errCode).
+		Msg("Not retrying on error because it is not transient nor is it in our additional codes list.")
 
-	return true
+	return false
 }
 
 // Use this method for aggregates which should take a UUID in new versions but not old ones.

From 293d7b4b9cd18b97a0bb5ecd675266a02bce1169 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 14:27:16 -0500
Subject: [PATCH 076/104] revert non-test changes to ensure that existing tests
 verify

---
 internal/verifier/change_stream.go      | 123 +++++++++---------------
 internal/verifier/change_stream_test.go |   6 +-
 internal/verifier/check.go              |  35 ++-----
 internal/verifier/migration_verifier.go |  46 ++++++---
 4 files changed, 91 insertions(+), 119 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index a853bea9..ddf8f5ba 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -12,6 +12,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+	"golang.org/x/exp/constraints"
 )
 
 const fauxDocSizeForDeleteEvents = 1024
@@ -104,11 +105,6 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 // GetChangeStreamFilter returns an aggregation pipeline that filters
 // namespaces as per configuration.
 //
-// Note that this omits verifier.globalFilter because we still need to
-// recheck any out-filter documents that may have changed in order to
-// account for filter traversals (i.e., updates that change whether a
-// document matches the filter).
-//
 // NB: Ideally we could make the change stream give $bsonSize(fullDocument)
 // and omit fullDocument, but $bsonSize was new in MongoDB 4.4, and we still
 // want to verify migrations from 4.2. fullDocument is unlikely to be a
@@ -126,60 +122,55 @@ func (verifier *Verifier) GetChangeStreamFilter() []bson.D {
 	return []bson.D{stage}
 }
 
-func (verifier *Verifier) readAndHandleOneChangeEventBatch(ctx context.Context, cs *mongo.ChangeStream) error {
-	eventsRead := 0
-	var changeEventBatch []ParsedEvent
-
-	for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
-		gotEvent := cs.TryNext(ctx)
-
-		if cs.Err() != nil {
-			return errors.Wrap(cs.Err(), "change stream iteration failed")
-		}
+func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.ChangeStream) {
+	defer cs.Close(ctx)
 
-		if !gotEvent {
-			break
-		}
+	var lastPersistedTime time.Time
 
-		if changeEventBatch == nil {
-			changeEventBatch = make([]ParsedEvent, cs.RemainingBatchLength()+1)
+	persistResumeTokenIfNeeded := func() error {
+		if time.Since(lastPersistedTime) <= minChangeStreamPersistInterval {
+			return nil
 		}
 
-		if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
-			return errors.Wrap(err, "failed to decode change event")
+		err := verifier.persistChangeStreamResumeToken(ctx, cs)
+		if err == nil {
+			lastPersistedTime = time.Now()
 		}
 
-		eventsRead++
+		return err
 	}
 
-	if eventsRead == 0 {
-		return nil
-	}
+	readAndHandleOneChangeEventBatch := func() (bool, error) {
+		eventsRead := 0
+		var changeEventBatch []ParsedEvent
 
-	err := verifier.HandleChangeStreamEvents(ctx, changeEventBatch)
-	if err != nil {
-		return errors.Wrap(err, "failed to handle change events")
-	}
+		for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
+			gotEvent := cs.TryNext(ctx)
 
-	return nil
-}
+			if !gotEvent || cs.Err() != nil {
+				break
+			}
 
-func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.ChangeStream) {
-	defer cs.Close(ctx)
+			if changeEventBatch == nil {
+				changeEventBatch = make([]ParsedEvent, cs.RemainingBatchLength()+1)
+			}
 
-	var lastPersistedTime time.Time
+			if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
+				return false, errors.Wrap(err, "failed to decode change event")
+			}
 
-	persistResumeTokenIfNeeded := func() error {
-		if time.Since(lastPersistedTime) <= minChangeStreamPersistInterval {
-			return nil
+			eventsRead++
 		}
 
-		err := verifier.persistChangeStreamResumeToken(ctx, cs)
-		if err == nil {
-			lastPersistedTime = time.Now()
+		if eventsRead > 0 {
+			verifier.logger.Debug().Int("eventsCount", eventsRead).Msgf("Received a batch of events.")
+			err := verifier.HandleChangeStreamEvents(ctx, changeEventBatch)
+			if err != nil {
+				return false, errors.Wrap(err, "failed to handle change events")
+			}
 		}
 
-		return err
+		return eventsRead > 0, errors.Wrap(cs.Err(), "change stream iteration failed")
 	}
 
 	for {
@@ -198,45 +189,29 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 		// If the changeStreamEnderChan has a message, the user has indicated that
 		// source writes are ended. This means we should exit rather than continue
 		// reading the change stream since there should be no more events.
-		case finalTs := <-verifier.changeStreamFinalTsChan:
+		case <-verifier.changeStreamEnderChan:
 			verifier.logger.Debug().
-				Interface("finalTimestamp", finalTs).
-				Msg("Change stream thread received final timestamp. Finalizing change stream.")
+				Msg("Change stream thread received shutdown request.")
 
 			changeStreamEnded = true
 
 			// Read all change events until the source reports no events.
 			// (i.e., the `getMore` call returns empty)
 			for {
-				var curTs primitive.Timestamp
-				curTs, err = extractTimestampFromResumeToken(cs.ResumeToken())
-				if err != nil {
-					err = errors.Wrap(err, "failed to extract timestamp from change stream's resume token")
-					break
-				}
-
-				if curTs == finalTs || curTs.After(finalTs) {
-					verifier.logger.Debug().
-						Interface("currentTimestamp", curTs).
-						Interface("finalTimestamp", finalTs).
-						Msg("Change stream has reached the final timestamp. Shutting down.")
+				var gotEvent bool
+				gotEvent, err = readAndHandleOneChangeEventBatch()
 
-					break
-				}
-
-				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
-
-				if err != nil {
+				if !gotEvent || err != nil {
 					break
 				}
 			}
 
 		default:
-			err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
+			_, err = readAndHandleOneChangeEventBatch()
+		}
 
-			if err == nil {
-				err = persistResumeTokenIfNeeded()
-			}
+		if err == nil {
+			err = persistResumeTokenIfNeeded()
 		}
 
 		if err != nil && !errors.Is(err, context.Canceled) {
@@ -267,9 +242,9 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 
 	infoLog := verifier.logger.Info()
 	if verifier.lastChangeEventTime == nil {
-		infoLog = infoLog.Str("lastEventTime", "none")
+		infoLog = infoLog.Str("changeStreamStopTime", "none")
 	} else {
-		infoLog = infoLog.Interface("lastEventTime", *verifier.lastChangeEventTime)
+		infoLog = infoLog.Interface("changeStreamStopTime", *verifier.lastChangeEventTime)
 	}
 
 	infoLog.Msg("Change stream is done.")
@@ -295,7 +270,7 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 
 		ts, err := extractTimestampFromResumeToken(savedResumeToken)
 		if err == nil {
-			logEvent = addTimestampToLogEvent(ts, logEvent)
+			logEvent = addUnixTimeToLogEvent(ts.T, logEvent)
 		} else {
 			verifier.logger.Warn().
 				Err(err).
@@ -348,10 +323,8 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 	return nil
 }
 
-func addTimestampToLogEvent(ts primitive.Timestamp, event *zerolog.Event) *zerolog.Event {
-	return event.
-		Interface("timestamp", ts).
-		Time("time", time.Unix(int64(ts.T), int64(0)))
+func addUnixTimeToLogEvent[T constraints.Integer](unixTime T, event *zerolog.Event) *zerolog.Event {
+	return event.Time("timestampTime", time.Unix(int64(unixTime), int64(0)))
 }
 
 func (v *Verifier) getChangeStreamMetadataCollection() *mongo.Collection {
@@ -390,7 +363,7 @@ func (verifier *Verifier) persistChangeStreamResumeToken(ctx context.Context, cs
 		logEvent := verifier.logger.Debug()
 
 		if err == nil {
-			logEvent = addTimestampToLogEvent(ts, logEvent)
+			logEvent = addUnixTimeToLogEvent(ts.T, logEvent)
 		} else {
 			verifier.logger.Warn().Err(err).
 				Msg("failed to extract resume token timestamp")
diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index 597fed8a..1a0b87c0 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -141,7 +141,8 @@ func (suite *IntegrationTestSuite) TestStartAtTimeNoChanges() {
 	err = verifier.StartChangeStream(ctx)
 	suite.Require().NoError(err)
 	suite.Require().Equal(verifier.srcStartAtTs, origStartTs)
-	verifier.changeStreamFinalTsChan <- *origStartTs
+	//verifier.changeStreamFinalTsChan <- *origStartTs
+	verifier.changeStreamEnderChan <- struct{}{}
 	<-verifier.changeStreamDoneChan
 	suite.Require().Equal(verifier.srcStartAtTs, origStartTs)
 }
@@ -183,7 +184,8 @@ func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
 		"session time after events should exceed the original",
 	)
 
-	verifier.changeStreamFinalTsChan <- *postEventsSessionTime
+	//verifier.changeStreamFinalTsChan <- *postEventsSessionTime
+	verifier.changeStreamEnderChan <- struct{}{}
 	<-verifier.changeStreamDoneChan
 
 	suite.Assert().Equal(
diff --git a/internal/verifier/check.go b/internal/verifier/check.go
index 3e4c238f..5a60bfb5 100644
--- a/internal/verifier/check.go
+++ b/internal/verifier/check.go
@@ -42,24 +42,13 @@ func (verifier *Verifier) Check(ctx context.Context, filter map[string]any) {
 	verifier.MaybeStartPeriodicHeapProfileCollection(ctx)
 }
 
-func (verifier *Verifier) waitForChangeStream(ctx context.Context) error {
+func (verifier *Verifier) waitForChangeStream() error {
 	verifier.mux.RLock()
 	csRunning := verifier.changeStreamRunning
 	verifier.mux.RUnlock()
 	if csRunning {
 		verifier.logger.Debug().Msg("Changestream still running, signalling that writes are done and waiting for change stream to exit")
-
-		finalTs, err := GetNewClusterTime(
-			ctx,
-			verifier.logger,
-			verifier.srcClient,
-		)
-
-		if err != nil {
-			return errors.Wrapf(err, "failed to fetch source's cluster time")
-		}
-
-		verifier.changeStreamFinalTsChan <- finalTs
+		verifier.changeStreamEnderChan <- struct{}{}
 		select {
 		case err := <-verifier.changeStreamErrChan:
 			verifier.logger.Warn().Err(err).
@@ -77,7 +66,6 @@ func (verifier *Verifier) waitForChangeStream(ctx context.Context) error {
 func (verifier *Verifier) CheckWorker(ctx context.Context) error {
 	verifier.logger.Debug().Msgf("Starting %d verification workers", verifier.numWorkers)
 	ctx, cancel := context.WithCancel(ctx)
-
 	wg := sync.WaitGroup{}
 	for i := 0; i < verifier.numWorkers; i++ {
 		wg.Add(1)
@@ -186,9 +174,7 @@ func (verifier *Verifier) CheckDriver(ctx context.Context, filter map[string]any
 	verifier.mux.RLock()
 	csRunning := verifier.changeStreamRunning
 	verifier.mux.RUnlock()
-	if csRunning {
-		verifier.logger.Debug().Msg("Check: Change stream already running.")
-	} else {
+	if !csRunning {
 		verifier.logger.Debug().Msg("Change stream not running; starting change stream")
 
 		err = verifier.StartChangeStream(ctx)
@@ -256,7 +242,7 @@ func (verifier *Verifier) CheckDriver(ctx context.Context, filter map[string]any
 			// It's necessary to wait for the change stream to finish before incrementing the
 			// generation number, or the last changes will not be checked.
 			verifier.mux.Unlock()
-			err := verifier.waitForChangeStream(ctx)
+			err := verifier.waitForChangeStream()
 			if err != nil {
 				return err
 			}
@@ -402,15 +388,12 @@ func (verifier *Verifier) Work(ctx context.Context, workerNum int, wg *sync.Wait
 			if errors.Is(err, mongo.ErrNoDocuments) {
 				duration := verifier.workerSleepDelayMillis * time.Millisecond
 
-				if duration > 0 {
-					verifier.logger.Debug().
-						Int("workerNum", workerNum).
-						Stringer("duration", duration).
-						Msg("No tasks found. Sleeping.")
-
-					time.Sleep(duration)
-				}
+				verifier.logger.Debug().
+					Int("workerNum", workerNum).
+					Stringer("duration", duration).
+					Msg("No tasks found. Sleeping.")
 
+				time.Sleep(duration)
 				continue
 			} else if err != nil {
 				panic(err)
diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go
index 39c4fbf8..809e4fe3 100644
--- a/internal/verifier/migration_verifier.go
+++ b/internal/verifier/migration_verifier.go
@@ -24,6 +24,7 @@ import (
 	"github.com/10gen/migration-verifier/internal/uuidutil"
 	"github.com/olekukonko/tablewriter"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -48,6 +49,7 @@ const (
 	SrcNamespaceField = "query_filter.namespace"
 	DstNamespaceField = "query_filter.to"
 	NumWorkers        = 10
+	refetch           = "TODO_CHANGE_ME_REFETCH"
 	Idle              = "idle"
 	Check             = "check"
 	Recheck           = "recheck"
@@ -121,12 +123,12 @@ type Verifier struct {
 	metaDBName    string
 	srcStartAtTs  *primitive.Timestamp
 
-	mux                     sync.RWMutex
-	changeStreamRunning     bool
-	changeStreamFinalTsChan chan primitive.Timestamp
-	changeStreamErrChan     chan error
-	changeStreamDoneChan    chan struct{}
-	lastChangeEventTime     *primitive.Timestamp
+	mux                   sync.RWMutex
+	changeStreamRunning   bool
+	changeStreamEnderChan chan struct{}
+	changeStreamErrChan   chan error
+	changeStreamDoneChan  chan struct{}
+	lastChangeEventTime   *primitive.Timestamp
 
 	readConcernSetting ReadConcernSetting
 
@@ -186,15 +188,15 @@ func NewVerifier(settings VerifierSettings) *Verifier {
 	}
 
 	return &Verifier{
-		phase:                   Idle,
-		numWorkers:              NumWorkers,
-		readPreference:          readpref.Primary(),
-		partitionSizeInBytes:    400 * 1024 * 1024,
-		failureDisplaySize:      DefaultFailureDisplaySize,
-		changeStreamFinalTsChan: make(chan primitive.Timestamp),
-		changeStreamErrChan:     make(chan error),
-		changeStreamDoneChan:    make(chan struct{}),
-		readConcernSetting:      readConcern,
+		phase:                 Idle,
+		numWorkers:            NumWorkers,
+		readPreference:        readpref.Primary(),
+		partitionSizeInBytes:  400 * 1024 * 1024,
+		failureDisplaySize:    DefaultFailureDisplaySize,
+		changeStreamEnderChan: make(chan struct{}),
+		changeStreamErrChan:   make(chan error),
+		changeStreamDoneChan:  make(chan struct{}),
+		readConcernSetting:    readConcern,
 
 		// This will get recreated once gen0 starts, but we want it
 		// here in case the change streams gets an event before then.
@@ -313,7 +315,15 @@ func (verifier *Verifier) SetPartitionSizeMB(partitionSizeMB uint32) {
 }
 
 func (verifier *Verifier) SetLogger(logPath string) {
-	verifier.logger, verifier.writer = getLoggerAndWriter(logPath)
+	writer := getLogWriter(logPath)
+	verifier.writer = writer
+
+	consoleWriter := zerolog.ConsoleWriter{
+		Out:        writer,
+		TimeFormat: timeFormat,
+	}
+	l := zerolog.New(consoleWriter).With().Timestamp().Logger()
+	verifier.logger = logger.NewLogger(&l, writer)
 }
 
 func (verifier *Verifier) SetSrcNamespaces(arg []string) {
@@ -1162,6 +1172,10 @@ func (verifier *Verifier) verificationTaskCollection() *mongo.Collection {
 	return verifier.verificationDatabase().Collection(verificationTasksCollection)
 }
 
+func (verifier *Verifier) refetchCollection() *mongo.Collection {
+	return verifier.verificationDatabase().Collection(refetch)
+}
+
 func (verifier *Verifier) srcClientDatabase(dbName string) *mongo.Database {
 	db := verifier.srcClient.Database(dbName)
 	// No need to check the write concern because we do not write to the source database.

From 0eda70480098813ae7cf3b71fe237127ade495eb Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 15:05:33 -0500
Subject: [PATCH 077/104] create test

---
 internal/verifier/change_stream_test.go | 66 +++++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index 1a0b87c0..6c1f0c20 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/samber/lo"
 	"github.com/stretchr/testify/require"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -241,3 +242,68 @@ func (suite *IntegrationTestSuite) TestWithChangeEventsBatching() {
 		"the verifier should flush a recheck doc after a batch",
 	)
 }
+
+func (suite *IntegrationTestSuite) TestEventBeforeWritesOff() {
+	ctx := suite.Context()
+
+	verifier := suite.BuildVerifier()
+
+	checkDoneChan := make(chan struct{})
+	checkContinueChan := make(chan struct{})
+
+	// start verifier
+	verifierDoneChan := make(chan struct{})
+	go func() {
+		err := verifier.CheckDriver(ctx, nil, checkDoneChan, checkContinueChan)
+		suite.Require().NoError(err)
+
+		close(verifierDoneChan)
+	}()
+
+	// wait for generation 1
+	<-checkDoneChan
+
+	db := suite.srcMongoClient.Database(suite.DBNameForTest())
+	coll := db.Collection("mycoll")
+
+	docsCount := 10_000
+	docs := lo.RepeatBy(docsCount, func(_ int) bson.D { return bson.D{} })
+	_, err := coll.InsertMany(
+		ctx,
+		lo.ToAnySlice(docs),
+	)
+	suite.Require().NoError(err)
+
+	verifier.WritesOff(ctx)
+
+	verifierDone := false
+	for !verifierDone {
+		select {
+		case <-verifierDoneChan:
+			verifierDone = true
+		case <-checkDoneChan:
+		case checkContinueChan <- struct{}{}:
+		}
+	}
+
+	generation := verifier.generation
+	failedTasks, incompleteTasks, err := FetchFailedAndIncompleteTasks(
+		ctx,
+		verifier.verificationTaskCollection(),
+		verificationTaskVerifyDocuments,
+		generation,
+	)
+	suite.Require().NoError(err)
+
+	suite.Require().Empty(incompleteTasks, "all tasks should be finished")
+
+	totalFailed := lo.Reduce(
+		failedTasks,
+		func(sofar int, task VerificationTask, _ int) int {
+			return sofar + len(task.Ids)
+		},
+		0,
+	)
+
+	suite.Assert().Equal(docsCount, totalFailed, "all source docs should be missing")
+}

From d25eb4290fa5fc757ddffa92a6b94b28e905992a Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 16:14:14 -0500
Subject: [PATCH 078/104] Revert "revert non-test changes to ensure that
 existing tests verify"

This reverts commit 293d7b4b9cd18b97a0bb5ecd675266a02bce1169.
---
 internal/verifier/change_stream.go      | 123 +++++++++++++++---------
 internal/verifier/change_stream_test.go |   6 +-
 internal/verifier/check.go              |  35 +++++--
 internal/verifier/migration_verifier.go |  46 +++------
 4 files changed, 119 insertions(+), 91 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index ddf8f5ba..a853bea9 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -12,7 +12,6 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
-	"golang.org/x/exp/constraints"
 )
 
 const fauxDocSizeForDeleteEvents = 1024
@@ -105,6 +104,11 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 // GetChangeStreamFilter returns an aggregation pipeline that filters
 // namespaces as per configuration.
 //
+// Note that this omits verifier.globalFilter because we still need to
+// recheck any out-filter documents that may have changed in order to
+// account for filter traversals (i.e., updates that change whether a
+// document matches the filter).
+//
 // NB: Ideally we could make the change stream give $bsonSize(fullDocument)
 // and omit fullDocument, but $bsonSize was new in MongoDB 4.4, and we still
 // want to verify migrations from 4.2. fullDocument is unlikely to be a
@@ -122,55 +126,60 @@ func (verifier *Verifier) GetChangeStreamFilter() []bson.D {
 	return []bson.D{stage}
 }
 
-func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.ChangeStream) {
-	defer cs.Close(ctx)
+func (verifier *Verifier) readAndHandleOneChangeEventBatch(ctx context.Context, cs *mongo.ChangeStream) error {
+	eventsRead := 0
+	var changeEventBatch []ParsedEvent
 
-	var lastPersistedTime time.Time
+	for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
+		gotEvent := cs.TryNext(ctx)
 
-	persistResumeTokenIfNeeded := func() error {
-		if time.Since(lastPersistedTime) <= minChangeStreamPersistInterval {
-			return nil
+		if cs.Err() != nil {
+			return errors.Wrap(cs.Err(), "change stream iteration failed")
 		}
 
-		err := verifier.persistChangeStreamResumeToken(ctx, cs)
-		if err == nil {
-			lastPersistedTime = time.Now()
+		if !gotEvent {
+			break
 		}
 
-		return err
+		if changeEventBatch == nil {
+			changeEventBatch = make([]ParsedEvent, cs.RemainingBatchLength()+1)
+		}
+
+		if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
+			return errors.Wrap(err, "failed to decode change event")
+		}
+
+		eventsRead++
 	}
 
-	readAndHandleOneChangeEventBatch := func() (bool, error) {
-		eventsRead := 0
-		var changeEventBatch []ParsedEvent
+	if eventsRead == 0 {
+		return nil
+	}
 
-		for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
-			gotEvent := cs.TryNext(ctx)
+	err := verifier.HandleChangeStreamEvents(ctx, changeEventBatch)
+	if err != nil {
+		return errors.Wrap(err, "failed to handle change events")
+	}
 
-			if !gotEvent || cs.Err() != nil {
-				break
-			}
+	return nil
+}
 
-			if changeEventBatch == nil {
-				changeEventBatch = make([]ParsedEvent, cs.RemainingBatchLength()+1)
-			}
+func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.ChangeStream) {
+	defer cs.Close(ctx)
 
-			if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
-				return false, errors.Wrap(err, "failed to decode change event")
-			}
+	var lastPersistedTime time.Time
 
-			eventsRead++
+	persistResumeTokenIfNeeded := func() error {
+		if time.Since(lastPersistedTime) <= minChangeStreamPersistInterval {
+			return nil
 		}
 
-		if eventsRead > 0 {
-			verifier.logger.Debug().Int("eventsCount", eventsRead).Msgf("Received a batch of events.")
-			err := verifier.HandleChangeStreamEvents(ctx, changeEventBatch)
-			if err != nil {
-				return false, errors.Wrap(err, "failed to handle change events")
-			}
+		err := verifier.persistChangeStreamResumeToken(ctx, cs)
+		if err == nil {
+			lastPersistedTime = time.Now()
 		}
 
-		return eventsRead > 0, errors.Wrap(cs.Err(), "change stream iteration failed")
+		return err
 	}
 
 	for {
@@ -189,29 +198,45 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 		// If the changeStreamEnderChan has a message, the user has indicated that
 		// source writes are ended. This means we should exit rather than continue
 		// reading the change stream since there should be no more events.
-		case <-verifier.changeStreamEnderChan:
+		case finalTs := <-verifier.changeStreamFinalTsChan:
 			verifier.logger.Debug().
-				Msg("Change stream thread received shutdown request.")
+				Interface("finalTimestamp", finalTs).
+				Msg("Change stream thread received final timestamp. Finalizing change stream.")
 
 			changeStreamEnded = true
 
 			// Read all change events until the source reports no events.
 			// (i.e., the `getMore` call returns empty)
 			for {
-				var gotEvent bool
-				gotEvent, err = readAndHandleOneChangeEventBatch()
+				var curTs primitive.Timestamp
+				curTs, err = extractTimestampFromResumeToken(cs.ResumeToken())
+				if err != nil {
+					err = errors.Wrap(err, "failed to extract timestamp from change stream's resume token")
+					break
+				}
+
+				if curTs == finalTs || curTs.After(finalTs) {
+					verifier.logger.Debug().
+						Interface("currentTimestamp", curTs).
+						Interface("finalTimestamp", finalTs).
+						Msg("Change stream has reached the final timestamp. Shutting down.")
 
-				if !gotEvent || err != nil {
+					break
+				}
+
+				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
+
+				if err != nil {
 					break
 				}
 			}
 
 		default:
-			_, err = readAndHandleOneChangeEventBatch()
-		}
+			err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
 
-		if err == nil {
-			err = persistResumeTokenIfNeeded()
+			if err == nil {
+				err = persistResumeTokenIfNeeded()
+			}
 		}
 
 		if err != nil && !errors.Is(err, context.Canceled) {
@@ -242,9 +267,9 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 
 	infoLog := verifier.logger.Info()
 	if verifier.lastChangeEventTime == nil {
-		infoLog = infoLog.Str("changeStreamStopTime", "none")
+		infoLog = infoLog.Str("lastEventTime", "none")
 	} else {
-		infoLog = infoLog.Interface("changeStreamStopTime", *verifier.lastChangeEventTime)
+		infoLog = infoLog.Interface("lastEventTime", *verifier.lastChangeEventTime)
 	}
 
 	infoLog.Msg("Change stream is done.")
@@ -270,7 +295,7 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 
 		ts, err := extractTimestampFromResumeToken(savedResumeToken)
 		if err == nil {
-			logEvent = addUnixTimeToLogEvent(ts.T, logEvent)
+			logEvent = addTimestampToLogEvent(ts, logEvent)
 		} else {
 			verifier.logger.Warn().
 				Err(err).
@@ -323,8 +348,10 @@ func (verifier *Verifier) StartChangeStream(ctx context.Context) error {
 	return nil
 }
 
-func addUnixTimeToLogEvent[T constraints.Integer](unixTime T, event *zerolog.Event) *zerolog.Event {
-	return event.Time("timestampTime", time.Unix(int64(unixTime), int64(0)))
+func addTimestampToLogEvent(ts primitive.Timestamp, event *zerolog.Event) *zerolog.Event {
+	return event.
+		Interface("timestamp", ts).
+		Time("time", time.Unix(int64(ts.T), int64(0)))
 }
 
 func (v *Verifier) getChangeStreamMetadataCollection() *mongo.Collection {
@@ -363,7 +390,7 @@ func (verifier *Verifier) persistChangeStreamResumeToken(ctx context.Context, cs
 		logEvent := verifier.logger.Debug()
 
 		if err == nil {
-			logEvent = addUnixTimeToLogEvent(ts.T, logEvent)
+			logEvent = addTimestampToLogEvent(ts, logEvent)
 		} else {
 			verifier.logger.Warn().Err(err).
 				Msg("failed to extract resume token timestamp")
diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index 6c1f0c20..8f495cd5 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -142,8 +142,7 @@ func (suite *IntegrationTestSuite) TestStartAtTimeNoChanges() {
 	err = verifier.StartChangeStream(ctx)
 	suite.Require().NoError(err)
 	suite.Require().Equal(verifier.srcStartAtTs, origStartTs)
-	//verifier.changeStreamFinalTsChan <- *origStartTs
-	verifier.changeStreamEnderChan <- struct{}{}
+	verifier.changeStreamFinalTsChan <- *origStartTs
 	<-verifier.changeStreamDoneChan
 	suite.Require().Equal(verifier.srcStartAtTs, origStartTs)
 }
@@ -185,8 +184,7 @@ func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
 		"session time after events should exceed the original",
 	)
 
-	//verifier.changeStreamFinalTsChan <- *postEventsSessionTime
-	verifier.changeStreamEnderChan <- struct{}{}
+	verifier.changeStreamFinalTsChan <- *postEventsSessionTime
 	<-verifier.changeStreamDoneChan
 
 	suite.Assert().Equal(
diff --git a/internal/verifier/check.go b/internal/verifier/check.go
index 5a60bfb5..3e4c238f 100644
--- a/internal/verifier/check.go
+++ b/internal/verifier/check.go
@@ -42,13 +42,24 @@ func (verifier *Verifier) Check(ctx context.Context, filter map[string]any) {
 	verifier.MaybeStartPeriodicHeapProfileCollection(ctx)
 }
 
-func (verifier *Verifier) waitForChangeStream() error {
+func (verifier *Verifier) waitForChangeStream(ctx context.Context) error {
 	verifier.mux.RLock()
 	csRunning := verifier.changeStreamRunning
 	verifier.mux.RUnlock()
 	if csRunning {
 		verifier.logger.Debug().Msg("Changestream still running, signalling that writes are done and waiting for change stream to exit")
-		verifier.changeStreamEnderChan <- struct{}{}
+
+		finalTs, err := GetNewClusterTime(
+			ctx,
+			verifier.logger,
+			verifier.srcClient,
+		)
+
+		if err != nil {
+			return errors.Wrapf(err, "failed to fetch source's cluster time")
+		}
+
+		verifier.changeStreamFinalTsChan <- finalTs
 		select {
 		case err := <-verifier.changeStreamErrChan:
 			verifier.logger.Warn().Err(err).
@@ -66,6 +77,7 @@ func (verifier *Verifier) waitForChangeStream() error {
 func (verifier *Verifier) CheckWorker(ctx context.Context) error {
 	verifier.logger.Debug().Msgf("Starting %d verification workers", verifier.numWorkers)
 	ctx, cancel := context.WithCancel(ctx)
+
 	wg := sync.WaitGroup{}
 	for i := 0; i < verifier.numWorkers; i++ {
 		wg.Add(1)
@@ -174,7 +186,9 @@ func (verifier *Verifier) CheckDriver(ctx context.Context, filter map[string]any
 	verifier.mux.RLock()
 	csRunning := verifier.changeStreamRunning
 	verifier.mux.RUnlock()
-	if !csRunning {
+	if csRunning {
+		verifier.logger.Debug().Msg("Check: Change stream already running.")
+	} else {
 		verifier.logger.Debug().Msg("Change stream not running; starting change stream")
 
 		err = verifier.StartChangeStream(ctx)
@@ -242,7 +256,7 @@ func (verifier *Verifier) CheckDriver(ctx context.Context, filter map[string]any
 			// It's necessary to wait for the change stream to finish before incrementing the
 			// generation number, or the last changes will not be checked.
 			verifier.mux.Unlock()
-			err := verifier.waitForChangeStream()
+			err := verifier.waitForChangeStream(ctx)
 			if err != nil {
 				return err
 			}
@@ -388,12 +402,15 @@ func (verifier *Verifier) Work(ctx context.Context, workerNum int, wg *sync.Wait
 			if errors.Is(err, mongo.ErrNoDocuments) {
 				duration := verifier.workerSleepDelayMillis * time.Millisecond
 
-				verifier.logger.Debug().
-					Int("workerNum", workerNum).
-					Stringer("duration", duration).
-					Msg("No tasks found. Sleeping.")
+				if duration > 0 {
+					verifier.logger.Debug().
+						Int("workerNum", workerNum).
+						Stringer("duration", duration).
+						Msg("No tasks found. Sleeping.")
+
+					time.Sleep(duration)
+				}
 
-				time.Sleep(duration)
 				continue
 			} else if err != nil {
 				panic(err)
diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go
index 809e4fe3..39c4fbf8 100644
--- a/internal/verifier/migration_verifier.go
+++ b/internal/verifier/migration_verifier.go
@@ -24,7 +24,6 @@ import (
 	"github.com/10gen/migration-verifier/internal/uuidutil"
 	"github.com/olekukonko/tablewriter"
 	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -49,7 +48,6 @@ const (
 	SrcNamespaceField = "query_filter.namespace"
 	DstNamespaceField = "query_filter.to"
 	NumWorkers        = 10
-	refetch           = "TODO_CHANGE_ME_REFETCH"
 	Idle              = "idle"
 	Check             = "check"
 	Recheck           = "recheck"
@@ -123,12 +121,12 @@ type Verifier struct {
 	metaDBName    string
 	srcStartAtTs  *primitive.Timestamp
 
-	mux                   sync.RWMutex
-	changeStreamRunning   bool
-	changeStreamEnderChan chan struct{}
-	changeStreamErrChan   chan error
-	changeStreamDoneChan  chan struct{}
-	lastChangeEventTime   *primitive.Timestamp
+	mux                     sync.RWMutex
+	changeStreamRunning     bool
+	changeStreamFinalTsChan chan primitive.Timestamp
+	changeStreamErrChan     chan error
+	changeStreamDoneChan    chan struct{}
+	lastChangeEventTime     *primitive.Timestamp
 
 	readConcernSetting ReadConcernSetting
 
@@ -188,15 +186,15 @@ func NewVerifier(settings VerifierSettings) *Verifier {
 	}
 
 	return &Verifier{
-		phase:                 Idle,
-		numWorkers:            NumWorkers,
-		readPreference:        readpref.Primary(),
-		partitionSizeInBytes:  400 * 1024 * 1024,
-		failureDisplaySize:    DefaultFailureDisplaySize,
-		changeStreamEnderChan: make(chan struct{}),
-		changeStreamErrChan:   make(chan error),
-		changeStreamDoneChan:  make(chan struct{}),
-		readConcernSetting:    readConcern,
+		phase:                   Idle,
+		numWorkers:              NumWorkers,
+		readPreference:          readpref.Primary(),
+		partitionSizeInBytes:    400 * 1024 * 1024,
+		failureDisplaySize:      DefaultFailureDisplaySize,
+		changeStreamFinalTsChan: make(chan primitive.Timestamp),
+		changeStreamErrChan:     make(chan error),
+		changeStreamDoneChan:    make(chan struct{}),
+		readConcernSetting:      readConcern,
 
 		// This will get recreated once gen0 starts, but we want it
 		// here in case the change streams gets an event before then.
@@ -315,15 +313,7 @@ func (verifier *Verifier) SetPartitionSizeMB(partitionSizeMB uint32) {
 }
 
 func (verifier *Verifier) SetLogger(logPath string) {
-	writer := getLogWriter(logPath)
-	verifier.writer = writer
-
-	consoleWriter := zerolog.ConsoleWriter{
-		Out:        writer,
-		TimeFormat: timeFormat,
-	}
-	l := zerolog.New(consoleWriter).With().Timestamp().Logger()
-	verifier.logger = logger.NewLogger(&l, writer)
+	verifier.logger, verifier.writer = getLoggerAndWriter(logPath)
 }
 
 func (verifier *Verifier) SetSrcNamespaces(arg []string) {
@@ -1172,10 +1162,6 @@ func (verifier *Verifier) verificationTaskCollection() *mongo.Collection {
 	return verifier.verificationDatabase().Collection(verificationTasksCollection)
 }
 
-func (verifier *Verifier) refetchCollection() *mongo.Collection {
-	return verifier.verificationDatabase().Collection(refetch)
-}
-
 func (verifier *Verifier) srcClientDatabase(dbName string) *mongo.Database {
 	db := verifier.srcClient.Database(dbName)
 	// No need to check the write concern because we do not write to the source database.

From 5df3ecc4335a78474cdafd57ef965846c972cecc Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 16:17:39 -0500
Subject: [PATCH 079/104] Revert "rework cluster-time-getting"

This reverts commit 45090916464505d398f0a701b4d37757b9507b59.
---
 internal/verifier/check.go            |   2 +-
 internal/verifier/clustertime.go      | 117 +++++++++++++++++++++-----
 internal/verifier/clustertime_test.go |   2 +-
 3 files changed, 97 insertions(+), 24 deletions(-)

diff --git a/internal/verifier/check.go b/internal/verifier/check.go
index 3e4c238f..ff2242cc 100644
--- a/internal/verifier/check.go
+++ b/internal/verifier/check.go
@@ -49,7 +49,7 @@ func (verifier *Verifier) waitForChangeStream(ctx context.Context) error {
 	if csRunning {
 		verifier.logger.Debug().Msg("Changestream still running, signalling that writes are done and waiting for change stream to exit")
 
-		finalTs, err := GetNewClusterTime(
+		finalTs, err := GetClusterTime(
 			ctx,
 			verifier.logger,
 			verifier.srcClient,
diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index 2fa75189..04ef9ff3 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/10gen/migration-verifier/internal/logger"
 	"github.com/10gen/migration-verifier/internal/retry"
+	"github.com/10gen/migration-verifier/internal/util"
 	"github.com/10gen/migration-verifier/mbson"
 	"github.com/pkg/errors"
 	"go.mongodb.org/mongo-driver/bson"
@@ -16,11 +17,11 @@ import (
 
 const opTimeKeyInServerResponse = "operationTime"
 
-// GetNewClusterTime advances the remote cluster’s cluster time an returns
-// that time. In sharded clusters this advancement happens across all shards,
-// which (usefully!) equalizes the shards’ cluster time and triggers them to
-// output all events before then.
-func GetNewClusterTime(
+// GetClusterTime returns the remote cluster’s cluster time.
+// In so doing it creates a new oplog entry across all shards.
+// The “note” will go into the cluster’s oplog, so keep it
+// short but meaningful.
+func GetClusterTime(
 	ctx context.Context,
 	logger *logger.Logger,
 	client *mongo.Client,
@@ -29,7 +30,31 @@ func GetNewClusterTime(
 
 	var optime primitive.Timestamp
 
+	// To get the cluster time, we submit a request to append an oplog note
+	// but set an unreasonably-early maxClusterTime. All the shards will fail
+	// the request but still send their current clusterTime to the mongos,
+	// which will return the most recent of those. Thus we can fetch the
+	// most recent cluster time without altering the oplog.
 	err := retryer.RunForTransientErrorsOnly(
+		ctx,
+		logger,
+		func(_ *retry.Info) error {
+			var err error
+			optime, err = fetchClusterTime(ctx, client)
+			return err
+		},
+	)
+
+	if err != nil {
+		return primitive.Timestamp{}, err
+	}
+
+	// OPTIMIZATION FOR SHARDED CLUSTERS: We append another oplog entry to
+	// bring all shards to a cluster time that’s *after* the optime that we’ll
+	// return. That way any events *at*
+	//
+	// Since this is just an optimization, failures here are nonfatal.
+	err = retryer.RunForTransientErrorsOnly(
 		ctx,
 		logger,
 		func(_ *retry.Info) error {
@@ -38,21 +63,73 @@ func GetNewClusterTime(
 			return err
 		},
 	)
+	if err != nil {
+		// This isn't serious enough even to warn on, so leave it at info-level.
+		logger.Info().Err(err).
+			Msg("Failed to append oplog note; change stream may need extra time to finish.")
+	}
+
+	return optime, nil
+}
+
+// Use this when we just need the correct cluster time without
+// actually changing any shards’ oplogs.
+func fetchClusterTime(
+	ctx context.Context,
+	client *mongo.Client,
+) (primitive.Timestamp, error) {
+	cmd, rawResponse, err := runAppendOplogNote(
+		ctx,
+		client,
+		"expect StaleClusterTime error",
+		bson.E{"maxClusterTime", primitive.Timestamp{1, 0}},
+	)
+
+	// We expect an error here; if we didn't get one then something is
+	// amiss on the server.
+	if err == nil {
+		return primitive.Timestamp{}, errors.Errorf("server request unexpectedly succeeded: %v", cmd)
+	}
+
+	if !util.IsStaleClusterTimeError(err) {
+		return primitive.Timestamp{}, errors.Wrap(
+			err,
+			"unexpected error (expected StaleClusterTime) from request",
+		)
+	}
 
-	return optime, err
+	return getOpTimeFromRawResponse(rawResponse)
 }
 
 func syncClusterTimeAcrossShards(
 	ctx context.Context,
 	client *mongo.Client,
 ) (primitive.Timestamp, error) {
-	cmd := bson.D{
-		{"appendOplogNote", 1},
-		{"data", bson.D{
-			{"migration-verifier", "syncing cluster time"},
-		}},
+	_, rawResponse, err := runAppendOplogNote(ctx, client, "syncing cluster time")
+
+	if err != nil {
+		return primitive.Timestamp{}, err
 	}
 
+	return getOpTimeFromRawResponse(rawResponse)
+}
+
+func runAppendOplogNote(
+	ctx context.Context,
+	client *mongo.Client,
+	note string,
+	extraPieces ...bson.E,
+) (bson.D, bson.Raw, error) {
+	cmd := append(
+		bson.D{
+			{"appendOplogNote", 1},
+			{"data", bson.D{
+				{"migration-verifier", note},
+			}},
+		},
+		extraPieces...,
+	)
+
 	resp := client.
 		Database(
 			"admin",
@@ -60,21 +137,17 @@ func syncClusterTimeAcrossShards(
 		).
 		RunCommand(ctx, cmd)
 
-	rawResponse, err := resp.Raw()
+	raw, err := resp.Raw()
 
-	if err != nil {
-		return primitive.Timestamp{}, errors.Wrapf(
-			err,
-			"command (%v) failed unexpectedly",
-			cmd,
-		)
-	}
-
-	return getOpTimeFromRawResponse(rawResponse)
+	return cmd, raw, errors.Wrapf(
+		err,
+		"command (%v) failed unexpectedly",
+		cmd,
+	)
 }
 
 func getOpTimeFromRawResponse(rawResponse bson.Raw) (primitive.Timestamp, error) {
-	// Return the response’s `operationTime`.
+	// Get the `operationTime` from the response and return it.
 	var optime primitive.Timestamp
 
 	found, err := mbson.RawLookup(rawResponse, &optime, opTimeKeyInServerResponse)
diff --git a/internal/verifier/clustertime_test.go b/internal/verifier/clustertime_test.go
index 6d2e5acb..eab0f61e 100644
--- a/internal/verifier/clustertime_test.go
+++ b/internal/verifier/clustertime_test.go
@@ -6,7 +6,7 @@ func (suite *IntegrationTestSuite) TestGetClusterTime() {
 	ctx := context.Background()
 	logger, _ := getLoggerAndWriter("stdout")
 
-	ts, err := GetNewClusterTime(ctx, logger, suite.srcMongoClient)
+	ts, err := GetClusterTime(ctx, logger, suite.srcMongoClient)
 	suite.Require().NoError(err)
 
 	suite.Assert().NotZero(ts, "timestamp should be nonzero")

From 9b0b88891ec2aae7535d14cf9982959a06a5b753 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 16:44:36 -0500
Subject: [PATCH 080/104] use fixed appendOplogNote logic

---
 internal/verifier/check.go            |  2 +-
 internal/verifier/clustertime.go      | 59 +++++++++++++--------------
 internal/verifier/clustertime_test.go |  2 +-
 3 files changed, 30 insertions(+), 33 deletions(-)

diff --git a/internal/verifier/check.go b/internal/verifier/check.go
index ff2242cc..3e4c238f 100644
--- a/internal/verifier/check.go
+++ b/internal/verifier/check.go
@@ -49,7 +49,7 @@ func (verifier *Verifier) waitForChangeStream(ctx context.Context) error {
 	if csRunning {
 		verifier.logger.Debug().Msg("Changestream still running, signalling that writes are done and waiting for change stream to exit")
 
-		finalTs, err := GetClusterTime(
+		finalTs, err := GetNewClusterTime(
 			ctx,
 			verifier.logger,
 			verifier.srcClient,
diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index 04ef9ff3..c0f5cf7a 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -17,30 +17,25 @@ import (
 
 const opTimeKeyInServerResponse = "operationTime"
 
-// GetClusterTime returns the remote cluster’s cluster time.
-// In so doing it creates a new oplog entry across all shards.
-// The “note” will go into the cluster’s oplog, so keep it
-// short but meaningful.
-func GetClusterTime(
+// GetNewClusterTime advances the cluster time and returns that time.
+// All shards’ cluster times will meet or exceed the returned time.
+func GetNewClusterTime(
 	ctx context.Context,
 	logger *logger.Logger,
 	client *mongo.Client,
 ) (primitive.Timestamp, error) {
 	retryer := retry.New(retry.DefaultDurationLimit)
 
-	var optime primitive.Timestamp
+	var clusterTime primitive.Timestamp
 
-	// To get the cluster time, we submit a request to append an oplog note
-	// but set an unreasonably-early maxClusterTime. All the shards will fail
-	// the request but still send their current clusterTime to the mongos,
-	// which will return the most recent of those. Thus we can fetch the
-	// most recent cluster time without altering the oplog.
+	// First we just fetch the latest cluster time without updating any
+	// shards’ oplogs.
 	err := retryer.RunForTransientErrorsOnly(
 		ctx,
 		logger,
 		func(_ *retry.Info) error {
 			var err error
-			optime, err = fetchClusterTime(ctx, client)
+			clusterTime, err = fetchClusterTime(ctx, client)
 			return err
 		},
 	)
@@ -49,17 +44,15 @@ func GetClusterTime(
 		return primitive.Timestamp{}, err
 	}
 
-	// OPTIMIZATION FOR SHARDED CLUSTERS: We append another oplog entry to
-	// bring all shards to a cluster time that’s *after* the optime that we’ll
-	// return. That way any events *at*
-	//
-	// Since this is just an optimization, failures here are nonfatal.
+	// fetchClusterTime() will have taught the mongos about the most current
+	// shard’s cluster time. Now we tell that mongos to update all lagging
+	// shards to that time.
 	err = retryer.RunForTransientErrorsOnly(
 		ctx,
 		logger,
 		func(_ *retry.Info) error {
 			var err error
-			optime, err = syncClusterTimeAcrossShards(ctx, client)
+			clusterTime, err = syncClusterTimeAcrossShards(ctx, client, clusterTime)
 			return err
 		},
 	)
@@ -69,7 +62,7 @@ func GetClusterTime(
 			Msg("Failed to append oplog note; change stream may need extra time to finish.")
 	}
 
-	return optime, nil
+	return clusterTime, nil
 }
 
 // Use this when we just need the correct cluster time without
@@ -82,7 +75,7 @@ func fetchClusterTime(
 		ctx,
 		client,
 		"expect StaleClusterTime error",
-		bson.E{"maxClusterTime", primitive.Timestamp{1, 0}},
+		primitive.Timestamp{1, 0},
 	)
 
 	// We expect an error here; if we didn't get one then something is
@@ -104,8 +97,14 @@ func fetchClusterTime(
 func syncClusterTimeAcrossShards(
 	ctx context.Context,
 	client *mongo.Client,
+	maxTime primitive.Timestamp,
 ) (primitive.Timestamp, error) {
-	_, rawResponse, err := runAppendOplogNote(ctx, client, "syncing cluster time")
+	_, rawResponse, err := runAppendOplogNote(
+		ctx,
+		client,
+		"syncing cluster time",
+		maxTime,
+	)
 
 	if err != nil {
 		return primitive.Timestamp{}, err
@@ -118,17 +117,15 @@ func runAppendOplogNote(
 	ctx context.Context,
 	client *mongo.Client,
 	note string,
-	extraPieces ...bson.E,
+	maxClusterTime primitive.Timestamp,
 ) (bson.D, bson.Raw, error) {
-	cmd := append(
-		bson.D{
-			{"appendOplogNote", 1},
-			{"data", bson.D{
-				{"migration-verifier", note},
-			}},
-		},
-		extraPieces...,
-	)
+	cmd := bson.D{
+		{"appendOplogNote", 1},
+		{"maxClusterTime", maxClusterTime},
+		{"data", bson.D{
+			{"migration-verifier", note},
+		}},
+	}
 
 	resp := client.
 		Database(
diff --git a/internal/verifier/clustertime_test.go b/internal/verifier/clustertime_test.go
index eab0f61e..6d2e5acb 100644
--- a/internal/verifier/clustertime_test.go
+++ b/internal/verifier/clustertime_test.go
@@ -6,7 +6,7 @@ func (suite *IntegrationTestSuite) TestGetClusterTime() {
 	ctx := context.Background()
 	logger, _ := getLoggerAndWriter("stdout")
 
-	ts, err := GetClusterTime(ctx, logger, suite.srcMongoClient)
+	ts, err := GetNewClusterTime(ctx, logger, suite.srcMongoClient)
 	suite.Require().NoError(err)
 
 	suite.Assert().NotZero(ts, "timestamp should be nonzero")

From 313b1fec3f0db30367e1c63ca0c80739d003303c Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 19:01:23 -0500
Subject: [PATCH 081/104] maybe fix sharding

---
 internal/verifier/change_stream_test.go | 9 ++++++---
 internal/verifier/clustertime.go        | 9 ++++++---
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index 8f495cd5..2307a6bf 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -249,6 +249,12 @@ func (suite *IntegrationTestSuite) TestEventBeforeWritesOff() {
 	checkDoneChan := make(chan struct{})
 	checkContinueChan := make(chan struct{})
 
+	db := suite.srcMongoClient.Database(suite.DBNameForTest())
+	coll := db.Collection("mycoll")
+	suite.Require().NoError(
+		db.CreateCollection(ctx, coll.Name()),
+	)
+
 	// start verifier
 	verifierDoneChan := make(chan struct{})
 	go func() {
@@ -261,9 +267,6 @@ func (suite *IntegrationTestSuite) TestEventBeforeWritesOff() {
 	// wait for generation 1
 	<-checkDoneChan
 
-	db := suite.srcMongoClient.Database(suite.DBNameForTest())
-	coll := db.Collection("mycoll")
-
 	docsCount := 10_000
 	docs := lo.RepeatBy(docsCount, func(_ int) bson.D { return bson.D{} })
 	_, err := coll.InsertMany(
diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index c0f5cf7a..552450e9 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -28,8 +28,8 @@ func GetNewClusterTime(
 
 	var clusterTime primitive.Timestamp
 
-	// First we just fetch the latest cluster time without updating any
-	// shards’ oplogs.
+	// First we just fetch the latest cluster time among all shards without
+	// updating any shards’ oplogs.
 	err := retryer.RunForTransientErrorsOnly(
 		ctx,
 		logger,
@@ -107,7 +107,10 @@ func syncClusterTimeAcrossShards(
 	)
 
 	if err != nil {
-		return primitive.Timestamp{}, err
+		return primitive.Timestamp{}, errors.Wrap(
+			err,
+			"failed to append note to oplog",
+		)
 	}
 
 	return getOpTimeFromRawResponse(rawResponse)

From 63f675481eaed0e985958c610e08aee8aa66de9e Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 19:04:51 -0500
Subject: [PATCH 082/104] switch to always getting new cluster time

---
 internal/verifier/clustertime.go | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index 552450e9..5d9295ca 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -35,7 +35,7 @@ func GetNewClusterTime(
 		logger,
 		func(_ *retry.Info) error {
 			var err error
-			clusterTime, err = fetchClusterTime(ctx, client)
+			clusterTime, err = fetchNewClusterTime(ctx, client)
 			return err
 		},
 	)
@@ -67,7 +67,7 @@ func GetNewClusterTime(
 
 // Use this when we just need the correct cluster time without
 // actually changing any shards’ oplogs.
-func fetchClusterTime(
+func fetchNewClusterTime(
 	ctx context.Context,
 	client *mongo.Client,
 ) (primitive.Timestamp, error) {
@@ -75,7 +75,6 @@ func fetchClusterTime(
 		ctx,
 		client,
 		"expect StaleClusterTime error",
-		primitive.Timestamp{1, 0},
 	)
 
 	// We expect an error here; if we didn't get one then something is
@@ -103,7 +102,7 @@ func syncClusterTimeAcrossShards(
 		ctx,
 		client,
 		"syncing cluster time",
-		maxTime,
+		bson.E{"maxClusterTime", maxTime},
 	)
 
 	if err != nil {
@@ -120,15 +119,17 @@ func runAppendOplogNote(
 	ctx context.Context,
 	client *mongo.Client,
 	note string,
-	maxClusterTime primitive.Timestamp,
+	extra ...bson.E,
 ) (bson.D, bson.Raw, error) {
-	cmd := bson.D{
-		{"appendOplogNote", 1},
-		{"maxClusterTime", maxClusterTime},
-		{"data", bson.D{
-			{"migration-verifier", note},
-		}},
-	}
+	cmd := append(
+		bson.D{
+			{"appendOplogNote", 1},
+			{"data", bson.D{
+				{"migration-verifier", note},
+			}},
+		},
+		extra...,
+	)
 
 	resp := client.
 		Database(

From 229ac9ad3424feab69bd0369e2af8912b4500005 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 19:05:34 -0500
Subject: [PATCH 083/104] tweak

---
 internal/verifier/clustertime.go | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index 5d9295ca..c9fec2f2 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -5,7 +5,6 @@ import (
 
 	"github.com/10gen/migration-verifier/internal/logger"
 	"github.com/10gen/migration-verifier/internal/retry"
-	"github.com/10gen/migration-verifier/internal/util"
 	"github.com/10gen/migration-verifier/mbson"
 	"github.com/pkg/errors"
 	"go.mongodb.org/mongo-driver/bson"
@@ -71,22 +70,15 @@ func fetchNewClusterTime(
 	ctx context.Context,
 	client *mongo.Client,
 ) (primitive.Timestamp, error) {
-	cmd, rawResponse, err := runAppendOplogNote(
+	_, rawResponse, err := runAppendOplogNote(
 		ctx,
 		client,
 		"expect StaleClusterTime error",
 	)
-
-	// We expect an error here; if we didn't get one then something is
-	// amiss on the server.
-	if err == nil {
-		return primitive.Timestamp{}, errors.Errorf("server request unexpectedly succeeded: %v", cmd)
-	}
-
-	if !util.IsStaleClusterTimeError(err) {
+	if err != nil {
 		return primitive.Timestamp{}, errors.Wrap(
 			err,
-			"unexpected error (expected StaleClusterTime) from request",
+			"failed to append note to oplog",
 		)
 	}
 

From 9ec9c698e3d6508cbbcfd1fb4cd5b3e26218b202 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 19:32:44 -0500
Subject: [PATCH 084/104] enforce read/write concern

---
 internal/verifier/integration_test_suite.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index ecb44783..d18073fd 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -11,6 +11,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.mongodb.org/mongo-driver/mongo/readconcern"
 	"go.mongodb.org/mongo-driver/mongo/writeconcern"
 )
 
@@ -47,17 +48,23 @@ func (suite *IntegrationTestSuite) Context() context.Context {
 
 func (suite *IntegrationTestSuite) SetupSuite() {
 	ctx := context.Background()
-	clientOpts := options.Client().ApplyURI(suite.srcConnStr).SetAppName("Verifier Test Suite").SetWriteConcern(writeconcern.Majority())
+	clientOpts := options.Client().ApplyURI(suite.srcConnStr).SetAppName("Verifier Test Suite").
+		SetWriteConcern(writeconcern.Majority()).
+		SetReadConcern(readconcern.Majority())
 	var err error
 
 	suite.srcMongoClient, err = mongo.Connect(ctx, clientOpts)
 	suite.Require().NoError(err)
 
-	clientOpts = options.Client().ApplyURI(suite.dstConnStr).SetAppName("Verifier Test Suite").SetWriteConcern(writeconcern.Majority())
+	clientOpts = options.Client().ApplyURI(suite.dstConnStr).SetAppName("Verifier Test Suite").
+		SetWriteConcern(writeconcern.Majority()).
+		SetReadConcern(readconcern.Majority())
 	suite.dstMongoClient, err = mongo.Connect(ctx, clientOpts)
 	suite.Require().NoError(err)
 
-	clientOpts = options.Client().ApplyURI(suite.metaConnStr).SetAppName("Verifier Test Suite")
+	clientOpts = options.Client().ApplyURI(suite.metaConnStr).SetAppName("Verifier Test Suite").
+		SetWriteConcern(writeconcern.Majority()).
+		SetReadConcern(readconcern.Majority())
 	suite.metaMongoClient, err = mongo.Connect(ctx, clientOpts)
 	suite.Require().NoError(err)
 

From 7126acd052c261728f8532637ea8cbf2d94dda1c Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 19:40:31 -0500
Subject: [PATCH 085/104] fix tyopo

---
 internal/verifier/clustertime.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index c9fec2f2..65294411 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -51,7 +51,7 @@ func GetNewClusterTime(
 		logger,
 		func(_ *retry.Info) error {
 			var err error
-			clusterTime, err = syncClusterTimeAcrossShards(ctx, client, clusterTime)
+			_, err = syncClusterTimeAcrossShards(ctx, client, clusterTime)
 			return err
 		},
 	)

From 640a8c2efa3b71ac57748129334c637c2931eac0 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 19:41:28 -0500
Subject: [PATCH 086/104] Revert "tweak"

This reverts commit 229ac9ad3424feab69bd0369e2af8912b4500005.
---
 internal/verifier/clustertime.go | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index 65294411..fc8b454a 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/10gen/migration-verifier/internal/logger"
 	"github.com/10gen/migration-verifier/internal/retry"
+	"github.com/10gen/migration-verifier/internal/util"
 	"github.com/10gen/migration-verifier/mbson"
 	"github.com/pkg/errors"
 	"go.mongodb.org/mongo-driver/bson"
@@ -70,15 +71,22 @@ func fetchNewClusterTime(
 	ctx context.Context,
 	client *mongo.Client,
 ) (primitive.Timestamp, error) {
-	_, rawResponse, err := runAppendOplogNote(
+	cmd, rawResponse, err := runAppendOplogNote(
 		ctx,
 		client,
 		"expect StaleClusterTime error",
 	)
-	if err != nil {
+
+	// We expect an error here; if we didn't get one then something is
+	// amiss on the server.
+	if err == nil {
+		return primitive.Timestamp{}, errors.Errorf("server request unexpectedly succeeded: %v", cmd)
+	}
+
+	if !util.IsStaleClusterTimeError(err) {
 		return primitive.Timestamp{}, errors.Wrap(
 			err,
-			"failed to append note to oplog",
+			"unexpected error (expected StaleClusterTime) from request",
 		)
 	}
 

From 173bc8c33ccd05418f9c288c8b370b9cfdcbde16 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 19:41:29 -0500
Subject: [PATCH 087/104] Revert "switch to always getting new cluster time"

This reverts commit 63f675481eaed0e985958c610e08aee8aa66de9e.
---
 internal/verifier/clustertime.go | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index fc8b454a..2d1e2572 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -35,7 +35,7 @@ func GetNewClusterTime(
 		logger,
 		func(_ *retry.Info) error {
 			var err error
-			clusterTime, err = fetchNewClusterTime(ctx, client)
+			clusterTime, err = fetchClusterTime(ctx, client)
 			return err
 		},
 	)
@@ -67,7 +67,7 @@ func GetNewClusterTime(
 
 // Use this when we just need the correct cluster time without
 // actually changing any shards’ oplogs.
-func fetchNewClusterTime(
+func fetchClusterTime(
 	ctx context.Context,
 	client *mongo.Client,
 ) (primitive.Timestamp, error) {
@@ -75,6 +75,7 @@ func fetchNewClusterTime(
 		ctx,
 		client,
 		"expect StaleClusterTime error",
+		primitive.Timestamp{1, 0},
 	)
 
 	// We expect an error here; if we didn't get one then something is
@@ -102,7 +103,7 @@ func syncClusterTimeAcrossShards(
 		ctx,
 		client,
 		"syncing cluster time",
-		bson.E{"maxClusterTime", maxTime},
+		maxTime,
 	)
 
 	if err != nil {
@@ -119,17 +120,15 @@ func runAppendOplogNote(
 	ctx context.Context,
 	client *mongo.Client,
 	note string,
-	extra ...bson.E,
+	maxClusterTime primitive.Timestamp,
 ) (bson.D, bson.Raw, error) {
-	cmd := append(
-		bson.D{
-			{"appendOplogNote", 1},
-			{"data", bson.D{
-				{"migration-verifier", note},
-			}},
-		},
-		extra...,
-	)
+	cmd := bson.D{
+		{"appendOplogNote", 1},
+		{"maxClusterTime", maxClusterTime},
+		{"data", bson.D{
+			{"migration-verifier", note},
+		}},
+	}
 
 	resp := client.
 		Database(

From c6e4db9c394489a00ef5d20d0883266a89c754d3 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 19:49:09 -0500
Subject: [PATCH 088/104] debug

---
 internal/verifier/clustertime.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index 2d1e2572..ce862538 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -57,8 +57,8 @@ func GetNewClusterTime(
 		},
 	)
 	if err != nil {
-		// This isn't serious enough even to warn on, so leave it at info-level.
-		logger.Info().Err(err).
+		// This isn't serious enough even for info-level.
+		logger.Debug().Err(err).
 			Msg("Failed to append oplog note; change stream may need extra time to finish.")
 	}
 

From f355d8e8fed9e86ea89b0113524908f1aa381582 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 20:48:24 -0500
Subject: [PATCH 089/104] change writesOff to send the finalTs

---
 internal/verifier/change_stream.go           | 31 ++++++++++++----
 internal/verifier/check.go                   | 39 ++++++--------------
 internal/verifier/migration_verifier.go      | 25 ++++++++++++-
 internal/verifier/migration_verifier_test.go | 13 ++++++-
 internal/verifier/web_server.go              |  8 +++-
 internal/verifier/web_server_test.go         |  4 +-
 6 files changed, 78 insertions(+), 42 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index a853bea9..679cce84 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -126,11 +126,26 @@ func (verifier *Verifier) GetChangeStreamFilter() []bson.D {
 	return []bson.D{stage}
 }
 
-func (verifier *Verifier) readAndHandleOneChangeEventBatch(ctx context.Context, cs *mongo.ChangeStream) error {
+func (verifier *Verifier) readAndHandleOneChangeEventBatch(
+	ctx context.Context,
+	cs *mongo.ChangeStream,
+	finalTs *primitive.Timestamp,
+) error {
 	eventsRead := 0
 	var changeEventBatch []ParsedEvent
 
 	for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
+		// Once the change stream reaches the final timestamp we should stop reading.
+		if finalTs != nil {
+			csTimestamp, err := extractTimestampFromResumeToken(cs.ResumeToken())
+			if err != nil {
+				return errors.Wrap(err, "failed to extract timestamp from change stream's resume token")
+			}
+			if !csTimestamp.Before(*finalTs) {
+				break
+			}
+		}
+
 		gotEvent := cs.TryNext(ctx)
 
 		if cs.Err() != nil {
@@ -184,7 +199,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 
 	for {
 		var err error
-		var changeStreamEnded bool
+		var gotFinalTimestamp bool
 
 		select {
 
@@ -203,7 +218,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 				Interface("finalTimestamp", finalTs).
 				Msg("Change stream thread received final timestamp. Finalizing change stream.")
 
-			changeStreamEnded = true
+			gotFinalTimestamp = true
 
 			// Read all change events until the source reports no events.
 			// (i.e., the `getMore` call returns empty)
@@ -224,7 +239,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 					break
 				}
 
-				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
+				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs, &finalTs)
 
 				if err != nil {
 					break
@@ -232,7 +247,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 			}
 
 		default:
-			err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
+			err = verifier.readAndHandleOneChangeEventBatch(ctx, cs, nil)
 
 			if err == nil {
 				err = persistResumeTokenIfNeeded()
@@ -246,12 +261,12 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 
 			verifier.changeStreamErrChan <- err
 
-			if !changeStreamEnded {
+			if !gotFinalTimestamp {
 				break
 			}
 		}
 
-		if changeStreamEnded {
+		if gotFinalTimestamp {
 			verifier.mux.Lock()
 			verifier.changeStreamRunning = false
 			if verifier.lastChangeEventTime != nil {
@@ -260,7 +275,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 			verifier.mux.Unlock()
 			// since we have started Recheck, we must signal that we have
 			// finished the change stream changes so that Recheck can continue.
-			verifier.changeStreamDoneChan <- struct{}{}
+			close(verifier.changeStreamDoneChan)
 			break
 		}
 	}
diff --git a/internal/verifier/check.go b/internal/verifier/check.go
index 3e4c238f..e868a68b 100644
--- a/internal/verifier/check.go
+++ b/internal/verifier/check.go
@@ -43,34 +43,19 @@ func (verifier *Verifier) Check(ctx context.Context, filter map[string]any) {
 }
 
 func (verifier *Verifier) waitForChangeStream(ctx context.Context) error {
-	verifier.mux.RLock()
-	csRunning := verifier.changeStreamRunning
-	verifier.mux.RUnlock()
-	if csRunning {
-		verifier.logger.Debug().Msg("Changestream still running, signalling that writes are done and waiting for change stream to exit")
-
-		finalTs, err := GetNewClusterTime(
-			ctx,
-			verifier.logger,
-			verifier.srcClient,
-		)
-
-		if err != nil {
-			return errors.Wrapf(err, "failed to fetch source's cluster time")
-		}
-
-		verifier.changeStreamFinalTsChan <- finalTs
-		select {
-		case err := <-verifier.changeStreamErrChan:
-			verifier.logger.Warn().Err(err).
-				Msg("Received error from change stream.")
-			return err
-		case <-verifier.changeStreamDoneChan:
-			verifier.logger.Debug().
-				Msg("Received completion signal from change stream.")
-			break
-		}
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case err := <-verifier.changeStreamErrChan:
+		verifier.logger.Warn().Err(err).
+			Msg("Received error from change stream.")
+		return err
+	case <-verifier.changeStreamDoneChan:
+		verifier.logger.Debug().
+			Msg("Received completion signal from change stream.")
+		break
 	}
+
 	return nil
 }
 
diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go
index 39c4fbf8..c6cea798 100644
--- a/internal/verifier/migration_verifier.go
+++ b/internal/verifier/migration_verifier.go
@@ -127,6 +127,7 @@ type Verifier struct {
 	changeStreamErrChan     chan error
 	changeStreamDoneChan    chan struct{}
 	lastChangeEventTime     *primitive.Timestamp
+	writesOffTimestamp      *primitive.Timestamp
 
 	readConcernSetting ReadConcernSetting
 
@@ -226,13 +227,33 @@ func (verifier *Verifier) SetFailureDisplaySize(size int64) {
 	verifier.failureDisplaySize = size
 }
 
-func (verifier *Verifier) WritesOff(ctx context.Context) {
+func (verifier *Verifier) WritesOff(ctx context.Context) error {
 	verifier.logger.Debug().
 		Msg("WritesOff called.")
 
 	verifier.mux.Lock()
+	defer verifier.mux.Unlock()
 	verifier.writesOff = true
-	verifier.mux.Unlock()
+
+	if verifier.writesOffTimestamp == nil {
+		verifier.logger.Debug().Msg("Change stream still running. Signalling that writes are done.")
+
+		finalTs, err := GetNewClusterTime(
+			ctx,
+			verifier.logger,
+			verifier.srcClient,
+		)
+
+		if err != nil {
+			return errors.Wrapf(err, "failed to fetch source's cluster time")
+		}
+
+		verifier.writesOffTimestamp = &finalTs
+
+		verifier.changeStreamFinalTsChan <- finalTs
+	}
+
+	return nil
 }
 
 func (verifier *Verifier) WritesOn(ctx context.Context) {
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index 3cdd55b2..e3055b5e 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1390,8 +1390,19 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 	// because of the calls to WritesOff
 	status, err = verifier.GetVerificationStatus()
 	suite.Require().NoError(err)
+
 	// there should be a failure from the src insert
-	suite.Require().Equal(VerificationStatus{TotalTasks: 1, FailedTasks: 1}, *status)
+	suite.T().Logf("status: %+v", *status)
+	suite.Assert().Equal(VerificationStatus{TotalTasks: 1, FailedTasks: 1}, *status)
+
+	failedTasks, incompleteTasks, err := FetchFailedAndIncompleteTasks(
+		ctx,
+		verifier.verificationTaskCollection(),
+		verificationTaskVerifyDocuments,
+		verifier.generation,
+	)
+	suite.T().Logf("failed: %+v", failedTasks)
+	suite.T().Logf("incomplete: %+v", incompleteTasks)
 
 	checkContinueChan <- struct{}{}
 	require.NoError(suite.T(), errGroup.Wait())
diff --git a/internal/verifier/web_server.go b/internal/verifier/web_server.go
index 9f3925f0..0c15f8be 100644
--- a/internal/verifier/web_server.go
+++ b/internal/verifier/web_server.go
@@ -22,7 +22,7 @@ const RequestInProgressErrorDescription = "Another request is currently in progr
 // MigrationVerifierAPI represents the interaction webserver with mongosync
 type MigrationVerifierAPI interface {
 	Check(ctx context.Context, filter map[string]any)
-	WritesOff(ctx context.Context)
+	WritesOff(ctx context.Context) error
 	WritesOn(ctx context.Context)
 	GetProgress(ctx context.Context) (Progress, error)
 }
@@ -214,7 +214,11 @@ func (server *WebServer) writesOffEndpoint(c *gin.Context) {
 		return
 	}
 
-	server.Mapi.WritesOff(context.Background())
+	err := server.Mapi.WritesOff(context.Background())
+	if err != nil {
+		c.JSON(http.StatusOK, gin.H{"error": err.Error()})
+		return
+	}
 	successResponse(c)
 }
 
diff --git a/internal/verifier/web_server_test.go b/internal/verifier/web_server_test.go
index efa586f6..c7c8b476 100644
--- a/internal/verifier/web_server_test.go
+++ b/internal/verifier/web_server_test.go
@@ -30,8 +30,8 @@ func NewMockVerifier() *MockVerifier {
 func (verifier *MockVerifier) Check(ctx context.Context, filter map[string]any) {
 	verifier.filter = filter
 }
-func (verifier *MockVerifier) WritesOff(ctx context.Context) {}
-func (verifier *MockVerifier) WritesOn(ctx context.Context)  {}
+func (verifier *MockVerifier) WritesOff(ctx context.Context) error { return nil }
+func (verifier *MockVerifier) WritesOn(ctx context.Context)        {}
 func (verifier *MockVerifier) GetProgress(ctx context.Context) (Progress, error) {
 	return Progress{}, nil
 }

From 725bc7c2aac0b2c8f259d8e7e2e852a651df0466 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 21:11:27 -0500
Subject: [PATCH 090/104] fix error

---
 internal/verifier/change_stream_test.go      |  2 +-
 internal/verifier/migration_verifier_test.go | 24 +++++++-------------
 main/migration_verifier.go                   |  6 ++++-
 3 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index 2307a6bf..75df99b4 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -275,7 +275,7 @@ func (suite *IntegrationTestSuite) TestEventBeforeWritesOff() {
 	)
 	suite.Require().NoError(err)
 
-	verifier.WritesOff(ctx)
+	suite.Require().NoError(verifier.WritesOff(ctx))
 
 	verifierDone := false
 	for !verifierDone {
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index e3055b5e..10860a7e 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1381,28 +1381,20 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 	suite.Require().Equal(VerificationStatus{TotalTasks: 1, CompletedTasks: 1}, *status)
 
 	// turn writes off
-	verifier.WritesOff(ctx)
+	suite.Require().NoError(verifier.WritesOff(ctx))
+
+	// now write to the source, this should not be seen by the change stream which should have ended
+	// because of the calls to WritesOff
 	_, err = srcColl.InsertOne(ctx, bson.M{"_id": 1019, "x": 1019})
 	suite.Require().NoError(err)
 	checkContinueChan <- struct{}{}
 	<-checkDoneChan
-	// now write to the source, this should not be seen by the change stream which should have ended
-	// because of the calls to WritesOff
+
 	status, err = verifier.GetVerificationStatus()
 	suite.Require().NoError(err)
 
-	// there should be a failure from the src insert
-	suite.T().Logf("status: %+v", *status)
-	suite.Assert().Equal(VerificationStatus{TotalTasks: 1, FailedTasks: 1}, *status)
-
-	failedTasks, incompleteTasks, err := FetchFailedAndIncompleteTasks(
-		ctx,
-		verifier.verificationTaskCollection(),
-		verificationTaskVerifyDocuments,
-		verifier.generation,
-	)
-	suite.T().Logf("failed: %+v", failedTasks)
-	suite.T().Logf("incomplete: %+v", incompleteTasks)
+	// there should be a no more tasks
+	suite.Assert().Equal(VerificationStatus{}, *status)
 
 	checkContinueChan <- struct{}{}
 	require.NoError(suite.T(), errGroup.Wait())
@@ -1523,7 +1515,7 @@ func (suite *IntegrationTestSuite) TestVerifierWithFilter() {
 	suite.Require().Equal(VerificationStatus{TotalTasks: 1, CompletedTasks: 1}, *status)
 
 	// Turn writes off.
-	verifier.WritesOff(ctx)
+	suite.Require().NoError(verifier.WritesOff(ctx))
 
 	// Tell CheckDriver to do one more pass. This should terminate the change stream.
 	checkContinueChan <- struct{}{}
diff --git a/main/migration_verifier.go b/main/migration_verifier.go
index cb5a6320..8863bbba 100644
--- a/main/migration_verifier.go
+++ b/main/migration_verifier.go
@@ -176,7 +176,11 @@ func main() {
 				zerolog.SetGlobalLevel(zerolog.DebugLevel)
 			}
 			if cCtx.Bool(checkOnly) {
-				verifier.WritesOff(ctx)
+				err := verifier.WritesOff(ctx)
+				if err != nil {
+					return errors.Wrap(err, "failed to set writes off")
+				}
+
 				return verifier.CheckDriver(ctx, nil)
 			} else {
 				return verifier.StartServer()

From fc00f0f98c8fa996a92350d270a5db3d623a1b07 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Thu, 21 Nov 2024 23:54:10 -0500
Subject: [PATCH 091/104] remove test

---
 internal/verifier/change_stream.go           |  2 +-
 internal/verifier/change_stream_test.go      |  4 +--
 internal/verifier/check.go                   |  4 +--
 internal/verifier/integration_test_suite.go  |  3 ++
 internal/verifier/migration_verifier.go      | 38 +++++++++++---------
 internal/verifier/migration_verifier_test.go | 21 +----------
 6 files changed, 29 insertions(+), 43 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 679cce84..ee9222db 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -213,7 +213,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 		// If the changeStreamEnderChan has a message, the user has indicated that
 		// source writes are ended. This means we should exit rather than continue
 		// reading the change stream since there should be no more events.
-		case finalTs := <-verifier.changeStreamFinalTsChan:
+		case finalTs := <-verifier.changeStreamWritesOffTsChan:
 			verifier.logger.Debug().
 				Interface("finalTimestamp", finalTs).
 				Msg("Change stream thread received final timestamp. Finalizing change stream.")
diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index 75df99b4..5a265a75 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -142,7 +142,7 @@ func (suite *IntegrationTestSuite) TestStartAtTimeNoChanges() {
 	err = verifier.StartChangeStream(ctx)
 	suite.Require().NoError(err)
 	suite.Require().Equal(verifier.srcStartAtTs, origStartTs)
-	verifier.changeStreamFinalTsChan <- *origStartTs
+	verifier.changeStreamWritesOffTsChan <- *origStartTs
 	<-verifier.changeStreamDoneChan
 	suite.Require().Equal(verifier.srcStartAtTs, origStartTs)
 }
@@ -184,7 +184,7 @@ func (suite *IntegrationTestSuite) TestStartAtTimeWithChanges() {
 		"session time after events should exceed the original",
 	)
 
-	verifier.changeStreamFinalTsChan <- *postEventsSessionTime
+	verifier.changeStreamWritesOffTsChan <- *postEventsSessionTime
 	<-verifier.changeStreamDoneChan
 
 	suite.Assert().Equal(
diff --git a/internal/verifier/check.go b/internal/verifier/check.go
index e868a68b..0ae007c5 100644
--- a/internal/verifier/check.go
+++ b/internal/verifier/check.go
@@ -25,8 +25,6 @@ var failedStatus = mapset.NewSet(
 	verificationTaskMetadataMismatch,
 )
 
-var verificationStatusCheckInterval time.Duration = 15 * time.Second
-
 // Check is the asynchronous entry point to Check, should only be called by the web server. Use
 // CheckDriver directly for synchronous run.
 // testChan is a pair of channels for coordinating generations in tests.
@@ -110,7 +108,7 @@ func (verifier *Verifier) CheckWorker(ctx context.Context) error {
 		//wait for task to be created, if none of the tasks found.
 		if verificationStatus.AddedTasks > 0 || verificationStatus.ProcessingTasks > 0 || verificationStatus.RecheckTasks > 0 {
 			waitForTaskCreation++
-			time.Sleep(verificationStatusCheckInterval)
+			time.Sleep(verifier.verificationStatusCheckInterval)
 		} else {
 			verifier.PrintVerificationSummary(ctx, GenerationComplete)
 			verifier.logger.Debug().Msg("Verification tasks complete")
diff --git a/internal/verifier/integration_test_suite.go b/internal/verifier/integration_test_suite.go
index d18073fd..33e3a336 100644
--- a/internal/verifier/integration_test_suite.go
+++ b/internal/verifier/integration_test_suite.go
@@ -3,6 +3,7 @@ package verifier
 import (
 	"context"
 	"strings"
+	"time"
 
 	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/pkg/errors"
@@ -166,6 +167,8 @@ func (suite *IntegrationTestSuite) BuildVerifier() *Verifier {
 	verifier.SetGenerationPauseDelayMillis(0)
 	verifier.SetWorkerSleepDelayMillis(0)
 
+	verifier.verificationStatusCheckInterval = 10 * time.Millisecond
+
 	ctx := suite.Context()
 
 	suite.Require().NoError(
diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go
index c6cea798..79443d73 100644
--- a/internal/verifier/migration_verifier.go
+++ b/internal/verifier/migration_verifier.go
@@ -121,13 +121,13 @@ type Verifier struct {
 	metaDBName    string
 	srcStartAtTs  *primitive.Timestamp
 
-	mux                     sync.RWMutex
-	changeStreamRunning     bool
-	changeStreamFinalTsChan chan primitive.Timestamp
-	changeStreamErrChan     chan error
-	changeStreamDoneChan    chan struct{}
-	lastChangeEventTime     *primitive.Timestamp
-	writesOffTimestamp      *primitive.Timestamp
+	mux                         sync.RWMutex
+	changeStreamRunning         bool
+	changeStreamWritesOffTsChan chan primitive.Timestamp
+	changeStreamErrChan         chan error
+	changeStreamDoneChan        chan struct{}
+	lastChangeEventTime         *primitive.Timestamp
+	writesOffTimestamp          *primitive.Timestamp
 
 	readConcernSetting ReadConcernSetting
 
@@ -137,6 +137,8 @@ type Verifier struct {
 	globalFilter map[string]any
 
 	pprofInterval time.Duration
+
+	verificationStatusCheckInterval time.Duration
 }
 
 // VerificationStatus holds the Verification Status
@@ -187,19 +189,21 @@ func NewVerifier(settings VerifierSettings) *Verifier {
 	}
 
 	return &Verifier{
-		phase:                   Idle,
-		numWorkers:              NumWorkers,
-		readPreference:          readpref.Primary(),
-		partitionSizeInBytes:    400 * 1024 * 1024,
-		failureDisplaySize:      DefaultFailureDisplaySize,
-		changeStreamFinalTsChan: make(chan primitive.Timestamp),
-		changeStreamErrChan:     make(chan error),
-		changeStreamDoneChan:    make(chan struct{}),
-		readConcernSetting:      readConcern,
+		phase:                       Idle,
+		numWorkers:                  NumWorkers,
+		readPreference:              readpref.Primary(),
+		partitionSizeInBytes:        400 * 1024 * 1024,
+		failureDisplaySize:          DefaultFailureDisplaySize,
+		changeStreamWritesOffTsChan: make(chan primitive.Timestamp),
+		changeStreamErrChan:         make(chan error),
+		changeStreamDoneChan:        make(chan struct{}),
+		readConcernSetting:          readConcern,
 
 		// This will get recreated once gen0 starts, but we want it
 		// here in case the change streams gets an event before then.
 		eventRecorder: NewEventRecorder(),
+
+		verificationStatusCheckInterval: 15 * time.Second,
 	}
 }
 
@@ -250,7 +254,7 @@ func (verifier *Verifier) WritesOff(ctx context.Context) error {
 
 		verifier.writesOffTimestamp = &finalTs
 
-		verifier.changeStreamFinalTsChan <- finalTs
+		verifier.changeStreamWritesOffTsChan <- finalTs
 	}
 
 	return nil
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index 10860a7e..6b2e6cbe 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1378,26 +1378,7 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 	status = waitForTasks()
 
 	// there should be no failures now, since they are are equivalent at this point in time
-	suite.Require().Equal(VerificationStatus{TotalTasks: 1, CompletedTasks: 1}, *status)
-
-	// turn writes off
-	suite.Require().NoError(verifier.WritesOff(ctx))
-
-	// now write to the source, this should not be seen by the change stream which should have ended
-	// because of the calls to WritesOff
-	_, err = srcColl.InsertOne(ctx, bson.M{"_id": 1019, "x": 1019})
-	suite.Require().NoError(err)
-	checkContinueChan <- struct{}{}
-	<-checkDoneChan
-
-	status, err = verifier.GetVerificationStatus()
-	suite.Require().NoError(err)
-
-	// there should be a no more tasks
-	suite.Assert().Equal(VerificationStatus{}, *status)
-
-	checkContinueChan <- struct{}{}
-	require.NoError(suite.T(), errGroup.Wait())
+	suite.Assert().Equal(VerificationStatus{TotalTasks: 1, CompletedTasks: 1}, *status)
 }
 
 func (suite *IntegrationTestSuite) TestVerifierWithFilter() {

From d87dc0feedce355ce7ef00ee7451005b774f76d7 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 00:28:02 -0500
Subject: [PATCH 092/104] test fixes

---
 internal/verifier/clustertime.go             | 5 ++++-
 internal/verifier/migration_verifier.go      | 4 +++-
 internal/verifier/migration_verifier_test.go | 5 +++++
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/internal/verifier/clustertime.go b/internal/verifier/clustertime.go
index ce862538..30a2c579 100644
--- a/internal/verifier/clustertime.go
+++ b/internal/verifier/clustertime.go
@@ -106,7 +106,10 @@ func syncClusterTimeAcrossShards(
 		maxTime,
 	)
 
-	if err != nil {
+	// If any shard’s cluster time >= maxTime, the mongos will return a
+	// StaleClusterTime error. This particular error doesn’t indicate a
+	// failure, so we ignore it.
+	if err != nil && !util.IsStaleClusterTimeError(err) {
 		return primitive.Timestamp{}, errors.Wrap(
 			err,
 			"failed to append note to oplog",
diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go
index 79443d73..b5da73bf 100644
--- a/internal/verifier/migration_verifier.go
+++ b/internal/verifier/migration_verifier.go
@@ -236,7 +236,6 @@ func (verifier *Verifier) WritesOff(ctx context.Context) error {
 		Msg("WritesOff called.")
 
 	verifier.mux.Lock()
-	defer verifier.mux.Unlock()
 	verifier.writesOff = true
 
 	if verifier.writesOffTimestamp == nil {
@@ -254,7 +253,10 @@ func (verifier *Verifier) WritesOff(ctx context.Context) error {
 
 		verifier.writesOffTimestamp = &finalTs
 
+		verifier.mux.Unlock()
 		verifier.changeStreamWritesOffTsChan <- finalTs
+	} else {
+		verifier.mux.Unlock()
 	}
 
 	return nil
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index 6b2e6cbe..e93aa7bb 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -1379,6 +1379,11 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 
 	// there should be no failures now, since they are are equivalent at this point in time
 	suite.Assert().Equal(VerificationStatus{TotalTasks: 1, CompletedTasks: 1}, *status)
+
+	suite.Require().NoError(verifier.WritesOff(ctx))
+
+	checkContinueChan <- struct{}{}
+	require.NoError(suite.T(), errGroup.Wait())
 }
 
 func (suite *IntegrationTestSuite) TestVerifierWithFilter() {

From ebc391fdd36e4b6aabb9b3ddadad0100238e4c46 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 00:58:53 -0500
Subject: [PATCH 093/104] check runner

---
 internal/verifier/change_stream_test.go      | 25 ++-------
 internal/verifier/check_runner.go            | 55 ++++++++++++++++++++
 internal/verifier/migration_verifier.go      |  4 ++
 internal/verifier/migration_verifier_test.go | 42 ++++++---------
 4 files changed, 78 insertions(+), 48 deletions(-)
 create mode 100644 internal/verifier/check_runner.go

diff --git a/internal/verifier/change_stream_test.go b/internal/verifier/change_stream_test.go
index 5a265a75..ce00be68 100644
--- a/internal/verifier/change_stream_test.go
+++ b/internal/verifier/change_stream_test.go
@@ -246,9 +246,6 @@ func (suite *IntegrationTestSuite) TestEventBeforeWritesOff() {
 
 	verifier := suite.BuildVerifier()
 
-	checkDoneChan := make(chan struct{})
-	checkContinueChan := make(chan struct{})
-
 	db := suite.srcMongoClient.Database(suite.DBNameForTest())
 	coll := db.Collection("mycoll")
 	suite.Require().NoError(
@@ -256,16 +253,10 @@ func (suite *IntegrationTestSuite) TestEventBeforeWritesOff() {
 	)
 
 	// start verifier
-	verifierDoneChan := make(chan struct{})
-	go func() {
-		err := verifier.CheckDriver(ctx, nil, checkDoneChan, checkContinueChan)
-		suite.Require().NoError(err)
+	verifierRunner := RunVerifierCheck(suite.Context(), suite.T(), verifier)
 
-		close(verifierDoneChan)
-	}()
-
-	// wait for generation 1
-	<-checkDoneChan
+	// wait for generation 0 to end
+	verifierRunner.AwaitGenerationEnd()
 
 	docsCount := 10_000
 	docs := lo.RepeatBy(docsCount, func(_ int) bson.D { return bson.D{} })
@@ -277,15 +268,7 @@ func (suite *IntegrationTestSuite) TestEventBeforeWritesOff() {
 
 	suite.Require().NoError(verifier.WritesOff(ctx))
 
-	verifierDone := false
-	for !verifierDone {
-		select {
-		case <-verifierDoneChan:
-			verifierDone = true
-		case <-checkDoneChan:
-		case checkContinueChan <- struct{}{}:
-		}
-	}
+	suite.Require().NoError(verifierRunner.Await())
 
 	generation := verifier.generation
 	failedTasks, incompleteTasks, err := FetchFailedAndIncompleteTasks(
diff --git a/internal/verifier/check_runner.go b/internal/verifier/check_runner.go
new file mode 100644
index 00000000..fe48fd26
--- /dev/null
+++ b/internal/verifier/check_runner.go
@@ -0,0 +1,55 @@
+package verifier
+
+import (
+	"context"
+	"testing"
+)
+
+type CheckRunner struct {
+	checkDoneChan        chan error
+	generationDoneChan   chan struct{}
+	doNextGenerationChan chan struct{}
+}
+
+func RunVerifierCheck(ctx context.Context, t *testing.T, verifier *Verifier) *CheckRunner {
+	verifierDoneChan := make(chan error)
+
+	generationDoneChan := make(chan struct{})
+	doNextGenerationChan := make(chan struct{})
+
+	go func() {
+		err := verifier.CheckDriver(ctx, nil, generationDoneChan, doNextGenerationChan)
+		verifierDoneChan <- err
+	}()
+
+	return &CheckRunner{
+		checkDoneChan:        verifierDoneChan,
+		generationDoneChan:   generationDoneChan,
+		doNextGenerationChan: doNextGenerationChan,
+	}
+}
+
+// AwaitGenerationEnd blocks until the check’s current generation ends.
+func (cr *CheckRunner) AwaitGenerationEnd() {
+	<-cr.generationDoneChan
+}
+
+// StartNextGeneration blocks until it can tell the check to start
+// the next generation.
+func (cr *CheckRunner) StartNextGeneration() {
+	cr.doNextGenerationChan <- struct{}{}
+}
+
+// Await will await generations and start new ones until the check
+// finishes. It returns the error that verifier.CheckDriver() returns.
+func (cr *CheckRunner) Await() error {
+	for {
+		select {
+		case err := <-cr.checkDoneChan:
+			return err
+
+		case <-cr.generationDoneChan:
+		case cr.doNextGenerationChan <- struct{}{}:
+		}
+	}
+}
diff --git a/internal/verifier/migration_verifier.go b/internal/verifier/migration_verifier.go
index b5da73bf..c37eea7d 100644
--- a/internal/verifier/migration_verifier.go
+++ b/internal/verifier/migration_verifier.go
@@ -254,6 +254,10 @@ func (verifier *Verifier) WritesOff(ctx context.Context) error {
 		verifier.writesOffTimestamp = &finalTs
 
 		verifier.mux.Unlock()
+
+		// This has to happen under the lock because the change stream
+		// might be inserting docs into the recheck queue, which happens
+		// under the lock.
 		verifier.changeStreamWritesOffTsChan <- finalTs
 	} else {
 		verifier.mux.Unlock()
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index e93aa7bb..d6e9d8e5 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -20,7 +20,6 @@ import (
 	"github.com/10gen/migration-verifier/internal/testutil"
 	"github.com/cespare/permute/v2"
 	"github.com/rs/zerolog"
-	"github.com/rs/zerolog/log"
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -29,7 +28,6 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
-	"golang.org/x/sync/errgroup"
 )
 
 func TestIntegration(t *testing.T) {
@@ -1292,7 +1290,7 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 	verifier.SetDstNamespaces([]string{"testDb2.testColl3"})
 	verifier.SetNamespaceMap()
 
-	ctx := context.Background()
+	ctx := suite.Context()
 
 	srcColl := suite.srcMongoClient.Database("testDb1").Collection("testColl1")
 	dstColl := suite.dstMongoClient.Database("testDb2").Collection("testColl3")
@@ -1303,18 +1301,7 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 	_, err = dstColl.InsertOne(ctx, bson.M{"_id": 1, "x": 42})
 	suite.Require().NoError(err)
 
-	checkDoneChan := make(chan struct{})
-	checkContinueChan := make(chan struct{})
-
-	errGroup, errGrpCtx := errgroup.WithContext(context.Background())
-	errGroup.Go(func() error {
-		checkDriverErr := verifier.CheckDriver(errGrpCtx, nil, checkDoneChan, checkContinueChan)
-		// Log this as fatal error so that the test doesn't hang.
-		if checkDriverErr != nil {
-			log.Fatal().Err(checkDriverErr).Msg("check driver error")
-		}
-		return checkDriverErr
-	})
+	runner := RunVerifierCheck(ctx, suite.T(), verifier)
 
 	waitForTasks := func() *VerificationStatus {
 		status, err := verifier.GetVerificationStatus()
@@ -1326,8 +1313,8 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 			suite.T().Logf("TotalTasks is 0 (generation=%d); waiting %s then will run another generation …", verifier.generation, delay)
 
 			time.Sleep(delay)
-			checkContinueChan <- struct{}{}
-			<-checkDoneChan
+			runner.StartNextGeneration()
+			runner.AwaitGenerationEnd()
 			status, err = verifier.GetVerificationStatus()
 			suite.Require().NoError(err)
 		}
@@ -1335,7 +1322,7 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 	}
 
 	// wait for one generation to finish
-	<-checkDoneChan
+	runner.AwaitGenerationEnd()
 	status := waitForTasks()
 	suite.Require().Equal(VerificationStatus{TotalTasks: 2, FailedTasks: 1, CompletedTasks: 1}, *status)
 
@@ -1344,10 +1331,10 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 	suite.Require().NoError(err)
 
 	// tell check to start the next generation
-	checkContinueChan <- struct{}{}
+	runner.StartNextGeneration()
 
 	// wait for generation to finish
-	<-checkDoneChan
+	runner.AwaitGenerationEnd()
 	status = waitForTasks()
 	// there should be no failures now, since they are are equivalent at this point in time
 	suite.Require().Equal(VerificationStatus{TotalTasks: 1, CompletedTasks: 1}, *status)
@@ -1357,10 +1344,10 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 	suite.Require().NoError(err)
 
 	// tell check to start the next generation
-	checkContinueChan <- struct{}{}
+	runner.StartNextGeneration()
 
 	// wait for one generation to finish
-	<-checkDoneChan
+	runner.AwaitGenerationEnd()
 	status = waitForTasks()
 
 	// there should be a failure from the src insert
@@ -1371,19 +1358,20 @@ func (suite *IntegrationTestSuite) TestGenerationalRechecking() {
 	suite.Require().NoError(err)
 
 	// continue
-	checkContinueChan <- struct{}{}
+	runner.StartNextGeneration()
 
 	// wait for it to finish again, this should be a clean run
-	<-checkDoneChan
+	runner.AwaitGenerationEnd()
 	status = waitForTasks()
 
 	// there should be no failures now, since they are are equivalent at this point in time
 	suite.Assert().Equal(VerificationStatus{TotalTasks: 1, CompletedTasks: 1}, *status)
 
+	// We could just abandon this verifier, but we might as well shut it down
+	// gracefully. That prevents a spurious error in the log from “drop”
+	// change events.
 	suite.Require().NoError(verifier.WritesOff(ctx))
-
-	checkContinueChan <- struct{}{}
-	require.NoError(suite.T(), errGroup.Wait())
+	suite.Require().NoError(runner.Await())
 }
 
 func (suite *IntegrationTestSuite) TestVerifierWithFilter() {

From 37268077f989a817fe2c8d317f9a2417a45611a1 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 01:16:34 -0500
Subject: [PATCH 094/104] diagnose 8.0 sharded issue

---
 .github/workflows/all.yml          | 44 +++++++++++++--------------
 internal/verifier/change_stream.go | 48 +++++++++++++++++-------------
 2 files changed, 49 insertions(+), 43 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index a452eb1c..94f3d0f5 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -14,32 +14,32 @@ jobs:
 
         # versions are: source, destination
         mongodb_versions:
-          - [ '4.2', '4.2' ]
-          - [ '4.2', '4.4' ]
-          - [ '4.2', '5.0' ]
-          - [ '4.2', '6.0' ]
-
-          - [ '4.4', '4.4' ]
-          - [ '4.4', '5.0' ]
-          - [ '4.4', '6.0' ]
-
-          - [ '5.0', '5.0' ]
-          - [ '5.0', '6.0' ]
-          - [ '5.0', '7.0' ]
-
-          - [ '6.0', '6.0' ]
-          - [ '6.0', '7.0' ]
-          - [ '6.0', '8.0' ]
-
-          - [ '7.0', '7.0' ]
-          - [ '7.0', '8.0' ]
+#          - [ '4.2', '4.2' ]
+#          - [ '4.2', '4.4' ]
+#          - [ '4.2', '5.0' ]
+#          - [ '4.2', '6.0' ]
+#
+#          - [ '4.4', '4.4' ]
+#          - [ '4.4', '5.0' ]
+#          - [ '4.4', '6.0' ]
+#
+#          - [ '5.0', '5.0' ]
+#          - [ '5.0', '6.0' ]
+#          - [ '5.0', '7.0' ]
+#
+#          - [ '6.0', '6.0' ]
+#          - [ '6.0', '7.0' ]
+#          - [ '6.0', '8.0' ]
+#
+#          - [ '7.0', '7.0' ]
+#          - [ '7.0', '8.0' ]
 
           - [ '8.0', '8.0' ]
 
         topology:
-          - name: replset
-            srcConnStr: mongodb://localhost:27020,localhost:27021,localhost:27022
-            dstConnStr: mongodb://localhost:27030,localhost:27031,localhost:27032
+#          - name: replset
+#            srcConnStr: mongodb://localhost:27020,localhost:27021,localhost:27022
+#            dstConnStr: mongodb://localhost:27030,localhost:27031,localhost:27032
 
           - name: sharded
             args: --sharded 2
diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index ee9222db..55a61b0d 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -42,15 +42,15 @@ const (
 )
 
 type UnknownEventError struct {
-	Event *ParsedEvent
+	RawEvent bson.Raw
 }
 
 func (uee UnknownEventError) Error() string {
-	return fmt.Sprintf("Received event with unknown optype: %+v", uee.Event)
+	return fmt.Sprintf("received event with unknown optype: %+v", uee.RawEvent)
 }
 
 // HandleChangeStreamEvents performs the necessary work for change stream events after receiving a batch.
-func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []ParsedEvent) error {
+func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []bson.Raw) error {
 	if len(batch) == 0 {
 		return nil
 	}
@@ -60,7 +60,13 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 	docIDs := make([]interface{}, len(batch))
 	dataSizes := make([]int, len(batch))
 
-	for i, changeEvent := range batch {
+	for i, rawChangeEvent := range batch {
+		var changeEvent ParsedEvent
+		err := bson.Unmarshal(rawChangeEvent, &changeEvent)
+		if err != nil {
+			return errors.Wrapf(err, "failed to unmarshal change event (%+v) to %T", rawChangeEvent, changeEvent)
+		}
+
 		if changeEvent.ClusterTime != nil &&
 			(verifier.lastChangeEventTime == nil ||
 				verifier.lastChangeEventTime.Compare(*changeEvent.ClusterTime) < 0) {
@@ -75,7 +81,7 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 			fallthrough
 		case "update":
 			if err := verifier.eventRecorder.AddEvent(&changeEvent); err != nil {
-				return errors.Wrapf(err, "failed to augment stats with change event: %+v", changeEvent)
+				return errors.Wrapf(err, "failed to augment stats with change event (%+v)", rawChangeEvent)
 			}
 			dbNames[i] = changeEvent.Ns.DB
 			collNames[i] = changeEvent.Ns.Coll
@@ -90,7 +96,7 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 				dataSizes[i] = len(changeEvent.FullDocument)
 			}
 		default:
-			return UnknownEventError{Event: &changeEvent}
+			return UnknownEventError{RawEvent: rawChangeEvent}
 		}
 	}
 
@@ -129,19 +135,19 @@ func (verifier *Verifier) GetChangeStreamFilter() []bson.D {
 func (verifier *Verifier) readAndHandleOneChangeEventBatch(
 	ctx context.Context,
 	cs *mongo.ChangeStream,
-	finalTs *primitive.Timestamp,
+	writesOffTs *primitive.Timestamp,
 ) error {
 	eventsRead := 0
 	var changeEventBatch []ParsedEvent
 
 	for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
-		// Once the change stream reaches the final timestamp we should stop reading.
-		if finalTs != nil {
+		// Once the change stream reaches the writesOff timestamp we should stop reading.
+		if writesOffTs != nil {
 			csTimestamp, err := extractTimestampFromResumeToken(cs.ResumeToken())
 			if err != nil {
 				return errors.Wrap(err, "failed to extract timestamp from change stream's resume token")
 			}
-			if !csTimestamp.Before(*finalTs) {
+			if !csTimestamp.Before(*writesOffTs) {
 				break
 			}
 		}
@@ -199,7 +205,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 
 	for {
 		var err error
-		var gotFinalTimestamp bool
+		var gotwritesOffTimestamp bool
 
 		select {
 
@@ -213,12 +219,12 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 		// If the changeStreamEnderChan has a message, the user has indicated that
 		// source writes are ended. This means we should exit rather than continue
 		// reading the change stream since there should be no more events.
-		case finalTs := <-verifier.changeStreamWritesOffTsChan:
+		case writesOffTs := <-verifier.changeStreamWritesOffTsChan:
 			verifier.logger.Debug().
-				Interface("finalTimestamp", finalTs).
-				Msg("Change stream thread received final timestamp. Finalizing change stream.")
+				Interface("writesOffTimestamp", writesOffTs).
+				Msg("Change stream thread received writesOff timestamp. Finalizing change stream.")
 
-			gotFinalTimestamp = true
+			gotwritesOffTimestamp = true
 
 			// Read all change events until the source reports no events.
 			// (i.e., the `getMore` call returns empty)
@@ -230,16 +236,16 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 					break
 				}
 
-				if curTs == finalTs || curTs.After(finalTs) {
+				if curTs == writesOffTs || curTs.After(writesOffTs) {
 					verifier.logger.Debug().
 						Interface("currentTimestamp", curTs).
-						Interface("finalTimestamp", finalTs).
-						Msg("Change stream has reached the final timestamp. Shutting down.")
+						Interface("writesOffTimestamp", writesOffTs).
+						Msg("Change stream has reached the writesOff timestamp. Shutting down.")
 
 					break
 				}
 
-				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs, &finalTs)
+				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs, &writesOffTs)
 
 				if err != nil {
 					break
@@ -261,12 +267,12 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 
 			verifier.changeStreamErrChan <- err
 
-			if !gotFinalTimestamp {
+			if !gotwritesOffTimestamp {
 				break
 			}
 		}
 
-		if gotFinalTimestamp {
+		if gotwritesOffTimestamp {
 			verifier.mux.Lock()
 			verifier.changeStreamRunning = false
 			if verifier.lastChangeEventTime != nil {

From 2213a724267eb6f43998294eb9d79f59735ef27d Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 01:25:07 -0500
Subject: [PATCH 095/104] update types etc.

---
 .github/workflows/all.yml                    | 17 ++++---
 internal/verifier/change_stream.go           | 14 +++---
 internal/verifier/migration_verifier_test.go | 52 +++++++++++---------
 internal/verifier/recheck_test.go            |  2 +-
 4 files changed, 48 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 94f3d0f5..0994ff50 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -60,8 +60,12 @@ jobs:
         with:
           go-version: stable
 
-      - name: Install m
-        run: npm install -g m mongosh
+      - name: Install m and mtools
+        run: |-
+            {
+                echo npm install -g m
+                echo pipx install 'mtools[all]'
+            } | parallel
 
       - name: Install MongoDB ${{ matrix.mongodb_versions[0] }} (source)
         run: yes | m ${{ matrix.mongodb_versions[0] }} && dirname $(readlink $(which mongod)) > .srcpath
@@ -72,18 +76,15 @@ jobs:
       - name: Install latest stable MongoDB (metadata)
         run: yes | m stable && dirname $(readlink $(which mongod)) > .metapath
 
-      - name: Install mtools
-        run: pipx install 'mtools[all]'
-
       - name: Build
         run: go build main/migration_verifier.go
 
       - name: Start clusters
         run: |-
             {
-                echo "mlaunch init --binarypath $(cat .srcpath) --port 27020 --dir src --replicaset ${{ matrix.topology.args }}"
-                echo "mlaunch init --binarypath $(cat .dstpath) --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}"
-                echo "mlaunch init --binarypath $(cat .metapath) --port 27040 --dir meta --replicaset --nodes 1"
+                echo mlaunch init --binarypath $(cat .srcpath) --port 27020 --dir src --replicaset ${{ matrix.topology.args }}
+                echo mlaunch init --binarypath $(cat .dstpath) --port 27030 --dir dst --replicaset ${{ matrix.topology.args }}
+                echo mlaunch init --binarypath $(cat .metapath) --port 27040 --dir meta --replicaset --nodes 1
             } | parallel
 
       - name: Test
diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 55a61b0d..bb1dddf1 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -42,6 +42,7 @@ const (
 )
 
 type UnknownEventError struct {
+	Event    ParsedEvent
 	RawEvent bson.Raw
 }
 
@@ -96,7 +97,10 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 				dataSizes[i] = len(changeEvent.FullDocument)
 			}
 		default:
-			return UnknownEventError{RawEvent: rawChangeEvent}
+			return UnknownEventError{
+				Event:    changeEvent,
+				RawEvent: rawChangeEvent,
+			}
 		}
 	}
 
@@ -138,7 +142,7 @@ func (verifier *Verifier) readAndHandleOneChangeEventBatch(
 	writesOffTs *primitive.Timestamp,
 ) error {
 	eventsRead := 0
-	var changeEventBatch []ParsedEvent
+	var changeEventBatch []bson.Raw
 
 	for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
 		// Once the change stream reaches the writesOff timestamp we should stop reading.
@@ -163,12 +167,10 @@ func (verifier *Verifier) readAndHandleOneChangeEventBatch(
 		}
 
 		if changeEventBatch == nil {
-			changeEventBatch = make([]ParsedEvent, cs.RemainingBatchLength()+1)
+			changeEventBatch = make([]bson.Raw, cs.RemainingBatchLength()+1)
 		}
 
-		if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
-			return errors.Wrap(err, "failed to decode change event")
-		}
+		copy(changeEventBatch[eventsRead], cs.Current)
 
 		eventsRead++
 	}
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index d6e9d8e5..1492f357 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -145,28 +145,36 @@ func (suite *IntegrationTestSuite) TestGetNamespaceStatistics_Recheck() {
 
 	err := verifier.HandleChangeStreamEvents(
 		ctx,
-		[]ParsedEvent{{
-			OpType: "insert",
-			Ns:     &Namespace{DB: "mydb", Coll: "coll2"},
-			DocKey: DocKey{
-				ID: "heyhey",
-			},
-		}},
+		[]bson.Raw{
+			testutil.MustMarshal(
+				ParsedEvent{
+					OpType: "insert",
+					Ns:     &Namespace{DB: "mydb", Coll: "coll2"},
+					DocKey: DocKey{
+						ID: "heyhey",
+					},
+				},
+			),
+		},
 	)
 	suite.Require().NoError(err)
 
 	err = verifier.HandleChangeStreamEvents(
 		ctx,
-		[]ParsedEvent{{
-			ID: bson.M{
-				"docID": "ID/docID",
-			},
-			OpType: "insert",
-			Ns:     &Namespace{DB: "mydb", Coll: "coll1"},
-			DocKey: DocKey{
-				ID: "hoohoo",
-			},
-		}},
+		[]bson.Raw{
+			testutil.MustMarshal(
+				ParsedEvent{
+					ID: bson.M{
+						"docID": "ID/docID",
+					},
+					OpType: "insert",
+					Ns:     &Namespace{DB: "mydb", Coll: "coll1"},
+					DocKey: DocKey{
+						ID: "hoohoo",
+					},
+				},
+			),
+		},
 	)
 	suite.Require().NoError(err)
 
@@ -408,19 +416,19 @@ func (suite *IntegrationTestSuite) TestFailedVerificationTaskInsertions() {
 		},
 	}
 
-	err = verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
+	err = verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
 	suite.Require().NoError(err)
 	event.OpType = "insert"
-	err = verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
+	err = verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
 	suite.Require().NoError(err)
 	event.OpType = "replace"
-	err = verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
+	err = verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
 	suite.Require().NoError(err)
 	event.OpType = "update"
-	err = verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
+	err = verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
 	suite.Require().NoError(err)
 	event.OpType = "flibbity"
-	err = verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
+	err = verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
 	badEventErr := UnknownEventError{}
 	suite.Require().ErrorAs(err, &badEventErr)
 	suite.Assert().Equal("flibbity", badEventErr.Event.OpType)
diff --git a/internal/verifier/recheck_test.go b/internal/verifier/recheck_test.go
index e4b14b50..bc1cfc36 100644
--- a/internal/verifier/recheck_test.go
+++ b/internal/verifier/recheck_test.go
@@ -52,7 +52,7 @@ func (suite *IntegrationTestSuite) TestFailedCompareThenReplace() {
 		FullDocument: testutil.MustMarshal(bson.D{{"foo", 1}}),
 	}
 
-	err := verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
+	err := verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
 	suite.Require().NoError(err)
 
 	recheckDocs = suite.fetchRecheckDocs(ctx, verifier)

From fc8bee168eeae316c1fa32731de9fc19926879a4 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 01:34:53 -0500
Subject: [PATCH 096/104] deep clone

---
 internal/verifier/change_stream.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index bb1dddf1..27ffa691 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -170,7 +170,10 @@ func (verifier *Verifier) readAndHandleOneChangeEventBatch(
 			changeEventBatch = make([]bson.Raw, cs.RemainingBatchLength()+1)
 		}
 
-		copy(changeEventBatch[eventsRead], cs.Current)
+		// NB: Decode() achieves a deep-clone of cs.Current.
+		if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
+			return errors.Wrap(err, "failed to decode change event")
+		}
 
 		eventsRead++
 	}

From 66dbb3100963f78100fd3df275ebf77c03bb3683 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 01:41:38 -0500
Subject: [PATCH 097/104] just one test

---
 .github/workflows/all.yml          | 2 +-
 internal/verifier/change_stream.go | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 0994ff50..170927a2 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -88,7 +88,7 @@ jobs:
             } | parallel
 
       - name: Test
-        run: go test -v ./... -race
+        run: go test -v ./... -race -testify.m TestEventBeforeWritesOff
         env:
           MVTEST_SRC: ${{matrix.topology.srcConnStr}}
           MVTEST_DST: ${{matrix.topology.dstConnStr}}
diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 27ffa691..ccc140b1 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -170,6 +170,8 @@ func (verifier *Verifier) readAndHandleOneChangeEventBatch(
 			changeEventBatch = make([]bson.Raw, cs.RemainingBatchLength()+1)
 		}
 
+		fmt.Printf("\n===== event: %+v\n", cs.Current)
+
 		// NB: Decode() achieves a deep-clone of cs.Current.
 		if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
 			return errors.Wrap(err, "failed to decode change event")

From 35bbfb44da60cfb6da8cbfff7b48e8ee97e1db36 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 01:46:21 -0500
Subject: [PATCH 098/104] more diag

---
 internal/verifier/change_stream.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index ccc140b1..4c173702 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -65,7 +65,7 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 		var changeEvent ParsedEvent
 		err := bson.Unmarshal(rawChangeEvent, &changeEvent)
 		if err != nil {
-			return errors.Wrapf(err, "failed to unmarshal change event (%+v) to %T", rawChangeEvent, changeEvent)
+			return errors.Wrapf(err, "failed to unmarshal change event %d of %d (%+v) to %T", 1+i, len(batch), rawChangeEvent, changeEvent)
 		}
 
 		if changeEvent.ClusterTime != nil &&

From 0c7f117a09c684492d1dc8d8290afb05b4c80879 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 01:48:54 -0500
Subject: [PATCH 099/104] always read full batch

---
 internal/verifier/change_stream.go | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 4c173702..bdd0d571 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -139,23 +139,11 @@ func (verifier *Verifier) GetChangeStreamFilter() []bson.D {
 func (verifier *Verifier) readAndHandleOneChangeEventBatch(
 	ctx context.Context,
 	cs *mongo.ChangeStream,
-	writesOffTs *primitive.Timestamp,
 ) error {
 	eventsRead := 0
 	var changeEventBatch []bson.Raw
 
 	for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
-		// Once the change stream reaches the writesOff timestamp we should stop reading.
-		if writesOffTs != nil {
-			csTimestamp, err := extractTimestampFromResumeToken(cs.ResumeToken())
-			if err != nil {
-				return errors.Wrap(err, "failed to extract timestamp from change stream's resume token")
-			}
-			if !csTimestamp.Before(*writesOffTs) {
-				break
-			}
-		}
-
 		gotEvent := cs.TryNext(ctx)
 
 		if cs.Err() != nil {
@@ -252,7 +240,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 					break
 				}
 
-				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs, &writesOffTs)
+				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
 
 				if err != nil {
 					break
@@ -260,7 +248,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 			}
 
 		default:
-			err = verifier.readAndHandleOneChangeEventBatch(ctx, cs, nil)
+			err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
 
 			if err == nil {
 				err = persistResumeTokenIfNeeded()

From dd48167a24c5493476ddd97e94b63e838ee998e1 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 01:49:28 -0500
Subject: [PATCH 100/104] enable full test matrix

---
 .github/workflows/all.yml | 46 +++++++++++++++++++--------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml
index 170927a2..458e6abd 100644
--- a/.github/workflows/all.yml
+++ b/.github/workflows/all.yml
@@ -14,32 +14,32 @@ jobs:
 
         # versions are: source, destination
         mongodb_versions:
-#          - [ '4.2', '4.2' ]
-#          - [ '4.2', '4.4' ]
-#          - [ '4.2', '5.0' ]
-#          - [ '4.2', '6.0' ]
-#
-#          - [ '4.4', '4.4' ]
-#          - [ '4.4', '5.0' ]
-#          - [ '4.4', '6.0' ]
-#
-#          - [ '5.0', '5.0' ]
-#          - [ '5.0', '6.0' ]
-#          - [ '5.0', '7.0' ]
-#
-#          - [ '6.0', '6.0' ]
-#          - [ '6.0', '7.0' ]
-#          - [ '6.0', '8.0' ]
-#
-#          - [ '7.0', '7.0' ]
-#          - [ '7.0', '8.0' ]
+          - [ '4.2', '4.2' ]
+          - [ '4.2', '4.4' ]
+          - [ '4.2', '5.0' ]
+          - [ '4.2', '6.0' ]
+
+          - [ '4.4', '4.4' ]
+          - [ '4.4', '5.0' ]
+          - [ '4.4', '6.0' ]
+
+          - [ '5.0', '5.0' ]
+          - [ '5.0', '6.0' ]
+          - [ '5.0', '7.0' ]
+
+          - [ '6.0', '6.0' ]
+          - [ '6.0', '7.0' ]
+          - [ '6.0', '8.0' ]
+
+          - [ '7.0', '7.0' ]
+          - [ '7.0', '8.0' ]
 
           - [ '8.0', '8.0' ]
 
         topology:
-#          - name: replset
-#            srcConnStr: mongodb://localhost:27020,localhost:27021,localhost:27022
-#            dstConnStr: mongodb://localhost:27030,localhost:27031,localhost:27032
+          - name: replset
+            srcConnStr: mongodb://localhost:27020,localhost:27021,localhost:27022
+            dstConnStr: mongodb://localhost:27030,localhost:27031,localhost:27032
 
           - name: sharded
             args: --sharded 2
@@ -88,7 +88,7 @@ jobs:
             } | parallel
 
       - name: Test
-        run: go test -v ./... -race -testify.m TestEventBeforeWritesOff
+        run: go test -v ./... -race
         env:
           MVTEST_SRC: ${{matrix.topology.srcConnStr}}
           MVTEST_DST: ${{matrix.topology.dstConnStr}}

From 164068a68cb7093ad896511895eac90ed66c9b04 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 01:59:23 -0500
Subject: [PATCH 101/104] revert

---
 internal/verifier/change_stream.go           | 45 +++++++++--------
 internal/verifier/migration_verifier_test.go | 52 +++++++++-----------
 internal/verifier/recheck_test.go            |  2 +-
 3 files changed, 45 insertions(+), 54 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index bdd0d571..375a56c2 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -42,16 +42,15 @@ const (
 )
 
 type UnknownEventError struct {
-	Event    ParsedEvent
-	RawEvent bson.Raw
+	Event ParsedEvent
 }
 
 func (uee UnknownEventError) Error() string {
-	return fmt.Sprintf("received event with unknown optype: %+v", uee.RawEvent)
+	return fmt.Sprintf("received event with unknown optype: %+v", uee.Event)
 }
 
 // HandleChangeStreamEvents performs the necessary work for change stream events after receiving a batch.
-func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []bson.Raw) error {
+func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []ParsedEvent) error {
 	if len(batch) == 0 {
 		return nil
 	}
@@ -61,13 +60,7 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 	docIDs := make([]interface{}, len(batch))
 	dataSizes := make([]int, len(batch))
 
-	for i, rawChangeEvent := range batch {
-		var changeEvent ParsedEvent
-		err := bson.Unmarshal(rawChangeEvent, &changeEvent)
-		if err != nil {
-			return errors.Wrapf(err, "failed to unmarshal change event %d of %d (%+v) to %T", 1+i, len(batch), rawChangeEvent, changeEvent)
-		}
-
+	for i, changeEvent := range batch {
 		if changeEvent.ClusterTime != nil &&
 			(verifier.lastChangeEventTime == nil ||
 				verifier.lastChangeEventTime.Compare(*changeEvent.ClusterTime) < 0) {
@@ -82,7 +75,7 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 			fallthrough
 		case "update":
 			if err := verifier.eventRecorder.AddEvent(&changeEvent); err != nil {
-				return errors.Wrapf(err, "failed to augment stats with change event (%+v)", rawChangeEvent)
+				return errors.Wrapf(err, "failed to augment stats with change event (%+v)", changeEvent)
 			}
 			dbNames[i] = changeEvent.Ns.DB
 			collNames[i] = changeEvent.Ns.Coll
@@ -97,10 +90,7 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 				dataSizes[i] = len(changeEvent.FullDocument)
 			}
 		default:
-			return UnknownEventError{
-				Event:    changeEvent,
-				RawEvent: rawChangeEvent,
-			}
+			return UnknownEventError{Event: changeEvent}
 		}
 	}
 
@@ -139,11 +129,23 @@ func (verifier *Verifier) GetChangeStreamFilter() []bson.D {
 func (verifier *Verifier) readAndHandleOneChangeEventBatch(
 	ctx context.Context,
 	cs *mongo.ChangeStream,
+	writesOffTs *primitive.Timestamp,
 ) error {
 	eventsRead := 0
-	var changeEventBatch []bson.Raw
+	var changeEventBatch []ParsedEvent
 
 	for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
+		// Once the change stream reaches the writesOff timestamp we should stop reading.
+		if writesOffTs != nil {
+			csTimestamp, err := extractTimestampFromResumeToken(cs.ResumeToken())
+			if err != nil {
+				return errors.Wrap(err, "failed to extract timestamp from change stream's resume token")
+			}
+			if !csTimestamp.Before(*writesOffTs) {
+				break
+			}
+		}
+
 		gotEvent := cs.TryNext(ctx)
 
 		if cs.Err() != nil {
@@ -155,12 +157,9 @@ func (verifier *Verifier) readAndHandleOneChangeEventBatch(
 		}
 
 		if changeEventBatch == nil {
-			changeEventBatch = make([]bson.Raw, cs.RemainingBatchLength()+1)
+			changeEventBatch = make([]ParsedEvent, cs.RemainingBatchLength()+1)
 		}
 
-		fmt.Printf("\n===== event: %+v\n", cs.Current)
-
-		// NB: Decode() achieves a deep-clone of cs.Current.
 		if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
 			return errors.Wrap(err, "failed to decode change event")
 		}
@@ -240,7 +239,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 					break
 				}
 
-				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
+				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs, &writesOffTs)
 
 				if err != nil {
 					break
@@ -248,7 +247,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 			}
 
 		default:
-			err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
+			err = verifier.readAndHandleOneChangeEventBatch(ctx, cs, nil)
 
 			if err == nil {
 				err = persistResumeTokenIfNeeded()
diff --git a/internal/verifier/migration_verifier_test.go b/internal/verifier/migration_verifier_test.go
index 1492f357..d6e9d8e5 100644
--- a/internal/verifier/migration_verifier_test.go
+++ b/internal/verifier/migration_verifier_test.go
@@ -145,36 +145,28 @@ func (suite *IntegrationTestSuite) TestGetNamespaceStatistics_Recheck() {
 
 	err := verifier.HandleChangeStreamEvents(
 		ctx,
-		[]bson.Raw{
-			testutil.MustMarshal(
-				ParsedEvent{
-					OpType: "insert",
-					Ns:     &Namespace{DB: "mydb", Coll: "coll2"},
-					DocKey: DocKey{
-						ID: "heyhey",
-					},
-				},
-			),
-		},
+		[]ParsedEvent{{
+			OpType: "insert",
+			Ns:     &Namespace{DB: "mydb", Coll: "coll2"},
+			DocKey: DocKey{
+				ID: "heyhey",
+			},
+		}},
 	)
 	suite.Require().NoError(err)
 
 	err = verifier.HandleChangeStreamEvents(
 		ctx,
-		[]bson.Raw{
-			testutil.MustMarshal(
-				ParsedEvent{
-					ID: bson.M{
-						"docID": "ID/docID",
-					},
-					OpType: "insert",
-					Ns:     &Namespace{DB: "mydb", Coll: "coll1"},
-					DocKey: DocKey{
-						ID: "hoohoo",
-					},
-				},
-			),
-		},
+		[]ParsedEvent{{
+			ID: bson.M{
+				"docID": "ID/docID",
+			},
+			OpType: "insert",
+			Ns:     &Namespace{DB: "mydb", Coll: "coll1"},
+			DocKey: DocKey{
+				ID: "hoohoo",
+			},
+		}},
 	)
 	suite.Require().NoError(err)
 
@@ -416,19 +408,19 @@ func (suite *IntegrationTestSuite) TestFailedVerificationTaskInsertions() {
 		},
 	}
 
-	err = verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
+	err = verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
 	suite.Require().NoError(err)
 	event.OpType = "insert"
-	err = verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
+	err = verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
 	suite.Require().NoError(err)
 	event.OpType = "replace"
-	err = verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
+	err = verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
 	suite.Require().NoError(err)
 	event.OpType = "update"
-	err = verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
+	err = verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
 	suite.Require().NoError(err)
 	event.OpType = "flibbity"
-	err = verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
+	err = verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
 	badEventErr := UnknownEventError{}
 	suite.Require().ErrorAs(err, &badEventErr)
 	suite.Assert().Equal("flibbity", badEventErr.Event.OpType)
diff --git a/internal/verifier/recheck_test.go b/internal/verifier/recheck_test.go
index bc1cfc36..e4b14b50 100644
--- a/internal/verifier/recheck_test.go
+++ b/internal/verifier/recheck_test.go
@@ -52,7 +52,7 @@ func (suite *IntegrationTestSuite) TestFailedCompareThenReplace() {
 		FullDocument: testutil.MustMarshal(bson.D{{"foo", 1}}),
 	}
 
-	err := verifier.HandleChangeStreamEvents(ctx, []bson.Raw{testutil.MustMarshal(event)})
+	err := verifier.HandleChangeStreamEvents(ctx, []ParsedEvent{event})
 	suite.Require().NoError(err)
 
 	recheckDocs = suite.fetchRecheckDocs(ctx, verifier)

From 51a823dec5431459db2ff6d453396fada63ac29d Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 02:03:29 -0500
Subject: [PATCH 102/104] revert unintended

---
 internal/verifier/change_stream.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 375a56c2..5c5e6614 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -42,7 +42,7 @@ const (
 )
 
 type UnknownEventError struct {
-	Event ParsedEvent
+	Event *ParsedEvent
 }
 
 func (uee UnknownEventError) Error() string {
@@ -90,7 +90,7 @@ func (verifier *Verifier) HandleChangeStreamEvents(ctx context.Context, batch []
 				dataSizes[i] = len(changeEvent.FullDocument)
 			}
 		default:
-			return UnknownEventError{Event: changeEvent}
+			return UnknownEventError{Event: &changeEvent}
 		}
 	}
 
@@ -161,7 +161,7 @@ func (verifier *Verifier) readAndHandleOneChangeEventBatch(
 		}
 
 		if err := cs.Decode(&changeEventBatch[eventsRead]); err != nil {
-			return errors.Wrap(err, "failed to decode change event")
+			return errors.Wrapf(err, "failed to decode change event to %T", changeEventBatch[eventsRead])
 		}
 
 		eventsRead++

From 0c59cbc762d4ccda0b577b7c58dd652c86ff1140 Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 02:09:01 -0500
Subject: [PATCH 103/104] comment

---
 internal/verifier/change_stream.go | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 5c5e6614..2b9c8f4f 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -126,26 +126,22 @@ func (verifier *Verifier) GetChangeStreamFilter() []bson.D {
 	return []bson.D{stage}
 }
 
+// This function reads a single `getMore` response into a slice.
+//
+// Note that this doesn’t care about the writesOff timestamp. Thus,
+// if writesOff has happened and a `getMore` response’s events straddle
+// the writesOff timestamp (i.e., some events precede it & others follow it),
+// the verifier will enqueue rechecks from those post-writesOff events. This
+// is unideal but shouldn’t impede correctness since post-writesOff events
+// shouldn’t really happen anyway by definition.
 func (verifier *Verifier) readAndHandleOneChangeEventBatch(
 	ctx context.Context,
 	cs *mongo.ChangeStream,
-	writesOffTs *primitive.Timestamp,
 ) error {
 	eventsRead := 0
 	var changeEventBatch []ParsedEvent
 
 	for hasEventInBatch := true; hasEventInBatch; hasEventInBatch = cs.RemainingBatchLength() > 0 {
-		// Once the change stream reaches the writesOff timestamp we should stop reading.
-		if writesOffTs != nil {
-			csTimestamp, err := extractTimestampFromResumeToken(cs.ResumeToken())
-			if err != nil {
-				return errors.Wrap(err, "failed to extract timestamp from change stream's resume token")
-			}
-			if !csTimestamp.Before(*writesOffTs) {
-				break
-			}
-		}
-
 		gotEvent := cs.TryNext(ctx)
 
 		if cs.Err() != nil {
@@ -239,7 +235,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 					break
 				}
 
-				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs, &writesOffTs)
+				err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
 
 				if err != nil {
 					break
@@ -247,7 +243,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 			}
 
 		default:
-			err = verifier.readAndHandleOneChangeEventBatch(ctx, cs, nil)
+			err = verifier.readAndHandleOneChangeEventBatch(ctx, cs)
 
 			if err == nil {
 				err = persistResumeTokenIfNeeded()

From 9dd0100684d3cb1dc2d2d422c96a31c3e35ff90c Mon Sep 17 00:00:00 2001
From: Felipe Gasper <felipe@felipegasper.com>
Date: Fri, 22 Nov 2024 02:21:50 -0500
Subject: [PATCH 104/104] partial revert

---
 internal/verifier/change_stream.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/verifier/change_stream.go b/internal/verifier/change_stream.go
index 2b9c8f4f..c4c4c9cc 100644
--- a/internal/verifier/change_stream.go
+++ b/internal/verifier/change_stream.go
@@ -271,7 +271,7 @@ func (verifier *Verifier) iterateChangeStream(ctx context.Context, cs *mongo.Cha
 			verifier.mux.Unlock()
 			// since we have started Recheck, we must signal that we have
 			// finished the change stream changes so that Recheck can continue.
-			close(verifier.changeStreamDoneChan)
+			verifier.changeStreamDoneChan <- struct{}{}
 			break
 		}
 	}