Fix $size error in statistics computation. (#139)

Commit b4467f7 broke statistics computation when a recheck generation includes a collection-metadata task, e.g., if an index or collection attribute mismatches.

This fixes that by refining the aggregation logic. Tests are added to shore up this functionality.

Author: FGasper

internal/verifier/migration_verifier_test.go

@@ -567,6 +567,113 @@ func (suite *IntegrationTestSuite) TestVerifierFetchDocuments() {
 	)
 }
 
+func (suite *IntegrationTestSuite) TestGetPersistedNamespaceStatistics_Metadata() {
+	ctx := suite.Context()
+	verifier := suite.BuildVerifier()
+	verifier.SetVerifyAll(true)
+
+	dbName := suite.DBNameForTest()
+
+	err := verifier.srcClient.Database(dbName).CreateCollection(
+		ctx,
+		"foo",
+	)
+	suite.Require().NoError(err)
+
+	runner := RunVerifierCheck(ctx, suite.T(), verifier)
+	suite.Require().NoError(runner.AwaitGenerationEnd())
+
+	stats, err := verifier.GetPersistedNamespaceStatistics(ctx)
+	suite.Require().NoError(err)
+
+	suite.Assert().Equal(
+		mslices.Of(NamespaceStats{
+			Namespace: dbName + ".foo",
+		}),
+		stats,
+		"stats should be as expected",
+	)
+
+	suite.Require().NoError(runner.StartNextGeneration())
+	suite.Require().NoError(runner.AwaitGenerationEnd())
+
+	stats, err = verifier.GetPersistedNamespaceStatistics(ctx)
+	suite.Require().NoError(err)
+
+	suite.Assert().Equal(
+		mslices.Of(NamespaceStats{
+			Namespace: dbName + ".foo",
+		}),
+		stats,
+		"stats should be as expected",
+	)
+}
+
+func (suite *IntegrationTestSuite) TestGetPersistedNamespaceStatistics_OneDoc() {
+	ctx := suite.Context()
+	verifier := suite.BuildVerifier()
+	verifier.SetVerifyAll(true)
+
+	bsonDoc := lo.Must(bson.Marshal(bson.D{{"_id", "foo"}}))
+
+	dbName := suite.DBNameForTest()
+	_, err := verifier.srcClient.Database(dbName).Collection("foo").
+		InsertOne(ctx, bsonDoc)
+	suite.Require().NoError(err)
+
+	err = verifier.dstClient.Database(dbName).CreateCollection(
+		ctx,
+		"foo",
+	)
+	suite.Require().NoError(err)
+
+	runner := RunVerifierCheck(ctx, suite.T(), verifier)
+	suite.Require().NoError(runner.AwaitGenerationEnd())
+
+	stats, err := verifier.GetPersistedNamespaceStatistics(ctx)
+	suite.Require().NoError(err)
+
+	suite.Require().NotEmpty(stats)
+	suite.Assert().NotZero(stats[0].BytesCompared, "bytes compared should be set")
+
+	suite.Assert().Equal(
+		mslices.Of(NamespaceStats{
+			Namespace:      dbName + ".foo",
+			DocsCompared:   1,
+			TotalDocs:      1,
+			BytesCompared:  stats[0].BytesCompared,
+			TotalBytes:     types.ByteCount(len(bsonDoc)),
+			PartitionsDone: 1,
+		}),
+		stats,
+		"stats should be as expected",
+	)
+
+	suite.Require().NoError(runner.StartNextGeneration())
+	suite.Require().NoError(runner.AwaitGenerationEnd())
+
+	stats, err = verifier.GetPersistedNamespaceStatistics(ctx)
+	suite.Require().NoError(err)
+
+	suite.Require().NotEmpty(stats)
+	suite.Assert().NotZero(stats[0].BytesCompared, "bytes compared should be set")
+
+	suite.Assert().Equal(
+		mslices.Of(NamespaceStats{
+			Namespace:      dbName + ".foo",
+			DocsCompared:   1,
+			TotalDocs:      1,
+			BytesCompared:  stats[0].BytesCompared,
+			PartitionsDone: 1,
+
+			// NB: TotalBytes is 0 because we can’t compute that from the
+			// change stream.
+		}),
+		stats,
+		"stats should be as expected",
+	)
+}
+
 func (suite *IntegrationTestSuite) TestGetPersistedNamespaceStatistics_Recheck() {
 	ctx := suite.Context()
 	verifier := suite.BuildVerifier()

internal/verifier/statistics.go

@@ -114,28 +114,26 @@ const perNsStatsPipelineTemplate = `[
 				In generation 0 we can get the total docs from the
 				verify-collection tasks.
 
-				In later generations we don’t have verify-collection tasks,
+				In later generations we may not have verify-collection tasks,
 				so we add up the individual recheck batch tasks. Note that,
 				in these tasks source_documents_count refers to the actual
 				number of docs found on the source, not all the documents
 				checked.
 			*/}}
 			"totalDocs": {
-				"$cond": [
-					{ "$or": [
-						{ "$eq": [ "$type", "{{.VerifyCollType}}" ] },
-						{ "$and": [
-							{ "$eq": [ "$type", "{{.VerifyDocsType}}" ] },
-							{ "$ne": [ "$generation", 0 ] }
-						] }
-					] },
-					{ "$cond": [
-						{"$eq": [ "$generation", 0 ]},
-						"$source_documents_count",
-						{ "$size": "$_ids" }
-					] },
-					0
-				]
+				"$cond": {
+					"if": {"$eq": [ "$generation", 0 ]},
+					"then": { "$cond": {
+						"if": { "$eq": [ "$type", "{{.VerifyCollType}}" ] },
+						"then": "$source_documents_count",
+						"else": 0
+					} },
+					"else": { "$cond": {
+						"if": { "$eq": [ "$type", "{{.VerifyDocsType}}" ] },
+						"then": { "$size": "$_ids" },
+						"else": 0
+					} }
+				}
 			},
 
 			{{/*