Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions internal/verifier/migration_verifier.go
Original file line number Diff line number Diff line change
Expand Up @@ -567,10 +567,14 @@ func (verifier *Verifier) compareOneDocument(srcClientDoc, dstClientDoc bson.Raw
func (verifier *Verifier) ProcessVerifyTask(ctx context.Context, workerNum int, task *VerificationTask) error {
start := time.Now()

verifier.logger.Debug().
debugLog := verifier.logger.Debug().
Int("workerNum", workerNum).
Interface("task", task.PrimaryKey).
Msg("Processing document comparison task.")
Str("namespace", task.QueryFilter.Namespace)

task.augmentLogWithDetails(debugLog)

debugLog.Msg("Processing document comparison task.")

problems, docsCount, bytesCount, err := verifier.FetchAndCompareDocuments(
ctx,
Expand Down Expand Up @@ -662,6 +666,7 @@ func (verifier *Verifier) ProcessVerifyTask(ctx context.Context, workerNum int,
verifier.logger.Debug().
Int("workerNum", workerNum).
Interface("task", task.PrimaryKey).
Str("namespace", task.QueryFilter.Namespace).
Stringer("timeElapsed", time.Since(start)).
Msg("Finished document comparison task.")

Expand Down
18 changes: 16 additions & 2 deletions internal/verifier/recheck.go
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,8 @@ func (verifier *Verifier) GenerateRecheckTasksWhileLocked(ctx context.Context) e
var prevDBName, prevCollName string
var idAccum []interface{}
var idLenAccum int
var dataSizeAccum int64
var totalDocs types.DocumentCount
var dataSizeAccum, totalRecheckData int64

maxDocsPerTask := rechecksCount / int64(verifier.numWorkers)

Expand Down Expand Up @@ -334,12 +335,25 @@ func (verifier *Verifier) GenerateRecheckTasksWhileLocked(ctx context.Context) e
idLenAccum += idLen
dataSizeAccum += int64(doc.DataSize)
idAccum = append(idAccum, doc.PrimaryKey.DocumentID)

totalRecheckData += int64(doc.DataSize)
totalDocs++
}

err = cursor.Err()
if err != nil {
return err
}

return persistBufferedRechecks()
err = persistBufferedRechecks()

if err == nil && totalDocs > 0 {
verifier.logger.Info().
Int("generation", 1+prevGeneration).
Int64("totalDocs", int64(totalDocs)).
Str("totalData", reportutils.FmtBytes(totalRecheckData)).
Msg("Scheduled documents for recheck in the new generation.")
}

return err
}
11 changes: 11 additions & 0 deletions internal/verifier/verification_task.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/10gen/migration-verifier/mslices"
"github.com/10gen/migration-verifier/option"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
Expand Down Expand Up @@ -87,6 +88,16 @@ type VerificationTask struct {
SourceByteCount types.ByteCount `bson:"source_bytes_count"`
}

func (t *VerificationTask) augmentLogWithDetails(evt *zerolog.Event) {
if len(t.Ids) > 0 {
evt.Int("documentCount", len(t.Ids))
} else {
evt.
Interface("minDocID", t.QueryFilter.Partition.Key.Lower).
Interface("maxDocID", t.QueryFilter.Partition.Upper)
}
}

// VerificationRange stores ID ranges for tasks that can be re-used between runs
type VerificationRange struct {
PrimaryKey primitive.ObjectID `bson:"_id"`
Expand Down
Loading