Skip to content

Commit 0f4b657

Browse files
committed
additional logs (hopefully helpful)
1 parent fa1264e commit 0f4b657

File tree

3 files changed

+39
-8
lines changed

3 files changed

+39
-8
lines changed

internal/verifier/migration_verifier.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -567,10 +567,14 @@ func (verifier *Verifier) compareOneDocument(srcClientDoc, dstClientDoc bson.Raw
567567
func (verifier *Verifier) ProcessVerifyTask(ctx context.Context, workerNum int, task *VerificationTask) error {
568568
start := time.Now()
569569

570-
verifier.logger.Debug().
570+
debugLog := verifier.logger.Debug().
571571
Int("workerNum", workerNum).
572572
Interface("task", task.PrimaryKey).
573-
Msg("Processing document comparison task.")
573+
Str("namespace", task.QueryFilter.Namespace)
574+
575+
task.augmentLogWithDetails(debugLog)
576+
577+
debugLog.Msg("Processing document comparison task.")
574578

575579
problems, docsCount, bytesCount, err := verifier.FetchAndCompareDocuments(
576580
ctx,
@@ -662,6 +666,7 @@ func (verifier *Verifier) ProcessVerifyTask(ctx context.Context, workerNum int,
662666
verifier.logger.Debug().
663667
Int("workerNum", workerNum).
664668
Interface("task", task.PrimaryKey).
669+
Str("namespace", task.QueryFilter.Namespace).
665670
Stringer("timeElapsed", time.Since(start)).
666671
Msg("Finished document comparison task.")
667672

internal/verifier/recheck.go

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,8 @@ func (verifier *Verifier) GenerateRecheckTasks(ctx context.Context) error {
232232
var prevDBName, prevCollName string
233233
var idAccum []interface{}
234234
var idLenAccum int
235-
var dataSizeAccum int64
235+
var totalDocs types.DocumentCount
236+
var dataSizeAccum, totalRecheckData int64
236237

237238
maxDocsPerTask := rechecksCount / int64(verifier.numWorkers)
238239

@@ -260,7 +261,7 @@ func (verifier *Verifier) GenerateRecheckTasks(ctx context.Context) error {
260261

261262
namespace := prevDBName + "." + prevCollName
262263

263-
err := verifier.InsertDocumentRecheckTask(
264+
task, err := verifier.InsertDocumentRecheckTask(
264265
ctx,
265266
idAccum,
266267
types.ByteCount(dataSizeAccum),
@@ -276,6 +277,7 @@ func (verifier *Verifier) GenerateRecheckTasks(ctx context.Context) error {
276277
}
277278

278279
verifier.logger.Debug().
280+
Interface("task", task.PrimaryKey).
279281
Str("namespace", namespace).
280282
Int("numDocuments", len(idAccum)).
281283
Str("dataSize", reportutils.FmtBytes(dataSizeAccum)).
@@ -323,12 +325,25 @@ func (verifier *Verifier) GenerateRecheckTasks(ctx context.Context) error {
323325
idLenAccum += idLen
324326
dataSizeAccum += int64(doc.DataSize)
325327
idAccum = append(idAccum, doc.PrimaryKey.DocumentID)
328+
329+
totalRecheckData += int64(doc.DataSize)
330+
totalDocs++
326331
}
327332

328333
err = cursor.Err()
329334
if err != nil {
330335
return err
331336
}
332337

333-
return persistBufferedRechecks()
338+
err = persistBufferedRechecks()
339+
340+
if err == nil && totalDocs > 0 {
341+
verifier.logger.Info().
342+
Int("generation", 1+prevGeneration).
343+
Int64("totalDocs", int64(totalDocs)).
344+
Str("totalData", reportutils.FmtBytes(totalRecheckData)).
345+
Msg("Scheduled documents for recheck in the new generation.")
346+
}
347+
348+
return err
334349
}

internal/verifier/verification_task.go

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414

1515
"github.com/10gen/migration-verifier/internal/partitions"
1616
"github.com/10gen/migration-verifier/internal/types"
17+
"github.com/rs/zerolog"
1718
"go.mongodb.org/mongo-driver/bson"
1819
"go.mongodb.org/mongo-driver/bson/primitive"
1920
"go.mongodb.org/mongo-driver/mongo/options"
@@ -82,6 +83,16 @@ type VerificationTask struct {
8283
SourceByteCount types.ByteCount `bson:"source_bytes_count"`
8384
}
8485

86+
func (t *VerificationTask) augmentLogWithDetails(evt *zerolog.Event) {
87+
if len(t.Ids) > 0 {
88+
evt.Int("documentCount", len(t.Ids))
89+
} else {
90+
evt.
91+
Interface("minDocID", t.QueryFilter.Partition.Key.Lower).
92+
Interface("maxDocID", t.QueryFilter.Partition.Upper)
93+
}
94+
}
95+
8596
// VerificationRange stores ID ranges for tasks that can be re-used between runs
8697
type VerificationRange struct {
8798
PrimaryKey primitive.ObjectID `bson:"_id"`
@@ -167,13 +178,13 @@ func (verifier *Verifier) InsertDocumentRecheckTask(
167178
ids []interface{},
168179
dataSize types.ByteCount,
169180
srcNamespace string,
170-
) error {
181+
) (VerificationTask, error) {
171182
dstNamespace := srcNamespace
172183
if verifier.nsMap.Len() != 0 {
173184
var ok bool
174185
dstNamespace, ok = verifier.nsMap.GetDstNamespace(srcNamespace)
175186
if !ok {
176-
return fmt.Errorf("Could not find Namespace %s", srcNamespace)
187+
return VerificationTask{}, fmt.Errorf("Could not find Namespace %s", srcNamespace)
177188
}
178189
}
179190

@@ -192,7 +203,7 @@ func (verifier *Verifier) InsertDocumentRecheckTask(
192203
}
193204

194205
_, err := verifier.verificationTaskCollection().InsertOne(ctx, &verificationTask)
195-
return err
206+
return verificationTask, err
196207
}
197208

198209
func (verifier *Verifier) FindNextVerifyTaskAndUpdate(ctx context.Context) (*VerificationTask, error) {

0 commit comments

Comments
 (0)