Skip to content

Commit 42b3937

Browse files
committed
fix(events-downloader): only consider a batch as complete if we downloaded all events from it
1 parent 78af376 commit 42b3937

File tree

2 files changed

+18
-321
lines changed

2 files changed

+18
-321
lines changed

packages/event-downloader/COE_HEIGHT_ZERO.md

Lines changed: 0 additions & 318 deletions
This file was deleted.

packages/event-downloader/src/orchestrator.ts

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,8 @@ function calculateBatches(latestEventId: number, batchSize: number): BatchInfo[]
113113

114114
/**
115115
* Merge calculated batches with existing progress from database.
116+
* Handles the case where a batch was completed with a smaller boundary
117+
* (e.g., latestEventId was lower during a previous run).
116118
*/
117119
function mergeBatchesWithProgress(
118120
batches: BatchInfo[],
@@ -125,12 +127,25 @@ function mergeBatchesWithProgress(
125127

126128
return batches.map((batch) => {
127129
const existing = progressMap.get(batch.start);
128-
if (existing && existing.status === 'completed') {
129-
return { ...batch, lastDownloaded: batch.end }; // Mark as complete
130+
if (!existing) {
131+
return batch;
130132
}
131-
if (existing && existing.last_downloaded !== null) {
133+
134+
// Only consider fully complete if the stored batch_end covers the calculated batch_end
135+
if (existing.status === 'completed' && existing.batch_end >= batch.end) {
136+
return { ...batch, lastDownloaded: batch.end };
137+
}
138+
139+
// Batch was "completed" but with a smaller boundary - resume from where it ended
140+
if (existing.status === 'completed') {
141+
return { ...batch, lastDownloaded: existing.batch_end };
142+
}
143+
144+
// For in-progress or failed batches, resume from last_downloaded
145+
if (existing.last_downloaded !== null) {
132146
return { ...batch, lastDownloaded: existing.last_downloaded };
133147
}
148+
134149
return batch;
135150
}).filter((batch) => {
136151
// Filter out completed batches

0 commit comments

Comments
 (0)