@@ -71,31 +71,33 @@ func (bf *BatchFinder) FindBatches(ctx context.Context) ([]LogBatch, error) {
7171 -- 2. insertedAt = offset.insertedAt AND timestamp > offset.timestamp, OR
7272 -- 3. insertedAt = offset.insertedAt AND timestamp = offset.timestamp AND reqID > offset.reqID
7373 --
74- -- For buckets with no offset, COALESCE defaults to epoch, so all logs are unprocessed .
74+ -- For buckets with no offset, LEFT JOIN returns NULL values which are handled explicitly in WHERE clause .
7575 new_logs_by_bucket AS (
7676 SELECT
7777 l.bucketName,
7878 l.raftSessionID,
7979 count() AS new_log_count,
8080 min(l.insertedAt) as min_ts,
81- COALESCE(o.lastProcessedInsertedAt, toDateTime('1970-01-01 00:00:00')) as lastProcessedInsertedAt,
82- COALESCE(o.lastProcessedTimestamp, toDateTime64('1970-01-01 00:00:00', 3)) as lastProcessedTimestamp,
83- COALESCE(o.lastProcessedReqId, '') as lastProcessedReqId
81+ o. lastProcessedInsertedAt,
82+ o. lastProcessedTimestamp,
83+ o. lastProcessedReqId
8484 FROM %s.%s AS l
8585 LEFT JOIN bucket_offsets AS o
8686 ON l.bucketName = o.bucketName
8787 AND l.raftSessionID = o.raftSessionID
8888 WHERE
8989 -- Triple composite offset comparison: log > offset
90- l.insertedAt > COALESCE(o.lastProcessedInsertedAt, toDateTime('1970-01-01 00:00:00'))
90+ -- When no offset exists (NULL from LEFT JOIN), include all logs
91+ o.lastProcessedInsertedAt IS NULL
92+ OR l.insertedAt > o.lastProcessedInsertedAt
9193 OR (
9294 l.insertedAt = o.lastProcessedInsertedAt
93- AND l.timestamp > COALESCE( o.lastProcessedTimestamp, toDateTime64('1970-01-01 00:00:00', 3))
95+ AND l.timestamp > o.lastProcessedTimestamp
9496 )
9597 OR (
9698 l.insertedAt = o.lastProcessedInsertedAt
9799 AND l.timestamp = o.lastProcessedTimestamp
98- AND l.req_id > COALESCE( o.lastProcessedReqId, '')
100+ AND l.req_id > o.lastProcessedReqId
99101 )
100102 GROUP BY l.bucketName, l.raftSessionID, o.lastProcessedInsertedAt, o.lastProcessedTimestamp, o.lastProcessedReqId
101103 )
0 commit comments