Skip to content

Commit a36fec4

Browse files
committed
LOGC-33: Remove COALESCE from BatchFinder query
Remove COALESCE functions from SELECT clause. Instead, handle NULL offsets explicitly in WHERE clause and let Go's database/sql scanner convert NULL DateTime values to zero time.
1 parent 6fc772f commit a36fec4

1 file changed

Lines changed: 9 additions & 7 deletions

File tree

pkg/logcourier/batchfinder.go

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -71,31 +71,33 @@ func (bf *BatchFinder) FindBatches(ctx context.Context) ([]LogBatch, error) {
7171
-- 2. insertedAt = offset.insertedAt AND timestamp > offset.timestamp, OR
7272
-- 3. insertedAt = offset.insertedAt AND timestamp = offset.timestamp AND reqID > offset.reqID
7373
--
74-
-- For buckets with no offset, COALESCE defaults to epoch, so all logs are unprocessed.
74+
-- For buckets with no offset, LEFT JOIN returns NULL values which are handled explicitly in WHERE clause.
7575
new_logs_by_bucket AS (
7676
SELECT
7777
l.bucketName,
7878
l.raftSessionID,
7979
count() AS new_log_count,
8080
min(l.insertedAt) as min_ts,
81-
COALESCE(o.lastProcessedInsertedAt, toDateTime('1970-01-01 00:00:00')) as lastProcessedInsertedAt,
82-
COALESCE(o.lastProcessedTimestamp, toDateTime64('1970-01-01 00:00:00', 3)) as lastProcessedTimestamp,
83-
COALESCE(o.lastProcessedReqId, '') as lastProcessedReqId
81+
o.lastProcessedInsertedAt,
82+
o.lastProcessedTimestamp,
83+
o.lastProcessedReqId
8484
FROM %s.%s AS l
8585
LEFT JOIN bucket_offsets AS o
8686
ON l.bucketName = o.bucketName
8787
AND l.raftSessionID = o.raftSessionID
8888
WHERE
8989
-- Triple composite offset comparison: log > offset
90-
l.insertedAt > COALESCE(o.lastProcessedInsertedAt, toDateTime('1970-01-01 00:00:00'))
90+
-- When no offset exists (NULL from LEFT JOIN), include all logs
91+
o.lastProcessedInsertedAt IS NULL
92+
OR l.insertedAt > o.lastProcessedInsertedAt
9193
OR (
9294
l.insertedAt = o.lastProcessedInsertedAt
93-
AND l.timestamp > COALESCE(o.lastProcessedTimestamp, toDateTime64('1970-01-01 00:00:00', 3))
95+
AND l.timestamp > o.lastProcessedTimestamp
9496
)
9597
OR (
9698
l.insertedAt = o.lastProcessedInsertedAt
9799
AND l.timestamp = o.lastProcessedTimestamp
98-
AND l.req_id > COALESCE(o.lastProcessedReqId, '')
100+
AND l.req_id > o.lastProcessedReqId
99101
)
100102
GROUP BY l.bucketName, l.raftSessionID, o.lastProcessedInsertedAt, o.lastProcessedTimestamp, o.lastProcessedReqId
101103
)

0 commit comments

Comments
 (0)