Skip to content

Commit ea22add

Browse files
authored
Merge pull request #1259 from hakman/kmsg-duplicate-message
fix(logwatchers/kmsg): prevent duplicate message replay after restart
2 parents e0a3f5b + b3379b0 commit ea22add

File tree

1 file changed

+16
-4
lines changed

1 file changed

+16
-4
lines changed

pkg/systemlogmonitor/logwatchers/kmsg/log_watcher_linux.go

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,8 @@ func (k *kernelLogWatcher) watchLoop() {
110110
klog.Errorf("Failed to close kmsg parser: %v", err)
111111
}
112112

113-
// Try to restart
113+
// Try to restart immediately. retryCreateParser() applies backoff only
114+
// after a failed NewParser() or SeekEnd() attempt.
114115
var restarted bool
115116
kmsgs, restarted = k.retryCreateParser()
116117
if !restarted {
@@ -140,18 +141,29 @@ func (k *kernelLogWatcher) watchLoop() {
140141

141142
// retryCreateParser attempts to create a new kmsg parser.
142143
// It tries immediately first, then waits retryDelay between subsequent failures.
144+
// On success, it seeks the new parser to the end of the kmsg ring buffer to
145+
// avoid replaying messages that were already processed before the restart.
146+
// Any messages written to kmsg between the old parser closing and the new
147+
// parser being seeked are not delivered; this is preferable to replaying an
148+
// entire ring buffer the watcher has already processed, especially when the
149+
// restart was triggered by a kmsg flood.
143150
// It returns the new message channel and true on success, or nil and false if stopping was signaled.
144151
func (k *kernelLogWatcher) retryCreateParser() (<-chan kmsgparser.Message, bool) {
145152
for {
146153
parser, err := kmsgparser.NewParser()
147-
if err == nil {
154+
if err != nil {
155+
klog.Errorf("Failed to create new kmsg parser, retrying in %v: %v", retryDelay, err)
156+
} else if seekErr := parser.SeekEnd(); seekErr != nil {
157+
klog.Errorf("Failed to seek new kmsg parser to end, retrying in %v: %v", retryDelay, seekErr)
158+
if closeErr := parser.Close(); closeErr != nil {
159+
klog.Errorf("Failed to close kmsg parser after seek failure: %v", closeErr)
160+
}
161+
} else {
148162
k.kmsgParser = parser
149163
klog.Infof("Successfully restarted kmsg parser")
150164
return parser.Parse(), true
151165
}
152166

153-
klog.Errorf("Failed to create new kmsg parser, retrying in %v: %v", retryDelay, err)
154-
155167
select {
156168
case <-k.tomb.Stopping():
157169
klog.Infof("Stop watching kernel log during restart attempt")

0 commit comments

Comments
 (0)