From e36f3b6f7c921b94f12a6a5a0543231040203efb Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Tue, 2 Dec 2025 09:26:10 +0100 Subject: [PATCH 01/28] tailerv2: implement first iteration --- .../source/file/internal/tailv2/config.go | 37 +++ .../internal/tailv2/fileext/file_posix.go | 29 ++ .../internal/tailv2/fileext/file_windows.go | 116 +++++++ .../loki/source/file/internal/tailv2/line.go | 9 + .../source/file/internal/tailv2/tailer.go | 214 ++++++++++++ .../file/internal/tailv2/tailer_test.go | 314 ++++++++++++++++++ .../file/internal/tailv2/testdata/mssql.log | Bin 0 -> 876 bytes .../loki/source/file/internal/tailv2/watch.go | 140 ++++++++ 8 files changed, 859 insertions(+) create mode 100644 internal/component/loki/source/file/internal/tailv2/config.go create mode 100644 internal/component/loki/source/file/internal/tailv2/fileext/file_posix.go create mode 100644 internal/component/loki/source/file/internal/tailv2/fileext/file_windows.go create mode 100644 internal/component/loki/source/file/internal/tailv2/line.go create mode 100644 internal/component/loki/source/file/internal/tailv2/tailer.go create mode 100644 internal/component/loki/source/file/internal/tailv2/tailer_test.go create mode 100644 internal/component/loki/source/file/internal/tailv2/testdata/mssql.log create mode 100644 internal/component/loki/source/file/internal/tailv2/watch.go diff --git a/internal/component/loki/source/file/internal/tailv2/config.go b/internal/component/loki/source/file/internal/tailv2/config.go new file mode 100644 index 0000000000..81a9136894 --- /dev/null +++ b/internal/component/loki/source/file/internal/tailv2/config.go @@ -0,0 +1,37 @@ +package tailv2 + +import ( + "time" + + "golang.org/x/text/encoding" +) + +type Config struct { + Filename string + Offset int64 + + // Change the decoder if the file is not UTF-8. + // If the tailer doesn't use the right decoding, the output text may be gibberish. + // For example, if the file is "UTF-16 LE" encoded, the tailer would not separate + // the new lines properly and the output could come out as chinese characters. + Decoder *encoding.Decoder + + WatcherConfig WatcherConfig +} + +type WatcherConfig struct { + // MinPollFrequency and MaxPollFrequency specify how frequently a + // PollingFileWatcher should poll the file. + // + // Watcher starts polling at MinPollFrequency, and will + // exponentially increase the polling frequency up to MaxPollFrequency if no + // new entries are found. The polling frequency is reset to MinPollFrequency + // whenever the file changes. + MinPollFrequency, MaxPollFrequency time.Duration +} + +// DefaultWatcherConfig holds default values for WatcherConfig +var DefaultWatcherConfig = WatcherConfig{ + MinPollFrequency: 250 * time.Millisecond, + MaxPollFrequency: 250 * time.Millisecond, +} diff --git a/internal/component/loki/source/file/internal/tailv2/fileext/file_posix.go b/internal/component/loki/source/file/internal/tailv2/fileext/file_posix.go new file mode 100644 index 0000000000..40642c02c9 --- /dev/null +++ b/internal/component/loki/source/file/internal/tailv2/fileext/file_posix.go @@ -0,0 +1,29 @@ +//go:build linux || darwin || freebsd || netbsd || openbsd +// +build linux darwin freebsd netbsd openbsd + +package fileext + +import ( + "os" + "path/filepath" +) + +func OpenFile(name string) (file *os.File, err error) { + filename := name + // Check if the path requested is a symbolic link + fi, err := os.Lstat(name) + if err != nil { + return nil, err + } + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + filename, err = filepath.EvalSymlinks(name) + if err != nil { + return nil, err + } + } + return os.Open(filename) +} + +func IsDeletePending(_ *os.File) (bool, error) { + return false, nil +} diff --git a/internal/component/loki/source/file/internal/tailv2/fileext/file_windows.go b/internal/component/loki/source/file/internal/tailv2/fileext/file_windows.go new file mode 100644 index 0000000000..fe34770680 --- /dev/null +++ b/internal/component/loki/source/file/internal/tailv2/fileext/file_windows.go @@ -0,0 +1,116 @@ +//go:build windows +// +build windows + +package fileext + +import ( + "os" + "runtime" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +// issue also described here +// https://codereview.appspot.com/8203043/ + +// https://github.com/jnwhiteh/golang/blob/master/src/pkg/os/file_windows.go#L133 +func OpenFile(name string) (file *os.File, err error) { + f, e := open(name, os.O_RDONLY|syscall.O_CLOEXEC, 0) + if e != nil { + return nil, e + } + return os.NewFile(uintptr(f), name), nil +} + +// https://github.com/jnwhiteh/golang/blob/master/src/pkg/syscall/syscall_windows.go#L218 +func open(path string, mode int, _ uint32) (fd syscall.Handle, err error) { + if len(path) == 0 { + return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return syscall.InvalidHandle, err + } + var access uint32 + switch mode & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) { + case syscall.O_RDONLY: + access = syscall.GENERIC_READ + case syscall.O_WRONLY: + access = syscall.GENERIC_WRITE + case syscall.O_RDWR: + access = syscall.GENERIC_READ | syscall.GENERIC_WRITE + } + if mode&syscall.O_CREAT != 0 { + access |= syscall.GENERIC_WRITE + } + if mode&syscall.O_APPEND != 0 { + access &^= syscall.GENERIC_WRITE + access |= syscall.FILE_APPEND_DATA + } + sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE) + var sa *syscall.SecurityAttributes + if mode&syscall.O_CLOEXEC == 0 { + sa = makeInheritSa() + } + var createmode uint32 + switch { + case mode&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL): + createmode = syscall.CREATE_NEW + case mode&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC): + createmode = syscall.CREATE_ALWAYS + case mode&syscall.O_CREAT == syscall.O_CREAT: + createmode = syscall.OPEN_ALWAYS + case mode&syscall.O_TRUNC == syscall.O_TRUNC: + createmode = syscall.TRUNCATE_EXISTING + default: + createmode = syscall.OPEN_EXISTING + } + h, e := syscall.CreateFile(pathp, access, sharemode, sa, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0) + return h, e +} + +// https://github.com/jnwhiteh/golang/blob/master/src/pkg/syscall/syscall_windows.go#L211 +func makeInheritSa() *syscall.SecurityAttributes { + var sa syscall.SecurityAttributes + sa.Length = uint32(unsafe.Sizeof(sa)) + sa.InheritHandle = 1 + return &sa +} + +func IsDeletePending(f *os.File) (bool, error) { + if f == nil { + return false, nil + } + + fi, err := getFileStandardInfo(f) + if err != nil { + return false, err + } + + return fi.DeletePending, nil +} + +// From: https://github.com/microsoft/go-winio/blob/main/fileinfo.go +// FileStandardInfo contains extended information for the file. +// FILE_STANDARD_INFO in WinBase.h +// https://docs.microsoft.com/en-us/windows/win32/api/winbase/ns-winbase-file_standard_info +type fileStandardInfo struct { + AllocationSize, EndOfFile int64 + NumberOfLinks uint32 + DeletePending, Directory bool +} + +// GetFileStandardInfo retrieves ended information for the file. +func getFileStandardInfo(f *os.File) (*fileStandardInfo, error) { + si := &fileStandardInfo{} + if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), + windows.FileStandardInfo, + (*byte)(unsafe.Pointer(si)), + uint32(unsafe.Sizeof(*si))); err != nil { + return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err} + } + runtime.KeepAlive(f) + return si, nil +} diff --git a/internal/component/loki/source/file/internal/tailv2/line.go b/internal/component/loki/source/file/internal/tailv2/line.go new file mode 100644 index 0000000000..fbf4fffde5 --- /dev/null +++ b/internal/component/loki/source/file/internal/tailv2/line.go @@ -0,0 +1,9 @@ +package tailv2 + +import "time" + +type Line struct { + Text string + Offset int64 + Time time.Time +} diff --git a/internal/component/loki/source/file/internal/tailv2/tailer.go b/internal/component/loki/source/file/internal/tailv2/tailer.go new file mode 100644 index 0000000000..21f1d8d2a7 --- /dev/null +++ b/internal/component/loki/source/file/internal/tailv2/tailer.go @@ -0,0 +1,214 @@ +package tailv2 + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "sync" + "time" + + "github.com/go-kit/log" + "github.com/grafana/alloy/internal/component/loki/source/file/internal/tailv2/fileext" + "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/dskit/backoff" +) + +func NewTailer(logger log.Logger, cfg *Config) (*Tailer, error) { + f, err := fileext.OpenFile(cfg.Filename) + if err != nil { + return nil, err + } + + if cfg.Offset != 0 { + // Seek to provided offset + if _, err := f.Seek(cfg.Offset, io.SeekStart); err != nil { + return nil, err + } + } + + watcher, err := newWatcher(cfg.Filename, cfg.WatcherConfig) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithCancel(context.Background()) + + return &Tailer{ + cfg: cfg, + logger: logger, + file: f, + reader: newReader(f, cfg), + watcher: watcher, + ctx: ctx, + cancel: cancel, + }, nil +} + +type Tailer struct { + cfg *Config + logger log.Logger + + mu sync.Mutex + file *os.File + reader *bufio.Reader + lastOffset int64 + + watcher *watcher + + ctx context.Context + cancel context.CancelFunc +} + +func (t *Tailer) Next() (*Line, error) { + t.mu.Lock() + defer t.mu.Unlock() + text, err := t.readLine() + + if err != nil { + if errors.Is(err, io.EOF) { + return nil, err + } + return nil, err + } + + offset, err := t.offset() + if err != nil { + return nil, err + } + + t.lastOffset = offset + + return &Line{ + Text: text, + Offset: offset, + Time: time.Now(), + }, nil +} + +func (t *Tailer) Wait() error { + t.mu.Lock() + defer t.mu.Unlock() + + offset, err := t.offset() + if err != nil { + return err + } + + event, err := t.watcher.blockUntilEvent(t.ctx, t.file, offset) + switch event { + case eventModified: + // We need to reset to last succeful offset because we could have consumed a partial line. + t.file.Seek(t.lastOffset, io.SeekStart) + t.reader.Reset(t.file) + return nil + case eventTruncated: + // We need to reopen the file when it was truncated. + return t.reopen(true) + case eventDeleted: + // In polling mode we could miss events when a file is deleted, so before we give up + // we try to reopen the file. + if err := t.reopen(false); err != nil { + return err + } + return nil + default: + return err + } +} + +func (t *Tailer) readLine() (string, error) { + line, err := t.reader.ReadString('\n') + if err != nil { + return line, err + } + + line = strings.TrimRight(line, "\n") + // Trim Windows line endings + line = strings.TrimSuffix(line, "\r") + return line, err +} + +func (t *Tailer) offset() (int64, error) { + offset, err := t.file.Seek(0, io.SeekCurrent) + if err != nil { + return 0, err + } + + return offset - int64(t.reader.Buffered()), nil +} + +func (t *Tailer) reopen(truncated bool) error { + // There are cases where the file is reopened so quickly it's still the same file + // which causes the poller to hang on an open file handle to a file no longer being written to + // and which eventually gets deleted. Save the current file handle info to make sure we only + // start tailing a different file. + cf, err := t.file.Stat() + if !truncated && err != nil { + level.Debug(t.logger).Log("msg", "stat of old file returned, this is not expected and may result in unexpected behavior") + // We don't action on this error but are logging it, not expecting to see it happen and not sure if we + // need to action on it, cf is checked for nil later on to accommodate this + } + + t.file.Close() + + backoff := backoff.New(t.ctx, backoff.Config{ + MinBackoff: DefaultWatcherConfig.MaxPollFrequency, + MaxBackoff: DefaultWatcherConfig.MaxPollFrequency, + MaxRetries: 20, + }) + + for backoff.Ongoing() { + file, err := fileext.OpenFile(t.cfg.Filename) + if err != nil { + if os.IsNotExist(err) { + level.Debug(t.logger).Log("msg", fmt.Sprintf("Waiting for %s to appear...", t.cfg.Filename)) + if err := t.watcher.blockUntilExists(t.ctx); err != nil { + return fmt.Errorf("Failed to detect creation of %s: %w", t.cfg.Filename, err) + } + backoff.Wait() + continue + } + return fmt.Errorf("Unable to open file %s: %s", t.cfg.Filename, err) + } + + // File exists and is opened, get information about it. + nf, err := file.Stat() + if err != nil { + level.Debug(t.logger).Log("msg", "Failed to stat new file to be tailed, will try to open it again") + file.Close() + backoff.Wait() + continue + } + + // Check to see if we are trying to reopen and tail the exact same file (and it was not truncated). + if !truncated && cf != nil && os.SameFile(cf, nf) { + file.Close() + backoff.Wait() + continue + } + + t.file = file + t.reader = newReader(t.file, t.cfg) + break + } + + return backoff.Err() +} + +func (t *Tailer) Stop() error { + t.cancel() + t.mu.Lock() + defer t.mu.Unlock() + return t.file.Close() +} + +func newReader(f *os.File, cfg *Config) *bufio.Reader { + if cfg.Decoder != nil { + return bufio.NewReader(cfg.Decoder.Reader(f)) + } + return bufio.NewReader(f) +} diff --git a/internal/component/loki/source/file/internal/tailv2/tailer_test.go b/internal/component/loki/source/file/internal/tailv2/tailer_test.go new file mode 100644 index 0000000000..f6bfe1f33d --- /dev/null +++ b/internal/component/loki/source/file/internal/tailv2/tailer_test.go @@ -0,0 +1,314 @@ +package tailv2 + +import ( + "context" + "io" + "os" + "strings" + "testing" + "time" + + "github.com/go-kit/log" + "github.com/stretchr/testify/require" + "golang.org/x/text/encoding/unicode" +) + +func TestTailTailer(t *testing.T) { + verify := func(t *testing.T, tailer *Tailer, expectedLine *Line, expectedErr error) { + t.Helper() + line, err := tailer.Next() + require.ErrorIs(t, err, expectedErr) + if expectedLine == nil { + require.Nil(t, line) + } else { + require.Equal(t, expectedLine.Text, line.Text) + require.Equal(t, expectedLine.Offset, line.Offset) + } + } + + t.Run("file must exist", func(t *testing.T) { + _, err := NewTailer(log.NewNopLogger(), &Config{ + Filename: "/no/such/file", + }) + require.ErrorIs(t, err, os.ErrNotExist) + + name := createFile(t, "exists", "") + defer removeFile(t, name) + + _, err = NewTailer(log.NewNopLogger(), &Config{ + Filename: name, + }) + require.NoError(t, err) + }) + + t.Run("over 4096 byte line", func(t *testing.T) { + testString := strings.Repeat("a", 4098) + + name := createFile(t, "over4096", "test\n"+testString+"\nhello\nworld\n") + defer removeFile(t, name) + + tailer, err := NewTailer(log.NewNopLogger(), &Config{ + Filename: name, + }) + require.NoError(t, err) + + verify(t, tailer, &Line{Text: "test", Offset: 5}, nil) + verify(t, tailer, &Line{Text: testString, Offset: 4104}, nil) + verify(t, tailer, &Line{Text: "hello", Offset: 4110}, nil) + verify(t, tailer, &Line{Text: "world", Offset: 4116}, nil) + verify(t, tailer, nil, io.EOF) + }) + + t.Run("read", func(t *testing.T) { + name := createFile(t, "read", "hello\nworld\ntest\n") + defer removeFile(t, name) + + const ( + first = 6 + middle = 12 + end = 17 + ) + + t.Run("start", func(t *testing.T) { + tailer, err := NewTailer(log.NewNopLogger(), &Config{ + Filename: name, + Offset: 0, + }) + require.NoError(t, err) + + verify(t, tailer, &Line{Text: "hello", Offset: first}, nil) + verify(t, tailer, &Line{Text: "world", Offset: middle}, nil) + verify(t, tailer, &Line{Text: "test", Offset: end}, nil) + verify(t, tailer, nil, io.EOF) + }) + + t.Run("skip first", func(t *testing.T) { + tailer, err := NewTailer(log.NewNopLogger(), &Config{ + Filename: name, + Offset: first, + }) + require.NoError(t, err) + + verify(t, tailer, &Line{Text: "world", Offset: middle}, nil) + verify(t, tailer, &Line{Text: "test", Offset: end}, nil) + verify(t, tailer, nil, io.EOF) + }) + + t.Run("end", func(t *testing.T) { + tailer, err := NewTailer(log.NewNopLogger(), &Config{ + Filename: name, + Offset: end, + }) + require.NoError(t, err) + verify(t, tailer, nil, io.EOF) + }) + }) + + t.Run("partail line", func(t *testing.T) { + name := createFile(t, "partial", "hello\nwo") + defer removeFile(t, name) + + tailer, err := NewTailer(log.NewNopLogger(), &Config{ + Offset: 0, + Filename: name, + }) + require.NoError(t, err) + + verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) + verify(t, tailer, nil, io.EOF) + + go appendToFile(t, name, "rld\n") + + require.NoError(t, tailer.Wait()) + verify(t, tailer, &Line{Text: "world", Offset: 12}, nil) + + verify(t, tailer, nil, io.EOF) + }) + + t.Run("wait", func(t *testing.T) { + name := createFile(t, "wait", "hello\nwo") + defer removeFile(t, name) + + tailer, err := NewTailer(log.NewNopLogger(), &Config{ + Offset: 0, + Filename: name, + }) + require.NoError(t, err) + + verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) + verify(t, tailer, nil, io.EOF) + + go func() { + <-time.After(200 * time.Millisecond) + appendToFile(t, name, "rld\n") + }() + require.NoError(t, tailer.Wait()) + verify(t, tailer, &Line{Text: "world", Offset: 12}, nil) + verify(t, tailer, nil, io.EOF) + }) + + t.Run("truncate", func(t *testing.T) { + name := createFile(t, "truncate", "a really long string goes here\nhello\nworld\n") + defer removeFile(t, name) + + tailer, err := NewTailer(log.NewNopLogger(), &Config{ + Filename: name, + WatcherConfig: WatcherConfig{ + MinPollFrequency: 5 * time.Millisecond, + MaxPollFrequency: 5 * time.Millisecond, + }, + }) + require.NoError(t, err) + + verify(t, tailer, &Line{Text: "a really long string goes here", Offset: 31}, nil) + verify(t, tailer, &Line{Text: "hello", Offset: 37}, nil) + verify(t, tailer, &Line{Text: "world", Offset: 43}, nil) + verify(t, tailer, nil, io.EOF) + + go func() { + // truncate now + <-time.After(100 * time.Millisecond) + truncateFile(t, name, "h311o\nw0r1d\nendofworld\n") + }() + + tailer.Wait() + verify(t, tailer, &Line{Text: "h311o", Offset: 6}, nil) + verify(t, tailer, &Line{Text: "w0r1d", Offset: 12}, nil) + verify(t, tailer, &Line{Text: "endofworld", Offset: 23}, nil) + verify(t, tailer, nil, io.EOF) + }) + + t.Run("stopped during wait", func(t *testing.T) { + name := createFile(t, "stopped", "hello\n") + defer removeFile(t, name) + + tailer, err := NewTailer(log.NewNopLogger(), &Config{ + Offset: 0, + Filename: name, + }) + require.NoError(t, err) + + verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) + verify(t, tailer, nil, io.EOF) + + go func() { + time.Sleep(100 * time.Millisecond) + require.NoError(t, tailer.Stop()) + }() + + require.ErrorIs(t, tailer.Wait(), context.Canceled) + }) + + t.Run("removed and created during wait", func(t *testing.T) { + name := createFile(t, "removed", "hello\n") + defer removeFile(t, name) + + tailer, err := NewTailer(log.NewNopLogger(), &Config{ + Offset: 0, + Filename: name, + WatcherConfig: WatcherConfig{ + MinPollFrequency: 5 * time.Millisecond, + MaxPollFrequency: 5 * time.Millisecond, + }, + }) + require.NoError(t, err) + + verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) + verify(t, tailer, nil, io.EOF) + + go func() { + time.Sleep(100 * time.Millisecond) + removeFile(t, name) + time.Sleep(100 * time.Millisecond) + recreateFile(t, name, "new\n") + }() + + require.NoError(t, tailer.Wait()) + + verify(t, tailer, &Line{Text: "new", Offset: 4}, nil) + verify(t, tailer, nil, io.EOF) + }) + + t.Run("stopped while waiting for file to be created", func(t *testing.T) { + name := createFile(t, "removed", "hello\n") + + tailer, err := NewTailer(log.NewNopLogger(), &Config{ + Offset: 0, + Filename: name, + WatcherConfig: WatcherConfig{ + MinPollFrequency: 5 * time.Millisecond, + MaxPollFrequency: 5 * time.Millisecond, + }, + }) + require.NoError(t, err) + + verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) + verify(t, tailer, nil, io.EOF) + + removeFile(t, name) + + go func() { + time.Sleep(100 * time.Millisecond) + tailer.Stop() + }() + + require.ErrorIs(t, tailer.Wait(), context.Canceled) + }) + + t.Run("UTF-16LE", func(t *testing.T) { + tailer, err := NewTailer(log.NewNopLogger(), &Config{ + Filename: "testdata/mssql.log", + Decoder: unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM).NewDecoder(), + }) + require.NoError(t, err) + + verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.58 Server Microsoft SQL Server 2019 (RTM) - 15.0.2000.5 (X64) ", Offset: 528}, nil) + verify(t, tailer, &Line{Text: " Sep 24 2019 13:48:23 ", Offset: 552}, nil) + verify(t, tailer, &Line{Text: " Copyright (C) 2019 Microsoft Corporation", Offset: 595}, nil) + verify(t, tailer, &Line{Text: " Enterprise Edition (64-bit) on Windows Server 2022 Standard 10.0 (Build 20348: ) (Hypervisor)", Offset: 697}, nil) + verify(t, tailer, &Line{Text: "", Offset: 699}, nil) + verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.71 Server UTC adjustment: 1:00", Offset: 756}, nil) + verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.71 Server (c) Microsoft Corporation.", Offset: 819}, nil) + verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.72 Server All rights reserved.", Offset: 876}, nil) + verify(t, tailer, nil, io.EOF) + }) + +} + +func createFile(t *testing.T, name, content string) string { + path := t.TempDir() + "/" + name + require.NoError(t, os.WriteFile(path, []byte(content), 0600)) + return path +} + +func recreateFile(t *testing.T, path, content string) { + require.NoError(t, os.WriteFile(path, []byte(content), 0600)) +} + +func appendToFile(t *testing.T, name, content string) { + f, err := os.OpenFile(name, os.O_APPEND|os.O_WRONLY, 0600) + require.NoError(t, err) + defer f.Close() + _, err = f.WriteString(content) + require.NoError(t, err) +} + +func truncateFile(t *testing.T, name, content string) { + f, err := os.OpenFile(name, os.O_TRUNC|os.O_WRONLY, 0600) + require.NoError(t, err) + defer f.Close() + _, err = f.WriteString(content) + require.NoError(t, err) +} + +/* +func renameFile(t *testing.T, oldname, newname string) { + oldname = t.TempDir() + "/" + oldname + newname = t.TempDir() + "/" + newname + require.NoError(t, os.Rename(oldname, newname)) +} +*/ + +func removeFile(t *testing.T, name string) { + require.NoError(t, os.Remove(name)) +} diff --git a/internal/component/loki/source/file/internal/tailv2/testdata/mssql.log b/internal/component/loki/source/file/internal/tailv2/testdata/mssql.log new file mode 100644 index 0000000000000000000000000000000000000000..0234db5bd33d23b377927f5ba62584bb3a3111b3 GIT binary patch literal 876 zcmb`GOH0F05QWcH!T)fVagorZK57v`OTmR*_(HeU`iRsvq*d|HtKUo#3bmkuT<&x- zuXFCq^z$QAOPRJ6^V>$IoZzxsOYiYYJXV(TC?^q?xf z2d6^s@XPg}M`stQ=M@*(tKMLlCAlVtdv2NUI zZ?y_RSA0*1o$8IAEqB&fWgN55LAJ;tC?hN>KI>D^e%+Y^^hif~q2}0QEWcfMBKSKa z9n*oLo?q^BD)Cr{>{f_B@4+tk%WRu12)bmVJ^VvhJi%uM`)4q%Q(f#S(q{XAh!J~d lUH$#^l0TwXQ&_$ChxB_4`eL(emL)H?ZK(b!Bc^5V{Q|PUcQyb3 literal 0 HcmV?d00001 diff --git a/internal/component/loki/source/file/internal/tailv2/watch.go b/internal/component/loki/source/file/internal/tailv2/watch.go new file mode 100644 index 0000000000..386c3020d3 --- /dev/null +++ b/internal/component/loki/source/file/internal/tailv2/watch.go @@ -0,0 +1,140 @@ +package tailv2 + +import ( + "context" + "fmt" + "os" + "runtime" + + "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/util" + "github.com/grafana/alloy/internal/component/loki/source/file/internal/tailv2/fileext" + "github.com/grafana/dskit/backoff" +) + +// watcher polls the file for changes. +type watcher struct { + filename string + cfg WatcherConfig + + ctx context.Context + cancel context.CancelFunc +} + +func newWatcher(filename string, cfg WatcherConfig) (*watcher, error) { + if cfg == (WatcherConfig{}) { + cfg = DefaultWatcherConfig + } + + if cfg.MinPollFrequency == 0 || cfg.MaxPollFrequency == 0 { + return nil, fmt.Errorf("MinPollFrequency and MaxPollFrequency must be greater than 0") + } else if cfg.MaxPollFrequency < cfg.MinPollFrequency { + return nil, fmt.Errorf("MaxPollFrequency must be larger than MinPollFrequency") + } + + return &watcher{ + filename: filename, + cfg: cfg, + }, nil +} + +// blockUntilExists will block until either file exists or context is canceled. +func (fw *watcher) blockUntilExists(ctx context.Context) error { + backoff := backoff.New(ctx, backoff.Config{ + MinBackoff: fw.cfg.MinPollFrequency, + MaxBackoff: fw.cfg.MaxPollFrequency, + }) + + for backoff.Ongoing() { + if _, err := os.Stat(fw.filename); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + + backoff.Wait() + } + + return backoff.Err() +} + +// blockUntilEvent will block until it detects a new event for file or context is canceled. +func (fw *watcher) blockUntilEvent(ctx context.Context, f *os.File, pos int64) (event, error) { + origFi, err := f.Stat() + if err != nil { + return eventNone, err + } + + backoff := backoff.New(ctx, backoff.Config{ + MinBackoff: fw.cfg.MinPollFrequency, + MaxBackoff: fw.cfg.MaxPollFrequency, + }) + + var ( + prevSize = pos + prevModTime = origFi.ModTime() + ) + for backoff.Ongoing() { + deletePending, err := fileext.IsDeletePending(f) + + // DeletePending is a windows state where the file has been queued + // for delete but won't actually get deleted until all handles are + // closed. It's a variation on the NotifyDeleted call below. + // + // IsDeletePending may fail in cases where the file handle becomes + // invalid, so we treat a failed call the same as a pending delete. + if err != nil || deletePending { + return eventDeleted, nil + } + + fi, err := os.Stat(fw.filename) + if err != nil { + // Windows cannot delete a file if a handle is still open (tail keeps one open) + // so it gives access denied to anything trying to read it until all handles are released. + if os.IsNotExist(err) || (runtime.GOOS == "windows" && os.IsPermission(err)) { + // File does not exist (has been deleted). + return eventDeleted, nil + } + + // XXX: report this error back to the user + util.Fatal("Failed to stat file %v: %v", fw.filename, err) + } + + // File got moved/renamed? + if !os.SameFile(origFi, fi) { + return eventDeleted, nil + } + + // File got truncated? + currentSize := fi.Size() + if prevSize > 0 && prevSize > currentSize { + return eventTruncated, nil + } + + // File got bigger? + if prevSize > 0 && prevSize < currentSize { + return eventModified, nil + } + prevSize = currentSize + + // File was appended to (changed)? + modTime := fi.ModTime() + if modTime != prevModTime { + prevModTime = modTime + return eventModified, nil + } + + // File hasn't changed; increase backoff for next sleep. + backoff.Wait() + } + + return eventNone, backoff.Err() +} + +type event int + +const ( + eventNone event = iota + eventTruncated + eventModified + eventDeleted +) From e11be397428714b5f761c89ac7934a234c067c20 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Tue, 2 Dec 2025 09:48:44 +0100 Subject: [PATCH 02/28] Refactor to handle wait in Next --- .../source/file/internal/tailv2/tailer.go | 52 +++++++------- .../file/internal/tailv2/tailer_test.go | 70 ++++++------------- .../loki/source/file/internal/tailv2/watch.go | 8 +-- 3 files changed, 51 insertions(+), 79 deletions(-) diff --git a/internal/component/loki/source/file/internal/tailv2/tailer.go b/internal/component/loki/source/file/internal/tailv2/tailer.go index 21f1d8d2a7..f7d9307765 100644 --- a/internal/component/loki/source/file/internal/tailv2/tailer.go +++ b/internal/component/loki/source/file/internal/tailv2/tailer.go @@ -12,9 +12,10 @@ import ( "time" "github.com/go-kit/log" + "github.com/grafana/dskit/backoff" + "github.com/grafana/alloy/internal/component/loki/source/file/internal/tailv2/fileext" "github.com/grafana/alloy/internal/runtime/logging/level" - "github.com/grafana/dskit/backoff" ) func NewTailer(logger log.Logger, cfg *Config) (*Tailer, error) { @@ -52,9 +53,10 @@ type Tailer struct { cfg *Config logger log.Logger - mu sync.Mutex - file *os.File - reader *bufio.Reader + mu sync.Mutex + file *os.File + reader *bufio.Reader + lastOffset int64 watcher *watcher @@ -63,14 +65,19 @@ type Tailer struct { cancel context.CancelFunc } +// FIXME: need clear exit signal func (t *Tailer) Next() (*Line, error) { t.mu.Lock() defer t.mu.Unlock() +read: text, err := t.readLine() if err != nil { if errors.Is(err, io.EOF) { - return nil, err + if err := t.wait(text != ""); err != nil { + return nil, err + } + goto read } return nil, err } @@ -89,10 +96,14 @@ func (t *Tailer) Next() (*Line, error) { }, nil } -func (t *Tailer) Wait() error { +func (t *Tailer) Stop() error { + t.cancel() t.mu.Lock() defer t.mu.Unlock() + return t.file.Close() +} +func (t *Tailer) wait(partial bool) error { offset, err := t.offset() if err != nil { return err @@ -101,9 +112,11 @@ func (t *Tailer) Wait() error { event, err := t.watcher.blockUntilEvent(t.ctx, t.file, offset) switch event { case eventModified: - // We need to reset to last succeful offset because we could have consumed a partial line. - t.file.Seek(t.lastOffset, io.SeekStart) - t.reader.Reset(t.file) + if partial { + // We need to reset to last succeful offset because we could have consumed a partial line. + t.file.Seek(t.lastOffset, io.SeekStart) + t.reader.Reset(t.file) + } return nil case eventTruncated: // We need to reopen the file when it was truncated. @@ -111,13 +124,11 @@ func (t *Tailer) Wait() error { case eventDeleted: // In polling mode we could miss events when a file is deleted, so before we give up // we try to reopen the file. - if err := t.reopen(false); err != nil { - return err - } - return nil + return t.reopen(false) default: return err } + } func (t *Tailer) readLine() (string, error) { @@ -148,9 +159,9 @@ func (t *Tailer) reopen(truncated bool) error { // start tailing a different file. cf, err := t.file.Stat() if !truncated && err != nil { - level.Debug(t.logger).Log("msg", "stat of old file returned, this is not expected and may result in unexpected behavior") // We don't action on this error but are logging it, not expecting to see it happen and not sure if we // need to action on it, cf is checked for nil later on to accommodate this + level.Debug(t.logger).Log("msg", "stat of old file returned, this is not expected and may result in unexpected behavior") } t.file.Close() @@ -165,9 +176,9 @@ func (t *Tailer) reopen(truncated bool) error { file, err := fileext.OpenFile(t.cfg.Filename) if err != nil { if os.IsNotExist(err) { - level.Debug(t.logger).Log("msg", fmt.Sprintf("Waiting for %s to appear...", t.cfg.Filename)) + level.Debug(t.logger).Log("msg", fmt.Sprintf("waiting for %s to appear...", t.cfg.Filename)) if err := t.watcher.blockUntilExists(t.ctx); err != nil { - return fmt.Errorf("Failed to detect creation of %s: %w", t.cfg.Filename, err) + return fmt.Errorf("failed to detect creation of %s: %w", t.cfg.Filename, err) } backoff.Wait() continue @@ -178,7 +189,7 @@ func (t *Tailer) reopen(truncated bool) error { // File exists and is opened, get information about it. nf, err := file.Stat() if err != nil { - level.Debug(t.logger).Log("msg", "Failed to stat new file to be tailed, will try to open it again") + level.Debug(t.logger).Log("msg", "failed to stat new file to be tailed, will try to open it again") file.Close() backoff.Wait() continue @@ -199,13 +210,6 @@ func (t *Tailer) reopen(truncated bool) error { return backoff.Err() } -func (t *Tailer) Stop() error { - t.cancel() - t.mu.Lock() - defer t.mu.Unlock() - return t.file.Close() -} - func newReader(f *os.File, cfg *Config) *bufio.Reader { if cfg.Decoder != nil { return bufio.NewReader(cfg.Decoder.Reader(f)) diff --git a/internal/component/loki/source/file/internal/tailv2/tailer_test.go b/internal/component/loki/source/file/internal/tailv2/tailer_test.go index f6bfe1f33d..572047d6c8 100644 --- a/internal/component/loki/source/file/internal/tailv2/tailer_test.go +++ b/internal/component/loki/source/file/internal/tailv2/tailer_test.go @@ -2,7 +2,6 @@ package tailv2 import ( "context" - "io" "os" "strings" "testing" @@ -51,12 +50,12 @@ func TestTailTailer(t *testing.T) { Filename: name, }) require.NoError(t, err) + defer tailer.Stop() verify(t, tailer, &Line{Text: "test", Offset: 5}, nil) verify(t, tailer, &Line{Text: testString, Offset: 4104}, nil) verify(t, tailer, &Line{Text: "hello", Offset: 4110}, nil) verify(t, tailer, &Line{Text: "world", Offset: 4116}, nil) - verify(t, tailer, nil, io.EOF) }) t.Run("read", func(t *testing.T) { @@ -75,11 +74,11 @@ func TestTailTailer(t *testing.T) { Offset: 0, }) require.NoError(t, err) + defer tailer.Stop() verify(t, tailer, &Line{Text: "hello", Offset: first}, nil) verify(t, tailer, &Line{Text: "world", Offset: middle}, nil) verify(t, tailer, &Line{Text: "test", Offset: end}, nil) - verify(t, tailer, nil, io.EOF) }) t.Run("skip first", func(t *testing.T) { @@ -88,19 +87,21 @@ func TestTailTailer(t *testing.T) { Offset: first, }) require.NoError(t, err) + defer tailer.Stop() verify(t, tailer, &Line{Text: "world", Offset: middle}, nil) verify(t, tailer, &Line{Text: "test", Offset: end}, nil) - verify(t, tailer, nil, io.EOF) }) - t.Run("end", func(t *testing.T) { + t.Run("last", func(t *testing.T) { tailer, err := NewTailer(log.NewNopLogger(), &Config{ Filename: name, - Offset: end, + Offset: middle, }) require.NoError(t, err) - verify(t, tailer, nil, io.EOF) + defer tailer.Stop() + + verify(t, tailer, &Line{Text: "test", Offset: end}, nil) }) }) @@ -113,38 +114,14 @@ func TestTailTailer(t *testing.T) { Filename: name, }) require.NoError(t, err) + defer tailer.Stop() verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) - verify(t, tailer, nil, io.EOF) - - go appendToFile(t, name, "rld\n") - - require.NoError(t, tailer.Wait()) - verify(t, tailer, &Line{Text: "world", Offset: 12}, nil) - - verify(t, tailer, nil, io.EOF) - }) - - t.Run("wait", func(t *testing.T) { - name := createFile(t, "wait", "hello\nwo") - defer removeFile(t, name) - - tailer, err := NewTailer(log.NewNopLogger(), &Config{ - Offset: 0, - Filename: name, - }) - require.NoError(t, err) - - verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) - verify(t, tailer, nil, io.EOF) - go func() { - <-time.After(200 * time.Millisecond) + time.Sleep(50 * time.Millisecond) appendToFile(t, name, "rld\n") }() - require.NoError(t, tailer.Wait()) verify(t, tailer, &Line{Text: "world", Offset: 12}, nil) - verify(t, tailer, nil, io.EOF) }) t.Run("truncate", func(t *testing.T) { @@ -159,11 +136,11 @@ func TestTailTailer(t *testing.T) { }, }) require.NoError(t, err) + defer tailer.Stop() verify(t, tailer, &Line{Text: "a really long string goes here", Offset: 31}, nil) verify(t, tailer, &Line{Text: "hello", Offset: 37}, nil) verify(t, tailer, &Line{Text: "world", Offset: 43}, nil) - verify(t, tailer, nil, io.EOF) go func() { // truncate now @@ -171,11 +148,10 @@ func TestTailTailer(t *testing.T) { truncateFile(t, name, "h311o\nw0r1d\nendofworld\n") }() - tailer.Wait() verify(t, tailer, &Line{Text: "h311o", Offset: 6}, nil) verify(t, tailer, &Line{Text: "w0r1d", Offset: 12}, nil) verify(t, tailer, &Line{Text: "endofworld", Offset: 23}, nil) - verify(t, tailer, nil, io.EOF) + }) t.Run("stopped during wait", func(t *testing.T) { @@ -189,14 +165,14 @@ func TestTailTailer(t *testing.T) { require.NoError(t, err) verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) - verify(t, tailer, nil, io.EOF) go func() { time.Sleep(100 * time.Millisecond) require.NoError(t, tailer.Stop()) }() - require.ErrorIs(t, tailer.Wait(), context.Canceled) + _, err = tailer.Next() + require.ErrorIs(t, err, context.Canceled) }) t.Run("removed and created during wait", func(t *testing.T) { @@ -212,21 +188,18 @@ func TestTailTailer(t *testing.T) { }, }) require.NoError(t, err) + defer tailer.Stop() verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) - verify(t, tailer, nil, io.EOF) go func() { - time.Sleep(100 * time.Millisecond) + time.Sleep(50 * time.Millisecond) removeFile(t, name) - time.Sleep(100 * time.Millisecond) + time.Sleep(50 * time.Millisecond) recreateFile(t, name, "new\n") }() - require.NoError(t, tailer.Wait()) - verify(t, tailer, &Line{Text: "new", Offset: 4}, nil) - verify(t, tailer, nil, io.EOF) }) t.Run("stopped while waiting for file to be created", func(t *testing.T) { @@ -243,16 +216,14 @@ func TestTailTailer(t *testing.T) { require.NoError(t, err) verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) - verify(t, tailer, nil, io.EOF) - removeFile(t, name) go func() { time.Sleep(100 * time.Millisecond) tailer.Stop() }() - - require.ErrorIs(t, tailer.Wait(), context.Canceled) + _, err = tailer.Next() + require.ErrorIs(t, err, context.Canceled) }) t.Run("UTF-16LE", func(t *testing.T) { @@ -261,6 +232,7 @@ func TestTailTailer(t *testing.T) { Decoder: unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM).NewDecoder(), }) require.NoError(t, err) + defer tailer.Stop() verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.58 Server Microsoft SQL Server 2019 (RTM) - 15.0.2000.5 (X64) ", Offset: 528}, nil) verify(t, tailer, &Line{Text: " Sep 24 2019 13:48:23 ", Offset: 552}, nil) @@ -270,9 +242,7 @@ func TestTailTailer(t *testing.T) { verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.71 Server UTC adjustment: 1:00", Offset: 756}, nil) verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.71 Server (c) Microsoft Corporation.", Offset: 819}, nil) verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.72 Server All rights reserved.", Offset: 876}, nil) - verify(t, tailer, nil, io.EOF) }) - } func createFile(t *testing.T, name, content string) string { diff --git a/internal/component/loki/source/file/internal/tailv2/watch.go b/internal/component/loki/source/file/internal/tailv2/watch.go index 386c3020d3..315b10f2a8 100644 --- a/internal/component/loki/source/file/internal/tailv2/watch.go +++ b/internal/component/loki/source/file/internal/tailv2/watch.go @@ -6,9 +6,9 @@ import ( "os" "runtime" - "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/util" - "github.com/grafana/alloy/internal/component/loki/source/file/internal/tailv2/fileext" "github.com/grafana/dskit/backoff" + + "github.com/grafana/alloy/internal/component/loki/source/file/internal/tailv2/fileext" ) // watcher polls the file for changes. @@ -94,9 +94,7 @@ func (fw *watcher) blockUntilEvent(ctx context.Context, f *os.File, pos int64) ( // File does not exist (has been deleted). return eventDeleted, nil } - - // XXX: report this error back to the user - util.Fatal("Failed to stat file %v: %v", fw.filename, err) + return eventNone, err } // File got moved/renamed? From 06a306a1f306b7046e31768d5d3de6e014be1d5f Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Tue, 2 Dec 2025 10:41:42 +0100 Subject: [PATCH 03/28] Use tailerv2 --- .../source/file/internal/tailv2/tailer.go | 13 +- internal/component/loki/source/file/tailer.go | 117 +++++------------- .../component/loki/source/file/tailer_test.go | 7 +- 3 files changed, 46 insertions(+), 91 deletions(-) diff --git a/internal/component/loki/source/file/internal/tailv2/tailer.go b/internal/component/loki/source/file/internal/tailv2/tailer.go index f7d9307765..82fdf720c2 100644 --- a/internal/component/loki/source/file/internal/tailv2/tailer.go +++ b/internal/component/loki/source/file/internal/tailv2/tailer.go @@ -96,6 +96,17 @@ read: }, nil } +func (t *Tailer) Size() (int64, error) { + t.mu.Lock() + defer t.mu.Unlock() + + fi, err := t.file.Stat() + if err != nil { + return 0, err + } + return fi.Size(), nil +} + func (t *Tailer) Stop() error { t.cancel() t.mu.Lock() @@ -203,7 +214,7 @@ func (t *Tailer) reopen(truncated bool) error { } t.file = file - t.reader = newReader(t.file, t.cfg) + t.reader.Reset(t.file) break } diff --git a/internal/component/loki/source/file/tailer.go b/internal/component/loki/source/file/tailer.go index 9e89ba3e17..738dc5b740 100644 --- a/internal/component/loki/source/file/tailer.go +++ b/internal/component/loki/source/file/tailer.go @@ -21,8 +21,7 @@ import ( "golang.org/x/text/encoding/ianaindex" "github.com/grafana/alloy/internal/component/common/loki" - "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail" - "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/watch" + "github.com/grafana/alloy/internal/component/loki/source/file/internal/tailv2" "github.com/grafana/alloy/internal/component/loki/source/internal/positions" "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/util" @@ -40,7 +39,7 @@ type tailer struct { tailFromEnd bool onPositionsFileError OnPositionsFileError - pollOptions watch.PollingFileWatcherOptions + pollOptions tailv2.WatcherConfig posAndSizeMtx sync.Mutex @@ -50,7 +49,7 @@ type tailer struct { report sync.Once - tail *tail.Tail + tail *tailv2.Tailer decoder *encoding.Decoder } @@ -79,7 +78,7 @@ func newTailer( tailFromEnd: opts.tailFromEnd, legacyPositionUsed: opts.legacyPositionUsed, onPositionsFileError: opts.onPositionsFileError, - pollOptions: watch.PollingFileWatcherOptions{ + pollOptions: tailv2.WatcherConfig{ MinPollFrequency: opts.fileWatch.MinPollFrequency, MaxPollFrequency: opts.fileWatch.MaxPollFrequency, }, @@ -243,11 +242,11 @@ func (t *tailer) initRun() (loki.EntryHandler, error) { } } - tail, err := tail.TailFile(t.key.Path, tail.Config{ - Location: &tail.SeekInfo{Offset: pos, Whence: 0}, - Logger: t.logger, - PollOptions: t.pollOptions, - Decoder: t.decoder, + tail, err := tailv2.NewTailer(t.logger, &tailv2.Config{ + Filename: t.key.Path, + Offset: pos, + Decoder: t.decoder, + WatcherConfig: t.pollOptions, }) if err != nil { @@ -274,39 +273,6 @@ func getDecoder(encoding string) (*encoding.Decoder, error) { return encoder.NewDecoder(), nil } -// updatePosition is run in a goroutine and checks the current size of the file -// and saves it to the positions file at a regular interval. If there is ever -// an error it stops the tailer and exits, the tailer will be re-opened by the -// backoff retry method if it still exists and will start reading from the -// last successful entry in the positions file. -func (t *tailer) updatePosition(posquit chan struct{}) { - positionSyncPeriod := t.positions.SyncPeriod() - positionWait := time.NewTicker(positionSyncPeriod) - defer func() { - positionWait.Stop() - level.Info(t.logger).Log("msg", "position timer: exited", "path", t.key.Path) - // NOTE: metrics must be cleaned up after the position timer exits, as markPositionAndSize() updates metrics. - t.cleanupMetrics() - }() - - for { - select { - case <-positionWait.C: - err := t.markPositionAndSize() - if err != nil { - level.Error(t.logger).Log("msg", "position timer: error getting tail position and/or size, stopping tailer", "path", t.key.Path, "error", err) - err := t.tail.Stop() - if err != nil { - level.Error(t.logger).Log("msg", "position timer: error stopping tailer", "path", t.key.Path, "error", err) - } - return - } - case <-posquit: - return - } - } -} - // readLines consumes the t.tail.Lines channel from the // underlying tailer. It will only exit when that channel is closed. This is // important to avoid a deadlock in the underlying tailer which can happen if @@ -315,28 +281,27 @@ func (t *tailer) updatePosition(posquit chan struct{}) { // the t.tail.Lines channel func (t *tailer) readLines(handler loki.EntryHandler, done chan struct{}) { level.Info(t.logger).Log("msg", "tail routine: started", "path", t.key.Path) + var ( + entries = handler.Chan() + lastOffset = int64(0) + positionInterval = t.positions.SyncPeriod() + lastUpdatedPosition = time.Time{} + ) - posquit, posdone := make(chan struct{}), make(chan struct{}) - go func() { - t.updatePosition(posquit) - close(posdone) - }() - - // This function runs in a goroutine, if it exits this tailer will never do any more tailing. - // Clean everything up. defer func() { level.Info(t.logger).Log("msg", "tail routine: exited", "path", t.key.Path) - // Shut down the position marker thread - close(posquit) - <-posdone + size, _ := t.tail.Size() + t.updateStats(lastOffset, size) close(done) }() - entries := handler.Chan() for { - line, ok := <-t.tail.Lines - if !ok { - level.Info(t.logger).Log("msg", "tail routine: tail channel closed, stopping tailer", "path", t.key.Path, "reason", t.tail.Tomb.Err()) + line, err := t.tail.Next() + if err != nil { + // Maybe we should use a better signal than context canceled to indicate normal stop... + if !errors.Is(err, context.Canceled) { + level.Info(t.logger).Log("msg", "tail routine: stopping tailer", "path", t.key.Path, "err", err) + } return } @@ -350,43 +315,25 @@ func (t *tailer) readLines(handler loki.EntryHandler, done chan struct{}) { Line: line.Text, }, } - } -} - -func (t *tailer) markPositionAndSize() error { - // Lock this update because it can be called in two different goroutines - t.posAndSizeMtx.Lock() - defer t.posAndSizeMtx.Unlock() - size, err := t.tail.Size() - if err != nil { - // If the file no longer exists, no need to save position information - if err == os.ErrNotExist { - level.Info(t.logger).Log("msg", "skipping update of position for a file which does not currently exist", "path", t.key.Path) - return nil + lastOffset = line.Offset + if time.Since(lastUpdatedPosition) >= positionInterval { + lastUpdatedPosition = time.Now() + size, _ := t.tail.Size() + t.updateStats(lastOffset, size) } - return err - } - - pos, err := t.tail.Tell() - if err != nil { - return err } +} +func (t *tailer) updateStats(offset int64, size int64) { // Update metrics and positions file all together to avoid race conditions when `t.tail` is stopped. t.metrics.totalBytes.WithLabelValues(t.key.Path).Set(float64(size)) - t.metrics.readBytes.WithLabelValues(t.key.Path).Set(float64(pos)) - t.positions.Put(t.key.Path, t.key.Labels, pos) + t.metrics.readBytes.WithLabelValues(t.key.Path).Set(float64(offset)) + t.positions.Put(t.key.Path, t.key.Labels, offset) - return nil } func (t *tailer) stop(done chan struct{}) { - // Save the current position before shutting down tailer to ensure that if the file is tailed again - // it start where it left off. - if err := t.markPositionAndSize(); err != nil { - level.Error(t.logger).Log("msg", "error marking file position when stopping tailer", "path", t.key.Path, "error", err) - } if err := t.tail.Stop(); err != nil { if util.IsEphemeralOrFileClosed(err) { // Don't log as error if the file is already closed, or we got an ephemeral error - it's a common case diff --git a/internal/component/loki/source/file/tailer_test.go b/internal/component/loki/source/file/tailer_test.go index 106e722124..03e8f4db2f 100644 --- a/internal/component/loki/source/file/tailer_test.go +++ b/internal/component/loki/source/file/tailer_test.go @@ -354,11 +354,8 @@ func TestTailerCorruptedPositions(t *testing.T) { close(done) }() - require.EventuallyWithT(t, func(c *assert.CollectT) { - assert.True(c, tailer.IsRunning()) - assert.Equal(c, "16", positionsFile.GetString(logFile.Name(), labels.String())) - }, time.Second, 50*time.Millisecond) - + // tailer needs some time to start + time.Sleep(50 * time.Millisecond) _, err = logFile.Write([]byte("writing some text\n")) require.NoError(t, err) select { From 53a4759cff1817027346b1fbc47d13b821290323 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Tue, 2 Dec 2025 10:45:38 +0100 Subject: [PATCH 04/28] Rename structure to file --- .../internal/tailv2/{tailer.go => file.go} | 88 ++++++------- .../tailv2/{tailer_test.go => file_test.go} | 116 +++++++++--------- internal/component/loki/source/file/tailer.go | 14 +-- 3 files changed, 109 insertions(+), 109 deletions(-) rename internal/component/loki/source/file/internal/tailv2/{tailer.go => file.go} (68%) rename internal/component/loki/source/file/internal/tailv2/{tailer_test.go => file_test.go} (58%) diff --git a/internal/component/loki/source/file/internal/tailv2/tailer.go b/internal/component/loki/source/file/internal/tailv2/file.go similarity index 68% rename from internal/component/loki/source/file/internal/tailv2/tailer.go rename to internal/component/loki/source/file/internal/tailv2/file.go index 82fdf720c2..b11acd4b41 100644 --- a/internal/component/loki/source/file/internal/tailv2/tailer.go +++ b/internal/component/loki/source/file/internal/tailv2/file.go @@ -18,7 +18,7 @@ import ( "github.com/grafana/alloy/internal/runtime/logging/level" ) -func NewTailer(logger log.Logger, cfg *Config) (*Tailer, error) { +func NewFile(logger log.Logger, cfg *Config) (*File, error) { f, err := fileext.OpenFile(cfg.Filename) if err != nil { return nil, err @@ -38,7 +38,7 @@ func NewTailer(logger log.Logger, cfg *Config) (*Tailer, error) { ctx, cancel := context.WithCancel(context.Background()) - return &Tailer{ + return &File{ cfg: cfg, logger: logger, file: f, @@ -49,7 +49,7 @@ func NewTailer(logger log.Logger, cfg *Config) (*Tailer, error) { }, nil } -type Tailer struct { +type File struct { cfg *Config logger log.Logger @@ -66,15 +66,15 @@ type Tailer struct { } // FIXME: need clear exit signal -func (t *Tailer) Next() (*Line, error) { - t.mu.Lock() - defer t.mu.Unlock() +func (f *File) Next() (*Line, error) { + f.mu.Lock() + defer f.mu.Unlock() read: - text, err := t.readLine() + text, err := f.readLine() if err != nil { if errors.Is(err, io.EOF) { - if err := t.wait(text != ""); err != nil { + if err := f.wait(text != ""); err != nil { return nil, err } goto read @@ -82,12 +82,12 @@ read: return nil, err } - offset, err := t.offset() + offset, err := f.offset() if err != nil { return nil, err } - t.lastOffset = offset + f.lastOffset = offset return &Line{ Text: text, @@ -96,54 +96,54 @@ read: }, nil } -func (t *Tailer) Size() (int64, error) { - t.mu.Lock() - defer t.mu.Unlock() +func (f *File) Size() (int64, error) { + f.mu.Lock() + defer f.mu.Unlock() - fi, err := t.file.Stat() + fi, err := f.file.Stat() if err != nil { return 0, err } return fi.Size(), nil } -func (t *Tailer) Stop() error { - t.cancel() - t.mu.Lock() - defer t.mu.Unlock() - return t.file.Close() +func (f *File) Stop() error { + f.cancel() + f.mu.Lock() + defer f.mu.Unlock() + return f.file.Close() } -func (t *Tailer) wait(partial bool) error { - offset, err := t.offset() +func (f *File) wait(partial bool) error { + offset, err := f.offset() if err != nil { return err } - event, err := t.watcher.blockUntilEvent(t.ctx, t.file, offset) + event, err := f.watcher.blockUntilEvent(f.ctx, f.file, offset) switch event { case eventModified: if partial { // We need to reset to last succeful offset because we could have consumed a partial line. - t.file.Seek(t.lastOffset, io.SeekStart) - t.reader.Reset(t.file) + f.file.Seek(f.lastOffset, io.SeekStart) + f.reader.Reset(f.file) } return nil case eventTruncated: // We need to reopen the file when it was truncated. - return t.reopen(true) + return f.reopen(true) case eventDeleted: // In polling mode we could miss events when a file is deleted, so before we give up // we try to reopen the file. - return t.reopen(false) + return f.reopen(false) default: return err } } -func (t *Tailer) readLine() (string, error) { - line, err := t.reader.ReadString('\n') +func (f *File) readLine() (string, error) { + line, err := f.reader.ReadString('\n') if err != nil { return line, err } @@ -154,53 +154,53 @@ func (t *Tailer) readLine() (string, error) { return line, err } -func (t *Tailer) offset() (int64, error) { - offset, err := t.file.Seek(0, io.SeekCurrent) +func (f *File) offset() (int64, error) { + offset, err := f.file.Seek(0, io.SeekCurrent) if err != nil { return 0, err } - return offset - int64(t.reader.Buffered()), nil + return offset - int64(f.reader.Buffered()), nil } -func (t *Tailer) reopen(truncated bool) error { +func (f *File) reopen(truncated bool) error { // There are cases where the file is reopened so quickly it's still the same file // which causes the poller to hang on an open file handle to a file no longer being written to // and which eventually gets deleted. Save the current file handle info to make sure we only // start tailing a different file. - cf, err := t.file.Stat() + cf, err := f.file.Stat() if !truncated && err != nil { // We don't action on this error but are logging it, not expecting to see it happen and not sure if we // need to action on it, cf is checked for nil later on to accommodate this - level.Debug(t.logger).Log("msg", "stat of old file returned, this is not expected and may result in unexpected behavior") + level.Debug(f.logger).Log("msg", "stat of old file returned, this is not expected and may result in unexpected behavior") } - t.file.Close() + f.file.Close() - backoff := backoff.New(t.ctx, backoff.Config{ + backoff := backoff.New(f.ctx, backoff.Config{ MinBackoff: DefaultWatcherConfig.MaxPollFrequency, MaxBackoff: DefaultWatcherConfig.MaxPollFrequency, MaxRetries: 20, }) for backoff.Ongoing() { - file, err := fileext.OpenFile(t.cfg.Filename) + file, err := fileext.OpenFile(f.cfg.Filename) if err != nil { if os.IsNotExist(err) { - level.Debug(t.logger).Log("msg", fmt.Sprintf("waiting for %s to appear...", t.cfg.Filename)) - if err := t.watcher.blockUntilExists(t.ctx); err != nil { - return fmt.Errorf("failed to detect creation of %s: %w", t.cfg.Filename, err) + level.Debug(f.logger).Log("msg", fmt.Sprintf("waiting for %s to appear...", f.cfg.Filename)) + if err := f.watcher.blockUntilExists(f.ctx); err != nil { + return fmt.Errorf("failed to detect creation of %s: %w", f.cfg.Filename, err) } backoff.Wait() continue } - return fmt.Errorf("Unable to open file %s: %s", t.cfg.Filename, err) + return fmt.Errorf("Unable to open file %s: %s", f.cfg.Filename, err) } // File exists and is opened, get information about it. nf, err := file.Stat() if err != nil { - level.Debug(t.logger).Log("msg", "failed to stat new file to be tailed, will try to open it again") + level.Debug(f.logger).Log("msg", "failed to stat new file to be tailed, will try to open it again") file.Close() backoff.Wait() continue @@ -213,8 +213,8 @@ func (t *Tailer) reopen(truncated bool) error { continue } - t.file = file - t.reader.Reset(t.file) + f.file = file + f.reader.Reset(f.file) break } diff --git a/internal/component/loki/source/file/internal/tailv2/tailer_test.go b/internal/component/loki/source/file/internal/tailv2/file_test.go similarity index 58% rename from internal/component/loki/source/file/internal/tailv2/tailer_test.go rename to internal/component/loki/source/file/internal/tailv2/file_test.go index 572047d6c8..93feee6bbe 100644 --- a/internal/component/loki/source/file/internal/tailv2/tailer_test.go +++ b/internal/component/loki/source/file/internal/tailv2/file_test.go @@ -12,10 +12,10 @@ import ( "golang.org/x/text/encoding/unicode" ) -func TestTailTailer(t *testing.T) { - verify := func(t *testing.T, tailer *Tailer, expectedLine *Line, expectedErr error) { +func TestFile(t *testing.T) { + verify := func(t *testing.T, f *File, expectedLine *Line, expectedErr error) { t.Helper() - line, err := tailer.Next() + line, err := f.Next() require.ErrorIs(t, err, expectedErr) if expectedLine == nil { require.Nil(t, line) @@ -26,7 +26,7 @@ func TestTailTailer(t *testing.T) { } t.Run("file must exist", func(t *testing.T) { - _, err := NewTailer(log.NewNopLogger(), &Config{ + _, err := NewFile(log.NewNopLogger(), &Config{ Filename: "/no/such/file", }) require.ErrorIs(t, err, os.ErrNotExist) @@ -34,7 +34,7 @@ func TestTailTailer(t *testing.T) { name := createFile(t, "exists", "") defer removeFile(t, name) - _, err = NewTailer(log.NewNopLogger(), &Config{ + _, err = NewFile(log.NewNopLogger(), &Config{ Filename: name, }) require.NoError(t, err) @@ -46,16 +46,16 @@ func TestTailTailer(t *testing.T) { name := createFile(t, "over4096", "test\n"+testString+"\nhello\nworld\n") defer removeFile(t, name) - tailer, err := NewTailer(log.NewNopLogger(), &Config{ + file, err := NewFile(log.NewNopLogger(), &Config{ Filename: name, }) require.NoError(t, err) - defer tailer.Stop() + defer file.Stop() - verify(t, tailer, &Line{Text: "test", Offset: 5}, nil) - verify(t, tailer, &Line{Text: testString, Offset: 4104}, nil) - verify(t, tailer, &Line{Text: "hello", Offset: 4110}, nil) - verify(t, tailer, &Line{Text: "world", Offset: 4116}, nil) + verify(t, file, &Line{Text: "test", Offset: 5}, nil) + verify(t, file, &Line{Text: testString, Offset: 4104}, nil) + verify(t, file, &Line{Text: "hello", Offset: 4110}, nil) + verify(t, file, &Line{Text: "world", Offset: 4116}, nil) }) t.Run("read", func(t *testing.T) { @@ -69,39 +69,39 @@ func TestTailTailer(t *testing.T) { ) t.Run("start", func(t *testing.T) { - tailer, err := NewTailer(log.NewNopLogger(), &Config{ + file, err := NewFile(log.NewNopLogger(), &Config{ Filename: name, Offset: 0, }) require.NoError(t, err) - defer tailer.Stop() + defer file.Stop() - verify(t, tailer, &Line{Text: "hello", Offset: first}, nil) - verify(t, tailer, &Line{Text: "world", Offset: middle}, nil) - verify(t, tailer, &Line{Text: "test", Offset: end}, nil) + verify(t, file, &Line{Text: "hello", Offset: first}, nil) + verify(t, file, &Line{Text: "world", Offset: middle}, nil) + verify(t, file, &Line{Text: "test", Offset: end}, nil) }) t.Run("skip first", func(t *testing.T) { - tailer, err := NewTailer(log.NewNopLogger(), &Config{ + file, err := NewFile(log.NewNopLogger(), &Config{ Filename: name, Offset: first, }) require.NoError(t, err) - defer tailer.Stop() + defer file.Stop() - verify(t, tailer, &Line{Text: "world", Offset: middle}, nil) - verify(t, tailer, &Line{Text: "test", Offset: end}, nil) + verify(t, file, &Line{Text: "world", Offset: middle}, nil) + verify(t, file, &Line{Text: "test", Offset: end}, nil) }) t.Run("last", func(t *testing.T) { - tailer, err := NewTailer(log.NewNopLogger(), &Config{ + file, err := NewFile(log.NewNopLogger(), &Config{ Filename: name, Offset: middle, }) require.NoError(t, err) - defer tailer.Stop() + defer file.Stop() - verify(t, tailer, &Line{Text: "test", Offset: end}, nil) + verify(t, file, &Line{Text: "test", Offset: end}, nil) }) }) @@ -109,26 +109,26 @@ func TestTailTailer(t *testing.T) { name := createFile(t, "partial", "hello\nwo") defer removeFile(t, name) - tailer, err := NewTailer(log.NewNopLogger(), &Config{ + file, err := NewFile(log.NewNopLogger(), &Config{ Offset: 0, Filename: name, }) require.NoError(t, err) - defer tailer.Stop() + defer file.Stop() - verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) + verify(t, file, &Line{Text: "hello", Offset: 6}, nil) go func() { time.Sleep(50 * time.Millisecond) appendToFile(t, name, "rld\n") }() - verify(t, tailer, &Line{Text: "world", Offset: 12}, nil) + verify(t, file, &Line{Text: "world", Offset: 12}, nil) }) t.Run("truncate", func(t *testing.T) { name := createFile(t, "truncate", "a really long string goes here\nhello\nworld\n") defer removeFile(t, name) - tailer, err := NewTailer(log.NewNopLogger(), &Config{ + file, err := NewFile(log.NewNopLogger(), &Config{ Filename: name, WatcherConfig: WatcherConfig{ MinPollFrequency: 5 * time.Millisecond, @@ -136,11 +136,11 @@ func TestTailTailer(t *testing.T) { }, }) require.NoError(t, err) - defer tailer.Stop() + defer file.Stop() - verify(t, tailer, &Line{Text: "a really long string goes here", Offset: 31}, nil) - verify(t, tailer, &Line{Text: "hello", Offset: 37}, nil) - verify(t, tailer, &Line{Text: "world", Offset: 43}, nil) + verify(t, file, &Line{Text: "a really long string goes here", Offset: 31}, nil) + verify(t, file, &Line{Text: "hello", Offset: 37}, nil) + verify(t, file, &Line{Text: "world", Offset: 43}, nil) go func() { // truncate now @@ -148,9 +148,9 @@ func TestTailTailer(t *testing.T) { truncateFile(t, name, "h311o\nw0r1d\nendofworld\n") }() - verify(t, tailer, &Line{Text: "h311o", Offset: 6}, nil) - verify(t, tailer, &Line{Text: "w0r1d", Offset: 12}, nil) - verify(t, tailer, &Line{Text: "endofworld", Offset: 23}, nil) + verify(t, file, &Line{Text: "h311o", Offset: 6}, nil) + verify(t, file, &Line{Text: "w0r1d", Offset: 12}, nil) + verify(t, file, &Line{Text: "endofworld", Offset: 23}, nil) }) @@ -158,20 +158,20 @@ func TestTailTailer(t *testing.T) { name := createFile(t, "stopped", "hello\n") defer removeFile(t, name) - tailer, err := NewTailer(log.NewNopLogger(), &Config{ + file, err := NewFile(log.NewNopLogger(), &Config{ Offset: 0, Filename: name, }) require.NoError(t, err) - verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) + verify(t, file, &Line{Text: "hello", Offset: 6}, nil) go func() { time.Sleep(100 * time.Millisecond) - require.NoError(t, tailer.Stop()) + require.NoError(t, file.Stop()) }() - _, err = tailer.Next() + _, err = file.Next() require.ErrorIs(t, err, context.Canceled) }) @@ -179,7 +179,7 @@ func TestTailTailer(t *testing.T) { name := createFile(t, "removed", "hello\n") defer removeFile(t, name) - tailer, err := NewTailer(log.NewNopLogger(), &Config{ + file, err := NewFile(log.NewNopLogger(), &Config{ Offset: 0, Filename: name, WatcherConfig: WatcherConfig{ @@ -188,9 +188,9 @@ func TestTailTailer(t *testing.T) { }, }) require.NoError(t, err) - defer tailer.Stop() + defer file.Stop() - verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) + verify(t, file, &Line{Text: "hello", Offset: 6}, nil) go func() { time.Sleep(50 * time.Millisecond) @@ -199,13 +199,13 @@ func TestTailTailer(t *testing.T) { recreateFile(t, name, "new\n") }() - verify(t, tailer, &Line{Text: "new", Offset: 4}, nil) + verify(t, file, &Line{Text: "new", Offset: 4}, nil) }) t.Run("stopped while waiting for file to be created", func(t *testing.T) { name := createFile(t, "removed", "hello\n") - tailer, err := NewTailer(log.NewNopLogger(), &Config{ + file, err := NewFile(log.NewNopLogger(), &Config{ Offset: 0, Filename: name, WatcherConfig: WatcherConfig{ @@ -215,33 +215,33 @@ func TestTailTailer(t *testing.T) { }) require.NoError(t, err) - verify(t, tailer, &Line{Text: "hello", Offset: 6}, nil) + verify(t, file, &Line{Text: "hello", Offset: 6}, nil) removeFile(t, name) go func() { time.Sleep(100 * time.Millisecond) - tailer.Stop() + file.Stop() }() - _, err = tailer.Next() + _, err = file.Next() require.ErrorIs(t, err, context.Canceled) }) t.Run("UTF-16LE", func(t *testing.T) { - tailer, err := NewTailer(log.NewNopLogger(), &Config{ + file, err := NewFile(log.NewNopLogger(), &Config{ Filename: "testdata/mssql.log", Decoder: unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM).NewDecoder(), }) require.NoError(t, err) - defer tailer.Stop() - - verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.58 Server Microsoft SQL Server 2019 (RTM) - 15.0.2000.5 (X64) ", Offset: 528}, nil) - verify(t, tailer, &Line{Text: " Sep 24 2019 13:48:23 ", Offset: 552}, nil) - verify(t, tailer, &Line{Text: " Copyright (C) 2019 Microsoft Corporation", Offset: 595}, nil) - verify(t, tailer, &Line{Text: " Enterprise Edition (64-bit) on Windows Server 2022 Standard 10.0 (Build 20348: ) (Hypervisor)", Offset: 697}, nil) - verify(t, tailer, &Line{Text: "", Offset: 699}, nil) - verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.71 Server UTC adjustment: 1:00", Offset: 756}, nil) - verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.71 Server (c) Microsoft Corporation.", Offset: 819}, nil) - verify(t, tailer, &Line{Text: "2025-03-11 11:11:02.72 Server All rights reserved.", Offset: 876}, nil) + defer file.Stop() + + verify(t, file, &Line{Text: "2025-03-11 11:11:02.58 Server Microsoft SQL Server 2019 (RTM) - 15.0.2000.5 (X64) ", Offset: 528}, nil) + verify(t, file, &Line{Text: " Sep 24 2019 13:48:23 ", Offset: 552}, nil) + verify(t, file, &Line{Text: " Copyright (C) 2019 Microsoft Corporation", Offset: 595}, nil) + verify(t, file, &Line{Text: " Enterprise Edition (64-bit) on Windows Server 2022 Standard 10.0 (Build 20348: ) (Hypervisor)", Offset: 697}, nil) + verify(t, file, &Line{Text: "", Offset: 699}, nil) + verify(t, file, &Line{Text: "2025-03-11 11:11:02.71 Server UTC adjustment: 1:00", Offset: 756}, nil) + verify(t, file, &Line{Text: "2025-03-11 11:11:02.71 Server (c) Microsoft Corporation.", Offset: 819}, nil) + verify(t, file, &Line{Text: "2025-03-11 11:11:02.72 Server All rights reserved.", Offset: 876}, nil) }) } diff --git a/internal/component/loki/source/file/tailer.go b/internal/component/loki/source/file/tailer.go index 738dc5b740..8bb00243f9 100644 --- a/internal/component/loki/source/file/tailer.go +++ b/internal/component/loki/source/file/tailer.go @@ -49,7 +49,7 @@ type tailer struct { report sync.Once - tail *tailv2.Tailer + file *tailv2.File decoder *encoding.Decoder } @@ -242,7 +242,7 @@ func (t *tailer) initRun() (loki.EntryHandler, error) { } } - tail, err := tailv2.NewTailer(t.logger, &tailv2.Config{ + tail, err := tailv2.NewFile(t.logger, &tailv2.Config{ Filename: t.key.Path, Offset: pos, Decoder: t.decoder, @@ -253,7 +253,7 @@ func (t *tailer) initRun() (loki.EntryHandler, error) { return nil, fmt.Errorf("failed to tail the file: %w", err) } - t.tail = tail + t.file = tail labelsMiddleware := t.labels.Merge(model.LabelSet{labelFilename: model.LabelValue(t.key.Path)}) handler := loki.AddLabelsMiddleware(labelsMiddleware).Wrap(loki.NewEntryHandler(t.receiver.Chan(), func() {})) @@ -290,13 +290,13 @@ func (t *tailer) readLines(handler loki.EntryHandler, done chan struct{}) { defer func() { level.Info(t.logger).Log("msg", "tail routine: exited", "path", t.key.Path) - size, _ := t.tail.Size() + size, _ := t.file.Size() t.updateStats(lastOffset, size) close(done) }() for { - line, err := t.tail.Next() + line, err := t.file.Next() if err != nil { // Maybe we should use a better signal than context canceled to indicate normal stop... if !errors.Is(err, context.Canceled) { @@ -319,7 +319,7 @@ func (t *tailer) readLines(handler loki.EntryHandler, done chan struct{}) { lastOffset = line.Offset if time.Since(lastUpdatedPosition) >= positionInterval { lastUpdatedPosition = time.Now() - size, _ := t.tail.Size() + size, _ := t.file.Size() t.updateStats(lastOffset, size) } } @@ -334,7 +334,7 @@ func (t *tailer) updateStats(offset int64, size int64) { } func (t *tailer) stop(done chan struct{}) { - if err := t.tail.Stop(); err != nil { + if err := t.file.Stop(); err != nil { if util.IsEphemeralOrFileClosed(err) { // Don't log as error if the file is already closed, or we got an ephemeral error - it's a common case // when files are rotating while being read and the tailer would have stopped correctly anyway. From 73829d1ecc344fb83001dd9bc24be586fb17f960 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Tue, 2 Dec 2025 10:50:22 +0100 Subject: [PATCH 05/28] remove old package and rename --- .../loki/source/file/internal/tail/README.md | 32 -- .../file/internal/{tailv2 => tail}/config.go | 2 +- .../file/internal/{tailv2 => tail}/file.go | 4 +- .../internal/{tailv2 => tail}/file_test.go | 2 +- .../{tailv2 => tail}/fileext/file_posix.go | 0 .../{tailv2 => tail}/fileext/file_windows.go | 0 .../file/internal/{tailv2 => tail}/line.go | 2 +- .../loki/source/file/internal/tail/tail.go | 401 ------------------ .../source/file/internal/tail/tail_posix.go | 24 -- .../source/file/internal/tail/tail_test.go | 285 ------------- .../source/file/internal/tail/tail_windows.go | 13 - .../source/file/internal/tail/util/util.go | 48 --- .../file/internal/{tailv2 => tail}/watch.go | 4 +- .../file/internal/tail/watch/file_posix.go | 9 - .../file/internal/tail/watch/file_windows.go | 47 -- .../file/internal/tail/watch/filechanges.go | 36 -- .../file/internal/tail/watch/polling.go | 227 ---------- .../source/file/internal/tail/watch/watch.go | 25 -- .../source/file/internal/tail/watch_test.go | 1 + .../file/internal/tail/winfile/winfile.go | 92 ---- .../file/internal/tailv2/testdata/mssql.log | Bin 876 -> 0 bytes internal/component/loki/source/file/tailer.go | 13 +- 22 files changed, 15 insertions(+), 1252 deletions(-) delete mode 100644 internal/component/loki/source/file/internal/tail/README.md rename internal/component/loki/source/file/internal/{tailv2 => tail}/config.go (98%) rename internal/component/loki/source/file/internal/{tailv2 => tail}/file.go (99%) rename internal/component/loki/source/file/internal/{tailv2 => tail}/file_test.go (99%) rename internal/component/loki/source/file/internal/{tailv2 => tail}/fileext/file_posix.go (100%) rename internal/component/loki/source/file/internal/{tailv2 => tail}/fileext/file_windows.go (100%) rename internal/component/loki/source/file/internal/{tailv2 => tail}/line.go (84%) delete mode 100644 internal/component/loki/source/file/internal/tail/tail.go delete mode 100644 internal/component/loki/source/file/internal/tail/tail_posix.go delete mode 100644 internal/component/loki/source/file/internal/tail/tail_test.go delete mode 100644 internal/component/loki/source/file/internal/tail/tail_windows.go delete mode 100644 internal/component/loki/source/file/internal/tail/util/util.go rename internal/component/loki/source/file/internal/{tailv2 => tail}/watch.go (98%) delete mode 100644 internal/component/loki/source/file/internal/tail/watch/file_posix.go delete mode 100644 internal/component/loki/source/file/internal/tail/watch/file_windows.go delete mode 100644 internal/component/loki/source/file/internal/tail/watch/filechanges.go delete mode 100644 internal/component/loki/source/file/internal/tail/watch/polling.go delete mode 100644 internal/component/loki/source/file/internal/tail/watch/watch.go create mode 100644 internal/component/loki/source/file/internal/tail/watch_test.go delete mode 100644 internal/component/loki/source/file/internal/tail/winfile/winfile.go delete mode 100644 internal/component/loki/source/file/internal/tailv2/testdata/mssql.log diff --git a/internal/component/loki/source/file/internal/tail/README.md b/internal/component/loki/source/file/internal/tail/README.md deleted file mode 100644 index a387d1da7b..0000000000 --- a/internal/component/loki/source/file/internal/tail/README.md +++ /dev/null @@ -1,32 +0,0 @@ - -**NOTE**: This is a fork of https://github.com/grafana/tail, which is a fork of https://github.com/hpcloud/tail. -The `grafana/tail` repo is no longer mainained because the Loki team has deprecated the Promtail project. -It is easier for the Alloy team to maintain this tail package inside the Alloy repo than to have a separate repository for it. - -Use outside of that context is not tested or supported. - -# Go package for tail-ing files - -A Go package striving to emulate the features of the BSD `tail` program. - -```Go -t, err := tail.TailFile("/var/log/nginx.log", tail.Config{Follow: true}) -for line := range t.Lines { - fmt.Println(line.Text) -} -``` - -See [API documentation](http://godoc.org/github.com/hpcloud/tail). - -## Log rotation - -Tail comes with full support for truncation/move detection as it is -designed to work with log rotation tools. - -## Installing - - go get github.com/hpcloud/tail/... - -## Windows support - -This package [needs assistance](https://github.com/hpcloud/tail/labels/Windows) for full Windows support. diff --git a/internal/component/loki/source/file/internal/tailv2/config.go b/internal/component/loki/source/file/internal/tail/config.go similarity index 98% rename from internal/component/loki/source/file/internal/tailv2/config.go rename to internal/component/loki/source/file/internal/tail/config.go index 81a9136894..20b0ca763f 100644 --- a/internal/component/loki/source/file/internal/tailv2/config.go +++ b/internal/component/loki/source/file/internal/tail/config.go @@ -1,4 +1,4 @@ -package tailv2 +package tail import ( "time" diff --git a/internal/component/loki/source/file/internal/tailv2/file.go b/internal/component/loki/source/file/internal/tail/file.go similarity index 99% rename from internal/component/loki/source/file/internal/tailv2/file.go rename to internal/component/loki/source/file/internal/tail/file.go index b11acd4b41..687dbd713a 100644 --- a/internal/component/loki/source/file/internal/tailv2/file.go +++ b/internal/component/loki/source/file/internal/tail/file.go @@ -1,4 +1,4 @@ -package tailv2 +package tail import ( "bufio" @@ -14,7 +14,7 @@ import ( "github.com/go-kit/log" "github.com/grafana/dskit/backoff" - "github.com/grafana/alloy/internal/component/loki/source/file/internal/tailv2/fileext" + "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/fileext" "github.com/grafana/alloy/internal/runtime/logging/level" ) diff --git a/internal/component/loki/source/file/internal/tailv2/file_test.go b/internal/component/loki/source/file/internal/tail/file_test.go similarity index 99% rename from internal/component/loki/source/file/internal/tailv2/file_test.go rename to internal/component/loki/source/file/internal/tail/file_test.go index 93feee6bbe..cb167afb71 100644 --- a/internal/component/loki/source/file/internal/tailv2/file_test.go +++ b/internal/component/loki/source/file/internal/tail/file_test.go @@ -1,4 +1,4 @@ -package tailv2 +package tail import ( "context" diff --git a/internal/component/loki/source/file/internal/tailv2/fileext/file_posix.go b/internal/component/loki/source/file/internal/tail/fileext/file_posix.go similarity index 100% rename from internal/component/loki/source/file/internal/tailv2/fileext/file_posix.go rename to internal/component/loki/source/file/internal/tail/fileext/file_posix.go diff --git a/internal/component/loki/source/file/internal/tailv2/fileext/file_windows.go b/internal/component/loki/source/file/internal/tail/fileext/file_windows.go similarity index 100% rename from internal/component/loki/source/file/internal/tailv2/fileext/file_windows.go rename to internal/component/loki/source/file/internal/tail/fileext/file_windows.go diff --git a/internal/component/loki/source/file/internal/tailv2/line.go b/internal/component/loki/source/file/internal/tail/line.go similarity index 84% rename from internal/component/loki/source/file/internal/tailv2/line.go rename to internal/component/loki/source/file/internal/tail/line.go index fbf4fffde5..4c4f8fc32f 100644 --- a/internal/component/loki/source/file/internal/tailv2/line.go +++ b/internal/component/loki/source/file/internal/tail/line.go @@ -1,4 +1,4 @@ -package tailv2 +package tail import "time" diff --git a/internal/component/loki/source/file/internal/tail/tail.go b/internal/component/loki/source/file/internal/tail/tail.go deleted file mode 100644 index 1eef9ac878..0000000000 --- a/internal/component/loki/source/file/internal/tail/tail.go +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright (c) 2015 HPE Software Inc. All rights reserved. -// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. - -package tail - -import ( - "bufio" - "errors" - "fmt" - "io" - "os" - "strings" - "sync" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "golang.org/x/text/encoding" - "gopkg.in/tomb.v1" - - "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/watch" -) - -var ( - ErrStop = errors.New("tail should now stop") -) - -type Line struct { - Text string - Time time.Time -} - -// SeekInfo represents arguments to `os.Seek` -type SeekInfo struct { - Offset int64 - Whence int // os.SEEK_* -} - -// Config is used to specify how a file must be tailed. -type Config struct { - Logger log.Logger - // Seek to this location before tailing - Location *SeekInfo - PollOptions watch.PollingFileWatcherOptions - - // Change the decoder if the file is not UTF-8. - // If the tailer doesn't use the right decoding, the output text may be gibberish. - // For example, if the file is "UTF-16 LE" encoded, the tailer would not separate - // the new lines properly and the output could come out as chinese characters. - Decoder *encoding.Decoder -} - -type Tail struct { - Filename string - Lines chan *Line - Config - - fileMut sync.Mutex - file *os.File - - readerMut sync.Mutex - reader *bufio.Reader - - watcher watch.FileWatcher - changes *watch.FileChanges - - tomb.Tomb // provides: Done, Kill, Dying -} - -// TailFile begins tailing the file. Output stream is made available -// via the `Tail.Lines` channel. To handle errors during tailing, -// invoke the `Wait` or `Err` method after finishing reading from the -// `Lines` channel. -func TailFile(filename string, config Config) (*Tail, error) { - t := &Tail{ - Filename: filename, - Lines: make(chan *Line), - Config: config, - } - - // when Logger was not specified in config, use default logger - if t.Logger == nil { - t.Logger = log.NewNopLogger() - } - - var err error - t.watcher, err = watch.NewPollingFileWatcher(filename, config.PollOptions) - if err != nil { - return nil, err - } - - t.file, err = OpenFile(t.Filename) - if err != nil { - return nil, err - } - - // Seek to requested location. - if t.Location != nil { - _, err := t.file.Seek(t.Location.Offset, t.Location.Whence) - if err != nil { - return nil, err - } - } - - t.watcher.SetFile(t.file) - - t.reader = t.getReader() - - go t.tailFileSync() - - return t, nil -} - -// Return the file's current position, like stdio's ftell(). -// But this value is not very accurate. -// it may readed one line in the chan(tail.Lines), -// so it may lost one line. -func (tail *Tail) Tell() (int64, error) { - tail.fileMut.Lock() - if tail.file == nil { - tail.fileMut.Unlock() - return 0, os.ErrNotExist - } - offset, err := tail.file.Seek(0, io.SeekCurrent) - tail.fileMut.Unlock() - if err != nil { - return 0, err - } - - tail.readerMut.Lock() - defer tail.readerMut.Unlock() - if tail.reader == nil { - return 0, nil - } - - offset -= int64(tail.reader.Buffered()) - return offset, nil -} - -// Size returns the length in bytes of the file being tailed, -// or 0 with an error if there was an error Stat'ing the file. -func (tail *Tail) Size() (int64, error) { - tail.fileMut.Lock() - f := tail.file - if f == nil { - tail.fileMut.Unlock() - return 0, os.ErrNotExist - } - fi, err := f.Stat() - tail.fileMut.Unlock() - - if err != nil { - return 0, err - } - size := fi.Size() - return size, nil -} - -// Stop stops the tailing activity. -func (tail *Tail) Stop() error { - tail.Kill(nil) - return tail.Wait() -} - -func (tail *Tail) close() { - close(tail.Lines) - tail.closeFile() -} - -func (tail *Tail) closeFile() { - tail.fileMut.Lock() - defer tail.fileMut.Unlock() - if tail.file != nil { - tail.file.Close() - tail.file = nil - } -} - -func (tail *Tail) reopen(truncated bool) error { - // There are cases where the file is reopened so quickly it's still the same file - // which causes the poller to hang on an open file handle to a file no longer being written to - // and which eventually gets deleted. Save the current file handle info to make sure we only - // start tailing a different file. - cf, err := tail.file.Stat() - if !truncated && err != nil { - level.Debug(tail.Logger).Log("msg", "stat of old file returned, this is not expected and may result in unexpected behavior") - // We don't action on this error but are logging it, not expecting to see it happen and not sure if we - // need to action on it, cf is checked for nil later on to accommodate this - } - - tail.closeFile() - retries := 20 - for { - var err error - tail.fileMut.Lock() - tail.file, err = OpenFile(tail.Filename) - tail.watcher.SetFile(tail.file) - tail.fileMut.Unlock() - if err != nil { - if os.IsNotExist(err) { - level.Debug(tail.Logger).Log("msg", fmt.Sprintf("Waiting for %s to appear...", tail.Filename)) - if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil { - if err == tomb.ErrDying { - return err - } - return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err) - } - continue - } - return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err) - } - - // File exists and is opened, get information about it. - nf, err := tail.file.Stat() - if err != nil { - level.Debug(tail.Logger).Log("msg", "Failed to stat new file to be tailed, will try to open it again") - tail.closeFile() - continue - } - - // Check to see if we are trying to reopen and tail the exact same file (and it was not truncated). - if !truncated && cf != nil && os.SameFile(cf, nf) { - retries-- - if retries <= 0 { - return errors.New("gave up trying to reopen log file with a different handle") - } - - select { - case <-time.After(watch.DefaultPollingFileWatcherOptions.MaxPollFrequency): - tail.closeFile() - continue - case <-tail.Tomb.Dying(): - return tomb.ErrDying - } - } - break - } - return nil -} - -func (tail *Tail) readLine() (string, error) { - tail.readerMut.Lock() - line, err := tail.reader.ReadString('\n') - tail.readerMut.Unlock() - if err != nil { - // Note ReadString "returns the data read before the error" in - // case of an error, including EOF, so we return it as is. The - // caller is expected to process it if err is EOF. - return line, err - } - - line = strings.TrimRight(line, "\n") - - // Trim Windows line endings - line = strings.TrimSuffix(line, "\r") - - return line, err -} - -func (tail *Tail) tailFileSync() { - defer tail.Done() - defer tail.close() - - var ( - err error - offset int64 - oneMoreRun bool - ) - - // Read line by line. - for { - // grab the position in case we need to back up in the event of a half-line - offset, err = tail.Tell() - if err != nil { - tail.Kill(err) - return - } - - line, err := tail.readLine() - - // Process `line` even if err is EOF. - switch err { - case nil: - select { - case tail.Lines <- &Line{line, time.Now()}: - case <-tail.Dying(): - return - } - - case io.EOF: - if line != "" { - // this has the potential to never return the last line if - // it's not followed by a newline; seems a fair trade here - err := tail.seekTo(SeekInfo{Offset: offset, Whence: 0}) - if err != nil { - tail.Kill(err) - return - } - } - - // oneMoreRun is set true when a file is deleted, - // this is to catch events which might get missed in polling mode. - // now that the last run is completed, finish deleting the file - if oneMoreRun { - oneMoreRun = false - err = tail.finishDelete() - if err != nil { - if err != ErrStop { - tail.Kill(err) - } - return - } - } - - // When EOF is reached, wait for more data to become available. - oneMoreRun, err = tail.waitForChanges() - if err != nil { - if err != ErrStop { - tail.Kill(err) - } - return - } - default: - // non-EOF error - tail.Killf("Error reading %s: %s", tail.Filename, err) - return - } - } -} - -// waitForChanges waits until the file has been appended, deleted, -// moved or truncated. When moved or deleted - the file will be -// reopened if ReOpen is true. Truncated files are always reopened. -func (tail *Tail) waitForChanges() (bool, error) { - if tail.changes == nil { - pos, err := tail.file.Seek(0, io.SeekCurrent) - if err != nil { - return false, err - } - tail.changes, err = tail.watcher.ChangeEvents(&tail.Tomb, pos) - if err != nil { - return false, err - } - } - - select { - case <-tail.changes.Modified: - return false, nil - case <-tail.changes.Deleted: - // In polling mode we could miss events when a file is deleted, so before we give up our file handle - // run the poll one more time to catch anything we may have missed since the last poll. - return true, nil - case <-tail.changes.Truncated: - // Always reopen truncated files. - level.Debug(tail.Logger).Log("msg", fmt.Sprintf("Re-opening truncated file %s ...", tail.Filename)) - if err := tail.reopen(true); err != nil { - return false, err - } - level.Debug(tail.Logger).Log("msg", fmt.Sprintf("Successfully reopened truncated %s", tail.Filename)) - tail.readerMut.Lock() - tail.reader = tail.getReader() - tail.readerMut.Unlock() - return false, nil - case <-tail.Dying(): - return false, ErrStop - } -} - -func (tail *Tail) finishDelete() error { - tail.changes = nil - level.Debug(tail.Logger).Log("msg", fmt.Sprintf("Re-opening moved/deleted file %s ...", tail.Filename)) - if err := tail.reopen(false); err != nil { - return err - } - level.Debug(tail.Logger).Log("msg", fmt.Sprintf("Successfully reopened %s", tail.Filename)) - - tail.readerMut.Lock() - tail.reader = tail.getReader() - tail.readerMut.Unlock() - return nil -} - -func (tail *Tail) getReader() *bufio.Reader { - if tail.Decoder != nil { - return bufio.NewReader(tail.Decoder.Reader(tail.file)) - } else { - return bufio.NewReader(tail.file) - } -} - -func (tail *Tail) seekTo(pos SeekInfo) error { - _, err := tail.file.Seek(pos.Offset, pos.Whence) - if err != nil { - return fmt.Errorf("Seek error on %s: %s", tail.Filename, err) - } - // Reset the read buffer whenever the file is re-seek'ed - tail.readerMut.Lock() - tail.reader.Reset(tail.file) - tail.readerMut.Unlock() - return nil -} diff --git a/internal/component/loki/source/file/internal/tail/tail_posix.go b/internal/component/loki/source/file/internal/tail/tail_posix.go deleted file mode 100644 index 871dc1b288..0000000000 --- a/internal/component/loki/source/file/internal/tail/tail_posix.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build linux || darwin || freebsd || netbsd || openbsd - -package tail - -import ( - "os" - "path/filepath" -) - -func OpenFile(name string) (file *os.File, err error) { - filename := name - // Check if the path requested is a symbolic link - fi, err := os.Lstat(name) - if err != nil { - return nil, err - } - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - filename, err = filepath.EvalSymlinks(name) - if err != nil { - return nil, err - } - } - return os.Open(filename) -} diff --git a/internal/component/loki/source/file/internal/tail/tail_test.go b/internal/component/loki/source/file/internal/tail/tail_test.go deleted file mode 100644 index 79943d81e9..0000000000 --- a/internal/component/loki/source/file/internal/tail/tail_test.go +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright (c) 2015 HPE Software Inc. All rights reserved. -// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. - -// TODO: -// * repeat all the tests with Poll:true - -package tail - -import ( - _ "fmt" - "io" - "os" - "strings" - "sync" - "testing" - "time" - - "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/watch" - "github.com/stretchr/testify/assert" - "golang.org/x/text/encoding/unicode" -) - -var testPollingOptions = watch.PollingFileWatcherOptions{ - // Use a smaller poll duration for faster test runs. Keep it below - // 100ms (which value is used as common delays for tests) - MinPollFrequency: 5 * time.Millisecond, - MaxPollFrequency: 5 * time.Millisecond, -} - -func TestTail(t *testing.T) { - verify := func(t *testing.T, tail *Tail, lines []string) { - got := make([]string, 0, len(lines)) - - var wg sync.WaitGroup - wg.Go(func() { - for { - line := <-tail.Lines - got = append(got, line.Text) - if len(got) == len(lines) { - return - } - } - }) - wg.Wait() - assert.Equal(t, lines, got) - } - - t.Run("file must exist", func(t *testing.T) { - _, err := TailFile("/no/such/file", Config{}) - assert.Error(t, err) - }) - - t.Run("should be able to stop", func(t *testing.T) { - tail, err := TailFile("README.md", Config{}) - assert.NoError(t, err) - assert.NoError(t, tail.Stop()) - }) - - t.Run("over 4096 byte line", func(t *testing.T) { - tailTest := NewTailTest("Over4096ByteLine", t) - testString := strings.Repeat("a", 4097) - tailTest.CreateFile("test.txt", "test\n"+testString+"\nhello\nworld\n") - defer tailTest.RemoveFile("test.txt") - tail := tailTest.StartTail("test.txt", Config{}) - defer tail.Stop() - - verify(t, tail, []string{"test", testString, "hello", "world"}) - }) - - t.Run("read full", func(t *testing.T) { - tailTest := NewTailTest("location-full", t) - tailTest.CreateFile("test.txt", "hello\nworld\n") - defer tailTest.RemoveFile("test.txt") - tail := tailTest.StartTail("test.txt", Config{Location: nil}) - defer tail.Stop() - - verify(t, tail, []string{"hello", "world"}) - }) - - t.Run("read end", func(t *testing.T) { - tailTest := NewTailTest("location-end", t) - tailTest.CreateFile("test.txt", "hello\nworld\n") - defer tailTest.RemoveFile("test.txt") - tail := tailTest.StartTail("test.txt", Config{Location: &SeekInfo{0, io.SeekEnd}}) - defer tail.Stop() - - go func() { - <-time.After(100 * time.Millisecond) - tailTest.AppendFile("test.txt", "more\ndata\n") - - <-time.After(100 * time.Millisecond) - tailTest.AppendFile("test.txt", "more\ndata\n") - }() - - verify(t, tail, []string{"more", "data", "more", "data"}) - }) - - t.Run("read middle", func(t *testing.T) { - tailTest := NewTailTest("location-middle", t) - tailTest.CreateFile("test.txt", "hello\nworld\n") - defer tailTest.RemoveFile("test.txt") - tail := tailTest.StartTail("test.txt", Config{Location: &SeekInfo{-6, io.SeekEnd}}) - defer tail.Stop() - - go func() { - <-time.After(100 * time.Millisecond) - tailTest.AppendFile("test.txt", "more\ndata\n") - - <-time.After(100 * time.Millisecond) - tailTest.AppendFile("test.txt", "more\ndata\n") - }() - - verify(t, tail, []string{"world", "more", "data", "more", "data"}) - }) - - t.Run("reseek", func(t *testing.T) { - tailTest := NewTailTest("reseek-polling", t) - tailTest.CreateFile("test.txt", "a really long string goes here\nhello\nworld\n") - defer tailTest.RemoveFile("test.txt") - tail := tailTest.StartTail("test.txt", Config{PollOptions: testPollingOptions}) - defer tail.Stop() - - go func() { - // truncate now - <-time.After(100 * time.Millisecond) - tailTest.TruncateFile("test.txt", "h311o\nw0r1d\nendofworld\n") - }() - - verify(t, tail, []string{"a really long string goes here", "hello", "world", "h311o", "w0r1d", "endofworld"}) - }) - - t.Run("tell", func(t *testing.T) { - tailTest := NewTailTest("tell-position", t) - tailTest.CreateFile("test.txt", "hello\nworld\nagain\nmore\n") - defer tailTest.RemoveFile("test.txt") - - tail := tailTest.StartTail("test.txt", Config{Location: &SeekInfo{0, io.SeekStart}}) - - // read one line - <-tail.Lines - offset, err := tail.Tell() - assert.NoError(t, err) - - tail.Stop() - - tail = tailTest.StartTail("test.txt", Config{Location: &SeekInfo{offset, io.SeekStart}}) - l := <-tail.Lines - assert.Equal(t, "again", l.Text) - - tail.Stop() - }) - - t.Run("UTF-16LE", func(t *testing.T) { - tail, err := TailFile("testdata/mssql.log", Config{Decoder: unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM).NewDecoder()}) - assert.NoError(t, err) - defer tail.Stop() - - expectedLines := []string{ - "2025-03-11 11:11:02.58 Server Microsoft SQL Server 2019 (RTM) - 15.0.2000.5 (X64) ", - " Sep 24 2019 13:48:23 ", - " Copyright (C) 2019 Microsoft Corporation", - " Enterprise Edition (64-bit) on Windows Server 2022 Standard 10.0 (Build 20348: ) (Hypervisor)", - "", - "2025-03-11 11:11:02.71 Server UTC adjustment: 1:00", - "2025-03-11 11:11:02.71 Server (c) Microsoft Corporation.", - "2025-03-11 11:11:02.72 Server All rights reserved.", - } - - verify(t, tail, expectedLines) - }) -} - -func TestTellRace(t *testing.T) { - tailTest := NewTailTest("tell-race", t) - tailTest.CreateFile("test.txt", "hello\nworld\n") - - tail := tailTest.StartTail("test.txt", Config{PollOptions: testPollingOptions}) - - <-tail.Lines - <-tail.Lines - - _, err := tail.Tell() - if err != nil { - t.Fatal("unexpected error", err) - } - - tailTest.TruncateFile("test.txt", "yay\nyay2\n") - - // wait for reopen to happen - time.Sleep(100 * time.Millisecond) - - _, err = tail.Tell() - if err != nil { - t.Fatal("unexpected error", err) - } - -} - -func TestSizeRace(t *testing.T) { - tailTest := NewTailTest("tell-race", t) - tailTest.CreateFile("test.txt", "hello\nworld\n") - tail := tailTest.StartTail("test.txt", Config{PollOptions: testPollingOptions}) - - <-tail.Lines - <-tail.Lines - - s1, err := tail.Size() - if err != nil { - t.Fatal("unexpected error", err) - } - - tailTest.TruncateFile("test.txt", "yay\nyay2\n") // smaller than before - - // wait for reopen to happen - time.Sleep(100 * time.Millisecond) - - s2, err := tail.Size() - if err != nil { - t.Fatal("unexpected error", err) - } - - if s2 == 0 || s2 > s1 { - t.Fatal("expected 0 < s2 < s1! s1:", s1, "s2:", s2) - } -} - -// Test library -type TailTest struct { - Name string - path string - t *testing.T -} - -func NewTailTest(name string, t *testing.T) TailTest { - tt := TailTest{name, t.TempDir() + "/" + name, t} - err := os.MkdirAll(tt.path, os.ModeTemporary|0700) - if err != nil { - tt.t.Fatal(err) - } - - return tt -} - -func (t TailTest) CreateFile(name string, contents string) { - assert.NoError(t.t, os.WriteFile(t.path+"/"+name, []byte(contents), 0600)) -} - -func (t TailTest) AppendToFile(name string, contents string) { - assert.NoError(t.t, os.WriteFile(t.path+"/"+name, []byte(contents), 0600|os.ModeAppend)) - -} - -func (t TailTest) RemoveFile(name string) { - err := os.Remove(t.path + "/" + name) - assert.NoError(t.t, err) - -} - -func (t TailTest) RenameFile(oldname string, newname string) { - oldname = t.path + "/" + oldname - newname = t.path + "/" + newname - assert.NoError(t.t, os.Rename(oldname, newname)) -} - -func (t TailTest) AppendFile(name string, contents string) { - f, err := os.OpenFile(t.path+"/"+name, os.O_APPEND|os.O_WRONLY, 0600) - assert.NoError(t.t, err) - defer f.Close() - _, err = f.WriteString(contents) - assert.NoError(t.t, err) -} - -func (t TailTest) TruncateFile(name string, contents string) { - f, err := os.OpenFile(t.path+"/"+name, os.O_TRUNC|os.O_WRONLY, 0600) - assert.NoError(t.t, err) - defer f.Close() - _, err = f.WriteString(contents) - assert.NoError(t.t, err) -} - -func (t TailTest) StartTail(name string, config Config) *Tail { - tail, err := TailFile(t.path+"/"+name, config) - assert.NoError(t.t, err) - return tail -} diff --git a/internal/component/loki/source/file/internal/tail/tail_windows.go b/internal/component/loki/source/file/internal/tail/tail_windows.go deleted file mode 100644 index 7593c7617c..0000000000 --- a/internal/component/loki/source/file/internal/tail/tail_windows.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build windows - -package tail - -import ( - "os" - - "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/winfile" -) - -func OpenFile(name string) (file *os.File, err error) { - return winfile.OpenFile(name, os.O_RDONLY, 0) -} diff --git a/internal/component/loki/source/file/internal/tail/util/util.go b/internal/component/loki/source/file/internal/tail/util/util.go deleted file mode 100644 index 54151fe39f..0000000000 --- a/internal/component/loki/source/file/internal/tail/util/util.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2015 HPE Software Inc. All rights reserved. -// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. - -package util - -import ( - "fmt" - "log" - "os" - "runtime/debug" -) - -type Logger struct { - *log.Logger -} - -var LOGGER = &Logger{log.New(os.Stderr, "", log.LstdFlags)} - -// fatal is like panic except it displays only the current goroutine's stack. -func Fatal(format string, v ...interface{}) { - // https://github.com/hpcloud/log/blob/master/log.go#L45 - LOGGER.Output(2, fmt.Sprintf("FATAL -- "+format, v...)+"\n"+string(debug.Stack())) - os.Exit(1) -} - -// partitionString partitions the string into chunks of given size, -// with the last chunk of variable size. -func PartitionString(s string, chunkSize int) []string { - if chunkSize <= 0 { - panic("invalid chunkSize") - } - length := len(s) - chunks := 1 + length/chunkSize - start := 0 - end := chunkSize - parts := make([]string, 0, chunks) - for { - if end > length { - end = length - } - parts = append(parts, s[start:end]) - if end == length { - break - } - start, end = end, end+chunkSize - } - return parts -} diff --git a/internal/component/loki/source/file/internal/tailv2/watch.go b/internal/component/loki/source/file/internal/tail/watch.go similarity index 98% rename from internal/component/loki/source/file/internal/tailv2/watch.go rename to internal/component/loki/source/file/internal/tail/watch.go index 315b10f2a8..f0098a50fd 100644 --- a/internal/component/loki/source/file/internal/tailv2/watch.go +++ b/internal/component/loki/source/file/internal/tail/watch.go @@ -1,4 +1,4 @@ -package tailv2 +package tail import ( "context" @@ -8,7 +8,7 @@ import ( "github.com/grafana/dskit/backoff" - "github.com/grafana/alloy/internal/component/loki/source/file/internal/tailv2/fileext" + "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/fileext" ) // watcher polls the file for changes. diff --git a/internal/component/loki/source/file/internal/tail/watch/file_posix.go b/internal/component/loki/source/file/internal/tail/watch/file_posix.go deleted file mode 100644 index dae82ae94d..0000000000 --- a/internal/component/loki/source/file/internal/tail/watch/file_posix.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build linux || darwin || freebsd || netbsd || openbsd - -package watch - -import "os" - -func IsDeletePending(_ *os.File) (bool, error) { - return false, nil -} diff --git a/internal/component/loki/source/file/internal/tail/watch/file_windows.go b/internal/component/loki/source/file/internal/tail/watch/file_windows.go deleted file mode 100644 index d1ca7c64d0..0000000000 --- a/internal/component/loki/source/file/internal/tail/watch/file_windows.go +++ /dev/null @@ -1,47 +0,0 @@ -//go:build windows - -package watch - -import ( - "os" - "runtime" - "unsafe" - - "golang.org/x/sys/windows" -) - -func IsDeletePending(f *os.File) (bool, error) { - if f == nil { - return false, nil - } - - fi, err := getFileStandardInfo(f) - if err != nil { - return false, err - } - - return fi.DeletePending, nil -} - -// From: https://github.com/microsoft/go-winio/blob/main/fileinfo.go -// FileStandardInfo contains extended information for the file. -// FILE_STANDARD_INFO in WinBase.h -// https://docs.microsoft.com/en-us/windows/win32/api/winbase/ns-winbase-file_standard_info -type fileStandardInfo struct { - AllocationSize, EndOfFile int64 - NumberOfLinks uint32 - DeletePending, Directory bool -} - -// GetFileStandardInfo retrieves ended information for the file. -func getFileStandardInfo(f *os.File) (*fileStandardInfo, error) { - si := &fileStandardInfo{} - if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), - windows.FileStandardInfo, - (*byte)(unsafe.Pointer(si)), - uint32(unsafe.Sizeof(*si))); err != nil { - return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err} - } - runtime.KeepAlive(f) - return si, nil -} diff --git a/internal/component/loki/source/file/internal/tail/watch/filechanges.go b/internal/component/loki/source/file/internal/tail/watch/filechanges.go deleted file mode 100644 index f80aead9ad..0000000000 --- a/internal/component/loki/source/file/internal/tail/watch/filechanges.go +++ /dev/null @@ -1,36 +0,0 @@ -package watch - -type FileChanges struct { - Modified chan bool // Channel to get notified of modifications - Truncated chan bool // Channel to get notified of truncations - Deleted chan bool // Channel to get notified of deletions/renames -} - -func NewFileChanges() *FileChanges { - return &FileChanges{ - make(chan bool, 1), make(chan bool, 1), make(chan bool, 1)} -} - -func (fc *FileChanges) NotifyModified() { - sendOnlyIfEmpty(fc.Modified) -} - -func (fc *FileChanges) NotifyTruncated() { - sendOnlyIfEmpty(fc.Truncated) -} - -func (fc *FileChanges) NotifyDeleted() { - sendOnlyIfEmpty(fc.Deleted) -} - -// sendOnlyIfEmpty sends on a bool channel only if the channel has no -// backlog to be read by other goroutines. This concurrency pattern -// can be used to notify other goroutines if and only if they are -// looking for it (i.e., subsequent notifications can be compressed -// into one). -func sendOnlyIfEmpty(ch chan bool) { - select { - case ch <- true: - default: - } -} diff --git a/internal/component/loki/source/file/internal/tail/watch/polling.go b/internal/component/loki/source/file/internal/tail/watch/polling.go deleted file mode 100644 index 688d77508a..0000000000 --- a/internal/component/loki/source/file/internal/tail/watch/polling.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright (c) 2015 HPE Software Inc. All rights reserved. -// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. - -package watch - -import ( - "fmt" - "os" - "runtime" - "sync" - "time" - - "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/util" - "gopkg.in/tomb.v1" -) - -// PollingFileWatcher polls the file for changes. -type PollingFileWatcher struct { - File *os.File - Filename string - Size int64 - Options PollingFileWatcherOptions - mtx sync.RWMutex // protects File and Size fields -} - -// PollingFileWatcherOptions customizes a PollingFileWatcher. -type PollingFileWatcherOptions struct { - // MinPollFrequency and MaxPollFrequency specify how frequently a - // PollingFileWatcher should poll the file. - // - // PollingFileWatcher starts polling at MinPollFrequency, and will - // exponentially increase the polling frequency up to MaxPollFrequency if no - // new entries are found. The polling frequency is reset to MinPollFrequency - // whenever a new log entry is found or if the polled file changes. - MinPollFrequency, MaxPollFrequency time.Duration -} - -// DefaultPollingFileWatcherOptions holds default values for -// PollingFileWatcherOptions. -var DefaultPollingFileWatcherOptions = PollingFileWatcherOptions{ - MinPollFrequency: 250 * time.Millisecond, - MaxPollFrequency: 250 * time.Millisecond, -} - -func NewPollingFileWatcher(filename string, opts PollingFileWatcherOptions) (*PollingFileWatcher, error) { - if opts == (PollingFileWatcherOptions{}) { - opts = DefaultPollingFileWatcherOptions - } - - if opts.MinPollFrequency == 0 || opts.MaxPollFrequency == 0 { - return nil, fmt.Errorf("MinPollFrequency and MaxPollFrequency must be greater than 0") - } else if opts.MaxPollFrequency < opts.MinPollFrequency { - return nil, fmt.Errorf("MaxPollFrequency must be larger than MinPollFrequency") - } - - return &PollingFileWatcher{ - File: nil, - Filename: filename, - Size: 0, - Options: opts, - }, nil -} - -func (fw *PollingFileWatcher) BlockUntilExists(t *tomb.Tomb) error { - bo := newPollBackoff(fw.Options) - - for { - if _, err := os.Stat(fw.Filename); err == nil { - return nil - } else if !os.IsNotExist(err) { - return err - } - select { - case <-time.After(bo.WaitTime()): - bo.Backoff() - continue - case <-t.Dying(): - return tomb.ErrDying - } - } -} - -func (fw *PollingFileWatcher) ChangeEvents(t *tomb.Tomb, pos int64) (*FileChanges, error) { - origFi, err := os.Stat(fw.Filename) - if err != nil { - return nil, err - } - - changes := NewFileChanges() - var prevModTime time.Time - - // XXX: use tomb.Tomb to cleanly manage these goroutines. replace - // the fatal (below) with tomb's Kill. - - fw.mtx.Lock() - fw.Size = pos - fw.mtx.Unlock() - - bo := newPollBackoff(fw.Options) - - go func() { - fw.mtx.RLock() - prevSize := fw.Size - fw.mtx.RUnlock() - for { - select { - case <-t.Dying(): - return - default: - } - - time.Sleep(bo.WaitTime()) - fw.mtx.RLock() - file := fw.File - fw.mtx.RUnlock() - deletePending, err := IsDeletePending(file) - - // DeletePending is a windows state where the file has been queued - // for delete but won't actually get deleted until all handles are - // closed. It's a variation on the NotifyDeleted call below. - // - // IsDeletePending may fail in cases where the file handle becomes - // invalid, so we treat a failed call the same as a pending delete. - if err != nil || deletePending { - fw.closeFile() - changes.NotifyDeleted() - return - } - - fi, err := os.Stat(fw.Filename) - if err != nil { - // Windows cannot delete a file if a handle is still open (tail keeps one open) - // so it gives access denied to anything trying to read it until all handles are released. - if os.IsNotExist(err) || (runtime.GOOS == "windows" && os.IsPermission(err)) { - // File does not exist (has been deleted). - changes.NotifyDeleted() - return - } - - // XXX: report this error back to the user - util.Fatal("Failed to stat file %v: %v", fw.Filename, err) - } - - // File got moved/renamed? - if !os.SameFile(origFi, fi) { - changes.NotifyDeleted() - return - } - - // File got truncated? - fw.mtx.Lock() - fw.Size = fi.Size() - currentSize := fw.Size - fw.mtx.Unlock() - - if prevSize > 0 && prevSize > currentSize { - changes.NotifyTruncated() - prevSize = currentSize - bo.Reset() - continue - } - // File got bigger? - if prevSize > 0 && prevSize < currentSize { - changes.NotifyModified() - prevSize = currentSize - bo.Reset() - continue - } - prevSize = currentSize - - // File was appended to (changed)? - modTime := fi.ModTime() - if modTime != prevModTime { - prevModTime = modTime - changes.NotifyModified() - bo.Reset() - continue - } - - // File hasn't changed; increase backoff for next sleep. - bo.Backoff() - } - }() - - return changes, nil -} - -func (fw *PollingFileWatcher) SetFile(f *os.File) { - fw.mtx.Lock() - fw.File = f - fw.mtx.Unlock() -} - -func (fw *PollingFileWatcher) closeFile() { - fw.mtx.Lock() - if fw.File != nil { - _ = fw.File.Close() // Best effort close - } - fw.mtx.Unlock() -} - -type pollBackoff struct { - current time.Duration - opts PollingFileWatcherOptions -} - -func newPollBackoff(opts PollingFileWatcherOptions) *pollBackoff { - return &pollBackoff{ - current: opts.MinPollFrequency, - opts: opts, - } -} - -func (pb *pollBackoff) WaitTime() time.Duration { - return pb.current -} - -func (pb *pollBackoff) Reset() { - pb.current = pb.opts.MinPollFrequency -} - -func (pb *pollBackoff) Backoff() { - pb.current = pb.current * 2 - if pb.current > pb.opts.MaxPollFrequency { - pb.current = pb.opts.MaxPollFrequency - } -} diff --git a/internal/component/loki/source/file/internal/tail/watch/watch.go b/internal/component/loki/source/file/internal/tail/watch/watch.go deleted file mode 100644 index 18d3045668..0000000000 --- a/internal/component/loki/source/file/internal/tail/watch/watch.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2015 HPE Software Inc. All rights reserved. -// Copyright (c) 2013 ActiveState Software Inc. All rights reserved. - -package watch - -import ( - "gopkg.in/tomb.v1" - "os" -) - -// FileWatcher monitors file-level events. -type FileWatcher interface { - // BlockUntilExists blocks until the file comes into existence. - BlockUntilExists(*tomb.Tomb) error - - // ChangeEvents reports on changes to a file, be it modification, - // deletion, renames or truncations. Returned FileChanges group of - // channels will be closed, thus become unusable, after a deletion - // or truncation event. - // In order to properly report truncations, ChangeEvents requires - // the caller to pass their current offset in the file. - ChangeEvents(*tomb.Tomb, int64) (*FileChanges, error) - - SetFile(f *os.File) -} diff --git a/internal/component/loki/source/file/internal/tail/watch_test.go b/internal/component/loki/source/file/internal/tail/watch_test.go new file mode 100644 index 0000000000..78e2dd5781 --- /dev/null +++ b/internal/component/loki/source/file/internal/tail/watch_test.go @@ -0,0 +1 @@ +package tail diff --git a/internal/component/loki/source/file/internal/tail/winfile/winfile.go b/internal/component/loki/source/file/internal/tail/winfile/winfile.go deleted file mode 100644 index a691207453..0000000000 --- a/internal/component/loki/source/file/internal/tail/winfile/winfile.go +++ /dev/null @@ -1,92 +0,0 @@ -//go:build windows - -package winfile - -import ( - "os" - "syscall" - "unsafe" -) - -// issue also described here -//https://codereview.appspot.com/8203043/ - -// https://github.com/jnwhiteh/golang/blob/master/src/pkg/syscall/syscall_windows.go#L218 -func Open(path string, mode int, perm uint32) (fd syscall.Handle, err error) { - if len(path) == 0 { - return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND - } - pathp, err := syscall.UTF16PtrFromString(path) - if err != nil { - return syscall.InvalidHandle, err - } - var access uint32 - switch mode & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) { - case syscall.O_RDONLY: - access = syscall.GENERIC_READ - case syscall.O_WRONLY: - access = syscall.GENERIC_WRITE - case syscall.O_RDWR: - access = syscall.GENERIC_READ | syscall.GENERIC_WRITE - } - if mode&syscall.O_CREAT != 0 { - access |= syscall.GENERIC_WRITE - } - if mode&syscall.O_APPEND != 0 { - access &^= syscall.GENERIC_WRITE - access |= syscall.FILE_APPEND_DATA - } - sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE) - var sa *syscall.SecurityAttributes - if mode&syscall.O_CLOEXEC == 0 { - sa = makeInheritSa() - } - var createmode uint32 - switch { - case mode&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL): - createmode = syscall.CREATE_NEW - case mode&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC): - createmode = syscall.CREATE_ALWAYS - case mode&syscall.O_CREAT == syscall.O_CREAT: - createmode = syscall.OPEN_ALWAYS - case mode&syscall.O_TRUNC == syscall.O_TRUNC: - createmode = syscall.TRUNCATE_EXISTING - default: - createmode = syscall.OPEN_EXISTING - } - h, e := syscall.CreateFile(pathp, access, sharemode, sa, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0) - return h, e -} - -// https://github.com/jnwhiteh/golang/blob/master/src/pkg/syscall/syscall_windows.go#L211 -func makeInheritSa() *syscall.SecurityAttributes { - var sa syscall.SecurityAttributes - sa.Length = uint32(unsafe.Sizeof(sa)) - sa.InheritHandle = 1 - return &sa -} - -// https://github.com/jnwhiteh/golang/blob/master/src/pkg/os/file_windows.go#L133 -func OpenFile(name string, flag int, perm os.FileMode) (file *os.File, err error) { - r, e := Open(name, flag|syscall.O_CLOEXEC, syscallMode(perm)) - if e != nil { - return nil, e - } - return os.NewFile(uintptr(r), name), nil -} - -// https://github.com/jnwhiteh/golang/blob/master/src/pkg/os/file_posix.go#L61 -func syscallMode(i os.FileMode) (o uint32) { - o |= uint32(i.Perm()) - if i&os.ModeSetuid != 0 { - o |= syscall.S_ISUID - } - if i&os.ModeSetgid != 0 { - o |= syscall.S_ISGID - } - if i&os.ModeSticky != 0 { - o |= syscall.S_ISVTX - } - // No mapping for Go's ModeTemporary (plan9 only). - return -} diff --git a/internal/component/loki/source/file/internal/tailv2/testdata/mssql.log b/internal/component/loki/source/file/internal/tailv2/testdata/mssql.log deleted file mode 100644 index 0234db5bd33d23b377927f5ba62584bb3a3111b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 876 zcmb`GOH0F05QWcH!T)fVagorZK57v`OTmR*_(HeU`iRsvq*d|HtKUo#3bmkuT<&x- zuXFCq^z$QAOPRJ6^V>$IoZzxsOYiYYJXV(TC?^q?xf z2d6^s@XPg}M`stQ=M@*(tKMLlCAlVtdv2NUI zZ?y_RSA0*1o$8IAEqB&fWgN55LAJ;tC?hN>KI>D^e%+Y^^hif~q2}0QEWcfMBKSKa z9n*oLo?q^BD)Cr{>{f_B@4+tk%WRu12)bmVJ^VvhJi%uM`)4q%Q(f#S(q{XAh!J~d lUH$#^l0TwXQ&_$ChxB_4`eL(emL)H?ZK(b!Bc^5V{Q|PUcQyb3 diff --git a/internal/component/loki/source/file/tailer.go b/internal/component/loki/source/file/tailer.go index 8bb00243f9..4e61dffed7 100644 --- a/internal/component/loki/source/file/tailer.go +++ b/internal/component/loki/source/file/tailer.go @@ -21,7 +21,7 @@ import ( "golang.org/x/text/encoding/ianaindex" "github.com/grafana/alloy/internal/component/common/loki" - "github.com/grafana/alloy/internal/component/loki/source/file/internal/tailv2" + "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail" "github.com/grafana/alloy/internal/component/loki/source/internal/positions" "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/util" @@ -39,7 +39,8 @@ type tailer struct { tailFromEnd bool onPositionsFileError OnPositionsFileError - pollOptions tailv2.WatcherConfig + tailConfig *tail.Config + watcherConfig tail.WatcherConfig posAndSizeMtx sync.Mutex @@ -49,7 +50,7 @@ type tailer struct { report sync.Once - file *tailv2.File + file *tail.File decoder *encoding.Decoder } @@ -78,7 +79,7 @@ func newTailer( tailFromEnd: opts.tailFromEnd, legacyPositionUsed: opts.legacyPositionUsed, onPositionsFileError: opts.onPositionsFileError, - pollOptions: tailv2.WatcherConfig{ + watcherConfig: tail.WatcherConfig{ MinPollFrequency: opts.fileWatch.MinPollFrequency, MaxPollFrequency: opts.fileWatch.MaxPollFrequency, }, @@ -242,11 +243,11 @@ func (t *tailer) initRun() (loki.EntryHandler, error) { } } - tail, err := tailv2.NewFile(t.logger, &tailv2.Config{ + tail, err := tail.NewFile(t.logger, &tail.Config{ Filename: t.key.Path, Offset: pos, Decoder: t.decoder, - WatcherConfig: t.pollOptions, + WatcherConfig: t.watcherConfig, }) if err != nil { From 4b66c5c924d493db07ed46dc6c74392afc568ebb Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Tue, 2 Dec 2025 11:52:13 +0100 Subject: [PATCH 06/28] Remove watcher and have free standing blocking functions --- .../file/internal/tail/{watch.go => block.go} | 61 ++++++------------- .../loki/source/file/internal/tail/config.go | 11 +--- .../loki/source/file/internal/tail/file.go | 31 +++++----- .../source/file/internal/tail/watch_test.go | 1 - internal/component/loki/source/file/tailer.go | 1 - 5 files changed, 34 insertions(+), 71 deletions(-) rename internal/component/loki/source/file/internal/tail/{watch.go => block.go} (68%) delete mode 100644 internal/component/loki/source/file/internal/tail/watch_test.go diff --git a/internal/component/loki/source/file/internal/tail/watch.go b/internal/component/loki/source/file/internal/tail/block.go similarity index 68% rename from internal/component/loki/source/file/internal/tail/watch.go rename to internal/component/loki/source/file/internal/tail/block.go index f0098a50fd..847f6761d1 100644 --- a/internal/component/loki/source/file/internal/tail/watch.go +++ b/internal/component/loki/source/file/internal/tail/block.go @@ -2,7 +2,6 @@ package tail import ( "context" - "fmt" "os" "runtime" @@ -11,41 +10,15 @@ import ( "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/fileext" ) -// watcher polls the file for changes. -type watcher struct { - filename string - cfg WatcherConfig - - ctx context.Context - cancel context.CancelFunc -} - -func newWatcher(filename string, cfg WatcherConfig) (*watcher, error) { - if cfg == (WatcherConfig{}) { - cfg = DefaultWatcherConfig - } - - if cfg.MinPollFrequency == 0 || cfg.MaxPollFrequency == 0 { - return nil, fmt.Errorf("MinPollFrequency and MaxPollFrequency must be greater than 0") - } else if cfg.MaxPollFrequency < cfg.MinPollFrequency { - return nil, fmt.Errorf("MaxPollFrequency must be larger than MinPollFrequency") - } - - return &watcher{ - filename: filename, - cfg: cfg, - }, nil -} - // blockUntilExists will block until either file exists or context is canceled. -func (fw *watcher) blockUntilExists(ctx context.Context) error { +func blockUntilExists(ctx context.Context, cfg *Config) error { backoff := backoff.New(ctx, backoff.Config{ - MinBackoff: fw.cfg.MinPollFrequency, - MaxBackoff: fw.cfg.MaxPollFrequency, + MinBackoff: cfg.WatcherConfig.MinPollFrequency, + MaxBackoff: cfg.WatcherConfig.MaxPollFrequency, }) for backoff.Ongoing() { - if _, err := os.Stat(fw.filename); err == nil { + if _, err := os.Stat(cfg.Filename); err == nil { return nil } else if !os.IsNotExist(err) { return err @@ -57,16 +30,25 @@ func (fw *watcher) blockUntilExists(ctx context.Context) error { return backoff.Err() } +type event int + +const ( + eventNone event = iota + eventTruncated + eventModified + eventDeleted +) + // blockUntilEvent will block until it detects a new event for file or context is canceled. -func (fw *watcher) blockUntilEvent(ctx context.Context, f *os.File, pos int64) (event, error) { +func blockUntilEvent(ctx context.Context, f *os.File, pos int64, cfg *Config) (event, error) { origFi, err := f.Stat() if err != nil { return eventNone, err } backoff := backoff.New(ctx, backoff.Config{ - MinBackoff: fw.cfg.MinPollFrequency, - MaxBackoff: fw.cfg.MaxPollFrequency, + MinBackoff: cfg.WatcherConfig.MinPollFrequency, + MaxBackoff: cfg.WatcherConfig.MaxPollFrequency, }) var ( @@ -86,7 +68,7 @@ func (fw *watcher) blockUntilEvent(ctx context.Context, f *os.File, pos int64) ( return eventDeleted, nil } - fi, err := os.Stat(fw.filename) + fi, err := os.Stat(cfg.Filename) if err != nil { // Windows cannot delete a file if a handle is still open (tail keeps one open) // so it gives access denied to anything trying to read it until all handles are released. @@ -127,12 +109,3 @@ func (fw *watcher) blockUntilEvent(ctx context.Context, f *os.File, pos int64) ( return eventNone, backoff.Err() } - -type event int - -const ( - eventNone event = iota - eventTruncated - eventModified - eventDeleted -) diff --git a/internal/component/loki/source/file/internal/tail/config.go b/internal/component/loki/source/file/internal/tail/config.go index 20b0ca763f..d7e4674014 100644 --- a/internal/component/loki/source/file/internal/tail/config.go +++ b/internal/component/loki/source/file/internal/tail/config.go @@ -21,17 +21,12 @@ type Config struct { type WatcherConfig struct { // MinPollFrequency and MaxPollFrequency specify how frequently a - // PollingFileWatcher should poll the file. - // - // Watcher starts polling at MinPollFrequency, and will - // exponentially increase the polling frequency up to MaxPollFrequency if no - // new entries are found. The polling frequency is reset to MinPollFrequency - // whenever the file changes. + // files are polled for events. MinPollFrequency, MaxPollFrequency time.Duration } -// DefaultWatcherConfig holds default values for WatcherConfig -var DefaultWatcherConfig = WatcherConfig{ +// defaultWatcherConfig holds default values for WatcherConfig +var defaultWatcherConfig = WatcherConfig{ MinPollFrequency: 250 * time.Millisecond, MaxPollFrequency: 250 * time.Millisecond, } diff --git a/internal/component/loki/source/file/internal/tail/file.go b/internal/component/loki/source/file/internal/tail/file.go index 687dbd713a..2e0aeef490 100644 --- a/internal/component/loki/source/file/internal/tail/file.go +++ b/internal/component/loki/source/file/internal/tail/file.go @@ -31,21 +31,21 @@ func NewFile(logger log.Logger, cfg *Config) (*File, error) { } } - watcher, err := newWatcher(cfg.Filename, cfg.WatcherConfig) - if err != nil { - return nil, err + if cfg.WatcherConfig == (WatcherConfig{}) { + cfg.WatcherConfig = defaultWatcherConfig } + cfg.WatcherConfig.MinPollFrequency = min(cfg.WatcherConfig.MinPollFrequency, cfg.WatcherConfig.MaxPollFrequency) + ctx, cancel := context.WithCancel(context.Background()) return &File{ - cfg: cfg, - logger: logger, - file: f, - reader: newReader(f, cfg), - watcher: watcher, - ctx: ctx, - cancel: cancel, + cfg: cfg, + logger: logger, + file: f, + reader: newReader(f, cfg), + ctx: ctx, + cancel: cancel, }, nil } @@ -59,8 +59,6 @@ type File struct { lastOffset int64 - watcher *watcher - ctx context.Context cancel context.CancelFunc } @@ -120,7 +118,7 @@ func (f *File) wait(partial bool) error { return err } - event, err := f.watcher.blockUntilEvent(f.ctx, f.file, offset) + event, err := blockUntilEvent(f.ctx, f.file, offset, f.cfg) switch event { case eventModified: if partial { @@ -139,7 +137,6 @@ func (f *File) wait(partial bool) error { default: return err } - } func (f *File) readLine() (string, error) { @@ -178,8 +175,8 @@ func (f *File) reopen(truncated bool) error { f.file.Close() backoff := backoff.New(f.ctx, backoff.Config{ - MinBackoff: DefaultWatcherConfig.MaxPollFrequency, - MaxBackoff: DefaultWatcherConfig.MaxPollFrequency, + MinBackoff: defaultWatcherConfig.MaxPollFrequency, + MaxBackoff: defaultWatcherConfig.MaxPollFrequency, MaxRetries: 20, }) @@ -188,7 +185,7 @@ func (f *File) reopen(truncated bool) error { if err != nil { if os.IsNotExist(err) { level.Debug(f.logger).Log("msg", fmt.Sprintf("waiting for %s to appear...", f.cfg.Filename)) - if err := f.watcher.blockUntilExists(f.ctx); err != nil { + if err := blockUntilExists(f.ctx, f.cfg); err != nil { return fmt.Errorf("failed to detect creation of %s: %w", f.cfg.Filename, err) } backoff.Wait() diff --git a/internal/component/loki/source/file/internal/tail/watch_test.go b/internal/component/loki/source/file/internal/tail/watch_test.go deleted file mode 100644 index 78e2dd5781..0000000000 --- a/internal/component/loki/source/file/internal/tail/watch_test.go +++ /dev/null @@ -1 +0,0 @@ -package tail diff --git a/internal/component/loki/source/file/tailer.go b/internal/component/loki/source/file/tailer.go index 4e61dffed7..ee8141b0c1 100644 --- a/internal/component/loki/source/file/tailer.go +++ b/internal/component/loki/source/file/tailer.go @@ -39,7 +39,6 @@ type tailer struct { tailFromEnd bool onPositionsFileError OnPositionsFileError - tailConfig *tail.Config watcherConfig tail.WatcherConfig posAndSizeMtx sync.Mutex From 957eae88046a96b42e2a2bd29054e0e5a360345b Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Wed, 3 Dec 2025 16:17:29 +0100 Subject: [PATCH 07/28] remove unused metrics and cleanup metrics when stopped --- internal/component/loki/source/file/tailer.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/component/loki/source/file/tailer.go b/internal/component/loki/source/file/tailer.go index ee8141b0c1..c99546759e 100644 --- a/internal/component/loki/source/file/tailer.go +++ b/internal/component/loki/source/file/tailer.go @@ -41,8 +41,6 @@ type tailer struct { onPositionsFileError OnPositionsFileError watcherConfig tail.WatcherConfig - posAndSizeMtx sync.Mutex - running *atomic.Bool componentStopping func() bool @@ -155,7 +153,6 @@ func (t *tailer) Run(ctx context.Context) { } handler, err := t.initRun() - if err != nil { // We are retrying tailers until the target has disappeared. // We are mostly interested in this log if this happens directly when @@ -330,7 +327,6 @@ func (t *tailer) updateStats(offset int64, size int64) { t.metrics.totalBytes.WithLabelValues(t.key.Path).Set(float64(size)) t.metrics.readBytes.WithLabelValues(t.key.Path).Set(float64(offset)) t.positions.Put(t.key.Path, t.key.Labels, offset) - } func (t *tailer) stop(done chan struct{}) { @@ -352,6 +348,9 @@ func (t *tailer) stop(done chan struct{}) { level.Info(t.logger).Log("msg", "stopped tailing file", "path", t.key.Path) + // We need to cleanup created metrics + t.cleanupMetrics() + // If the component is not stopping, then it means that the target for this component is gone and that // we should clear the entry from the positions file. if !t.componentStopping() { From f92e0362648ecf2331e6540298657851918c023a Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:27:31 +0100 Subject: [PATCH 08/28] Add comments --- .../loki/source/file/internal/tail/block.go | 18 ++++++--- .../loki/source/file/internal/tail/config.go | 25 +++++++++---- .../loki/source/file/internal/tail/file.go | 37 ++++++++++++++++--- .../loki/source/file/internal/tail/line.go | 9 ++++- 4 files changed, 67 insertions(+), 22 deletions(-) diff --git a/internal/component/loki/source/file/internal/tail/block.go b/internal/component/loki/source/file/internal/tail/block.go index 847f6761d1..05fd98cc8b 100644 --- a/internal/component/loki/source/file/internal/tail/block.go +++ b/internal/component/loki/source/file/internal/tail/block.go @@ -10,7 +10,9 @@ import ( "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/fileext" ) -// blockUntilExists will block until either file exists or context is canceled. +// blockUntilExists blocks until the file specified in cfg exists or the context is canceled. +// It polls the file system at intervals defined by WatcherConfig polling frequencies. +// Returns an error if the context is canceled or an unrecoverable error occurs. func blockUntilExists(ctx context.Context, cfg *Config) error { backoff := backoff.New(ctx, backoff.Config{ MinBackoff: cfg.WatcherConfig.MinPollFrequency, @@ -30,16 +32,20 @@ func blockUntilExists(ctx context.Context, cfg *Config) error { return backoff.Err() } +// event represents a file system event detected during polling. type event int const ( - eventNone event = iota - eventTruncated - eventModified - eventDeleted + eventNone event = iota // no event detected + eventTruncated // file was truncated (size decreased) + eventModified // file was modified (size increased or modification time changed) + eventDeleted // file was deleted, moved, or renamed ) -// blockUntilEvent will block until it detects a new event for file or context is canceled. +// blockUntilEvent blocks until it detects a file system event for the given file or the context is canceled. +// It polls the file system to detect modifications, truncations, deletions, or renames. +// The pos parameter is the current file position and is used to detect truncation events. +// Returns the detected event type and any error encountered. Returns eventNone if the context is canceled. func blockUntilEvent(ctx context.Context, f *os.File, pos int64, cfg *Config) (event, error) { origFi, err := f.Stat() if err != nil { diff --git a/internal/component/loki/source/file/internal/tail/config.go b/internal/component/loki/source/file/internal/tail/config.go index d7e4674014..2decfa3a73 100644 --- a/internal/component/loki/source/file/internal/tail/config.go +++ b/internal/component/loki/source/file/internal/tail/config.go @@ -6,26 +6,35 @@ import ( "golang.org/x/text/encoding" ) +// Config holds configuration for tailing a file. type Config struct { + // Filename is the path to the file to tail. Filename string - Offset int64 + // Offset is the byte offset in the file where tailing should start. + // If 0, tailing starts from the beginning of the file. + Offset int64 - // Change the decoder if the file is not UTF-8. - // If the tailer doesn't use the right decoding, the output text may be gibberish. - // For example, if the file is "UTF-16 LE" encoded, the tailer would not separate - // the new lines properly and the output could come out as chinese characters. + // Decoder is an optional text decoder for non-UTF-8 encoded files. + // If the file is not UTF-8, the tailer must use the correct decoder + // or the output text may be corrupted. For example, if the file is + // "UTF-16 LE" encoded, the tailer would not separate new lines properly + // and the output could appear as garbled characters. Decoder *encoding.Decoder + // WatcherConfig controls how the file system is polled for changes. WatcherConfig WatcherConfig } +// WatcherConfig controls the polling behavior for detecting file system events. type WatcherConfig struct { - // MinPollFrequency and MaxPollFrequency specify how frequently a - // files are polled for events. + // MinPollFrequency and MaxPollFrequency specify the polling frequency range + // for detecting file system events. The actual polling frequency will vary + // within this range based on backoff behavior. MinPollFrequency, MaxPollFrequency time.Duration } -// defaultWatcherConfig holds default values for WatcherConfig +// defaultWatcherConfig holds the default polling configuration used when +// WatcherConfig is not explicitly provided in Config. var defaultWatcherConfig = WatcherConfig{ MinPollFrequency: 250 * time.Millisecond, MaxPollFrequency: 250 * time.Millisecond, diff --git a/internal/component/loki/source/file/internal/tail/file.go b/internal/component/loki/source/file/internal/tail/file.go index 2e0aeef490..392e08c5a6 100644 --- a/internal/component/loki/source/file/internal/tail/file.go +++ b/internal/component/loki/source/file/internal/tail/file.go @@ -18,6 +18,10 @@ import ( "github.com/grafana/alloy/internal/runtime/logging/level" ) +// NewFile creates a new File tailer for the specified file path. +// It opens the file and seeks to the provided offset if one is specified. +// The returned File can be used to read lines from the file as they are appended. +// The caller is responsible for calling Stop() when done to close the file and clean up resources. func NewFile(logger log.Logger, cfg *Config) (*File, error) { f, err := fileext.OpenFile(cfg.Filename) if err != nil { @@ -49,10 +53,14 @@ func NewFile(logger log.Logger, cfg *Config) (*File, error) { }, nil } +// File represents a file being tailed. It provides methods to read lines +// from the file as they are appended, handling file events such as truncation, +// deletion, and modification. File is safe for concurrent use. type File struct { cfg *Config logger log.Logger + // protects file, reader, and lastOffset. mu sync.Mutex file *os.File reader *bufio.Reader @@ -63,7 +71,8 @@ type File struct { cancel context.CancelFunc } -// FIXME: need clear exit signal +// Next reads and returns the next line from the file. +// It blocks until a line is available, file is closed or unrecoverable error occurs. func (f *File) Next() (*Line, error) { f.mu.Lock() defer f.mu.Unlock() @@ -94,6 +103,8 @@ read: }, nil } +// Size returns the current size of the file in bytes. +// It is safe to call concurrently with other File methods. func (f *File) Size() (int64, error) { f.mu.Lock() defer f.mu.Unlock() @@ -105,6 +116,9 @@ func (f *File) Size() (int64, error) { return fi.Size(), nil } +// Stop closes the file and cancels any ongoing wait operations. +// After Stop is called, Next() will return errors for any subsequent calls. +// It is safe to call Stop multiple times. func (f *File) Stop() error { f.cancel() f.mu.Lock() @@ -112,6 +126,8 @@ func (f *File) Stop() error { return f.file.Close() } +// wait blocks until a file event is detected (modification, truncation, or deletion). +// Returns an error if the context is canceled or an unrecoverable error occurs. func (f *File) wait(partial bool) error { offset, err := f.offset() if err != nil { @@ -122,7 +138,7 @@ func (f *File) wait(partial bool) error { switch event { case eventModified: if partial { - // We need to reset to last succeful offset because we could have consumed a partial line. + // We need to reset to last succeful offset because we consumed a partial line. f.file.Seek(f.lastOffset, io.SeekStart) f.reader.Reset(f.file) } @@ -139,6 +155,8 @@ func (f *File) wait(partial bool) error { } } +// readLine reads a single line from the file, including the newline character. +// The newline and any trailing carriage return (for Windows line endings) are stripped. func (f *File) readLine() (string, error) { line, err := f.reader.ReadString('\n') if err != nil { @@ -151,6 +169,8 @@ func (f *File) readLine() (string, error) { return line, err } +// offset returns the current byte offset in the file where the next read will occur. +// It accounts for buffered data in the reader. func (f *File) offset() (int64, error) { offset, err := f.file.Seek(0, io.SeekCurrent) if err != nil { @@ -160,11 +180,16 @@ func (f *File) offset() (int64, error) { return offset - int64(f.reader.Buffered()), nil } +// reopen closes the current file handle and opens a new one for the same file path. +// If truncated is true, it indicates the file was truncated and we should reopen immediately. +// If truncated is false, it indicates the file was deleted or moved, and we should wait +// for it to be recreated before reopening. +// +// reopen handles the case where a file is reopened so quickly it's still the same file, +// which could cause the poller to hang on an open file handle to a file no longer being +// written to. It saves the current file handle info to ensure we only start tailing a +// different file instance. func (f *File) reopen(truncated bool) error { - // There are cases where the file is reopened so quickly it's still the same file - // which causes the poller to hang on an open file handle to a file no longer being written to - // and which eventually gets deleted. Save the current file handle info to make sure we only - // start tailing a different file. cf, err := f.file.Stat() if !truncated && err != nil { // We don't action on this error but are logging it, not expecting to see it happen and not sure if we diff --git a/internal/component/loki/source/file/internal/tail/line.go b/internal/component/loki/source/file/internal/tail/line.go index 4c4f8fc32f..b52fa1842d 100644 --- a/internal/component/loki/source/file/internal/tail/line.go +++ b/internal/component/loki/source/file/internal/tail/line.go @@ -2,8 +2,13 @@ package tail import "time" +// Line represents a single line read from a tailed file. type Line struct { - Text string + // Text is the content of the line, with line endings stripped. + Text string + // Offset is the byte offset in the file immediately after this line, + // which is where the next read will occur. Offset int64 - Time time.Time + // Time is the timestamp when the line was read from the file. + Time time.Time } From c49f3f9ba3504206cb78b3bbc197e0ec462f168f Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:41:29 +0100 Subject: [PATCH 09/28] remove legacy build tags --- .../loki/source/file/internal/tail/fileext/file_posix.go | 1 - .../loki/source/file/internal/tail/fileext/file_windows.go | 1 - 2 files changed, 2 deletions(-) diff --git a/internal/component/loki/source/file/internal/tail/fileext/file_posix.go b/internal/component/loki/source/file/internal/tail/fileext/file_posix.go index 40642c02c9..de1f0fd648 100644 --- a/internal/component/loki/source/file/internal/tail/fileext/file_posix.go +++ b/internal/component/loki/source/file/internal/tail/fileext/file_posix.go @@ -1,5 +1,4 @@ //go:build linux || darwin || freebsd || netbsd || openbsd -// +build linux darwin freebsd netbsd openbsd package fileext diff --git a/internal/component/loki/source/file/internal/tail/fileext/file_windows.go b/internal/component/loki/source/file/internal/tail/fileext/file_windows.go index fe34770680..266ffb8e2c 100644 --- a/internal/component/loki/source/file/internal/tail/fileext/file_windows.go +++ b/internal/component/loki/source/file/internal/tail/fileext/file_windows.go @@ -1,5 +1,4 @@ //go:build windows -// +build windows package fileext From 42402cf6dc9f9519cca5e6a370af5ae346ec0a30 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:55:28 +0100 Subject: [PATCH 10/28] Update comment --- internal/component/loki/source/file/internal/tail/file.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/component/loki/source/file/internal/tail/file.go b/internal/component/loki/source/file/internal/tail/file.go index 392e08c5a6..1eda309f55 100644 --- a/internal/component/loki/source/file/internal/tail/file.go +++ b/internal/component/loki/source/file/internal/tail/file.go @@ -73,6 +73,7 @@ type File struct { // Next reads and returns the next line from the file. // It blocks until a line is available, file is closed or unrecoverable error occurs. +// If file was closed context.Canceled is returned. func (f *File) Next() (*Line, error) { f.mu.Lock() defer f.mu.Unlock() From 6bda270f334dff9fae7cc0bdf6481ae94e475e4c Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 12:04:54 +0100 Subject: [PATCH 11/28] remove unused function --- .../component/loki/source/file/internal/tail/file_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/component/loki/source/file/internal/tail/file_test.go b/internal/component/loki/source/file/internal/tail/file_test.go index cb167afb71..f759f5c7f6 100644 --- a/internal/component/loki/source/file/internal/tail/file_test.go +++ b/internal/component/loki/source/file/internal/tail/file_test.go @@ -271,14 +271,6 @@ func truncateFile(t *testing.T, name, content string) { require.NoError(t, err) } -/* -func renameFile(t *testing.T, oldname, newname string) { - oldname = t.TempDir() + "/" + oldname - newname = t.TempDir() + "/" + newname - require.NoError(t, os.Rename(oldname, newname)) -} -*/ - func removeFile(t *testing.T, name string) { require.NoError(t, os.Remove(name)) } From 2fca395557f702369469464b9836b421ed4d240b Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 13:40:07 +0100 Subject: [PATCH 12/28] Remove usage of Entry handler and set labels directly, this is done to avoid running 1 extra goroutine per file --- internal/component/loki/source/file/tailer.go | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/internal/component/loki/source/file/tailer.go b/internal/component/loki/source/file/tailer.go index c99546759e..9aacc8390e 100644 --- a/internal/component/loki/source/file/tailer.go +++ b/internal/component/loki/source/file/tailer.go @@ -71,7 +71,7 @@ func newTailer( receiver: receiver, positions: pos, key: positions.Entry{Path: opts.path, Labels: opts.labels.String()}, - labels: opts.labels, + labels: opts.labels.Merge(model.LabelSet{labelFilename: model.LabelValue(opts.path)}), running: atomic.NewBool(false), tailFromEnd: opts.tailFromEnd, legacyPositionUsed: opts.legacyPositionUsed, @@ -152,7 +152,7 @@ func (t *tailer) Run(ctx context.Context) { default: } - handler, err := t.initRun() + err := t.initRun() if err != nil { // We are retrying tailers until the target has disappeared. // We are mostly interested in this log if this happens directly when @@ -162,7 +162,6 @@ func (t *tailer) Run(ctx context.Context) { }) return } - defer handler.Stop() // We call report so that retries won't log. t.report.Do(func() {}) @@ -173,7 +172,7 @@ func (t *tailer) Run(ctx context.Context) { ctx, cancel := context.WithCancel(ctx) go func() { // readLines closes done on exit - t.readLines(handler, done) + t.readLines(done) cancel() }() @@ -184,21 +183,21 @@ func (t *tailer) Run(ctx context.Context) { t.stop(done) } -func (t *tailer) initRun() (loki.EntryHandler, error) { +func (t *tailer) initRun() error { fi, err := os.Stat(t.key.Path) if err != nil { - return nil, fmt.Errorf("failed to tail file: %w", err) + return fmt.Errorf("failed to tail file: %w", err) } pos, err := t.positions.Get(t.key.Path, t.key.Labels) if err != nil { switch t.onPositionsFileError { case OnPositionsFileErrorSkip: - return nil, fmt.Errorf("failed to get file position: %w", err) + return fmt.Errorf("failed to get file position: %w", err) case OnPositionsFileErrorRestartEnd: pos, err = getLastLinePosition(t.key.Path) if err != nil { - return nil, fmt.Errorf("failed to get last line position after positions error: %w", err) + return fmt.Errorf("failed to get last line position after positions error: %w", err) } level.Info(t.logger).Log("msg", "retrieved the position of the last line after positions error") default: @@ -215,7 +214,7 @@ func (t *tailer) initRun() (loki.EntryHandler, error) { if pos == 0 && t.legacyPositionUsed { pos, err = t.positions.Get(t.key.Path, "{}") if err != nil { - return nil, fmt.Errorf("failed to get file position with empty labels: %w", err) + return fmt.Errorf("failed to get file position with empty labels: %w", err) } } @@ -247,15 +246,12 @@ func (t *tailer) initRun() (loki.EntryHandler, error) { }) if err != nil { - return nil, fmt.Errorf("failed to tail the file: %w", err) + return fmt.Errorf("failed to tail the file: %w", err) } t.file = tail - labelsMiddleware := t.labels.Merge(model.LabelSet{labelFilename: model.LabelValue(t.key.Path)}) - handler := loki.AddLabelsMiddleware(labelsMiddleware).Wrap(loki.NewEntryHandler(t.receiver.Chan(), func() {})) - - return handler, nil + return nil } func getDecoder(encoding string) (*encoding.Decoder, error) { @@ -276,10 +272,10 @@ func getDecoder(encoding string) (*encoding.Decoder, error) { // there are unread lines in this channel and the Stop method on the tailer is // called, the underlying tailer will never exit if there are unread lines in // the t.tail.Lines channel -func (t *tailer) readLines(handler loki.EntryHandler, done chan struct{}) { +func (t *tailer) readLines(done chan struct{}) { level.Info(t.logger).Log("msg", "tail routine: started", "path", t.key.Path) var ( - entries = handler.Chan() + entries = t.receiver.Chan() lastOffset = int64(0) positionInterval = t.positions.SyncPeriod() lastUpdatedPosition = time.Time{} @@ -304,9 +300,7 @@ func (t *tailer) readLines(handler loki.EntryHandler, done chan struct{}) { t.metrics.readLines.WithLabelValues(t.key.Path).Inc() entries <- loki.Entry{ - // Allocate the expected size of labels. This matches the number of labels added by the middleware - // as configured in initRun(). - Labels: make(model.LabelSet, len(t.labels)+1), + Labels: t.labels, Entry: push.Entry{ Timestamp: line.Time, Line: line.Text, From 4b3e1b2b3090934e8013a964d4a291e2d365a481 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 13:46:31 +0100 Subject: [PATCH 13/28] Fix comment and update log level --- internal/component/loki/source/file/tailer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/component/loki/source/file/tailer.go b/internal/component/loki/source/file/tailer.go index 9aacc8390e..ff99c8560f 100644 --- a/internal/component/loki/source/file/tailer.go +++ b/internal/component/loki/source/file/tailer.go @@ -291,9 +291,9 @@ func (t *tailer) readLines(done chan struct{}) { for { line, err := t.file.Next() if err != nil { - // Maybe we should use a better signal than context canceled to indicate normal stop... + // We get context.Canceled if tail.File was stopped so we don't have to log it. if !errors.Is(err, context.Canceled) { - level.Info(t.logger).Log("msg", "tail routine: stopping tailer", "path", t.key.Path, "err", err) + level.Error(t.logger).Log("msg", "tail routine: stopping tailer", "path", t.key.Path, "err", err) } return } From 22894165aee1be81d51d99c11091079bccc751e4 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 14:02:06 +0100 Subject: [PATCH 14/28] remove sleep in test --- internal/component/loki/source/file/internal/tail/file_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/component/loki/source/file/internal/tail/file_test.go b/internal/component/loki/source/file/internal/tail/file_test.go index f759f5c7f6..49f3fed43b 100644 --- a/internal/component/loki/source/file/internal/tail/file_test.go +++ b/internal/component/loki/source/file/internal/tail/file_test.go @@ -193,7 +193,6 @@ func TestFile(t *testing.T) { verify(t, file, &Line{Text: "hello", Offset: 6}, nil) go func() { - time.Sleep(50 * time.Millisecond) removeFile(t, name) time.Sleep(50 * time.Millisecond) recreateFile(t, name, "new\n") From 75a4a409ff52f583c99f1c94b90badaccc3c30d7 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:07:33 +0100 Subject: [PATCH 15/28] fix issue where file got deleted before next was called and blocking is triggered --- internal/component/loki/source/file/internal/tail/block.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/component/loki/source/file/internal/tail/block.go b/internal/component/loki/source/file/internal/tail/block.go index 05fd98cc8b..b1e2228915 100644 --- a/internal/component/loki/source/file/internal/tail/block.go +++ b/internal/component/loki/source/file/internal/tail/block.go @@ -2,6 +2,7 @@ package tail import ( "context" + "errors" "os" "runtime" @@ -47,8 +48,12 @@ const ( // The pos parameter is the current file position and is used to detect truncation events. // Returns the detected event type and any error encountered. Returns eventNone if the context is canceled. func blockUntilEvent(ctx context.Context, f *os.File, pos int64, cfg *Config) (event, error) { - origFi, err := f.Stat() + origFi, err := os.Stat(cfg.Filename) if err != nil { + // If file no longer exists we treat it as a delete event. + if errors.Is(err, os.ErrNotExist) { + return eventDeleted, nil + } return eventNone, err } From b68fa134ab3ccf9f055167e721e076c3cc32f950 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:30:00 +0100 Subject: [PATCH 16/28] use exported function to check for error --- .../loki/source/file/internal/tail/block.go | 3 +- .../source/file/internal/tail/file_test.go | 34 +++---------------- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/internal/component/loki/source/file/internal/tail/block.go b/internal/component/loki/source/file/internal/tail/block.go index b1e2228915..e50ef16ef5 100644 --- a/internal/component/loki/source/file/internal/tail/block.go +++ b/internal/component/loki/source/file/internal/tail/block.go @@ -2,7 +2,6 @@ package tail import ( "context" - "errors" "os" "runtime" @@ -51,7 +50,7 @@ func blockUntilEvent(ctx context.Context, f *os.File, pos int64, cfg *Config) (e origFi, err := os.Stat(cfg.Filename) if err != nil { // If file no longer exists we treat it as a delete event. - if errors.Is(err, os.ErrNotExist) { + if os.IsNotExist(err) { return eventDeleted, nil } return eventNone, err diff --git a/internal/component/loki/source/file/internal/tail/file_test.go b/internal/component/loki/source/file/internal/tail/file_test.go index 49f3fed43b..06093c27d5 100644 --- a/internal/component/loki/source/file/internal/tail/file_test.go +++ b/internal/component/loki/source/file/internal/tail/file_test.go @@ -131,8 +131,8 @@ func TestFile(t *testing.T) { file, err := NewFile(log.NewNopLogger(), &Config{ Filename: name, WatcherConfig: WatcherConfig{ - MinPollFrequency: 5 * time.Millisecond, - MaxPollFrequency: 5 * time.Millisecond, + MinPollFrequency: 50 * time.Millisecond, + MaxPollFrequency: 50 * time.Millisecond, }, }) require.NoError(t, err) @@ -175,32 +175,6 @@ func TestFile(t *testing.T) { require.ErrorIs(t, err, context.Canceled) }) - t.Run("removed and created during wait", func(t *testing.T) { - name := createFile(t, "removed", "hello\n") - defer removeFile(t, name) - - file, err := NewFile(log.NewNopLogger(), &Config{ - Offset: 0, - Filename: name, - WatcherConfig: WatcherConfig{ - MinPollFrequency: 5 * time.Millisecond, - MaxPollFrequency: 5 * time.Millisecond, - }, - }) - require.NoError(t, err) - defer file.Stop() - - verify(t, file, &Line{Text: "hello", Offset: 6}, nil) - - go func() { - removeFile(t, name) - time.Sleep(50 * time.Millisecond) - recreateFile(t, name, "new\n") - }() - - verify(t, file, &Line{Text: "new", Offset: 4}, nil) - }) - t.Run("stopped while waiting for file to be created", func(t *testing.T) { name := createFile(t, "removed", "hello\n") @@ -208,8 +182,8 @@ func TestFile(t *testing.T) { Offset: 0, Filename: name, WatcherConfig: WatcherConfig{ - MinPollFrequency: 5 * time.Millisecond, - MaxPollFrequency: 5 * time.Millisecond, + MinPollFrequency: 50 * time.Millisecond, + MaxPollFrequency: 50 * time.Millisecond, }, }) require.NoError(t, err) From 4b519a06a761747f30726ac0dce93251a00def53 Mon Sep 17 00:00:00 2001 From: Karl Persson <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:41:07 +0100 Subject: [PATCH 17/28] Update internal/component/loki/source/file/internal/tail/file_test.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- internal/component/loki/source/file/internal/tail/file_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/component/loki/source/file/internal/tail/file_test.go b/internal/component/loki/source/file/internal/tail/file_test.go index 06093c27d5..6638579ace 100644 --- a/internal/component/loki/source/file/internal/tail/file_test.go +++ b/internal/component/loki/source/file/internal/tail/file_test.go @@ -105,7 +105,7 @@ func TestFile(t *testing.T) { }) }) - t.Run("partail line", func(t *testing.T) { + t.Run("partial line", func(t *testing.T) { name := createFile(t, "partial", "hello\nwo") defer removeFile(t, name) From 87132c034b8c8bfe0f201fdd426b9d6fa036b124 Mon Sep 17 00:00:00 2001 From: Karl Persson <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:41:30 +0100 Subject: [PATCH 18/28] Update internal/component/loki/source/file/internal/tail/file.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- internal/component/loki/source/file/internal/tail/file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/component/loki/source/file/internal/tail/file.go b/internal/component/loki/source/file/internal/tail/file.go index 1eda309f55..1de6ec7035 100644 --- a/internal/component/loki/source/file/internal/tail/file.go +++ b/internal/component/loki/source/file/internal/tail/file.go @@ -139,7 +139,7 @@ func (f *File) wait(partial bool) error { switch event { case eventModified: if partial { - // We need to reset to last succeful offset because we consumed a partial line. + // We need to reset to last successful offset because we consumed a partial line. f.file.Seek(f.lastOffset, io.SeekStart) f.reader.Reset(f.file) } From b299b70ac2afd3475d9de40bd1c8d7696834e33d Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:49:53 +0100 Subject: [PATCH 19/28] Update comment --- .../component/loki/source/file/internal/tail/file.go | 1 - internal/component/loki/source/file/tailer.go | 10 ++++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/internal/component/loki/source/file/internal/tail/file.go b/internal/component/loki/source/file/internal/tail/file.go index 1de6ec7035..fd81196b4e 100644 --- a/internal/component/loki/source/file/internal/tail/file.go +++ b/internal/component/loki/source/file/internal/tail/file.go @@ -214,7 +214,6 @@ func (f *File) reopen(truncated bool) error { if err := blockUntilExists(f.ctx, f.cfg); err != nil { return fmt.Errorf("failed to detect creation of %s: %w", f.cfg.Filename, err) } - backoff.Wait() continue } return fmt.Errorf("Unable to open file %s: %s", f.cfg.Filename, err) diff --git a/internal/component/loki/source/file/tailer.go b/internal/component/loki/source/file/tailer.go index ff99c8560f..886784ebcf 100644 --- a/internal/component/loki/source/file/tailer.go +++ b/internal/component/loki/source/file/tailer.go @@ -266,12 +266,10 @@ func getDecoder(encoding string) (*encoding.Decoder, error) { return encoder.NewDecoder(), nil } -// readLines consumes the t.tail.Lines channel from the -// underlying tailer. It will only exit when that channel is closed. This is -// important to avoid a deadlock in the underlying tailer which can happen if -// there are unread lines in this channel and the Stop method on the tailer is -// called, the underlying tailer will never exit if there are unread lines in -// the t.tail.Lines channel +// readLines reads lines from the tailed file by calling Next() in a loop. +// It processes each line by sending it to the receiver's channel and updates +// position tracking periodically. It exits when Next() returns an error, +// this happens when the tail.File is stopped or or we have a unrecoverable error. func (t *tailer) readLines(done chan struct{}) { level.Info(t.logger).Log("msg", "tail routine: started", "path", t.key.Path) var ( From 20228d32d6376ed998c0833474a9d7744eb5c5c1 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 16:00:05 +0100 Subject: [PATCH 20/28] If tail.File have been stopped all calls to Next are now returning context.Canceled --- .../loki/source/file/internal/tail/file.go | 6 +++++- .../loki/source/file/internal/tail/file_test.go | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/internal/component/loki/source/file/internal/tail/file.go b/internal/component/loki/source/file/internal/tail/file.go index fd81196b4e..204f21f52a 100644 --- a/internal/component/loki/source/file/internal/tail/file.go +++ b/internal/component/loki/source/file/internal/tail/file.go @@ -77,9 +77,13 @@ type File struct { func (f *File) Next() (*Line, error) { f.mu.Lock() defer f.mu.Unlock() + + if f.ctx.Err() != nil { + return nil, f.ctx.Err() + } + read: text, err := f.readLine() - if err != nil { if errors.Is(err, io.EOF) { if err := f.wait(text != ""); err != nil { diff --git a/internal/component/loki/source/file/internal/tail/file_test.go b/internal/component/loki/source/file/internal/tail/file_test.go index 6638579ace..5bd19280de 100644 --- a/internal/component/loki/source/file/internal/tail/file_test.go +++ b/internal/component/loki/source/file/internal/tail/file_test.go @@ -216,6 +216,20 @@ func TestFile(t *testing.T) { verify(t, file, &Line{Text: "2025-03-11 11:11:02.71 Server (c) Microsoft Corporation.", Offset: 819}, nil) verify(t, file, &Line{Text: "2025-03-11 11:11:02.72 Server All rights reserved.", Offset: 876}, nil) }) + + t.Run("calls to next after stop", func(t *testing.T) { + name := createFile(t, "stopped", "hello\n") + defer removeFile(t, name) + + file, err := NewFile(log.NewNopLogger(), &Config{ + Offset: 0, + Filename: name, + }) + require.NoError(t, err) + file.Stop() + + verify(t, file, nil, context.Canceled) + }) } func createFile(t *testing.T, name, content string) string { From 8faf64e0c0186daf202557912b5d6b318421386a Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Thu, 4 Dec 2025 16:15:21 +0100 Subject: [PATCH 21/28] update comment --- .../component/loki/source/file/internal/tail/block.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/internal/component/loki/source/file/internal/tail/block.go b/internal/component/loki/source/file/internal/tail/block.go index e50ef16ef5..c1fa2aa2a1 100644 --- a/internal/component/loki/source/file/internal/tail/block.go +++ b/internal/component/loki/source/file/internal/tail/block.go @@ -46,7 +46,7 @@ const ( // It polls the file system to detect modifications, truncations, deletions, or renames. // The pos parameter is the current file position and is used to detect truncation events. // Returns the detected event type and any error encountered. Returns eventNone if the context is canceled. -func blockUntilEvent(ctx context.Context, f *os.File, pos int64, cfg *Config) (event, error) { +func blockUntilEvent(ctx context.Context, f *os.File, prevSize int64, cfg *Config) (event, error) { origFi, err := os.Stat(cfg.Filename) if err != nil { // If file no longer exists we treat it as a delete event. @@ -61,10 +61,8 @@ func blockUntilEvent(ctx context.Context, f *os.File, pos int64, cfg *Config) (e MaxBackoff: cfg.WatcherConfig.MaxPollFrequency, }) - var ( - prevSize = pos - prevModTime = origFi.ModTime() - ) + prevModTime := origFi.ModTime() + for backoff.Ongoing() { deletePending, err := fileext.IsDeletePending(f) @@ -104,7 +102,6 @@ func blockUntilEvent(ctx context.Context, f *os.File, pos int64, cfg *Config) (e if prevSize > 0 && prevSize < currentSize { return eventModified, nil } - prevSize = currentSize // File was appended to (changed)? modTime := fi.ModTime() @@ -113,7 +110,7 @@ func blockUntilEvent(ctx context.Context, f *os.File, pos int64, cfg *Config) (e return eventModified, nil } - // File hasn't changed; increase backoff for next sleep. + // File hasn't changed so wait until next retry. backoff.Wait() } From e8683892ba0e6d3b7f96d5c62e69750b154676ec Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Fri, 5 Dec 2025 09:13:35 +0100 Subject: [PATCH 22/28] stat open file so we get correct comparison --- internal/component/loki/source/file/internal/tail/block.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/component/loki/source/file/internal/tail/block.go b/internal/component/loki/source/file/internal/tail/block.go index c1fa2aa2a1..838a65226a 100644 --- a/internal/component/loki/source/file/internal/tail/block.go +++ b/internal/component/loki/source/file/internal/tail/block.go @@ -47,7 +47,9 @@ const ( // The pos parameter is the current file position and is used to detect truncation events. // Returns the detected event type and any error encountered. Returns eventNone if the context is canceled. func blockUntilEvent(ctx context.Context, f *os.File, prevSize int64, cfg *Config) (event, error) { - origFi, err := os.Stat(cfg.Filename) + // NOTE: it is important that we stat the open file here. Later we do os.Stat(cfg.Filename) + // and use os.IsSameFile to detect if file was rotated. + origFi, err := f.Stat() if err != nil { // If file no longer exists we treat it as a delete event. if os.IsNotExist(err) { @@ -106,7 +108,6 @@ func blockUntilEvent(ctx context.Context, f *os.File, prevSize int64, cfg *Confi // File was appended to (changed)? modTime := fi.ModTime() if modTime != prevModTime { - prevModTime = modTime return eventModified, nil } From ef75b267fa5df2746b07da023d8679e6e3cf63aa Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Fri, 5 Dec 2025 09:45:58 +0100 Subject: [PATCH 23/28] Add tests --- .../loki/source/file/internal/tail/block.go | 5 +- .../source/file/internal/tail/block_test.go | 106 ++++++++++++++++++ .../source/file/internal/tail/file_test.go | 14 ++- 3 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 internal/component/loki/source/file/internal/tail/block_test.go diff --git a/internal/component/loki/source/file/internal/tail/block.go b/internal/component/loki/source/file/internal/tail/block.go index 838a65226a..43bd046c9d 100644 --- a/internal/component/loki/source/file/internal/tail/block.go +++ b/internal/component/loki/source/file/internal/tail/block.go @@ -101,13 +101,12 @@ func blockUntilEvent(ctx context.Context, f *os.File, prevSize int64, cfg *Confi } // File got bigger? - if prevSize > 0 && prevSize < currentSize { + if prevSize < currentSize { return eventModified, nil } // File was appended to (changed)? - modTime := fi.ModTime() - if modTime != prevModTime { + if fi.ModTime() != prevModTime { return eventModified, nil } diff --git a/internal/component/loki/source/file/internal/tail/block_test.go b/internal/component/loki/source/file/internal/tail/block_test.go new file mode 100644 index 0000000000..05ae8b5484 --- /dev/null +++ b/internal/component/loki/source/file/internal/tail/block_test.go @@ -0,0 +1,106 @@ +package tail + +import ( + "context" + "fmt" + "io" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestBlockUntilEvent(t *testing.T) { + watcherConfig := WatcherConfig{ + MinPollFrequency: 5 * time.Millisecond, + MaxPollFrequency: 5 * time.Millisecond, + } + + t.Run("should return modified event when file is written to", func(t *testing.T) { + f := createEmptyFile(t, "startempty") + defer f.Close() + + go func() { + time.Sleep(50 * time.Millisecond) + _, err := f.WriteString("updated") + require.NoError(t, err) + }() + + event, err := blockUntilEvent(context.Background(), f, 0, &Config{ + Filename: f.Name(), + WatcherConfig: watcherConfig, + }) + require.NoError(t, err) + require.Equal(t, eventModified, event) + }) + + t.Run("should return modified event if mod time is updated", func(t *testing.T) { + f := createEmptyFile(t, "startempty") + defer f.Close() + + go func() { + time.Sleep(50 * time.Millisecond) + require.NoError(t, os.Chtimes(f.Name(), time.Now(), time.Now())) + }() + + event, err := blockUntilEvent(context.Background(), f, 0, &Config{ + Filename: f.Name(), + WatcherConfig: watcherConfig, + }) + require.NoError(t, err) + require.Equal(t, eventModified, event) + }) + + t.Run("should return deleted event if file is deleted", func(t *testing.T) { + f := createEmptyFile(t, "startempty") + defer f.Close() + + go func() { + time.Sleep(50 * time.Millisecond) + removeFile(t, f.Name()) + }() + + event, err := blockUntilEvent(context.Background(), f, 0, &Config{ + Filename: f.Name(), + WatcherConfig: watcherConfig, + }) + require.NoError(t, err) + require.Equal(t, eventDeleted, event) + }) + + t.Run("should return deleted event if file is deleted before", func(t *testing.T) { + f := createEmptyFile(t, "startempty") + defer f.Close() + + removeFile(t, f.Name()) + + event, err := blockUntilEvent(context.Background(), f, 0, &Config{ + Filename: f.Name(), + WatcherConfig: watcherConfig, + }) + require.NoError(t, err) + require.Equal(t, eventDeleted, event) + }) + + t.Run("should return truncated event", func(t *testing.T) { + f := createFileWithContent(t, "truncate", "content") + defer f.Close() + + offset, err := f.Seek(0, io.SeekEnd) + require.NoError(t, err) + + go func() { + time.Sleep(50 * time.Millisecond) + err := f.Truncate(0) + fmt.Println(err) + }() + + event, err := blockUntilEvent(context.Background(), f, offset, &Config{ + Filename: f.Name(), + WatcherConfig: watcherConfig, + }) + require.NoError(t, err) + require.Equal(t, eventTruncated, event) + }) +} diff --git a/internal/component/loki/source/file/internal/tail/file_test.go b/internal/component/loki/source/file/internal/tail/file_test.go index 5bd19280de..d2c511715d 100644 --- a/internal/component/loki/source/file/internal/tail/file_test.go +++ b/internal/component/loki/source/file/internal/tail/file_test.go @@ -238,8 +238,18 @@ func createFile(t *testing.T, name, content string) string { return path } -func recreateFile(t *testing.T, path, content string) { - require.NoError(t, os.WriteFile(path, []byte(content), 0600)) +func createEmptyFile(t *testing.T, name string) *os.File { + path := t.TempDir() + "/" + name + f, err := os.Create(path) + require.NoError(t, err) + return f +} + +func createFileWithContent(t *testing.T, name, content string) *os.File { + path := createFile(t, name, content) + f, err := os.OpenFile(path, os.O_RDWR, 0) + require.NoError(t, err) + return f } func appendToFile(t *testing.T, name, content string) { From 295243f437d219b10f345139627fbf787c19e8f8 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Fri, 5 Dec 2025 10:00:20 +0100 Subject: [PATCH 24/28] add tests for blockUntilExists --- .../loki/source/file/internal/tail/block.go | 1 - .../source/file/internal/tail/block_test.go | 85 +++++++++++++++++-- .../source/file/internal/tail/file_test.go | 14 --- 3 files changed, 78 insertions(+), 22 deletions(-) diff --git a/internal/component/loki/source/file/internal/tail/block.go b/internal/component/loki/source/file/internal/tail/block.go index 43bd046c9d..083d2c0790 100644 --- a/internal/component/loki/source/file/internal/tail/block.go +++ b/internal/component/loki/source/file/internal/tail/block.go @@ -25,7 +25,6 @@ func blockUntilExists(ctx context.Context, cfg *Config) error { } else if !os.IsNotExist(err) { return err } - backoff.Wait() } diff --git a/internal/component/loki/source/file/internal/tail/block_test.go b/internal/component/loki/source/file/internal/tail/block_test.go index 05ae8b5484..efcad8e7de 100644 --- a/internal/component/loki/source/file/internal/tail/block_test.go +++ b/internal/component/loki/source/file/internal/tail/block_test.go @@ -2,15 +2,51 @@ package tail import ( "context" - "fmt" "io" "os" + "path/filepath" "testing" "time" "github.com/stretchr/testify/require" ) +func TestBlockUntilExists(t *testing.T) { + watcherConfig := WatcherConfig{ + MinPollFrequency: 5 * time.Millisecond, + MaxPollFrequency: 5 * time.Millisecond, + } + + t.Run("should block until file exists", func(t *testing.T) { + filename := filepath.Join(t.TempDir(), "eventually") + + go func() { + time.Sleep(10 * time.Millisecond) + createFileWithPath(t, filename, "") + }() + + err := blockUntilExists(context.Background(), &Config{ + Filename: filename, + WatcherConfig: watcherConfig, + }) + require.NoError(t, err) + }) + + t.Run("should exit when context is canceled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(10 * time.Millisecond) + cancel() + }() + + err := blockUntilExists(ctx, &Config{ + Filename: filepath.Join(t.TempDir(), "never"), + WatcherConfig: watcherConfig, + }) + require.ErrorIs(t, err, context.Canceled) + }) +} + func TestBlockUntilEvent(t *testing.T) { watcherConfig := WatcherConfig{ MinPollFrequency: 5 * time.Millisecond, @@ -22,7 +58,7 @@ func TestBlockUntilEvent(t *testing.T) { defer f.Close() go func() { - time.Sleep(50 * time.Millisecond) + time.Sleep(10 * time.Millisecond) _, err := f.WriteString("updated") require.NoError(t, err) }() @@ -40,7 +76,7 @@ func TestBlockUntilEvent(t *testing.T) { defer f.Close() go func() { - time.Sleep(50 * time.Millisecond) + time.Sleep(10 * time.Millisecond) require.NoError(t, os.Chtimes(f.Name(), time.Now(), time.Now())) }() @@ -57,7 +93,7 @@ func TestBlockUntilEvent(t *testing.T) { defer f.Close() go func() { - time.Sleep(50 * time.Millisecond) + time.Sleep(10 * time.Millisecond) removeFile(t, f.Name()) }() @@ -91,9 +127,8 @@ func TestBlockUntilEvent(t *testing.T) { require.NoError(t, err) go func() { - time.Sleep(50 * time.Millisecond) - err := f.Truncate(0) - fmt.Println(err) + time.Sleep(10 * time.Millisecond) + require.NoError(t, f.Truncate(0)) }() event, err := blockUntilEvent(context.Background(), f, offset, &Config{ @@ -103,4 +138,40 @@ func TestBlockUntilEvent(t *testing.T) { require.NoError(t, err) require.Equal(t, eventTruncated, event) }) + + t.Run("should exit when context is canceled", func(t *testing.T) { + f := createEmptyFile(t, "startempty") + defer f.Close() + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(10 * time.Millisecond) + cancel() + }() + + event, err := blockUntilEvent(ctx, f, 0, &Config{ + Filename: f.Name(), + WatcherConfig: watcherConfig, + }) + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, eventNone, event) + }) +} + +func createEmptyFile(t *testing.T, name string) *os.File { + path := filepath.Join(t.TempDir(), name) + f, err := os.Create(path) + require.NoError(t, err) + return f +} + +func createFileWithContent(t *testing.T, name, content string) *os.File { + path := createFile(t, name, content) + f, err := os.OpenFile(path, os.O_RDWR, 0) + require.NoError(t, err) + return f +} + +func createFileWithPath(t *testing.T, path, content string) { + require.NoError(t, os.WriteFile(path, []byte(content), 0600)) } diff --git a/internal/component/loki/source/file/internal/tail/file_test.go b/internal/component/loki/source/file/internal/tail/file_test.go index d2c511715d..a7265542c0 100644 --- a/internal/component/loki/source/file/internal/tail/file_test.go +++ b/internal/component/loki/source/file/internal/tail/file_test.go @@ -238,20 +238,6 @@ func createFile(t *testing.T, name, content string) string { return path } -func createEmptyFile(t *testing.T, name string) *os.File { - path := t.TempDir() + "/" + name - f, err := os.Create(path) - require.NoError(t, err) - return f -} - -func createFileWithContent(t *testing.T, name, content string) *os.File { - path := createFile(t, name, content) - f, err := os.OpenFile(path, os.O_RDWR, 0) - require.NoError(t, err) - return f -} - func appendToFile(t *testing.T, name, content string) { f, err := os.OpenFile(name, os.O_APPEND|os.O_WRONLY, 0600) require.NoError(t, err) From 2cbf1eed21d10f63539b196086b1c1e4adb53535 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Mon, 8 Dec 2025 10:03:50 +0100 Subject: [PATCH 25/28] Open file with correct flags on windows --- .../loki/source/file/internal/tail/block_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/component/loki/source/file/internal/tail/block_test.go b/internal/component/loki/source/file/internal/tail/block_test.go index efcad8e7de..8baa555613 100644 --- a/internal/component/loki/source/file/internal/tail/block_test.go +++ b/internal/component/loki/source/file/internal/tail/block_test.go @@ -9,6 +9,8 @@ import ( "time" "github.com/stretchr/testify/require" + + "github.com/grafana/alloy/internal/component/loki/source/file/internal/tail/fileext" ) func TestBlockUntilExists(t *testing.T) { @@ -90,6 +92,11 @@ func TestBlockUntilEvent(t *testing.T) { t.Run("should return deleted event if file is deleted", func(t *testing.T) { f := createEmptyFile(t, "startempty") + require.NoError(t, f.Close()) + + // NOTE: important for windows that we open with correct flags. + f, err := fileext.OpenFile(f.Name()) + require.NoError(t, err) defer f.Close() go func() { @@ -107,6 +114,11 @@ func TestBlockUntilEvent(t *testing.T) { t.Run("should return deleted event if file is deleted before", func(t *testing.T) { f := createEmptyFile(t, "startempty") + require.NoError(t, f.Close()) + + // NOTE: important for windows that we open with correct flags. + f, err := fileext.OpenFile(f.Name()) + require.NoError(t, err) defer f.Close() removeFile(t, f.Name()) From 9e7e6b3a5b2607434f8e475f4507be26c2f25ede Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Wed, 10 Dec 2025 09:41:27 +0100 Subject: [PATCH 26/28] trim both carriage return and newline --- internal/component/loki/source/file/internal/tail/file.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/internal/component/loki/source/file/internal/tail/file.go b/internal/component/loki/source/file/internal/tail/file.go index 204f21f52a..ad4b99e110 100644 --- a/internal/component/loki/source/file/internal/tail/file.go +++ b/internal/component/loki/source/file/internal/tail/file.go @@ -167,11 +167,7 @@ func (f *File) readLine() (string, error) { if err != nil { return line, err } - - line = strings.TrimRight(line, "\n") - // Trim Windows line endings - line = strings.TrimSuffix(line, "\r") - return line, err + return strings.TrimRight(line, "\r\n"), err } // offset returns the current byte offset in the file where the next read will occur. From 8666510477aee811eb4e3e7e8f425ef64f0a418f Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Wed, 10 Dec 2025 09:43:15 +0100 Subject: [PATCH 27/28] Use configured watcher config --- internal/component/loki/source/file/internal/tail/file.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/component/loki/source/file/internal/tail/file.go b/internal/component/loki/source/file/internal/tail/file.go index ad4b99e110..4046acea75 100644 --- a/internal/component/loki/source/file/internal/tail/file.go +++ b/internal/component/loki/source/file/internal/tail/file.go @@ -201,8 +201,8 @@ func (f *File) reopen(truncated bool) error { f.file.Close() backoff := backoff.New(f.ctx, backoff.Config{ - MinBackoff: defaultWatcherConfig.MaxPollFrequency, - MaxBackoff: defaultWatcherConfig.MaxPollFrequency, + MinBackoff: f.cfg.WatcherConfig.MinPollFrequency, + MaxBackoff: f.cfg.WatcherConfig.MaxPollFrequency, MaxRetries: 20, }) From 9f08d057cde08c792e76743f81a6e4ac0d259278 Mon Sep 17 00:00:00 2001 From: Kalle <23356117+kalleep@users.noreply.github.com> Date: Wed, 10 Dec 2025 09:52:59 +0100 Subject: [PATCH 28/28] remove unused dependency --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index a0787643f2..e09b510394 100644 --- a/go.mod +++ b/go.mod @@ -305,7 +305,6 @@ require ( google.golang.org/api v0.254.0 google.golang.org/grpc v1.76.0 google.golang.org/protobuf v1.36.10 - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools v2.2.0+incompatible