fix(op-acceptor): ensure logs are written to disk for timeout cases (#540)

When tests timeout and CI kills the process (SIGKILL, exit code 137),
buffered writes in AsyncFile were being lost because the background
goroutine never had a chance to flush them.

This adds:
- AsyncFile.Flush() to block until queued writes are synced to disk
- FileLogger.FlushAll() to flush all async writers
- FileLogger.LogTestResultSync() for synchronous logging of timeout results
- Graceful shutdown flush in nat.Stop() to persist logs before exit

This ensures per-test log files exist in logs/testrun-*/failed/ even
when tests are terminated due to timeout.

Author: scharissis

op-acceptor/logging/filelogger.go

@@ -129,6 +129,30 @@ func (af *AsyncFile) Close() error {
 	return af.file.Close()
 }
 
+// Flush blocks until all queued writes are written to disk and synced.
+// It sends an empty marker to the queue and waits for the queue to drain,
+// ensuring all previously queued writes have been processed.
+func (af *AsyncFile) Flush() error {
+	af.mu.Lock()
+	if af.stopped {
+		af.mu.Unlock()
+		return nil
+	}
+	af.mu.Unlock()
+
+	// Send an empty marker to the queue. Once the queue drains to empty,
+	// we know all writes (including the marker) have been processed.
+	af.queue <- []byte{}
+
+	// Wait for the queue to drain. This spin-wait is acceptable for flush
+	// operations which only occur during shutdown or timeout handling.
+	for len(af.queue) > 0 {
+		time.Sleep(time.Millisecond)
+	}
+
+	return af.file.Sync()
+}
+
 // NewFileLogger creates a new FileLogger with given configuration
 func NewFileLogger(baseDir string, runID string, networkName string, gateRuns []string) (*FileLogger, error) {
 	if runID == "" {
@@ -246,6 +270,20 @@ func (l *FileLogger) closeAllWriters() {
 	l.asyncWriters = make(map[string]*AsyncFile)
 }
 
+// FlushAll flushes all async writers to disk without closing them
+func (l *FileLogger) FlushAll() error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	var firstErr error
+	for _, writer := range l.asyncWriters {
+		if err := writer.Flush(); err != nil && firstErr == nil {
+			firstErr = err
+		}
+	}
+	return firstErr
+}
+
 // GetDirectoryForRunID returns the path for a specific runID
 // The runID must be provided, otherwise an error is returned
 func (l *FileLogger) GetDirectoryForRunID(runID string) (string, error) {
@@ -277,6 +315,15 @@ func (l *FileLogger) LogTestResult(result *types.TestResult, runID string) error
 	return nil
 }
 
+// LogTestResultSync logs a test result and flushes immediately to disk.
+// Use this for timeout cases to ensure logs are written before process termination.
+func (l *FileLogger) LogTestResultSync(result *types.TestResult, runID string) error {
+	if err := l.LogTestResult(result, runID); err != nil {
+		return err
+	}
+	return l.FlushAll()
+}
+
 // LogSummary writes a summary of the test run to a file
 // The runID must be provided, otherwise an error is returned
 func (l *FileLogger) LogSummary(summary string, runID string) error {

op-acceptor/nat.go

@@ -784,6 +784,13 @@ func (n *nat) writeRunArtifacts(runID string) error {
 func (n *nat) Stop(ctx context.Context) error {
 	n.config.Log.Info("Stopping op-acceptor")
 
+	// Flush logs before shutdown to ensure all buffered writes are persisted
+	if n.fileLogger != nil {
+		if err := n.fileLogger.FlushAll(); err != nil {
+			n.config.Log.Error("Error flushing logs during shutdown", "error", err)
+		}
+	}
+
 	// Check if we're already stopped
 	if !n.running.Load() {
 		n.config.Log.Debug("Service already stopped, nothing to do")

op-acceptor/runner/runner.go

@@ -976,8 +976,9 @@ func (r *runner) runSingleTest(ctx context.Context, metadata types.ValidatorMeta
 		}
 
 		// Force logging of timeout result to ensure it's captured
+		// Use synchronous logging to ensure writes are flushed to disk before potential process termination
 		if r.fileLogger != nil {
-			if logErr := r.fileLogger.LogTestResult(parsed, r.runID); logErr != nil {
+			if logErr := r.fileLogger.LogTestResultSync(parsed, r.runID); logErr != nil {
 				r.log.Error("Failed to log timeout result", "error", logErr, "test", metadata.FuncName)
 			}
 		}