Fix SQLite locked errors and cleanup crash with busy_timeout and retry logic

Author: carsso

src/wigo/global.go

@@ -211,18 +211,44 @@ Options:
 		log.Fatalf("Fail to create table in sqlite database : %s\n", err)
 	}
 
+	// Configure SQLite busy timeout to handle concurrent access
+	_, err = LocalWigo.sqlLiteConn.Exec("PRAGMA busy_timeout = 5000")
+	if err != nil {
+		log.Fatalf("Fail to configure sqlite busy_timeout : %s\n", err)
+	}
+
 	// Launch cleaning routing
 	go func() {
 		for {
 			ts := time.Now().Unix() - 86400*30
 			sqlStmt := `DELETE FROM logs WHERE date < ?;`
 
-			LocalWigo.sqlLiteLock.Lock()
-			_, err = LocalWigo.sqlLiteConn.Exec(sqlStmt, ts)
-			if err != nil {
-				log.Fatalf("Fail to clean logs in database : %s\n", err)
+			maxRetries := 3
+			backoffDelays := []time.Duration{50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond}
+
+			var err error
+			for attempt := 0; attempt < maxRetries; attempt++ {
+				LocalWigo.sqlLiteLock.Lock()
+				_, err = LocalWigo.sqlLiteConn.Exec(sqlStmt, ts)
+				LocalWigo.sqlLiteLock.Unlock()
+
+				if err == nil {
+					break
+				}
+
+				// Check if error is SQLITE_BUSY or database locked
+				errStr := err.Error()
+				isBusyError := strings.Contains(errStr, "SQLITE_BUSY")
+
+				if !isBusyError || attempt == maxRetries-1 {
+					// Not a busy error or last attempt, log and break
+					log.Printf("Fail to clean logs in database : %s\n", err)
+					break
+				}
+
+				// Wait before retry
+				time.Sleep(backoffDelays[attempt])
 			}
-			LocalWigo.sqlLiteLock.Unlock()
 
 			time.Sleep(time.Hour)
 		}

src/wigo/log.go

@@ -3,6 +3,7 @@ package wigo
 import (
 	"log"
 	"os"
+	"strings"
 	"time"
 )
 
@@ -44,15 +45,34 @@ func (this *Log) SetGroup(group string) {
 
 // Persist on disk
 func (this *Log) Persist() {
-	LocalWigo.sqlLiteLock.Lock()
-	defer LocalWigo.sqlLiteLock.Unlock()
-
 	sqlStmt := `INSERT INTO logs(date,level,grp,host,probe,message) VALUES(?,?,?,?,?,?);`
-	_, err := LocalWigo.sqlLiteConn.Exec(sqlStmt, this.Timestamp, this.Level, this.Group, this.Host, this.Probe, this.Message)
-	if err != nil {
-		log.Printf("Fail to insert log in sqlLite : %s", err)
+
+	maxRetries := 3
+	backoffDelays := []time.Duration{50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond}
+
+	var err error
+	for attempt := 0; attempt < maxRetries; attempt++ {
+		LocalWigo.sqlLiteLock.Lock()
+		_, err = LocalWigo.sqlLiteConn.Exec(sqlStmt, this.Timestamp, this.Level, this.Group, this.Host, this.Probe, this.Message)
+		LocalWigo.sqlLiteLock.Unlock()
+
+		if err == nil {
+			return
+		}
+
+		// Check if error is SQLITE_BUSY or database locked
+		errStr := err.Error()
+		isBusyError := strings.Contains(errStr, "SQLITE_BUSY")
+
+		if !isBusyError || attempt == maxRetries-1 {
+			// Not a busy error or last attempt, log and return
+			log.Printf("Fail to insert log in sqlLite : %s", err)
+			return
+		}
+
+		// Wait before retry
+		time.Sleep(backoffDelays[attempt])
 	}
-	return
 }
 
 // Log levels