refactor(database): Refactor database connection management to avoid sqlite database is locked

internal/bootstrap/db.go

@@ -2,6 +2,8 @@ package bootstrap
 
 import (
 	"fmt"
+	"github.com/OpenListTeam/OpenList/v4/internal/bootstrap/dbengine"
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
 	stdlog "log"
 	"strings"
 	"time"
@@ -10,9 +12,6 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/conf"
 	"github.com/OpenListTeam/OpenList/v4/internal/db"
 	log "github.com/sirupsen/logrus"
-	"gorm.io/driver/mysql"
-	"gorm.io/driver/postgres"
-	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 	"gorm.io/gorm/schema"
@@ -38,10 +37,10 @@ func InitDB() {
 		},
 		Logger: newLogger,
 	}
-	var dB *gorm.DB
+	var dB model.Connection
 	var err error
 	if flags.Dev {
-		dB, err = gorm.Open(sqlite.Open("file::memory:?cache=shared"), gormConfig)
+		dB, err = dbengine.CreateSqliteCon("file::memory:?cache=shared", gormConfig)
 		conf.Conf.Database.Type = "sqlite3"
 	} else {
 		database := conf.Conf.Database
@@ -51,18 +50,18 @@ func InitDB() {
 				if !(strings.HasSuffix(database.DBFile, ".db") && len(database.DBFile) > 3) {
 					log.Fatalf("db name error.")
 				}
-				dB, err = gorm.Open(sqlite.Open(fmt.Sprintf("%s?_journal=WAL&_vacuum=incremental",
-					database.DBFile)), gormConfig)
+				dB, err = dbengine.CreateSqliteCon(fmt.Sprintf("%s?_journal=WAL&_vacuum=incremental&_txlock=immediate",
+					database.DBFile), gormConfig)
 			}
 		case "mysql":
 			{
 				dsn := database.DSN
 				if dsn == "" {
-					//[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
+					// [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
 					dsn = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local&tls=%s",
 						database.User, database.Password, database.Host, database.Port, database.Name, database.SSLMode)
 				}
-				dB, err = gorm.Open(mysql.Open(dsn), gormConfig)
+				dB, err = dbengine.CreateMysqlCon(dsn, gormConfig)
 			}
 		case "postgres":
 			{
@@ -76,7 +75,7 @@ func InitDB() {
 							database.Host, database.User, database.Name, database.Port, database.SSLMode)
 					}
 				}
-				dB, err = gorm.Open(postgres.Open(dsn), gormConfig)
+				dB, err = dbengine.CreatePostgresCon(dsn, gormConfig)
 			}
 		default:
 			log.Fatalf("not supported database type: %s", database.Type)

internal/bootstrap/dbengine/mysql.go

@@ -0,0 +1,38 @@
+package dbengine
+
+import (
+	"fmt"
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+)
+
+// CreateMysqlCon creates MySQL database connections
+func CreateMysqlCon(dsn string, gormConfig *gorm.Config) (con model.Connection, err error) {
+	var (
+		db *gorm.DB
+	)
+
+	// Create MySQL database connection
+	db, err = gorm.Open(mysql.Open(dsn), gormConfig)
+	if err != nil {
+		return model.Connection{}, fmt.Errorf("cannot create MySQL database connection: %w", err)
+	}
+
+	// Get underlying database connection for configuration
+	sqlDB, err := db.DB()
+	if err != nil {
+		return model.Connection{}, fmt.Errorf("cannot access underlying MySQL database connection: %w", err)
+	}
+
+	// Set connection pool parameters
+	sqlDB.SetMaxOpenConns(100)
+	sqlDB.SetMaxIdleConns(10)
+	sqlDB.SetConnMaxLifetime(0)
+
+	// For MySQL, both read and write connections point to the same database instance
+	return model.Connection{
+		Read:  db, // Read connection
+		Write: db, // Write connection
+	}, nil
+}

internal/bootstrap/dbengine/postgres.go

@@ -0,0 +1,38 @@
+package dbengine
+
+import (
+	"fmt"
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+// CreatePostgresCon creates PostgreSQL database connections
+func CreatePostgresCon(dsn string, gormConfig *gorm.Config) (con model.Connection, err error) {
+	var (
+		db *gorm.DB
+	)
+
+	// Create PostgreSQL database connection
+	db, err = gorm.Open(postgres.Open(dsn), gormConfig)
+	if err != nil {
+		return model.Connection{}, fmt.Errorf("cannot create PostgreSQL database connection: %w", err)
+	}
+
+	// Get underlying database connection for configuration
+	sqlDB, err := db.DB()
+	if err != nil {
+		return model.Connection{}, fmt.Errorf("cannot access underlying PostgreSQL database connection: %w", err)
+	}
+
+	// Set connection pool parameters
+	sqlDB.SetMaxOpenConns(100)
+	sqlDB.SetMaxIdleConns(10)
+	sqlDB.SetConnMaxLifetime(0)
+
+	// For PostgreSQL, both read and write connections point to the same database instance
+	return model.Connection{
+		Read:  db, // Read connection
+		Write: db, // Write connection
+	}, nil
+}

internal/bootstrap/dbengine/sqlite.go

@@ -0,0 +1,102 @@
+package dbengine
+
+import (
+	"database/sql"
+	"fmt"
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"runtime"
+	"strings"
+)
+
+// CreateSqliteCon uses best practices for sqlite and creates two connections
+// One optimized for reading and the other optimized for writing
+// found the information here: https://kerkour.com/sqlite-for-servers
+// copied from https://github.com/bihe/monorepo/blob/477e534bd4c0814cdca73fea774b518148cebd3f/pkg/persistence/sqlite.go#L59
+// with little edit.
+// it should solve "database is locked error", and make better performance.
+func CreateSqliteCon(dsn string, gormConfig *gorm.Config) (con model.Connection, err error) {
+	var (
+		read  *gorm.DB
+		write *gorm.DB
+	)
+
+	// Read DB
+	read, err = gorm.Open(sqlite.Open(dsn), gormConfig)
+	if err != nil {
+		return model.Connection{}, fmt.Errorf("cannot create read database connection: %w", err)
+	}
+	readDB, err := read.DB()
+	if err != nil {
+		return model.Connection{}, fmt.Errorf("cannot access underlying read database connection: %w", err)
+	}
+	if !strings.Contains(dsn, ":memory:") && !strings.Contains(dsn, "mode=memory") {
+		err = setDefaultPragmas(readDB)
+	}
+	if err != nil {
+		return model.Connection{}, err
+	}
+	readDB.SetMaxOpenConns(max(4, runtime.NumCPU())) // read in parallel with open connection per core
+
+	// WriteDB
+	write, err = gorm.Open(sqlite.Open(dsn), gormConfig)
+	if err != nil {
+		return model.Connection{}, fmt.Errorf("cannot create write database connection: %w", err)
+	}
+	writeDB, err := write.DB()
+	if err != nil {
+		return model.Connection{}, fmt.Errorf("cannot access underlying write database connection: %w", err)
+	}
+	if !strings.Contains(dsn, ":memory:") && !strings.Contains(dsn, "mode=memory") {
+		err = setDefaultPragmas(readDB)
+	}
+	if err != nil {
+		return model.Connection{}, err
+	}
+	writeDB.SetMaxOpenConns(1) // only use one active connection for writing
+
+	return model.Connection{
+		Read:  read,
+		Write: write,
+	}, nil
+}
+
+// SetDefaultPragmas defines some sqlite pragmas for good performance and litestream compatibility
+// https://highperformancesqlite.com/articles/sqlite-recommended-pragmas
+// https://litestream.io/tips/
+func setDefaultPragmas(db *sql.DB) error {
+	var (
+		stmt string
+		val  string
+	)
+	defaultPragmas := map[string]string{
+		"journal_mode": "wal",   // https://www.sqlite.org/pragma.html#pragma_journal_mode
+		"busy_timeout": "5000",  // https://www.sqlite.org/pragma.html#pragma_busy_timeout
+		"synchronous":  "1",     // NORMAL --> https://www.sqlite.org/pragma.html#pragma_synchronous
+		"cache_size":   "10000", // 10000 pages = 40MB --> https://www.sqlite.org/pragma.html#pragma_cache_size
+		"foreign_keys": "1",     // 1(bool) --> https://www.sqlite.org/pragma.html#pragma_foreign_keys
+	}
+
+	// set the pragmas
+	for k := range defaultPragmas {
+		stmt = fmt.Sprintf("pragma %s = %s", k, defaultPragmas[k])
+		if _, err := db.Exec(stmt); err != nil {
+			return err
+		}
+	}
+
+	// validate the pragmas
+	for k := range defaultPragmas {
+		row := db.QueryRow(fmt.Sprintf("pragma %s", k))
+		err := row.Scan(&val)
+		if err != nil {
+			return err
+		}
+		if val != defaultPragmas[k] {
+			return fmt.Errorf("could not set pragma %s to %s", k, defaultPragmas[k])
+		}
+	}
+
+	return nil
+}

internal/db/db.go

@@ -5,13 +5,12 @@ import (
 
 	"github.com/OpenListTeam/OpenList/v4/internal/conf"
 	"github.com/OpenListTeam/OpenList/v4/internal/model"
-	"gorm.io/gorm"
 )
 
-var db *gorm.DB
+var rwDb model.Connection
 
-func Init(d *gorm.DB) {
-	db = d
+func Init(d model.Connection) {
+	rwDb = d
 	err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.SharingDB))
 	if err != nil {
 		log.Fatalf("failed migrate database: %s", err.Error())
@@ -21,27 +20,28 @@ func Init(d *gorm.DB) {
 func AutoMigrate(dst ...interface{}) error {
 	var err error
 	if conf.Conf.Database.Type == "mysql" {
-		err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(dst...)
+		err = rwDb.W().Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(dst...)
 	} else {
-		err = db.AutoMigrate(dst...)
+		err = rwDb.W().AutoMigrate(dst...)
 	}
 	return err
 }
 
-func GetDb() *gorm.DB {
-	return db
+func GetDb() model.Connection {
+	return rwDb
 }
 
 func Close() {
 	log.Info("closing db")
-	sqlDB, err := db.DB()
-	if err != nil {
-		log.Errorf("failed to get db: %s", err.Error())
-		return
+	var err error
+	switch conf.Conf.Database.Type {
+	case "sqlite3":
+		err = rwDb.Close(true)
+	default:
+		err = rwDb.Close(false)
 	}
-	err = sqlDB.Close()
 	if err != nil {
-		log.Errorf("failed to close db: %s", err.Error())
+		log.Errorf(err.Error())
 		return
 	}
 }

internal/db/meta.go

@@ -7,30 +7,30 @@ import (
 
 func GetMetaByPath(path string) (*model.Meta, error) {
 	meta := model.Meta{Path: path}
-	if err := db.Where(meta).First(&meta).Error; err != nil {
+	if err := rwDb.R().Where(meta).First(&meta).Error; err != nil {
 		return nil, errors.Wrapf(err, "failed select meta")
 	}
 	return &meta, nil
 }
 
 func GetMetaById(id uint) (*model.Meta, error) {
 	var u model.Meta
-	if err := db.First(&u, id).Error; err != nil {
+	if err := rwDb.R().First(&u, id).Error; err != nil {
 		return nil, errors.Wrapf(err, "failed get old meta")
 	}
 	return &u, nil
 }
 
 func CreateMeta(u *model.Meta) error {
-	return errors.WithStack(db.Create(u).Error)
+	return errors.WithStack(rwDb.W().Create(u).Error)
 }
 
 func UpdateMeta(u *model.Meta) error {
-	return errors.WithStack(db.Save(u).Error)
+	return errors.WithStack(rwDb.W().Save(u).Error)
 }
 
 func GetMetas(pageIndex, pageSize int) (metas []model.Meta, count int64, err error) {
-	metaDB := db.Model(&model.Meta{})
+	metaDB := rwDb.W().Model(&model.Meta{})
 	if err = metaDB.Count(&count).Error; err != nil {
 		return nil, 0, errors.Wrapf(err, "failed get metas count")
 	}
@@ -41,5 +41,5 @@ func GetMetas(pageIndex, pageSize int) (metas []model.Meta, count int64, err err
 }
 
 func DeleteMetaById(id uint) error {
-	return errors.WithStack(db.Delete(&model.Meta{}, id).Error)
+	return errors.WithStack(rwDb.W().Delete(&model.Meta{}, id).Error)
 }