feat(be): sqlite support

Author: mgukov

.dredd/hooks/helpers.go

@@ -70,7 +70,7 @@ func truncateAll() {
 	case *bolt.BoltDb:
 		// Do nothing
 	case *sql.SqlDb:
-		switch store.(*sql.SqlDb).Sql().Dialect.(type) {
+		switch store.(*sql.SqlDb).Sql().Dialect().(type) {
 		case gorp.PostgresDialect:
 			// Do nothing
 		case gorp.MySQLDialect:

db/Migration.go

@@ -3,6 +3,7 @@ package db
 import (
 	"fmt"
 	"github.com/semaphoreui/semaphore/pkg/tz"
+	"github.com/semaphoreui/semaphore/util"
 	"slices"
 	"strconv"
 	"strings"
@@ -21,7 +22,14 @@ func (m Migration) HumanoidVersion() string {
 	return "v" + m.Version
 }
 
-func GetMigrations() []Migration {
+func GetMigrations(dialect string) []Migration {
+	if dialect == util.DbDriverSQLite {
+		return []Migration{
+			{Version: "2.15.1.sqlite"},
+			{Version: "2.16.0.sqlite"},
+		}
+	}
+
 	return []Migration{
 		{Version: "0.0.0"},
 		{Version: "1.0.0"},
@@ -187,7 +195,7 @@ func Rollback(d Store, targetVersion string) error {
 
 	didRun := false
 
-	migrations := GetMigrations()
+	migrations := GetMigrations(d.GetDialect())
 	slices.Reverse(migrations)
 
 	for _, version := range migrations {
@@ -220,7 +228,7 @@ func Rollback(d Store, targetVersion string) error {
 func Migrate(d Store, targetVersion *string) error {
 	didRun := false
 
-	for _, version := range GetMigrations() {
+	for _, version := range GetMigrations(d.GetDialect()) {
 
 		if targetVersion != nil && version.Compare(Migration{Version: *targetVersion}) > 0 {
 			break

db/Store.go

@@ -194,6 +194,7 @@ type ConnectionManager interface {
 
 // MigrationManager handles database migrations
 type MigrationManager interface {
+	GetDialect() string
 	// IsInitialized indicates is database already initialized, or it is empty.
 	// The method is useful for creating required entities in database during first run.
 	IsInitialized() (bool, error)

db/bolt/BoltDb.go

@@ -44,6 +44,10 @@ type BoltDb struct {
 	terraformAlias   publicAlias
 }
 
+func (d *BoltDb) GetDialect() string {
+	return util.DbDriverBolt
+}
+
 var terraformAliasProps = db.ObjectProps{
 	TableName:         "terraform_alias",
 	Type:              reflect.TypeOf(db.TerraformInventoryAlias{}),

db/factory/store.go

@@ -14,11 +14,13 @@ func CreateStore() db.Store {
 	}
 	switch config.Dialect {
 	case util.DbDriverMySQL:
-		return &sql.SqlDb{}
+		return sql.CreateDb(config.Dialect)
 	case util.DbDriverBolt:
 		return bolt.CreateBoltDB()
 	case util.DbDriverPostgres:
-		return &sql.SqlDb{}
+		return sql.CreateDb(config.Dialect)
+	case util.DbDriverSQLite:
+		return sql.CreateDb(config.Dialect)
 	default:
 		panic("Unsupported database dialect: " + config.Dialect)
 	}

db/sql/SqlDb.go

@@ -18,10 +18,12 @@ import (
 	"github.com/semaphoreui/semaphore/pkg/task_logger"
 	"github.com/semaphoreui/semaphore/util"
 	log "github.com/sirupsen/logrus"
+	_ "modernc.org/sqlite" // Import the driver
 )
 
 type SqlDb struct {
-	sql *gorp.DbMap
+	sql     Sql
+	dialect string
 }
 
 var initialSQL = `
@@ -35,6 +37,14 @@ create table ` + "`migrations`" + ` (
 //go:embed migrations/*.sql
 var dbAssets embed.FS
 
+func CreateDb(dialect string) *SqlDb {
+	return &SqlDb{dialect: dialect}
+}
+
+func (d *SqlDb) GetDialect() string {
+	return d.dialect
+}
+
 func getQueryForParams(q squirrel.SelectBuilder, prefix string, props db.ObjectProps, params db.RetrieveQueryParams) (res squirrel.SelectBuilder, err error) {
 	pp, err := params.Validate(props)
 	if err != nil {
@@ -114,27 +124,31 @@ func (d *SqlDb) prepareQueryWithDialect(query string, dialect gorp.Dialect) stri
 }
 
 func (d *SqlDb) PrepareQuery(query string) string {
-	return d.prepareQueryWithDialect(query, d.sql.Dialect)
+	return d.prepareQueryWithDialect(query, d.sql.Dialect())
 }
 
 func (d *SqlDb) insert(primaryKeyColumnName string, query string, args ...any) (int, error) {
+	return d.insertBy(d.sql, primaryKeyColumnName, query, args...)
+}
+
+func (d *SqlDb) insertBy(executor gorp.SqlExecutor, primaryKeyColumnName string, query string, args ...any) (int, error) {
 	var insertId int64
 
-	switch d.sql.Dialect.(type) {
+	switch d.sql.Dialect().(type) {
 	case gorp.PostgresDialect:
 		var err error
 		if primaryKeyColumnName != "" {
 			query += " returning " + primaryKeyColumnName
-			err = d.sql.QueryRow(d.PrepareQuery(query), args...).Scan(&insertId)
+			err = executor.QueryRow(d.PrepareQuery(query), args...).Scan(&insertId)
 		} else {
-			_, err = d.sql.Exec(d.PrepareQuery(query), args...)
+			_, err = executor.Exec(d.PrepareQuery(query), args...)
 		}
 
 		if err != nil {
 			return 0, err
 		}
 	default:
-		res, err := d.exec(query, args...)
+		res, err := executor.Exec(d.PrepareQuery(query), args...)
 		if err != nil {
 			return 0, err
 		}
@@ -323,7 +337,7 @@ func (d *SqlDb) deleteObject(projectID int, props db.ObjectProps, objectID any)
 }
 
 func (d *SqlDb) Close(token string) {
-	err := d.sql.Db.Close()
+	err := d.sql.Close()
 	if err != nil {
 		panic(err)
 	}
@@ -364,16 +378,11 @@ func (d *SqlDb) Connect(_ string) {
 		panic(err)
 	}
 
-	var dialect gorp.Dialect
-
-	switch cfg.Dialect {
-	case util.DbDriverMySQL:
-		dialect = gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"}
-	case util.DbDriverPostgres:
-		dialect = gorp.PostgresDialect{}
-	}
+	d.sql = Create(cfg.Dialect, sqlDb) //&gorp.DbMap{Db: sqlDb, Dialect: dialect}
 
-	d.sql = &gorp.DbMap{Db: sqlDb, Dialect: dialect}
+	//if d.GetDialect() == util.DbDriverSQLite {
+	//	sqlDb.SetMaxOpenConns(1)
+	//}
 
 	d.sql.AddTableWithName(db.APIToken{}, "user__token").SetKeys(false, "id")
 	d.sql.AddTableWithName(db.AccessKey{}, "access_key").SetKeys(true, "id")
@@ -486,7 +495,7 @@ func (d *SqlDb) getObjectRefsFrom(
 	return
 }
 
-func (d *SqlDb) Sql() *gorp.DbMap {
+func (d *SqlDb) Sql() Sql {
 	return d.sql
 }
 

db/sql/common_sql.go

@@ -0,0 +1,190 @@
+package sql
+
+import (
+	"context"
+	"database/sql"
+	"github.com/go-gorp/gorp/v3"
+	"github.com/semaphoreui/semaphore/util"
+	"sync"
+)
+
+type FakeLocker struct {
+}
+
+func (f FakeLocker) Lock() {
+}
+
+func (f FakeLocker) Unlock() {
+}
+
+type CommonSql struct {
+	db    *gorp.DbMap
+	mutex sync.Locker
+}
+
+func Create(dialectStr string, sqlDb *sql.DB) *CommonSql {
+	var dialect gorp.Dialect
+
+	switch dialectStr {
+	case util.DbDriverMySQL:
+		dialect = gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"}
+	case util.DbDriverPostgres:
+		dialect = gorp.PostgresDialect{}
+	case util.DbDriverSQLite:
+		dialect = gorp.SqliteDialect{}
+	}
+
+	var mutex sync.Locker
+	if dialectStr == util.DbDriverSQLite {
+		mutex = &sync.Mutex{}
+	} else {
+		mutex = &FakeLocker{}
+	}
+
+	return &CommonSql{db: &gorp.DbMap{Db: sqlDb, Dialect: dialect}, mutex: mutex}
+}
+
+func (c CommonSql) Lock() {
+	c.mutex.Lock()
+}
+
+func (c CommonSql) Unlock() {
+	c.mutex.Unlock()
+}
+
+func (c CommonSql) QueryRow(query string, args ...interface{}) *sql.Row {
+	c.Lock()
+	defer c.Unlock()
+	return c.db.QueryRow(query, args...)
+}
+
+func (c CommonSql) Insert(list ...interface{}) error {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.Insert(list...)
+}
+
+func (c CommonSql) SelectOne(holder interface{}, query string, args ...interface{}) error {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.SelectOne(holder, query, args...)
+}
+
+func (c CommonSql) PrepareQuery(query string) string {
+	return query
+}
+
+func (c CommonSql) Exec(query string, args ...interface{}) (sql.Result, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.Exec(query, args...)
+}
+
+func (c CommonSql) Select(i interface{}, query string, args ...interface{}) ([]interface{}, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.Select(i, query, args...)
+}
+
+func (c CommonSql) SelectInt(query string, args ...interface{}) (int64, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.SelectInt(query, args...)
+}
+
+func (c CommonSql) AddTableWithName(i interface{}, name string) *gorp.TableMap {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.AddTableWithName(i, name)
+}
+
+func (c CommonSql) Dialect() gorp.Dialect {
+	return c.db.Dialect
+}
+
+func (c CommonSql) Close() error {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.Db.Close()
+}
+
+func (c CommonSql) Begin() (*gorp.Transaction, error) {
+	return c.db.Begin()
+}
+
+func (c CommonSql) WithContext(ctx context.Context) gorp.SqlExecutor {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.WithContext(ctx)
+}
+
+func (c CommonSql) Get(i interface{}, keys ...interface{}) (interface{}, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.Get(i, keys...)
+}
+
+func (c CommonSql) Update(list ...interface{}) (int64, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.Update(list...)
+}
+
+func (c CommonSql) Delete(list ...interface{}) (int64, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.Delete(list...)
+}
+
+func (c CommonSql) SelectNullInt(query string, args ...interface{}) (sql.NullInt64, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.SelectNullInt(query, args...)
+}
+
+func (c CommonSql) SelectFloat(query string, args ...interface{}) (float64, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.SelectFloat(query, args...)
+}
+
+func (c CommonSql) SelectNullFloat(query string, args ...interface{}) (sql.NullFloat64, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.SelectNullFloat(query, args...)
+}
+
+func (c CommonSql) SelectStr(query string, args ...interface{}) (string, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.SelectStr(query, args...)
+}
+
+func (c CommonSql) SelectNullStr(query string, args ...interface{}) (sql.NullString, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.SelectNullStr(query, args...)
+}
+
+func (c CommonSql) Query(query string, args ...interface{}) (*sql.Rows, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.db.Query(query, args...)
+}

db/sql/migration.go

@@ -55,7 +55,7 @@ func getVersionSQL(name string, ignoreErrors bool) (queries []string) {
 // prepareMigration converts migration SQLite-query to current dialect.
 // Supported MySQL and Postgres dialects.
 func (d *SqlDb) prepareMigration(query string) string {
-	switch d.sql.Dialect.(type) {
+	switch d.sql.Dialect().(type) {
 	case gorp.MySQLDialect:
 		query = autoIncrementRE.ReplaceAllString(query, "auto_increment")
 		query = ifExistsRE.ReplaceAllString(query, "")

db/sql/migration_2_10_24.go

@@ -7,7 +7,7 @@ type migration_2_10_24 struct {
 }
 
 func (m migration_2_10_24) PreApply(tx *gorp.Transaction) error {
-	switch m.db.sql.Dialect.(type) {
+	switch m.db.sql.Dialect().(type) {
 	case gorp.MySQLDialect:
 		_, _ = tx.Exec(m.db.PrepareQuery("alter table `project__template` drop foreign key `project__template_ibfk_6`"))
 	case gorp.PostgresDialect: