Merge pull request #3129 from semaphoreui/sqlite

feat(be): sqlite support

Author: fiftin

.github/workflows/dev.yml

@@ -225,6 +225,34 @@ jobs:
         run: |
           chmod +x ./semaphore && ./semaphore migrate --config config.json
 
+  migrate-sqlite:
+    runs-on: ubuntu-latest
+
+    needs:
+      - build-local
+
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: semaphore
+
+      - name: Write config
+        run: |
+          cat > config.json <<EOF
+            {
+              "sqlite": {
+                "host": "/tmp/db.sqlite"
+              },
+              "dialect": "sqlite",
+              "email_alert": false
+            }
+          EOF
+
+      - name: Migrate database
+        run: |
+          chmod +x ./semaphore && ./semaphore migrate --config config.json
+
   integrate-boltdb:
     runs-on: ubuntu-latest
 
@@ -532,6 +560,68 @@ jobs:
           task dredd:hooks
           task dredd:test
 
+  integrate-sqlite:
+    runs-on: ubuntu-latest
+
+    needs:
+      - migrate-sqlite
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Setup golang
+        uses: actions/setup-go@v5
+        with:
+          go-version: '^1.21.0'
+
+      - name: Setup nodejs
+        uses: actions/setup-node@v4
+        with:
+          node-version: '16'
+          cache: 'npm'
+          cache-dependency-path: web/package-lock.json
+
+      - name: Install go-task
+        run: |
+          go install github.com/go-task/task/v3/cmd/task@latest
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: semaphore
+
+      - name: Write config
+        run: |
+          cat > config.stdin <<EOF
+          4
+          /tmp/database.sqlite
+          /tmp/semaphore
+          http://localhost:3000
+          no
+          no
+          no
+          no
+          no
+          no
+          $(pwd)/.dredd
+          admin
+          admin@localhost
+          Developer
+          password
+          EOF
+
+      - name: Execute setup
+        run: |
+          chmod +x ./semaphore && ./semaphore setup - < config.stdin
+
+      - name: Launch dredd
+        run: |
+          task dredd:goodman
+          task dredd:deps
+          task dredd:hooks
+          task dredd:test
+
   deploy-server:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'semaphoreui'

.gitignore

@@ -13,6 +13,7 @@ api/public/**/*
 /config.runner.key
 /.dredd/config.json
 /database.boltdb
+/database.sqlite
 /database.boltdb.lock
 /database_test.boltdb
 .DS_Store

cli/setup/setup.go

@@ -115,6 +115,7 @@ func InteractiveSetup(conf *util.ConfigType) {
    1 - MySQL
    2 - BoltDB
    3 - PostgreSQL
+   4 - SQLite
 `
 
 	var db int
@@ -130,6 +131,9 @@ func InteractiveSetup(conf *util.ConfigType) {
 	case 3:
 		conf.Dialect = util.DbDriverPostgres
 		scanPostgres(conf)
+	case 4:
+		conf.Dialect = util.DbDriverSQLite
+		scanSQLite(conf)
 	}
 
 	defaultPlaybookPath := filepath.Join(os.TempDir(), "semaphore")
@@ -183,13 +187,11 @@ func InteractiveSetup(conf *util.ConfigType) {
 }
 
 func scanBoltDb(conf *util.ConfigType) {
-	workingDirectory, err := os.Getwd()
-	if err != nil {
-		workingDirectory = os.TempDir()
-	}
-	defaultBoltDBPath := filepath.Join(workingDirectory, "database.boltdb")
-	conf.BoltDb = &util.DbConfig{}
-	askValue("db filename", defaultBoltDBPath, &conf.BoltDb.Hostname)
+	conf.BoltDb = scanFileDB("database.boltdb")
+}
+
+func scanSQLite(conf *util.ConfigType) {
+	conf.SQLite = scanFileDB("database.sqlite")
 }
 
 func scanMySQL(conf *util.ConfigType) {
@@ -214,6 +216,17 @@ func scanPostgres(conf *util.ConfigType) {
 	}
 }
 
+func scanFileDB(defaultDbFile string) *util.DbConfig {
+	workingDirectory, err := os.Getwd()
+	if err != nil {
+		workingDirectory = os.TempDir()
+	}
+	defaultDBPath := filepath.Join(workingDirectory, defaultDbFile)
+	conf := &util.DbConfig{}
+	askValue("db Hostname", defaultDBPath, &conf.Hostname)
+	return conf
+}
+
 func scanErrorChecker(n int, err error) {
 	if err != nil && err.Error() != "unexpected newline" {
 		log.Warn("An input error occurred: " + err.Error())

db/Migration.go

@@ -3,6 +3,7 @@ package db
 import (
 	"fmt"
 	"github.com/semaphoreui/semaphore/pkg/tz"
+	"github.com/semaphoreui/semaphore/util"
 	"slices"
 	"strconv"
 	"strings"
@@ -21,7 +22,14 @@ func (m Migration) HumanoidVersion() string {
 	return "v" + m.Version
 }
 
-func GetMigrations() []Migration {
+func GetMigrations(dialect string) []Migration {
+	if dialect == util.DbDriverSQLite {
+		return []Migration{
+			{Version: "2.15.1.sqlite"},
+			{Version: "2.16.0.sqlite"},
+		}
+	}
+
 	return []Migration{
 		{Version: "0.0.0"},
 		{Version: "1.0.0"},
@@ -187,7 +195,7 @@ func Rollback(d Store, targetVersion string) error {
 
 	didRun := false
 
-	migrations := GetMigrations()
+	migrations := GetMigrations(d.GetDialect())
 	slices.Reverse(migrations)
 
 	for _, version := range migrations {
@@ -220,7 +228,7 @@ func Rollback(d Store, targetVersion string) error {
 func Migrate(d Store, targetVersion *string) error {
 	didRun := false
 
-	for _, version := range GetMigrations() {
+	for _, version := range GetMigrations(d.GetDialect()) {
 
 		if targetVersion != nil && version.Compare(Migration{Version: *targetVersion}) > 0 {
 			break

db/Store.go

@@ -194,6 +194,7 @@ type ConnectionManager interface {
 
 // MigrationManager handles database migrations
 type MigrationManager interface {
+	GetDialect() string
 	// IsInitialized indicates is database already initialized, or it is empty.
 	// The method is useful for creating required entities in database during first run.
 	IsInitialized() (bool, error)

db/bolt/BoltDb.go

@@ -44,6 +44,10 @@ type BoltDb struct {
 	terraformAlias   publicAlias
 }
 
+func (d *BoltDb) GetDialect() string {
+	return util.DbDriverBolt
+}
+
 var terraformAliasProps = db.ObjectProps{
 	TableName:         "terraform_alias",
 	Type:              reflect.TypeOf(db.TerraformInventoryAlias{}),

db/factory/store.go

@@ -14,11 +14,13 @@ func CreateStore() db.Store {
 	}
 	switch config.Dialect {
 	case util.DbDriverMySQL:
-		return &sql.SqlDb{}
+		return sql.CreateDb(config.Dialect)
 	case util.DbDriverBolt:
 		return bolt.CreateBoltDB()
 	case util.DbDriverPostgres:
-		return &sql.SqlDb{}
+		return sql.CreateDb(config.Dialect)
+	case util.DbDriverSQLite:
+		return sql.CreateDb(config.Dialect)
 	default:
 		panic("Unsupported database dialect: " + config.Dialect)
 	}

db/sql/SqlDb.go

@@ -18,10 +18,12 @@ import (
 	"github.com/semaphoreui/semaphore/pkg/task_logger"
 	"github.com/semaphoreui/semaphore/util"
 	log "github.com/sirupsen/logrus"
+	_ "modernc.org/sqlite" // Import the driver
 )
 
 type SqlDb struct {
-	sql *gorp.DbMap
+	sql     *gorp.DbMap
+	dialect string
 }
 
 var initialSQL = `
@@ -35,6 +37,14 @@ create table ` + "`migrations`" + ` (
 //go:embed migrations/*.sql
 var dbAssets embed.FS
 
+func CreateDb(dialect string) *SqlDb {
+	return &SqlDb{dialect: dialect}
+}
+
+func (d *SqlDb) GetDialect() string {
+	return d.dialect
+}
+
 func getQueryForParams(q squirrel.SelectBuilder, prefix string, props db.ObjectProps, params db.RetrieveQueryParams) (res squirrel.SelectBuilder, err error) {
 	pp, err := params.Validate(props)
 	if err != nil {
@@ -134,7 +144,7 @@ func (d *SqlDb) insert(primaryKeyColumnName string, query string, args ...any) (
 			return 0, err
 		}
 	default:
-		res, err := d.exec(query, args...)
+		res, err := d.sql.Exec(d.PrepareQuery(query), args...)
 		if err != nil {
 			return 0, err
 		}
@@ -371,10 +381,16 @@ func (d *SqlDb) Connect(_ string) {
 		dialect = gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8"}
 	case util.DbDriverPostgres:
 		dialect = gorp.PostgresDialect{}
+	case util.DbDriverSQLite:
+		dialect = gorp.SqliteDialect{}
 	}
 
 	d.sql = &gorp.DbMap{Db: sqlDb, Dialect: dialect}
 
+	if d.GetDialect() == util.DbDriverSQLite {
+		sqlDb.SetMaxOpenConns(1)
+	}
+
 	d.sql.AddTableWithName(db.APIToken{}, "user__token").SetKeys(false, "id")
 	d.sql.AddTableWithName(db.AccessKey{}, "access_key").SetKeys(true, "id")
 	d.sql.AddTableWithName(db.Environment{}, "project__environment").SetKeys(true, "id")