Add MySQL database support with YAML and environment configuration (#214)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: dkhalife <1736645+dkhalife@users.noreply.github.com>
Co-authored-by: Dany Khalife <dev@dkhalife.com>

Author: Copilot

README.md

@@ -80,17 +80,64 @@ Make sure to replace `/path/to/host` with your preferred root directory for conf
 
 ## ⚙️ Configuration
 
-In the [config](./config/) directory are a couple of starter configuration files for prod and dev environments. The server expects a config.yaml in the config directory and will load settings from it when started.
+In the [config](./apiserver/config/) directory are a couple of starter configuration files for prod and dev environments. The server expects a config.yaml in the config directory and will load settings from it when started.
 
-**Note:** You can set `email.host`, `email.port`, `email.email`, `email.password` and `jwt.secret` using environment variables `TW_EMAIL_HOST`, `TW_EMAIL_PORT`, `TW_EMAIL_SENDER`, `TW_EMAIL_PASSWORD` and `TW_JWT_SECRET` for improved security and flexibility. The server will fail to start if `jwt.secret` is left as `"secret"`, so be sure to set `TW_JWT_SECRET` or edit `config.yaml`.
+**Note:** You can set `email.host`, `email.port`, `email.email`, `email.password`, `jwt.secret`, and database credentials using environment variables for improved security and flexibility. The server will fail to start if `jwt.secret` is left as `"secret"`, so be sure to set `TW_JWT_SECRET` or edit `config.yaml`.
+
+### Database Configuration
+
+Task Wizard supports both SQLite and MySQL databases. By default, it uses SQLite.
+
+#### SQLite (default)
+
+To use SQLite, set `database.type` to `sqlite` (or leave it unset) in your `config.yaml`:
+
+```yaml
+database:
+  type: sqlite
+  path: /config/task-wizard.db
+  migration: true
+```
+
+#### MySQL
+
+To use MySQL, configure the database section:
+
+```yaml
+database:
+  type: mysql
+  host: localhost
+  port: 3306
+  database: taskwizard
+  username: taskuser
+  password: taskpass
+  migration: true
+```
+
+You can also use environment variables for database configuration:
+
+- `TW_DATABASE_TYPE` - Database type (sqlite or mysql)
+- `TW_DATABASE_HOST` - Database host
+- `TW_DATABASE_PORT` - Database port
+- `TW_DATABASE_NAME` - Database name
+- `TW_DATABASE_USERNAME` - Database username
+- `TW_DATABASE_PASSWORD` - Database password
+
+### Configuration Reference
 
 The configuration files are yaml mappings with the following values:
 
 | Configuration Entry                      | Default Value                                       | Description                                                                 |
 |------------------------------------------|-----------------------------------------------------|-----------------------------------------------------------------------------|
 | `name`                                   | `"prod"`                                            | The name of the environment configuration.                                  |
+| `database.type`                          | `sqlite`                                            | Database type: `sqlite` or `mysql`.                                         |
 | `database.migration`                     | `true`                                              | Indicates if database migration should be performed.                        |
-| `database.path`                          | `/config/task-wizard.db`                            | The path at which to store the SQLite database.                             |
+| `database.path`                          | `/config/task-wizard.db`                            | The path at which to store the SQLite database (SQLite only).               |
+| `database.host`                          | (empty)                                             | Database host (MySQL only).                                                 |
+| `database.port`                          | `3306`                                              | Database port (MySQL only).                                                 |
+| `database.database`                      | (empty)                                             | Database name (MySQL only).                                                 |
+| `database.username`                      | (empty)                                             | Database username (MySQL only).                                             |
+| `database.password`                      | (empty)                                             | Database password (MySQL only).                                             |
 | `jwt.secret`                             | `"secret"`                                          | The secret key used for signing JWT tokens. **Make sure to change that or set `TW_JWT_SECRET`.**   |
 | `jwt.session_time`                       | `168h`                                              | The duration for which a JWT session is valid.                              |
 | `jwt.max_refresh`                        | `168h`                                              | The maximum duration for refreshing a JWT session.                          |

apiserver/config/config.dev.yaml

@@ -1,4 +1,5 @@
 database:
+  type: sqlite
   migration: true
   path: task-wizard.db
 jwt:

apiserver/config/config.go

@@ -16,7 +16,13 @@ type Config struct {
 }
 
 type DatabaseConfig struct {
+	Type      string `mapstructure:"type" yaml:"type" default:"sqlite"`
 	FilePath  string `mapstructure:"path" yaml:"path" default:"/config/task-wizard.db"`
+	Host      string `mapstructure:"host" yaml:"host"`
+	Port      int    `mapstructure:"port" yaml:"port" default:"3306"`
+	Database  string `mapstructure:"database" yaml:"database"`
+	Username  string `mapstructure:"username" yaml:"username"`
+	Password  string `mapstructure:"password" yaml:"password"`
 	Migration bool   `mapstructure:"migration" yaml:"migration"`
 }
 
@@ -79,6 +85,12 @@ func LoadConfig(configFile string) *Config {
 	_ = viper.BindEnv("email.port", "TW_EMAIL_PORT")
 	_ = viper.BindEnv("email.email", "TW_EMAIL_SENDER")
 	_ = viper.BindEnv("email.password", "TW_EMAIL_PASSWORD")
+	_ = viper.BindEnv("database.type", "TW_DATABASE_TYPE")
+	_ = viper.BindEnv("database.host", "TW_DATABASE_HOST")
+	_ = viper.BindEnv("database.port", "TW_DATABASE_PORT")
+	_ = viper.BindEnv("database.database", "TW_DATABASE_NAME")
+	_ = viper.BindEnv("database.username", "TW_DATABASE_USERNAME")
+	_ = viper.BindEnv("database.password", "TW_DATABASE_PASSWORD")
 
 	err := viper.ReadInConfig()
 	if err != nil {

apiserver/config/config.yaml

@@ -1,4 +1,5 @@
 database:
+  type: sqlite
   migration: true
   path: /config/task-wizard.db
 jwt:

apiserver/config/config_test.go

@@ -24,6 +24,7 @@ func TestLoadConfig_Success(t *testing.T) {
   read_timeout: 10s
   write_timeout: 10s
 database:
+  type: sqlite
   path: test.db
   migration: true
 jwt:
@@ -53,6 +54,7 @@ email:
 	assert.Equal(t, 10*time.Second, cfg.Server.ReadTimeout)
 	assert.Equal(t, 10*time.Second, cfg.Server.WriteTimeout)
 
+	assert.Equal(t, "sqlite", cfg.Database.Type)
 	assert.Equal(t, "test.db", cfg.Database.FilePath)
 	assert.Equal(t, true, cfg.Database.Migration)
 
@@ -157,3 +159,79 @@ func TestLoadConfig_CLIOverridesEnv(t *testing.T) {
 	assert.Equal(t, 2222, cfg.Server.Port)
 	os.Unsetenv("TW_CONFIG_FILE")
 }
+
+func TestLoadConfig_DatabaseEnvOverride(t *testing.T) {
+	_ = os.MkdirAll("./config", 0755)
+	f, err := os.Create("./config/config.yaml")
+	assert.NoError(t, err)
+	defer os.Remove("./config/config.yaml")
+	defer f.Close()
+
+	_, err = f.WriteString(`database:
+  type: sqlite
+  path: /config/task-wizard.db
+server:
+  port: 1234
+jwt:
+  secret: testsecret
+`)
+	assert.NoError(t, err)
+
+	os.Setenv("TW_DATABASE_TYPE", "mysql")
+	os.Setenv("TW_DATABASE_HOST", "localhost")
+	os.Setenv("TW_DATABASE_PORT", "3307")
+	os.Setenv("TW_DATABASE_NAME", "taskwizard")
+	os.Setenv("TW_DATABASE_USERNAME", "dbuser")
+	os.Setenv("TW_DATABASE_PASSWORD", "dbpass")
+
+	viper.Reset()
+	cfg := LoadConfig("./config/config.yaml")
+
+	assert.Equal(t, "mysql", cfg.Database.Type)
+	assert.Equal(t, "localhost", cfg.Database.Host)
+	assert.Equal(t, 3307, cfg.Database.Port)
+	assert.Equal(t, "taskwizard", cfg.Database.Database)
+	assert.Equal(t, "dbuser", cfg.Database.Username)
+	assert.Equal(t, "dbpass", cfg.Database.Password)
+
+	os.Unsetenv("TW_DATABASE_TYPE")
+	os.Unsetenv("TW_DATABASE_HOST")
+	os.Unsetenv("TW_DATABASE_PORT")
+	os.Unsetenv("TW_DATABASE_NAME")
+	os.Unsetenv("TW_DATABASE_USERNAME")
+	os.Unsetenv("TW_DATABASE_PASSWORD")
+}
+
+func TestLoadConfig_MySQLConfig(t *testing.T) {
+	_ = os.MkdirAll("./config", 0755)
+	f, err := os.Create("./config/config.yaml")
+	assert.NoError(t, err)
+	defer os.Remove("./config/config.yaml")
+	defer f.Close()
+
+	_, err = f.WriteString(`database:
+  type: mysql
+  host: mysql.example.com
+  port: 3306
+  database: taskwizard
+  username: testuser
+  password: testpass
+  migration: true
+server:
+  port: 1234
+jwt:
+  secret: testsecret
+`)
+	assert.NoError(t, err)
+
+	viper.Reset()
+	cfg := LoadConfig("./config/config.yaml")
+
+	assert.Equal(t, "mysql", cfg.Database.Type)
+	assert.Equal(t, "mysql.example.com", cfg.Database.Host)
+	assert.Equal(t, 3306, cfg.Database.Port)
+	assert.Equal(t, "taskwizard", cfg.Database.Database)
+	assert.Equal(t, "testuser", cfg.Database.Username)
+	assert.Equal(t, "testpass", cfg.Database.Password)
+	assert.Equal(t, true, cfg.Database.Migration)
+}

apiserver/go.mod

@@ -13,17 +13,20 @@ require (
 	go.uber.org/fx v1.22.0
 	go.uber.org/zap v1.26.0
 	golang.org/x/crypto v0.41.0
-	gorm.io/gorm v1.25.10
+	gorm.io/gorm v1.30.0
 )
 
 require (
 	github.com/stretchr/testify v1.10.0
 	github.com/wneessen/go-mail v0.7.1
+	gorm.io/driver/mysql v1.6.0
 	gorm.io/driver/sqlite v1.5.7
 )
 
 require (
+	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/mattn/go-sqlite3 v1.14.22 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect

apiserver/go.sum

@@ -1,3 +1,5 @@
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/appleboy/gin-jwt/v2 v2.9.2 h1:GeS3lm9mb9HMmj7+GNjYUtpp3V1DAQ1TkUFa5poiZ7Y=
 github.com/appleboy/gin-jwt/v2 v2.9.2/go.mod h1:mxGjKt9Lrx9Xusy1SrnmsCJMZG6UJwmdHN9bN27/QDw=
 github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4=
@@ -40,6 +42,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
 github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
 github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
@@ -183,10 +187,12 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
+gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
 gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I=
 gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4=
-gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s=
-gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs=
+gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
 modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
 modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
 modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=

apiserver/internal/utils/database/database.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"fmt"
 	"log"
 	"os"
 	"strings"
@@ -9,6 +10,7 @@ import (
 	"dkhalife.com/tasks/core/config"
 	"dkhalife.com/tasks/core/internal/services/logging"
 	"github.com/glebarez/sqlite"
+	"gorm.io/driver/mysql"
 	"gorm.io/gorm"
 	gormLogger "gorm.io/gorm/logger"
 )
@@ -38,16 +40,49 @@ func NewDatabase(cfg *config.Config) (*gorm.DB, error) {
 		},
 	)
 
-	db, err := gorm.Open(sqlite.Open(cfg.Database.FilePath), &gorm.Config{Logger: logger})
-	if err != nil {
-		return nil, err
+	var dialector gorm.Dialector
+	dbType := strings.ToLower(cfg.Database.Type)
+	
+	switch dbType {
+	case "mysql":
+		// Validate required fields for MySQL
+		if cfg.Database.Host == "" {
+			return nil, fmt.Errorf("database.host is required for %s", dbType)
+		}
+		if cfg.Database.Database == "" {
+			return nil, fmt.Errorf("database.database is required for %s", dbType)
+		}
+		if cfg.Database.Username == "" {
+			return nil, fmt.Errorf("database.username is required for %s", dbType)
+		}
+		
+		dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local",
+			cfg.Database.Username,
+			cfg.Database.Password,
+			cfg.Database.Host,
+			cfg.Database.Port,
+			cfg.Database.Database,
+		)
+		dialector = mysql.Open(dsn)
+	case "sqlite", "":
+		dialector = sqlite.Open(cfg.Database.FilePath)
+	default:
+		return nil, fmt.Errorf("unsupported database type: %s (supported: sqlite, mysql)", cfg.Database.Type)
 	}
 
-	if err := db.Exec("PRAGMA journal_mode=WAL;").Error; err != nil {
+	db, err := gorm.Open(dialector, &gorm.Config{Logger: logger})
+	if err != nil {
 		return nil, err
 	}
-	if err := db.Exec("PRAGMA busy_timeout=5000;").Error; err != nil {
-		return nil, err
+
+	// Only apply SQLite-specific settings if using SQLite
+	if dbType == "sqlite" || dbType == "" {
+		if err := db.Exec("PRAGMA journal_mode=WAL;").Error; err != nil {
+			return nil, err
+		}
+		if err := db.Exec("PRAGMA busy_timeout=5000;").Error; err != nil {
+			return nil, err
+		}
 	}
 
 	return db, nil

apiserver/internal/utils/database/database_test.go

@@ -46,3 +46,57 @@ func (s *DatabaseTestSuite) TestNewDatabaseConnection() {
 	s.Require().NoError(err)
 	s.NoError(sqlDB.Ping())
 }
+
+func (s *DatabaseTestSuite) TestNewDatabase_MySQLMissingHost() {
+	cfg := &config.Config{
+		Database: config.DatabaseConfig{
+			Type:     "mysql",
+			Database: "testdb",
+			Username: "testuser",
+		},
+	}
+
+	_, err := NewDatabase(cfg)
+	s.Require().Error(err)
+	s.Contains(err.Error(), "database.host is required")
+}
+
+func (s *DatabaseTestSuite) TestNewDatabase_MySQLMissingDatabase() {
+	cfg := &config.Config{
+		Database: config.DatabaseConfig{
+			Type:     "mysql",
+			Host:     "localhost",
+			Username: "testuser",
+		},
+	}
+
+	_, err := NewDatabase(cfg)
+	s.Require().Error(err)
+	s.Contains(err.Error(), "database.database is required")
+}
+
+func (s *DatabaseTestSuite) TestNewDatabase_MySQLMissingUsername() {
+	cfg := &config.Config{
+		Database: config.DatabaseConfig{
+			Type:     "mysql",
+			Host:     "localhost",
+			Database: "testdb",
+		},
+	}
+
+	_, err := NewDatabase(cfg)
+	s.Require().Error(err)
+	s.Contains(err.Error(), "database.username is required")
+}
+
+func (s *DatabaseTestSuite) TestNewDatabase_UnsupportedType() {
+	cfg := &config.Config{
+		Database: config.DatabaseConfig{
+			Type: "postgresql",
+		},
+	}
+
+	_, err := NewDatabase(cfg)
+	s.Require().Error(err)
+	s.Contains(err.Error(), "unsupported database type")
+}