Skip to content

Commit ef99929

Browse files
slaoutslaout
authored
Add MySQL connection pool configuration (#1141)
1 parent eb16fc9 commit ef99929

File tree

10 files changed

+96
-10
lines changed

10 files changed

+96
-10
lines changed

‎cmd/gorse-in-one/main.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,10 @@ func setup(m *master.Master) {
145145

146146
// connect database
147147
m.DataClient, err = data.Open(m.Config.Database.DataStore, m.Config.Database.DataTablePrefix,
148-
storage.WithIsolationLevel(m.Config.Database.MySQL.IsolationLevel))
148+
storage.WithIsolationLevel(m.Config.Database.MySQL.IsolationLevel),
149+
storage.WithMaxOpenConns(m.Config.Database.MySQL.MaxOpenConns),
150+
storage.WithMaxIdleConns(m.Config.Database.MySQL.MaxIdleConns),
151+
storage.WithConnMaxLifetime(m.Config.Database.MySQL.ConnMaxLifetime))
149152
if err != nil {
150153
log.Logger().Fatal("failed to connect data database", zap.Error(err),
151154
zap.String("database", log.RedactDBURL(m.Config.Database.DataStore)))

‎config/config.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,10 @@ type DatabaseConfig struct {
8282
}
8383

8484
type MySQLConfig struct {
85-
IsolationLevel string `mapstructure:"isolation_level" validate:"oneof=READ-UNCOMMITTED READ-COMMITTED REPEATABLE-READ SERIALIZABLE"`
85+
IsolationLevel string `mapstructure:"isolation_level" validate:"oneof=READ-UNCOMMITTED READ-COMMITTED REPEATABLE-READ SERIALIZABLE"`
86+
MaxOpenConns int `mapstructure:"max_open_conns" validate:"gte=0"`
87+
MaxIdleConns int `mapstructure:"max_idle_conns" validate:"gte=0"`
88+
ConnMaxLifetime time.Duration `mapstructure:"conn_max_lifetime" validate:"gte=0"`
8689
}
8790

8891
// MasterConfig is the configuration for the master.
@@ -392,7 +395,10 @@ func GetDefaultConfig() *Config {
392395
return &Config{
393396
Database: DatabaseConfig{
394397
MySQL: MySQLConfig{
395-
IsolationLevel: "READ-UNCOMMITTED",
398+
IsolationLevel: "READ-UNCOMMITTED",
399+
MaxOpenConns: 0,
400+
MaxIdleConns: 0,
401+
ConnMaxLifetime: 0,
396402
},
397403
},
398404
Master: MasterConfig{
@@ -524,6 +530,9 @@ func setDefault() {
524530
defaultConfig := GetDefaultConfig()
525531
// [database.mysql]
526532
viper.SetDefault("database.mysql.isolation_level", defaultConfig.Database.MySQL.IsolationLevel)
533+
viper.SetDefault("database.mysql.max_open_conns", defaultConfig.Database.MySQL.MaxOpenConns)
534+
viper.SetDefault("database.mysql.max_idle_conns", defaultConfig.Database.MySQL.MaxIdleConns)
535+
viper.SetDefault("database.mysql.conn_max_lifetime", defaultConfig.Database.MySQL.ConnMaxLifetime)
527536
// [master]
528537
viper.SetDefault("master.port", defaultConfig.Master.Port)
529538
viper.SetDefault("master.host", defaultConfig.Master.Host)

‎config/config.toml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,15 @@ data_table_prefix = ""
3939
# Transaction isolation level. The default value is "READ-UNCOMMITTED".
4040
isolation_level = "READ-UNCOMMITTED"
4141

42+
# Maximum number of open connections to the database. Set to 0 to keep the driver default.
43+
max_open_conns = 0
44+
45+
# Maximum number of idle connections to the database. Set to 0 to keep the driver default.
46+
max_idle_conns = 0
47+
48+
# Maximum amount of time a connection may be reused. Set to "0s" to keep the driver default.
49+
conn_max_lifetime = "0s"
50+
4251
[master]
4352

4453
# GRPC port of the master node. The default value is 8086.

‎config/config_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ func TestUnmarshal(t *testing.T) {
7070
assert.Equal(t, "gorse_cache_", config.Database.CacheTablePrefix)
7171
assert.Equal(t, "gorse_data_", config.Database.DataTablePrefix)
7272
assert.Equal(t, "READ-UNCOMMITTED", config.Database.MySQL.IsolationLevel)
73+
assert.Equal(t, 0, config.Database.MySQL.MaxOpenConns)
74+
assert.Equal(t, 0, config.Database.MySQL.MaxIdleConns)
75+
assert.Equal(t, time.Duration(0), config.Database.MySQL.ConnMaxLifetime)
7376
// [master]
7477
assert.Equal(t, 8086, config.Master.Port)
7578
assert.Equal(t, "0.0.0.0", config.Master.Host)

‎master/master.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,10 @@ func (m *Master) Serve() {
174174

175175
// connect data database
176176
m.DataClient, err = data.Open(m.Config.Database.DataStore, m.Config.Database.DataTablePrefix,
177-
storage.WithIsolationLevel(m.Config.Database.MySQL.IsolationLevel))
177+
storage.WithIsolationLevel(m.Config.Database.MySQL.IsolationLevel),
178+
storage.WithMaxOpenConns(m.Config.Database.MySQL.MaxOpenConns),
179+
storage.WithMaxIdleConns(m.Config.Database.MySQL.MaxIdleConns),
180+
storage.WithConnMaxLifetime(m.Config.Database.MySQL.ConnMaxLifetime))
178181
if err != nil {
179182
log.Logger().Fatal("failed to connect data database", zap.Error(err),
180183
zap.String("database", log.RedactDBURL(m.Config.Database.DataStore)))
@@ -185,7 +188,10 @@ func (m *Master) Serve() {
185188

186189
// connect cache database
187190
m.CacheClient, err = cache.Open(m.Config.Database.CacheStore, m.Config.Database.CacheTablePrefix,
188-
storage.WithIsolationLevel(m.Config.Database.MySQL.IsolationLevel))
191+
storage.WithIsolationLevel(m.Config.Database.MySQL.IsolationLevel),
192+
storage.WithMaxOpenConns(m.Config.Database.MySQL.MaxOpenConns),
193+
storage.WithMaxIdleConns(m.Config.Database.MySQL.MaxIdleConns),
194+
storage.WithConnMaxLifetime(m.Config.Database.MySQL.ConnMaxLifetime))
189195
if err != nil {
190196
log.Logger().Fatal("failed to connect cache database", zap.Error(err),
191197
zap.String("database", log.RedactDBURL(m.Config.Database.CacheStore)))

‎server/server.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,11 @@ func (s *Server) Sync() {
178178
} else {
179179
log.Logger().Info("connect data store",
180180
zap.String("database", log.RedactDBURL(s.Config.Database.DataStore)))
181-
if s.DataClient, err = data.Open(s.Config.Database.DataStore, s.Config.Database.DataTablePrefix); err != nil {
181+
if s.DataClient, err = data.Open(s.Config.Database.DataStore, s.Config.Database.DataTablePrefix,
182+
storage.WithIsolationLevel(s.Config.Database.MySQL.IsolationLevel),
183+
storage.WithMaxOpenConns(s.Config.Database.MySQL.MaxOpenConns),
184+
storage.WithMaxIdleConns(s.Config.Database.MySQL.MaxIdleConns),
185+
storage.WithConnMaxLifetime(s.Config.Database.MySQL.ConnMaxLifetime)); err != nil {
182186
log.Logger().Error("failed to connect data store", zap.Error(err))
183187
goto sleep
184188
}
@@ -195,7 +199,11 @@ func (s *Server) Sync() {
195199
} else {
196200
log.Logger().Info("connect cache store",
197201
zap.String("database", log.RedactDBURL(s.Config.Database.CacheStore)))
198-
if s.CacheClient, err = cache.Open(s.Config.Database.CacheStore, s.Config.Database.CacheTablePrefix); err != nil {
202+
if s.CacheClient, err = cache.Open(s.Config.Database.CacheStore, s.Config.Database.CacheTablePrefix,
203+
storage.WithIsolationLevel(s.Config.Database.MySQL.IsolationLevel),
204+
storage.WithMaxOpenConns(s.Config.Database.MySQL.MaxOpenConns),
205+
storage.WithMaxIdleConns(s.Config.Database.MySQL.MaxIdleConns),
206+
storage.WithConnMaxLifetime(s.Config.Database.MySQL.ConnMaxLifetime)); err != nil {
199207
log.Logger().Error("failed to connect cache store", zap.Error(err))
200208
goto sleep
201209
}

‎storage/cache/sql.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ func init() {
8383
); err != nil {
8484
return nil, errors.Trace(err)
8585
}
86+
storage.ApplySQLPool(database.client, option)
8687
database.gormDB, err = gorm.Open(gormmysql.New(gormmysql.Config{Conn: database.client}), storage.NewGORMConfig(tablePrefix))
8788
if err != nil {
8889
return nil, errors.Trace(err)

‎storage/data/sql.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ func init() {
7878
); err != nil {
7979
return nil, errors.Trace(err)
8080
}
81+
storage.ApplySQLPool(database.client, option)
8182
database.gormDB, err = gorm.Open(mysql.New(mysql.Config{Conn: database.client}), storage.NewGORMConfig(tablePrefix))
8283
if err != nil {
8384
return nil, errors.Trace(err)

‎storage/options.go‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,15 @@
11
package storage
22

3+
import (
4+
"database/sql"
5+
"time"
6+
)
7+
38
type Options struct {
4-
IsolationLevel string
9+
IsolationLevel string
10+
MaxOpenConns int
11+
MaxIdleConns int
12+
ConnMaxLifetime time.Duration
513
}
614

715
type Option func(*Options)
@@ -12,6 +20,36 @@ func WithIsolationLevel(isolationLevel string) Option {
1220
}
1321
}
1422

23+
func WithMaxOpenConns(maxOpenConns int) Option {
24+
return func(o *Options) {
25+
o.MaxOpenConns = maxOpenConns
26+
}
27+
}
28+
29+
func WithMaxIdleConns(maxIdleConns int) Option {
30+
return func(o *Options) {
31+
o.MaxIdleConns = maxIdleConns
32+
}
33+
}
34+
35+
func WithConnMaxLifetime(connMaxLifetime time.Duration) Option {
36+
return func(o *Options) {
37+
o.ConnMaxLifetime = connMaxLifetime
38+
}
39+
}
40+
41+
func ApplySQLPool(db *sql.DB, opt Options) {
42+
if opt.MaxOpenConns > 0 {
43+
db.SetMaxOpenConns(opt.MaxOpenConns)
44+
}
45+
if opt.MaxIdleConns > 0 {
46+
db.SetMaxIdleConns(opt.MaxIdleConns)
47+
}
48+
if opt.ConnMaxLifetime > 0 {
49+
db.SetConnMaxLifetime(opt.ConnMaxLifetime)
50+
}
51+
}
52+
1553
func NewOptions(opts ...Option) Options {
1654
opt := Options{
1755
IsolationLevel: "READ-UNCOMMITTED",

‎worker/worker.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,11 @@ func (w *Worker) Sync() {
165165
} else {
166166
log.Logger().Info("connect data store",
167167
zap.String("database", log.RedactDBURL(w.Config.Database.DataStore)))
168-
if w.DataClient, err = data.Open(w.Config.Database.DataStore, w.Config.Database.DataTablePrefix); err != nil {
168+
if w.DataClient, err = data.Open(w.Config.Database.DataStore, w.Config.Database.DataTablePrefix,
169+
storage.WithIsolationLevel(w.Config.Database.MySQL.IsolationLevel),
170+
storage.WithMaxOpenConns(w.Config.Database.MySQL.MaxOpenConns),
171+
storage.WithMaxIdleConns(w.Config.Database.MySQL.MaxIdleConns),
172+
storage.WithConnMaxLifetime(w.Config.Database.MySQL.ConnMaxLifetime)); err != nil {
169173
log.Logger().Error("failed to connect data store", zap.Error(err))
170174
goto sleep
171175
}
@@ -182,7 +186,11 @@ func (w *Worker) Sync() {
182186
} else {
183187
log.Logger().Info("connect cache store",
184188
zap.String("database", log.RedactDBURL(w.Config.Database.CacheStore)))
185-
if w.CacheClient, err = cache.Open(w.Config.Database.CacheStore, w.Config.Database.CacheTablePrefix); err != nil {
189+
if w.CacheClient, err = cache.Open(w.Config.Database.CacheStore, w.Config.Database.CacheTablePrefix,
190+
storage.WithIsolationLevel(w.Config.Database.MySQL.IsolationLevel),
191+
storage.WithMaxOpenConns(w.Config.Database.MySQL.MaxOpenConns),
192+
storage.WithMaxIdleConns(w.Config.Database.MySQL.MaxIdleConns),
193+
storage.WithConnMaxLifetime(w.Config.Database.MySQL.ConnMaxLifetime)); err != nil {
186194
log.Logger().Error("failed to connect cache store", zap.Error(err))
187195
goto sleep
188196
}

0 commit comments

Comments
 (0)