diff --git a/cmd/root.go b/cmd/root.go index 2092033..61ea029 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -79,6 +79,8 @@ func init() { rootCmd.PersistentFlags().String("storage-orchestrator-clickhouse-password", "", "Clickhouse password for orchestrator storage") rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-asyncInsert", false, "Clickhouse async insert for orchestrator storage") rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for orchestrator storage") + rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for orchestrator storage") + rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for orchestrator storage") rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-disableTLS", false, "Clickhouse disableTLS for orchestrator storage") rootCmd.PersistentFlags().Int("storage-orchestrator-memory-maxItems", 0, "Max items for orchestrator memory storage") rootCmd.PersistentFlags().Int("storage-orchestrator-redis-poolSize", 0, "Redis pool size for orchestrator storage") @@ -91,11 +93,15 @@ func init() { rootCmd.PersistentFlags().String("storage-main-clickhouse-password", "", "Clickhouse password for main storage") rootCmd.PersistentFlags().Bool("storage-main-clickhouse-asyncInsert", false, "Clickhouse async insert for main storage") rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for main storage") + rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for main storage") + rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for main storage") rootCmd.PersistentFlags().Bool("storage-main-clickhouse-disableTLS", false, "Clickhouse disableTLS for main storage") rootCmd.PersistentFlags().String("storage-staging-clickhouse-username", "", "Clickhouse username for staging storage") rootCmd.PersistentFlags().String("storage-staging-clickhouse-password", "", "Clickhouse password for staging storage") rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-asyncInsert", false, "Clickhouse async insert for staging storage") rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for staging storage") + rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for staging storage") + rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for staging storage") rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-disableTLS", false, "Clickhouse disableTLS for staging storage") rootCmd.PersistentFlags().String("api-host", "localhost:3000", "API host") rootCmd.PersistentFlags().String("api-basicAuth-username", "", "API basic auth username") @@ -154,6 +160,8 @@ func init() { viper.BindPFlag("storage.staging.clickhouse.password", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-password")) viper.BindPFlag("storage.staging.clickhouse.asyncInsert", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-asyncInsert")) viper.BindPFlag("storage.staging.clickhouse.maxRowsPerInsert", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-maxRowsPerInsert")) + viper.BindPFlag("storage.staging.clickhouse.maxOpenConns", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-maxOpenConns")) + viper.BindPFlag("storage.staging.clickhouse.maxIdleConns", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-maxIdleConns")) viper.BindPFlag("storage.staging.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-disableTLS")) viper.BindPFlag("storage.main.clickhouse.database", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-database")) viper.BindPFlag("storage.main.clickhouse.host", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-host")) @@ -162,6 +170,8 @@ func init() { viper.BindPFlag("storage.main.clickhouse.password", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-password")) viper.BindPFlag("storage.main.clickhouse.asyncInsert", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-asyncInsert")) viper.BindPFlag("storage.main.clickhouse.maxRowsPerInsert", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-maxRowsPerInsert")) + viper.BindPFlag("storage.main.clickhouse.maxOpenConns", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-maxOpenConns")) + viper.BindPFlag("storage.main.clickhouse.maxIdleConns", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-maxIdleConns")) viper.BindPFlag("storage.main.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-disableTLS")) viper.BindPFlag("storage.orchestrator.clickhouse.database", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-database")) viper.BindPFlag("storage.orchestrator.clickhouse.host", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-host")) @@ -170,6 +180,8 @@ func init() { viper.BindPFlag("storage.orchestrator.clickhouse.password", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-password")) viper.BindPFlag("storage.orchestrator.clickhouse.asyncInsert", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-asyncInsert")) viper.BindPFlag("storage.orchestrator.clickhouse.maxRowsPerInsert", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-maxRowsPerInsert")) + viper.BindPFlag("storage.orchestrator.clickhouse.maxOpenConns", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-maxOpenConns")) + viper.BindPFlag("storage.orchestrator.clickhouse.maxIdleConns", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-maxIdleConns")) viper.BindPFlag("storage.orchestrator.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-disableTLS")) viper.BindPFlag("storage.orchestrator.memory.maxItems", rootCmd.PersistentFlags().Lookup("storage-orchestrator-memory-maxItems")) viper.BindPFlag("storage.orchestrator.redis.poolSize", rootCmd.PersistentFlags().Lookup("storage-orchestrator-redis-poolSize")) diff --git a/configs/config.go b/configs/config.go index c812c87..1ee1f2d 100644 --- a/configs/config.go +++ b/configs/config.go @@ -81,6 +81,8 @@ type ClickhouseConfig struct { DisableTLS bool `mapstructure:"disableTLS"` AsyncInsert bool `mapstructure:"asyncInsert"` MaxRowsPerInsert int `mapstructure:"maxRowsPerInsert"` + MaxOpenConns int `mapstructure:"maxOpenConns"` + MaxIdleConns int `mapstructure:"maxIdleConns"` ChainBasedConfig map[string]TableOverrideConfig `mapstructure:"chainBasedConfig"` } diff --git a/internal/storage/clickhouse.go b/internal/storage/clickhouse.go index a96a3f7..af30cfe 100644 --- a/internal/storage/clickhouse.go +++ b/internal/storage/clickhouse.go @@ -95,6 +95,8 @@ func connectDB(cfg *config.ClickhouseConfig) (clickhouse.Conn, error) { Username: cfg.Username, Password: cfg.Password, }, + MaxOpenConns: cfg.MaxOpenConns, + MaxIdleConns: cfg.MaxIdleConns, Settings: func() clickhouse.Settings { settings := clickhouse.Settings{ "do_not_merge_across_partitions_select_final": "1",