Skip to content

Commit 1a3ef92

Browse files
nischitpranischitpra
committed
max memory usage
1 parent 9ca9e74 commit 1a3ef92

File tree

3 files changed

+17
-1
lines changed

3 files changed

+17
-1
lines changed

‎cmd/root.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ func init() {
8484
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-disableTLS", false, "Clickhouse disableTLS for orchestrator storage")
8585
rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for orchestrator storage")
8686
rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxQueryTime", 60, "Clickhouse max query time for orchestrator storage")
87+
rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxMemoryUsage", 1000000000, "Clickhouse max memory usage in bytes for orchestrator storage")
8788
rootCmd.PersistentFlags().String("storage-orchestrator-postgres-host", "", "PostgreSQL host for orchestrator storage")
8889
rootCmd.PersistentFlags().Int("storage-orchestrator-postgres-port", 5432, "PostgreSQL port for orchestrator storage")
8990
rootCmd.PersistentFlags().String("storage-orchestrator-postgres-username", "", "PostgreSQL username for orchestrator storage")
@@ -105,6 +106,7 @@ func init() {
105106
rootCmd.PersistentFlags().Bool("storage-main-clickhouse-disableTLS", false, "Clickhouse disableTLS for main storage")
106107
rootCmd.PersistentFlags().Bool("storage-main-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for main storage")
107108
rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxQueryTime", 60, "Clickhouse max query time for main storage")
109+
rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxMemoryUsage", 1000000000, "Clickhouse max memory usage in bytes for main storage")
108110
rootCmd.PersistentFlags().String("storage-staging-clickhouse-username", "", "Clickhouse username for staging storage")
109111
rootCmd.PersistentFlags().String("storage-staging-clickhouse-password", "", "Clickhouse password for staging storage")
110112
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-asyncInsert", false, "Clickhouse async insert for staging storage")
@@ -114,6 +116,7 @@ func init() {
114116
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-disableTLS", false, "Clickhouse disableTLS for staging storage")
115117
rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for staging storage")
116118
rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxQueryTime", 60, "Clickhouse max query time for staging storage")
119+
rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxMemoryUsage", 1000000000, "Clickhouse max memory usage in bytes for staging storage")
117120
rootCmd.PersistentFlags().String("storage-staging-postgres-host", "", "PostgreSQL host for staging storage")
118121
rootCmd.PersistentFlags().Int("storage-staging-postgres-port", 5432, "PostgreSQL port for staging storage")
119122
rootCmd.PersistentFlags().String("storage-staging-postgres-username", "", "PostgreSQL username for staging storage")
@@ -196,6 +199,7 @@ func init() {
196199
viper.BindPFlag("storage.staging.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-disableTLS"))
197200
viper.BindPFlag("storage.staging.clickhouse.enableParallelViewProcessing", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-enableParallelViewProcessing"))
198201
viper.BindPFlag("storage.staging.clickhouse.maxQueryTime", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-maxQueryTime"))
202+
viper.BindPFlag("storage.staging.clickhouse.maxMemoryUsage", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-maxMemoryUsage"))
199203
viper.BindPFlag("storage.main.clickhouse.database", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-database"))
200204
viper.BindPFlag("storage.main.clickhouse.host", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-host"))
201205
viper.BindPFlag("storage.main.clickhouse.port", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-port"))
@@ -208,6 +212,7 @@ func init() {
208212
viper.BindPFlag("storage.main.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-disableTLS"))
209213
viper.BindPFlag("storage.main.clickhouse.enableParallelViewProcessing", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-enableParallelViewProcessing"))
210214
viper.BindPFlag("storage.main.clickhouse.maxQueryTime", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-maxQueryTime"))
215+
viper.BindPFlag("storage.main.clickhouse.maxMemoryUsage", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-maxMemoryUsage"))
211216
viper.BindPFlag("storage.orchestrator.clickhouse.database", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-database"))
212217
viper.BindPFlag("storage.orchestrator.clickhouse.host", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-host"))
213218
viper.BindPFlag("storage.orchestrator.clickhouse.port", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-port"))
@@ -220,6 +225,7 @@ func init() {
220225
viper.BindPFlag("storage.orchestrator.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-disableTLS"))
221226
viper.BindPFlag("storage.orchestrator.clickhouse.enableParallelViewProcessing", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-enableParallelViewProcessing"))
222227
viper.BindPFlag("storage.orchestrator.clickhouse.maxQueryTime", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-maxQueryTime"))
228+
viper.BindPFlag("storage.orchestrator.clickhouse.maxMemoryUsage", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-maxMemoryUsage"))
223229
viper.BindPFlag("storage.orchestrator.postgres.host", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-host"))
224230
viper.BindPFlag("storage.orchestrator.postgres.port", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-port"))
225231
viper.BindPFlag("storage.orchestrator.postgres.username", rootCmd.PersistentFlags().Lookup("storage-orchestrator-postgres-username"))

‎configs/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ type ClickhouseConfig struct {
8585
ChainBasedConfig map[string]TableOverrideConfig `mapstructure:"chainBasedConfig"`
8686
EnableParallelViewProcessing bool `mapstructure:"enableParallelViewProcessing"`
8787
MaxQueryTime int `mapstructure:"maxQueryTime"`
88+
MaxMemoryUsage int `mapstructure:"maxMemoryUsage"`
8889
}
8990

9091
type PostgresConfig struct {

‎internal/storage/clickhouse.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -670,8 +670,17 @@ func (c *ClickHouseConnector) addPostQueryClauses(query string, qf QueryFilter)
670670
}
671671

672672
// Add settings at the very end
673+
// Build settings string for ClickHouse query optimization
674+
var settings []string
673675
if c.cfg.MaxQueryTime > 0 {
674-
query += fmt.Sprintf(" SETTINGS max_execution_time = %d", c.cfg.MaxQueryTime)
676+
settings = append(settings, fmt.Sprintf("max_execution_time = %d", c.cfg.MaxQueryTime))
677+
}
678+
if c.cfg.MaxMemoryUsage > 0 {
679+
settings = append(settings, fmt.Sprintf("max_memory_usage = %d", c.cfg.MaxMemoryUsage))
680+
}
681+
682+
if len(settings) > 0 {
683+
query += " SETTINGS " + strings.Join(settings, ", ")
675684
}
676685

677686
return query

0 commit comments

Comments
 (0)