From 05d10710e04bd18b1de361e1226dce7ce2e125f6 Mon Sep 17 00:00:00 2001 From: iuwqyir Date: Tue, 10 Jun 2025 13:33:14 +0300 Subject: [PATCH] make ch parallel view processing configurable --- cmd/root.go | 6 ++++++ configs/config.go | 23 ++++++++++++----------- internal/storage/clickhouse.go | 4 +++- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index fabd4c6..4fa4c18 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -82,6 +82,7 @@ func init() { rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for orchestrator storage") rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for orchestrator storage") rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-disableTLS", false, "Clickhouse disableTLS for orchestrator storage") + rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for orchestrator storage") rootCmd.PersistentFlags().String("storage-staging-clickhouse-host", "", "Clickhouse host for staging storage") rootCmd.PersistentFlags().String("storage-main-clickhouse-host", "", "Clickhouse host for main storage") rootCmd.PersistentFlags().String("storage-main-clickhouse-username", "", "Clickhouse username for main storage") @@ -91,6 +92,7 @@ func init() { rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for main storage") rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for main storage") rootCmd.PersistentFlags().Bool("storage-main-clickhouse-disableTLS", false, "Clickhouse disableTLS for main storage") + rootCmd.PersistentFlags().Bool("storage-main-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for main storage") rootCmd.PersistentFlags().String("storage-staging-clickhouse-username", "", "Clickhouse username for staging storage") rootCmd.PersistentFlags().String("storage-staging-clickhouse-password", "", "Clickhouse password for staging storage") rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-asyncInsert", false, "Clickhouse async insert for staging storage") @@ -98,6 +100,7 @@ func init() { rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for staging storage") rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for staging storage") rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-disableTLS", false, "Clickhouse disableTLS for staging storage") + rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for staging storage") rootCmd.PersistentFlags().String("api-host", "localhost:3000", "API host") rootCmd.PersistentFlags().String("api-basicAuth-username", "", "API basic auth username") rootCmd.PersistentFlags().String("api-basicAuth-password", "", "API basic auth password") @@ -164,6 +167,7 @@ func init() { viper.BindPFlag("storage.staging.clickhouse.maxOpenConns", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-maxOpenConns")) viper.BindPFlag("storage.staging.clickhouse.maxIdleConns", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-maxIdleConns")) viper.BindPFlag("storage.staging.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-disableTLS")) + viper.BindPFlag("storage.staging.clickhouse.enableParallelViewProcessing", rootCmd.PersistentFlags().Lookup("storage-staging-clickhouse-enableParallelViewProcessing")) viper.BindPFlag("storage.main.clickhouse.database", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-database")) viper.BindPFlag("storage.main.clickhouse.host", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-host")) viper.BindPFlag("storage.main.clickhouse.port", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-port")) @@ -174,6 +178,7 @@ func init() { viper.BindPFlag("storage.main.clickhouse.maxOpenConns", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-maxOpenConns")) viper.BindPFlag("storage.main.clickhouse.maxIdleConns", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-maxIdleConns")) viper.BindPFlag("storage.main.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-disableTLS")) + viper.BindPFlag("storage.main.clickhouse.enableParallelViewProcessing", rootCmd.PersistentFlags().Lookup("storage-main-clickhouse-enableParallelViewProcessing")) viper.BindPFlag("storage.orchestrator.clickhouse.database", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-database")) viper.BindPFlag("storage.orchestrator.clickhouse.host", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-host")) viper.BindPFlag("storage.orchestrator.clickhouse.port", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-port")) @@ -184,6 +189,7 @@ func init() { viper.BindPFlag("storage.orchestrator.clickhouse.maxOpenConns", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-maxOpenConns")) viper.BindPFlag("storage.orchestrator.clickhouse.maxIdleConns", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-maxIdleConns")) viper.BindPFlag("storage.orchestrator.clickhouse.disableTLS", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-disableTLS")) + viper.BindPFlag("storage.orchestrator.clickhouse.enableParallelViewProcessing", rootCmd.PersistentFlags().Lookup("storage-orchestrator-clickhouse-enableParallelViewProcessing")) viper.BindPFlag("api.host", rootCmd.PersistentFlags().Lookup("api-host")) viper.BindPFlag("api.basicAuth.username", rootCmd.PersistentFlags().Lookup("api-basicAuth-username")) viper.BindPFlag("api.basicAuth.password", rootCmd.PersistentFlags().Lookup("api-basicAuth-password")) diff --git a/configs/config.go b/configs/config.go index fd84014..ecc5610 100644 --- a/configs/config.go +++ b/configs/config.go @@ -71,17 +71,18 @@ type TableConfig struct { type TableOverrideConfig map[string]TableConfig type ClickhouseConfig struct { - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - Database string `mapstructure:"database"` - DisableTLS bool `mapstructure:"disableTLS"` - AsyncInsert bool `mapstructure:"asyncInsert"` - MaxRowsPerInsert int `mapstructure:"maxRowsPerInsert"` - MaxOpenConns int `mapstructure:"maxOpenConns"` - MaxIdleConns int `mapstructure:"maxIdleConns"` - ChainBasedConfig map[string]TableOverrideConfig `mapstructure:"chainBasedConfig"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` + DisableTLS bool `mapstructure:"disableTLS"` + AsyncInsert bool `mapstructure:"asyncInsert"` + MaxRowsPerInsert int `mapstructure:"maxRowsPerInsert"` + MaxOpenConns int `mapstructure:"maxOpenConns"` + MaxIdleConns int `mapstructure:"maxIdleConns"` + ChainBasedConfig map[string]TableOverrideConfig `mapstructure:"chainBasedConfig"` + EnableParallelViewProcessing bool `mapstructure:"enableParallelViewProcessing"` } type RPCBatchRequestConfig struct { diff --git a/internal/storage/clickhouse.go b/internal/storage/clickhouse.go index 417bcdb..9630fea 100644 --- a/internal/storage/clickhouse.go +++ b/internal/storage/clickhouse.go @@ -103,7 +103,9 @@ func connectDB(cfg *config.ClickhouseConfig) (clickhouse.Conn, error) { "do_not_merge_across_partitions_select_final": "1", "use_skip_indexes_if_final": "1", "optimize_move_to_prewhere_if_final": "1", - "parallel_view_processing": "1", + } + if cfg.EnableParallelViewProcessing { + settings["parallel_view_processing"] = "1" } if cfg.AsyncInsert { settings["async_insert"] = "1"