diff --git a/cmd/root.go b/cmd/root.go index 2a9ac47..0140b4c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -80,13 +80,13 @@ func init() { rootCmd.PersistentFlags().String("storage-orchestrator-clickhouse-host", "", "Clickhouse host for orchestrator storage") rootCmd.PersistentFlags().String("storage-orchestrator-clickhouse-username", "", "Clickhouse username for orchestrator storage") rootCmd.PersistentFlags().String("storage-orchestrator-clickhouse-password", "", "Clickhouse password for orchestrator storage") - rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-asyncInsert", false, "Clickhouse async insert for orchestrator storage") + rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-asyncInsert", true, "Clickhouse async insert for orchestrator storage") rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for orchestrator storage") rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for orchestrator storage") rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for orchestrator storage") rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-disableTLS", false, "Clickhouse disableTLS for orchestrator storage") rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for orchestrator storage") - rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-enableCompression", false, "Clickhouse enableCompression for orchestrator storage") + rootCmd.PersistentFlags().Bool("storage-orchestrator-clickhouse-enableCompression", true, "Clickhouse enableCompression for orchestrator storage") rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxQueryTime", 60, "Clickhouse max query time for orchestrator storage") rootCmd.PersistentFlags().Int("storage-orchestrator-clickhouse-maxMemoryUsage", 1000000000, "Clickhouse max memory usage in bytes for orchestrator storage") rootCmd.PersistentFlags().String("storage-orchestrator-postgres-host", "", "PostgreSQL host for orchestrator storage") @@ -103,24 +103,24 @@ func init() { rootCmd.PersistentFlags().String("storage-main-clickhouse-host", "", "Clickhouse host for main storage") rootCmd.PersistentFlags().String("storage-main-clickhouse-username", "", "Clickhouse username for main storage") rootCmd.PersistentFlags().String("storage-main-clickhouse-password", "", "Clickhouse password for main storage") - rootCmd.PersistentFlags().Bool("storage-main-clickhouse-asyncInsert", false, "Clickhouse async insert for main storage") + rootCmd.PersistentFlags().Bool("storage-main-clickhouse-asyncInsert", true, "Clickhouse async insert for main storage") rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for main storage") rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for main storage") rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for main storage") rootCmd.PersistentFlags().Bool("storage-main-clickhouse-disableTLS", false, "Clickhouse disableTLS for main storage") rootCmd.PersistentFlags().Bool("storage-main-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for main storage") - rootCmd.PersistentFlags().Bool("storage-main-clickhouse-enableCompression", false, "Clickhouse enableCompression for main storage") + rootCmd.PersistentFlags().Bool("storage-main-clickhouse-enableCompression", true, "Clickhouse enableCompression for main storage") rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxQueryTime", 60, "Clickhouse max query time for main storage") rootCmd.PersistentFlags().Int("storage-main-clickhouse-maxMemoryUsage", 1000000000, "Clickhouse max memory usage in bytes for main storage") rootCmd.PersistentFlags().String("storage-staging-clickhouse-username", "", "Clickhouse username for staging storage") rootCmd.PersistentFlags().String("storage-staging-clickhouse-password", "", "Clickhouse password for staging storage") - rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-asyncInsert", false, "Clickhouse async insert for staging storage") + rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-asyncInsert", true, "Clickhouse async insert for staging storage") rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for staging storage") rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for staging storage") rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for staging storage") rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-disableTLS", false, "Clickhouse disableTLS for staging storage") rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-enableParallelViewProcessing", false, "Clickhouse enableParallelViewProcessing for staging storage") - rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-enableCompression", false, "Clickhouse enableCompression for staging storage") + rootCmd.PersistentFlags().Bool("storage-staging-clickhouse-enableCompression", true, "Clickhouse enableCompression for staging storage") rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxQueryTime", 60, "Clickhouse max query time for staging storage") rootCmd.PersistentFlags().Int("storage-staging-clickhouse-maxMemoryUsage", 1000000000, "Clickhouse max memory usage in bytes for staging storage") rootCmd.PersistentFlags().String("storage-staging-postgres-host", "", "PostgreSQL host for staging storage") @@ -198,7 +198,7 @@ func init() { rootCmd.PersistentFlags().String("migrator-destination-clickhouse-password", "", "Clickhouse password for migrator destination") rootCmd.PersistentFlags().String("migrator-destination-clickhouse-database", "", "Clickhouse database for migrator destination") rootCmd.PersistentFlags().Bool("migrator-destination-clickhouse-disableTLS", false, "Clickhouse disableTLS for migrator destination") - rootCmd.PersistentFlags().Bool("migrator-destination-clickhouse-asyncInsert", false, "Clickhouse async insert for migrator destination") + rootCmd.PersistentFlags().Bool("migrator-destination-clickhouse-asyncInsert", true, "Clickhouse async insert for migrator destination") rootCmd.PersistentFlags().Int("migrator-destination-clickhouse-maxRowsPerInsert", 100000, "Clickhouse max rows per insert for migrator destination") rootCmd.PersistentFlags().Int("migrator-destination-clickhouse-maxOpenConns", 30, "Clickhouse max open connections for migrator destination") rootCmd.PersistentFlags().Int("migrator-destination-clickhouse-maxIdleConns", 30, "Clickhouse max idle connections for migrator destination") diff --git a/internal/storage/clickhouse.go b/internal/storage/clickhouse.go index c6dbb15..6ef0c09 100644 --- a/internal/storage/clickhouse.go +++ b/internal/storage/clickhouse.go @@ -127,6 +127,9 @@ func connectDB(cfg *config.ClickhouseConfig) (clickhouse.Conn, error) { if cfg.AsyncInsert { settings["async_insert"] = "1" settings["wait_for_async_insert"] = "1" + settings["async_insert_busy_timeout_ms"] = "3000" // 3 seconds + settings["async_insert_max_data_size"] = "104857600" // 100MB + settings["async_insert_max_query_number"] = "100000" // 100k } return settings }(),