From 4873720a3cb1df9c3b759d2e0d4fc76af2050b0a Mon Sep 17 00:00:00 2001 From: Jake Loo <2171134+jakeloo@users.noreply.github.com> Date: Fri, 29 Aug 2025 17:00:47 +0000 Subject: [PATCH] Update s3 connector timeout --- cmd/root.go | 6 +++--- internal/storage/s3.go | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index b7607f3..02c669c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -229,10 +229,10 @@ func init() { rootCmd.PersistentFlags().String("migrator-destination-s3-secretAccessKey", "", "S3 secret access key for migrator destination") rootCmd.PersistentFlags().String("migrator-destination-s3-endpoint", "", "S3 endpoint URL for migrator destination") rootCmd.PersistentFlags().String("migrator-destination-s3-format", "parquet", "S3 storage format for migrator destination") - rootCmd.PersistentFlags().Int64("migrator-destination-s3-bufferSizeMB", 512, "S3 buffer size in MB before flush for migrator destination") - rootCmd.PersistentFlags().Int("migrator-destination-s3-bufferTimeoutSeconds", 300, "S3 buffer timeout in seconds before flush for migrator destination") + rootCmd.PersistentFlags().Int64("migrator-destination-s3-bufferSizeMB", 0, "S3 buffer size in MB before flush for migrator destination") + rootCmd.PersistentFlags().Int("migrator-destination-s3-bufferTimeoutSeconds", 0, "S3 buffer timeout in seconds before flush for migrator destination") rootCmd.PersistentFlags().Int("migrator-destination-s3-maxBlocksPerFile", 0, "S3 max blocks per file for migrator destination") - rootCmd.PersistentFlags().Uint("migrator-batchSize", 2000, "Batch size for storage operations in migrator") + rootCmd.PersistentFlags().Uint("migrator-batchSize", 0, "Batch size for storage operations in migrator") rootCmd.PersistentFlags().Uint("migrator-startBlock", 0, "Start block for migration") rootCmd.PersistentFlags().Uint("migrator-endBlock", 0, "End block for migration") rootCmd.PersistentFlags().Uint("migrator-workerCount", 0, "Worker count for migration") diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 2e37aa6..a0b51ae 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -97,12 +97,11 @@ func NewS3Connector(cfg *config.S3StorageConfig) (*S3Connector, error) { } } - // Set buffer defaults if cfg.BufferSize == 0 { - cfg.BufferSize = 1024 // 1GB default + cfg.BufferSize = 512 // 512MB default } if cfg.BufferTimeout == 0 { - cfg.BufferTimeout = 300 // 5 minutes default + cfg.BufferTimeout = 1 * 60 * 60 // 1 hour in seconds default } // Create formatter based on format