Lower s3 chunking default from 33GB to 10GB (#3149)

serprex · web-flow · commit 1d464df68af6 · 2025-07-07T16:46:01.000Z
seeing errors on smaller instances over memory consumption
&amp; taking too long to process 30GB
diff --git a/flow/internal/dynamicconf.go b/flow/internal/dynamicconf.go
@@ -164,7 +164,7 @@ var DynamicSettings = [...]*protos.DynamicSetting{
 	{
 		Name:         "PEERDB_S3_BYTES_PER_AVRO_FILE",
 		Description:  "S3 upload chunk size in bytes, needed for large unpartitioned initial loads.",
-		DefaultValue: "33333333333", // chosen to be round 30GB, but align with about half of 64MiB because S3 part size
+		DefaultValue: "10000000000", // 10GB, not GiB so to not align with 64MiB to avoid always having tiny part at end
 		ValueType:    protos.DynconfValueType_INT,
 		ApplyMode:    protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,
 	},

Original file line number	Diff line number	Diff line change
`@@ -164,7 +164,7 @@ var DynamicSettings = [...]*protos.DynamicSetting{`
`164`	`164`	`{`
`165`	`165`	`Name: "PEERDB_S3_BYTES_PER_AVRO_FILE",`
`166`	`166`	`Description: "S3 upload chunk size in bytes, needed for large unpartitioned initial loads.",`
`167`		`- DefaultValue: "33333333333", // chosen to be round 30GB, but align with about half of 64MiB because S3 part size`
	`167`	`+ DefaultValue: "10000000000", // 10GB, not GiB so to not align with 64MiB to avoid always having tiny part at end`
`168`	`168`	`ValueType: protos.DynconfValueType_INT,`
`169`	`169`	`ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,`
`170`	`170`	`},`