Add PollerInitCount and lower default PollerMaxCount (#1433)

What changed?

When autoscaling is enabled, initial poller count should start from MaxConcurrentActivityTaskPollers so it's same behavior when autoscaling is not enabled.
change default max number to 20
Why?

Jump to max would potentially congest network connections.

Author: shijiesheng

internal/internal_poller_autoscaler.go

@@ -28,7 +28,8 @@ import (
 const (
 	defaultPollerAutoScalerCooldown           = 10 * time.Second
 	defaultMinPollerSize                      = 2
-	defaultMaxPollerSize                      = 200
+	defaultMaxPollerSize                      = 20
+	defaultInitPollerSize                     = 2
 	defaultPollerAutoScalerWaitTimeUpperBound = 256 * time.Millisecond
 	defaultPollerAutoScalerWaitTimeLowerBound = 16 * time.Millisecond
 )
@@ -48,9 +49,13 @@ type (
 		PollerMinCount int
 
 		// Optional: The maximum number of pollers to start with.
-		// default: 200
+		// default: 20
 		PollerMaxCount int
 
+		// Optional: The initial number of pollers to start with.
+		// default: 2
+		PollerInitCount int
+
 		// Optional: The upper bound of poller wait time for poller autoscaler to scale down.
 		// default: 256ms
 		// NOTE: This is normally not needed to be set by user.

internal/internal_worker.go

@@ -276,9 +276,13 @@ func newWorkflowTaskWorkerInternal(
 		domain,
 		params,
 	)
+	pollerCount := params.MaxConcurrentDecisionTaskPollers
+	if params.AutoScalerOptions.Enabled {
+		pollerCount = params.AutoScalerOptions.PollerInitCount
+	}
 	worker := newBaseWorker(baseWorkerOptions{
 		pollerAutoScaler:  params.AutoScalerOptions,
-		pollerCount:       params.MaxConcurrentDecisionTaskPollers,
+		pollerCount:       pollerCount,
 		pollerRate:        defaultPollerRate,
 		maxConcurrentTask: params.MaxConcurrentDecisionTaskExecutionSize,
 		maxTaskPerSecond:  params.WorkerDecisionTasksPerSecond,
@@ -470,7 +474,7 @@ func newActivityTaskWorker(
 	ensureRequiredParams(&workerParams)
 	pollerCount := workerParams.MaxConcurrentActivityTaskPollers
 	if workerParams.AutoScalerOptions.Enabled {
-		pollerCount = workerParams.AutoScalerOptions.PollerMaxCount
+		pollerCount = workerParams.AutoScalerOptions.PollerInitCount
 	}
 	base := newBaseWorker(
 		baseWorkerOptions{
@@ -1301,6 +1305,9 @@ func augmentAutoScalerOptions(options AutoScalerOptions) AutoScalerOptions {
 	if options.PollerMaxCount <= 1 {
 		options.PollerMaxCount = defaultMaxPollerSize
 	}
+	if options.PollerInitCount <= 1 {
+		options.PollerInitCount = defaultInitPollerSize
+	}
 	if options.Cooldown == 0 {
 		options.Cooldown = defaultPollerAutoScalerCooldown
 	}

internal/internal_worker_test.go

@@ -1326,6 +1326,7 @@ func Test_augmentWorkerOptions(t *testing.T) {
 					Enabled:                  true,
 					PollerMinCount:           10,
 					PollerMaxCount:           20,
+					PollerInitCount:          5,
 					Cooldown:                 time.Minute * 3,
 					PollerWaitTimeUpperBound: time.Millisecond * 200,
 					PollerWaitTimeLowerBound: time.Millisecond * 100,
@@ -1366,6 +1367,7 @@ func Test_augmentWorkerOptions(t *testing.T) {
 					Enabled:                  true,
 					PollerMinCount:           10,
 					PollerMaxCount:           20,
+					PollerInitCount:          5,
 					Cooldown:                 time.Minute * 3,
 					PollerWaitTimeUpperBound: time.Millisecond * 200,
 					PollerWaitTimeLowerBound: time.Millisecond * 100,
@@ -1409,7 +1411,8 @@ func Test_augmentWorkerOptions(t *testing.T) {
 				AutoScalerOptions: AutoScalerOptions{
 					Enabled:                  false,
 					PollerMinCount:           2,
-					PollerMaxCount:           200,
+					PollerMaxCount:           20,
+					PollerInitCount:          2,
 					Cooldown:                 time.Second * 10,
 					PollerWaitTimeUpperBound: time.Millisecond * 256,
 					PollerWaitTimeLowerBound: time.Millisecond * 16,