changing dynamic poller scaling strategy. (#1197)

* changing dynamic poller scaling strategy.

Author: mindaugasbarcauskas

internal/common/autoscaler/recommender.go

@@ -51,7 +51,11 @@ func (l *linearRecommender) Recommend(currentResource ResourceUnit, currentUsage
 		if l.targetUsages[usageType] == 0 { // avoid division by zero
 			r = math.MaxFloat64
 		} else {
-			r = currentResource.Value() * currentUsages[usageType].Value() / l.targetUsages[usageType].Value()
+			if currentUsages[usageType].Value() == float64(1000) {
+				r = l.upper.Value()
+			} else {
+				r = currentResource.Value() * currentUsages[usageType].Value() / l.targetUsages[usageType].Value()
+			}
 		}
 		// boundary check
 		r = math.Min(l.upper.Value(), math.Max(l.lower.Value(), r))

internal/common/autoscaler/recommender_test.go

@@ -41,6 +41,14 @@ func Test_linearRecommender_Recommend(t *testing.T) {
 		},
 	}
 
+	highUpperValue := fields{
+		lower: 5,
+		upper: 100,
+		targetUsages: map[UsageType]MilliUsage{
+			PollerUtilizationRate: 500,
+		},
+	}
+
 	tests := []struct {
 		name   string
 		fields fields
@@ -113,6 +121,17 @@ func Test_linearRecommender_Recommend(t *testing.T) {
 			},
 			want: ResourceUnit(15),
 		},
+		{
+			name:   "over utilized, since we do not how many tasks are in the queue (because poller usage at 100%), scale up to max",
+			fields: highUpperValue,
+			args: args{
+				currentResource: 10,
+				currentUsages: map[UsageType]MilliUsage{
+					PollerUtilizationRate: 1000,
+				},
+			},
+			want: ResourceUnit(100),
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

internal/internal_poller_autoscaler_test.go

@@ -21,15 +21,16 @@
 package internal
 
 import (
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+
 	"github.com/stretchr/testify/assert"
 	"go.uber.org/atomic"
 	s "go.uber.org/cadence/.gen/go/shared"
 	"go.uber.org/cadence/internal/common/autoscaler"
 	"go.uber.org/zap/zaptest"
-	"math/rand"
-	"sync"
-	"testing"
-	"time"
 )
 
 func Test_pollerAutoscaler(t *testing.T) {
@@ -114,7 +115,7 @@ func Test_pollerAutoscaler(t *testing.T) {
 				autoScalerEpoch:    1,
 				isDryRun:           false,
 			},
-			want: 4,
+			want: 10,
 		},
 		{
 			name: "over utilized, scale up to max",