feat(policy): add fixed buffer autoscaling policy (#671)

* feat(policy): add new fixedBuffer autoscaling

The new policy will add a buffer of rooms of fixed size/amount
on top of the occupied rooms. Instead of relying on relative
occupancy to determine the amount of rooms in the system

* feat(api): new fixedBuffer policy to API

Managament API can receive a new policy type for scheduler autoscaling
which is the 'fixedBuffer' one. The validations were moved to a struct-level
validation so we can infere nested field values based on policy type

* chore(docker): remove no_cache from local dev
* fix(policy): refactor fixedBuffer map
* fix(metrics): default zero on game room metrics
* chore(worker): reduce log ingestion

Author: hspedro

cmd/commom/commom.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/topfreegames/maestro/internal/config"
 	"github.com/topfreegames/maestro/internal/config/viper"
+	"github.com/topfreegames/maestro/internal/core/entities/autoscaling"
 	"github.com/topfreegames/maestro/internal/service"
 	"github.com/topfreegames/maestro/internal/validations"
 	"go.uber.org/zap"
@@ -48,6 +49,12 @@ func ServiceSetup(ctx context.Context, cancelFn context.CancelFunc, logConfig, c
 		return fmt.Errorf("unable to register validations: %w", err), nil, nil
 	}
 
+	// Register autoscaling policy validation
+	err = autoscaling.RegisterPolicyValidation(validations.Validate)
+	if err != nil {
+		return fmt.Errorf("unable to register autoscaling policy validation: %w", err), nil, nil
+	}
+
 	viperConfig, err := viper.NewViperConfig(configPath)
 	if err != nil {
 		return fmt.Errorf("unable to load config: %w", err), nil, nil

cmd/worker/wire.go

@@ -53,6 +53,7 @@ func initializeWorker(c config.Config, builder *worker.WorkerBuilder) (*workerss
 		service.NewCreateSchedulerVersionConfig,
 		service.NewHealthControllerConfig,
 		service.NewOperationRoomsAddConfig,
+		service.NewOperationRoomsRemoveConfig,
 		service.NewRoomManagerConfig,
 		service.NewRoomManager,
 		service.NewOperationManagerConfig,

cmd/worker/worker.go

@@ -74,7 +74,7 @@ func runWorker() {
 	}
 
 	go func() {
-		zap.L().Info("starting operation cancellation request watcher")
+		zap.L().Debug("starting operation cancellation request watcher")
 		err = operationExecutionWorkerManager.WorkerOptions.OperationManager.WatchOperationCancellationRequests(ctx)
 		if err != nil {
 			zap.L().With(zap.Error(err)).Info("operation cancellation watcher stopped with error")

docs/tutorials/Autoscaling.md

@@ -111,6 +111,8 @@ autoscaling:
 Maestro has a set of predefined policy types that can be used to configure the autoscaling, each policy will implement
 a specific strategy for calculating the desired number of rooms and will have its configurable parameters.
 
+**Note:** Policy types are mutually exclusive. The parameters used will depend on the value of `autoscaling.policy.type`.
+
 ### Room Occupancy Policy
 The basic concept of this policy is to scale the scheduler up or down based on the actual room occupancy rate, by defining a "buffer" percentage
 of ready rooms that Maestro must keep. The desired number of rooms will be given by the following formula:
@@ -190,3 +192,124 @@ Below are some simulated examples of how the room occupancy policy will behave:
 |     1      |       1       |     0.3     |           2          |    Scale Up: +1     |
 |     2      |       2       |     0.9     |          20          |    Scale Up: +18    |
 
+### Fixed Buffer Amount Policy
+The Fixed Buffer Amount policy maintains a fixed number of rooms on top of the currently occupied rooms. This policy is useful when you want to ensure a consistent buffer of available rooms regardless of the occupancy rate.
+
+The desired number of rooms will be given by the following formula:
+
+```
+desiredNumberOfRooms = numberOfOccupiedRooms + fixedBufferAmount
+```
+
+Respecting autoscaling boundaries (min and max):
+
+```go
+if desiredNumberOfRooms < autoscaling.Min {
+  desiredNumberOfRooms = autoscaling.Min
+}
+
+if autoscaling.Max != -1 && desiredNumberOfRooms > autoscaling.Max {
+  desiredNumberOfRooms = autoscaling.Max
+}
+```
+
+Maestro will constantly try to maintain the specified fixed amount of rooms in addition to the occupied rooms, ensuring there are always enough rooms available for new players.
+
+#### Fixed Buffer Policy Parameters
+
+- **fixedBufferAmount** [integer]: The fixed number of rooms that Maestro should maintain on top of the occupied rooms. Must be a value greater than 0 and less than the maximum number of rooms (max) when max is greater than 0.
+
+#### Example
+
+[comment]: <> (YAML version)
+<details>
+    <summary>YAML version</summary>
+    <div class="highlight highlight-source-yaml position-relative overflow-auto">
+        <pre>
+name: String
+game: String
+...
+autoscaling:
+  enabled: true
+  min: 1
+  max: 100
+  policy:
+    type: fixedBuffer
+    parameters:
+      fixedBuffer:
+        amount: 50
+        </pre>
+    </div>
+</details>
+
+[comment]: <> (JSON version)
+<details>
+    <summary>JSON version</summary>
+    <div class="highlight highlight-source-yaml position-relative overflow-auto">
+        <pre>
+{
+  "autoscaling": {
+    "enabled": true,
+    "min": 1,
+    "max": 100,
+    "policy": {
+      "type": "fixedBuffer",
+      "parameters": {
+        "fixedBuffer": {
+          "amount": 50
+        }
+      }
+    }
+  }
+}
+        </pre>
+    </div>
+</details>
+
+[comment]: <> (Coexisting with RoomOccupancy)
+<details>
+    <summary>Coexisting with RoomOccupancy</summary>
+    <div class="highlight highlight-source-yaml position-relative overflow-auto">
+        <pre>
+{
+  "autoscaling": {
+    "enabled": true,
+    "min": 1,
+    "max": 100,
+    "policy": {
+      "type": "fixedBuffer",
+      "parameters": {
+        "fixedBuffer": {
+          "amount": 50
+        },
+        "roomOccupancy": {
+          "readyTarget": 0.2,
+          "downThreshold": 0.9
+        }
+      }
+    }
+  }
+}
+        </pre>
+    </div>
+</details>
+
+Below are some simulated examples of how the fixed buffer amount policy will behave:
+
+> Note that the autoscaling decision will always be limited by the min-max values!
+
+| totalRooms | occupiedRooms | fixedBufferAmount | desiredNumberOfRooms | autoscalingDecision |
+|:----------:|:-------------:|:-----------------:|:--------------------:|:-------------------:|
+|    100     |      80       |        50         |          130         |    Scale Up: +30    |
+|    100     |      50       |        50         |          100         |    Do Nothing: 0    |
+|    100     |      30       |        50         |           80         |   Scale Down: -20   |
+|     50     |      40       |        20         |           60         |    Scale Up: +10    |
+|     50     |      30       |        20         |           50         |    Do Nothing: 0    |
+|     50     |      10       |        20         |           30         |   Scale Down: -20   |
+|     10     |       5       |        10         |           15         |    Scale Up: +5     |
+|     10     |       0       |        10         |           10         |    Do Nothing: 0    |
+|     10     |       0       |         5         |            5         |   Scale Down: -5    |
+|     5      |       5       |         5         |           10         |    Scale Up: +5     |
+|     1      |       1       |         3         |            4         |    Scale Up: +3     |
+|     2      |       2       |         8         |           10         |    Scale Up: +8     |
+

e2e/framework/maestro/docker-compose.yml

@@ -4,7 +4,6 @@ services:
     build:
       context: ../../..
       dockerfile: ./e2e/framework/maestro/Dockerfile
-      no_cache: true
     environment:
       - MAESTRO_ADAPTERS_ROOMSTORAGE_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_SCHEDULERSTORAGE_POSTGRES_URL=postgres://maestro:maestro@postgres:5432/maestro?sslmode=disable
@@ -31,7 +30,6 @@ services:
     build:
       context: ../../..
       dockerfile: ./e2e/framework/maestro/Dockerfile
-      no_cache: true
     environment:
       - MAESTRO_ADAPTERS_ROOMSTORAGE_REDIS_URL=redis://redis:6379/0
       - MAESTRO_ADAPTERS_INSTANCESTORAGE_REDIS_URL=redis://redis:6379/0
@@ -60,7 +58,6 @@ services:
     build:
       context: ../../..
       dockerfile: ./e2e/framework/maestro/Dockerfile
-      no_cache: true
     environment:
       - MAESTRO_ADAPTERS_SCHEDULERSTORAGE_POSTGRES_URL=postgres://maestro:maestro@postgres:5432/maestro?sslmode=disable
       - MAESTRO_ADAPTERS_ROOMSTORAGE_REDIS_URL=redis://redis:6379/0
@@ -82,7 +79,6 @@ services:
     build:
       context: ../../..
       dockerfile: ./e2e/framework/maestro/Dockerfile
-      no_cache: true
     environment:
       - MAESTRO_ADAPTERS_SCHEDULERSTORAGE_POSTGRES_URL=postgres://maestro:maestro@postgres:5432/maestro?sslmode=disable
       - MAESTRO_ADAPTERS_OPERATIONFLOW_REDIS_URL=redis://redis:6379/0

internal/adapters/runtime/kubernetes/scheduler.go

@@ -238,7 +238,7 @@ func (k *kubernetes) MitigateDisruption(
 		// If current PDB already matches the desired state regarding MinAvailable and MaxUnavailable being nil,
 		// no update is needed.
 		if currentPdbMinAvailable == desiredMinAvailable && pdb.Spec.MaxUnavailable == nil {
-			k.logger.Info("PDB already in desired state", zap.String(logs.LogFieldSchedulerName, scheduler.Name))
+			k.logger.Debug("PDB already in desired state", zap.String(logs.LogFieldSchedulerName, scheduler.Name))
 			return nil // No update needed, success.
 		}
 

internal/api/handlers/requestadapters/schedulers.go

@@ -385,6 +385,11 @@ func fromApiAutoscalingPolicyParameters(apiPolicyParameters *api.PolicyParameter
 	if roomOccupancy := apiPolicyParameters.GetRoomOccupancy(); roomOccupancy != nil {
 		policyParameters.RoomOccupancy = fromApiRoomOccupancyPolicyToEntity(roomOccupancy)
 	}
+	if fixedBuffer := apiPolicyParameters.GetFixedBuffer(); fixedBuffer != nil {
+		policyParameters.FixedBuffer = &autoscaling.FixedBufferParams{
+			Amount: int(fixedBuffer.GetAmount()),
+		}
+	}
 	return policyParameters
 }
 
@@ -569,6 +574,17 @@ func getAutoscalingPolicy(autoscalingPolicy autoscaling.Policy) *api.Autoscaling
 func getPolicyParameters(parameters autoscaling.PolicyParameters) *api.PolicyParameters {
 	return &api.PolicyParameters{
 		RoomOccupancy: getRoomOccupancy(parameters.RoomOccupancy),
+		FixedBuffer:   getFixedBuffer(parameters.FixedBuffer),
+	}
+}
+
+func getFixedBuffer(fixedBuffer *autoscaling.FixedBufferParams) *api.FixedBuffer {
+	if fixedBuffer == nil {
+		return nil
+	}
+	fixedBufferAmount := int32(fixedBuffer.Amount)
+	return &api.FixedBuffer{
+		Amount: &fixedBufferAmount,
 	}
 }
 

internal/core/entities/autoscaling/autoscaling.go

@@ -30,7 +30,10 @@ type PolicyType string
 const (
 	// RoomOccupancy is an implemented policy in maestro autoscaler,
 	// it uses the number of occupied rooms and a ready rooms target percentage to calculate the desired number of rooms in a scheduler.
-	RoomOccupancy        PolicyType = "roomOccupancy"
+	RoomOccupancy PolicyType = "roomOccupancy"
+	// FixedBuffer is an implemented policy in maestro autoscaler,
+	// it uses a fixed amount of ready rooms to calculate the desired number of rooms in a scheduler.
+	FixedBuffer          PolicyType = "fixedBuffer"
 	DefaultDownThreshold float32    = 0.99
 )
 
@@ -70,17 +73,20 @@ func NewAutoscaling(enabled bool, min, max, cooldown int, policy Policy) (*Autos
 // Policy represents the autoscaling policy configuration.
 type Policy struct {
 	// Type indicates the autoscaling policy type.
-	Type PolicyType `validate:"oneof=roomOccupancy"`
+	Type PolicyType `validate:"oneof=roomOccupancy fixedBuffer"`
 	// Parameters indicates the autoscaling policy parameters.
-	Parameters PolicyParameters
+	Parameters PolicyParameters `validate:"required"`
 }
 
 // PolicyParameters represents the autoscaling policy parameters, its fields validations will
 // vary according to the policy type.
 type PolicyParameters struct {
 	// RoomOccupancy represents the parameters for RoomOccupancy policy type, it must be provided if Policy Type is RoomOccupancy.
 	// +optional
-	RoomOccupancy *RoomOccupancyParams `validate:"required_for_room_occupancy=Type"`
+	RoomOccupancy *RoomOccupancyParams
+	// Amount represents an integer of the amount of rooms that a scheduler should maintain at ready state at all times, it must be provided if Policy Type is Fixed.
+	// +optional
+	FixedBuffer *FixedBufferParams
 }
 
 // RoomOccupancyParams represents the parameters accepted by rooms occupancy autoscaling properties.
@@ -90,3 +96,9 @@ type RoomOccupancyParams struct {
 	// DownThreshold indicates the percentage of occupied rooms a scheduler should have to trigger a downscale event.
 	DownThreshold float64 `validate:"gt=0,lt=1"`
 }
+
+// FixedBufferParams represents the parameters accepted by fixed buffer autoscaling properties.
+type FixedBufferParams struct {
+	// Amount indicates the amount of rooms that a scheduler should maintain at ready state at all times.
+	Amount int `validate:"gt=0"`
+}

internal/core/entities/autoscaling/autoscaling_test.go

@@ -39,6 +39,13 @@ func TestNewAutoscaling(t *testing.T) {
 	if err != nil {
 		t.Errorf("unexpected error %d'", err)
 	}
+
+	// Register struct-level validation for Policy
+	err = RegisterPolicyValidation(validations.Validate)
+	if err != nil {
+		t.Errorf("unexpected error registering policy validation: %v", err)
+	}
+
 	translator := validations.GetDefaultTranslator()
 
 	validRoomOccupancyPolicy := Policy{
@@ -51,6 +58,15 @@ func TestNewAutoscaling(t *testing.T) {
 		},
 	}
 
+	validFixedBufferPolicy := Policy{
+		Type: FixedBuffer,
+		Parameters: PolicyParameters{
+			FixedBuffer: &FixedBufferParams{
+				Amount: 5,
+			},
+		},
+	}
+
 	t.Run("invalid scenarios", func(t *testing.T) {
 		t.Run("fails when try to create autoscaling with invalid Min", func(t *testing.T) {
 			_, err := NewAutoscaling(true, -1, 10, 10, validRoomOccupancyPolicy)
@@ -111,6 +127,22 @@ func TestNewAutoscaling(t *testing.T) {
 			validationErrs = err.(validator.ValidationErrors)
 			assert.Equal(t, "DownThreshold must be less than 1", validationErrs[0].Translate(translator))
 		})
+
+		t.Run("fails when try to create autoscaling with invalid fixedBuffer Policy", func(t *testing.T) {
+			_, err := NewAutoscaling(true, 1, 10, 10, Policy{Type: "fixedBuffer", Parameters: PolicyParameters{}})
+			validationErrs := err.(validator.ValidationErrors)
+			assert.Equal(t, "FixedBuffer must not be nil for FixedBuffer policy type", validationErrs[0].Translate(translator))
+
+			zeroValue := 0
+			_, err = NewAutoscaling(true, 1, 10, 10, Policy{Type: "fixedBuffer", Parameters: PolicyParameters{FixedBuffer: &FixedBufferParams{Amount: zeroValue}}})
+			validationErrs = err.(validator.ValidationErrors)
+			assert.Equal(t, "Amount must be greater than 0", validationErrs[0].Translate(translator))
+
+			negativeValue := -1
+			_, err = NewAutoscaling(true, 1, 10, 10, Policy{Type: "fixedBuffer", Parameters: PolicyParameters{FixedBuffer: &FixedBufferParams{Amount: negativeValue}}})
+			validationErrs = err.(validator.ValidationErrors)
+			assert.Equal(t, "Amount must be greater than 0", validationErrs[0].Translate(translator))
+		})
 	})
 
 	t.Run("valid scenarios", func(t *testing.T) {
@@ -124,6 +156,17 @@ func TestNewAutoscaling(t *testing.T) {
 			_, err = NewAutoscaling(false, 50, 100, 100, validRoomOccupancyPolicy)
 			assert.NoError(t, err)
 		})
+
+		t.Run("success when try to create valid autoscaling with fixedBuffer type", func(t *testing.T) {
+			_, err := NewAutoscaling(true, 10, -1, 0, validFixedBufferPolicy)
+			assert.NoError(t, err)
+
+			_, err = NewAutoscaling(false, 1, 1, 10, validFixedBufferPolicy)
+			assert.NoError(t, err)
+
+			_, err = NewAutoscaling(false, 50, 100, 100, validFixedBufferPolicy)
+			assert.NoError(t, err)
+		})
 	})
 
 }