Allow validation room to report as occupied in scheduler update (configurable) (#697)

Author: joaobologna

config/config.yaml

@@ -94,6 +94,7 @@ services:
     roomDeletionTimeoutMillis: 120000
     roomAllocationTTLMillis: 300000
     roomValidationAttempts: 3
+    roomValidationAcceptOccupiedStatus: false
   operationManager:
     operationLeaseTTLMillis: 5000
   eventsForwarder:

internal/core/operations/schedulers/newversion/executor.go

@@ -44,9 +44,10 @@ import (
 
 // Config defines configurations for the Executor.
 type Config struct {
-	RoomInitializationTimeout time.Duration
-	RoomValidationTimeout     time.Duration
-	RoomValidationAttempts    int
+	RoomInitializationTimeout          time.Duration
+	RoomValidationTimeout              time.Duration
+	RoomValidationAttempts             int
+	RoomValidationAcceptOccupiedStatus bool
 }
 
 // Executor holds the dependecies to execute the operation to create a new scheduler version.
@@ -196,10 +197,18 @@ func (ex *Executor) validateGameRoomCreation(ctx context.Context, scheduler *ent
 	timeoutContext, cancelFunc := context.WithTimeout(ctx, duration)
 	defer cancelFunc()
 
+	status := []game_room.GameRoomStatus{game_room.GameStatusReady, game_room.GameStatusError}
+	if ex.config.RoomValidationAcceptOccupiedStatus {
+		// WARNING: Accepting occupied as valid is dangerous. Use only when validation
+		// rooms may be picked up by the matchmaker before reporting ready status. If a new room
+		// version incorrectly transitions to occupied, Maestro may scale up indefinitely.
+		status = append(status, game_room.GameStatusOccupied)
+	}
+
 	roomStatus, waitRoomErr := ex.roomManager.WaitRoomStatus(
 		timeoutContext,
 		gameRoom,
-		[]game_room.GameRoomStatus{game_room.GameStatusReady, game_room.GameStatusError},
+		status,
 	)
 
 	if waitRoomErr != nil {
@@ -214,6 +223,9 @@ func (ex *Executor) validateGameRoomCreation(ctx context.Context, scheduler *ent
 	case game_room.GameStatusReady:
 		logger.Sugar().Infof("validation room with ID: %s is ready", gameRoom.ID)
 		return nil
+	case game_room.GameStatusOccupied:
+		logger.Sugar().Infof("validation room with ID: %s is occupied", gameRoom.ID)
+		return nil
 	case game_room.GameStatusError:
 		logger.Sugar().Infof("validation room with ID: %s is in error state", gameRoom.ID)
 		var statusDescription string

internal/core/operations/schedulers/newversion/executor_test.go

@@ -160,6 +160,59 @@ func TestExecutor_Execute(t *testing.T) {
 		require.Nil(t, result)
 	})
 
+	t.Run("should succeed - major version update, game room status is occupied with config enabled, returns no error -> enqueue switch active version op", func(t *testing.T) {
+		mockCtrl := gomock.NewController(t)
+
+		currentActiveScheduler := newValidSchedulerWithImageVersion("image-v1")
+		newScheduler := *newValidSchedulerWithImageVersion("image-v2")
+		newSchedulerExpectedVersion := "v2.0.0"
+		op := &operation.Operation{
+			ID:             "123",
+			Status:         operation.StatusInProgress,
+			DefinitionName: newversion.OperationName,
+			SchedulerName:  newScheduler.Name,
+		}
+		operationDef := &newversion.Definition{NewScheduler: &newScheduler}
+		roomManager := mockports.NewMockRoomManager(mockCtrl)
+		schedulerManager := mockports.NewMockSchedulerManager(mockCtrl)
+		operationsManager := mockports.NewMockOperationManager(mockCtrl)
+		switchOpID := "switch-active-version-op-id"
+		config := newversion.Config{
+			RoomInitializationTimeout:          time.Duration(120000),
+			RoomValidationTimeout:              time.Duration(120000),
+			RoomValidationAttempts:             1,
+			RoomValidationAcceptOccupiedStatus: true,
+		}
+
+		executor := newversion.NewExecutor(roomManager, schedulerManager, operationsManager, config)
+
+		schedulerVersions := []*entities.SchedulerVersion{{Version: "v1.0.0"}, {Version: "v1.1.0"}, {Version: "v1.2.0"}}
+		gameRoom := &game_room.GameRoom{ID: "id-1"}
+
+		roomManager.EXPECT().CreateRoom(gomock.Any(), gomock.Any(), true).Return(gameRoom, nil, nil)
+		roomManager.EXPECT().WaitRoomStatus(gomock.Any(), gameRoom, []game_room.GameRoomStatus{game_room.GameStatusReady, game_room.GameStatusError, game_room.GameStatusOccupied}).Return(game_room.GameStatusOccupied, nil)
+		roomManager.EXPECT().DeleteRoom(gomock.Any(), gomock.Any(), remove.NewVersionValidationFinished).Return(nil)
+
+		schedulerManager.
+			EXPECT().
+			CreateNewSchedulerVersionAndEnqueueSwitchVersion(gomock.Any(), gomock.Any()).
+			DoAndReturn(
+				func(ctx context.Context, scheduler *entities.Scheduler) (string, error) {
+					require.Equal(t, newSchedulerExpectedVersion, scheduler.Spec.Version)
+					return switchOpID, nil
+				})
+		schedulerManager.EXPECT().GetActiveScheduler(gomock.Any(), newScheduler.Name).Return(currentActiveScheduler, nil)
+		schedulerManager.EXPECT().GetSchedulerVersions(gomock.Any(), newScheduler.Name).Return(schedulerVersions, nil)
+		schedulerManager.EXPECT().UpdateScheduler(gomock.Any(), currentActiveScheduler).Return(nil)
+		operationsManager.EXPECT().AppendOperationEventToExecutionHistory(gomock.Any(), op, "Major version detected, starting game room validation process...")
+		operationsManager.EXPECT().AppendOperationEventToExecutionHistory(gomock.Any(), op, "1º Attempt: Game room validation success!")
+		operationsManager.EXPECT().AppendOperationEventToExecutionHistory(gomock.Any(), op, fmt.Sprintf("enqueued switch active version operation with id: %s", switchOpID))
+
+		result := executor.Execute(context.Background(), op, operationDef)
+
+		require.Nil(t, result)
+	})
+
 	t.Run("should succeed - major version update, game room is valid, greatest major version is v3, returns no error -> enqueue switch active version op", func(t *testing.T) {
 		mockCtrl := gomock.NewController(t)
 

internal/service/config.go

@@ -45,6 +45,7 @@ const (
 	roomInitializationTimeoutMillisConfigPath    = "services.roomManager.roomInitializationTimeoutMillis"
 	roomValidationTimeoutMillisConfigPath        = "services.roomManager.roomValidationTimeoutMillis"
 	roomRoomValidationAttemptsConfigPath         = "services.roomManager.roomValidationAttempts"
+	roomValidationAcceptOccupiedStatusConfigPath = "services.roomManager.roomValidationAcceptOccupiedStatus"
 	roomPingTimeoutMillisConfigPath              = "services.roomManager.roomPingTimeoutMillis"
 	roomDeletionTimeoutMillisConfigPath          = "services.roomManager.roomDeletionTimeoutMillis"
 	roomAllocationTTLMillisConfigPath            = "services.roomManager.roomAllocationTTLMillis"
@@ -64,11 +65,13 @@ func NewCreateSchedulerVersionConfig(c config.Config) newversion.Config {
 	if roomValidationAttempts < 1 {
 		roomValidationAttempts = 1
 	}
+	acceptOccupiedStatus := c.GetBool(roomValidationAcceptOccupiedStatusConfigPath)
 
 	createSchedulerVersionConfig := newversion.Config{
-		RoomInitializationTimeout: initializationTimeout,
-		RoomValidationTimeout:     validationTimeout,
-		RoomValidationAttempts:    roomValidationAttempts,
+		RoomInitializationTimeout:          initializationTimeout,
+		RoomValidationTimeout:              validationTimeout,
+		RoomValidationAttempts:             roomValidationAttempts,
+		RoomValidationAcceptOccupiedStatus: acceptOccupiedStatus,
 	}
 
 	return createSchedulerVersionConfig