Max in flight (#3907)

Adds max_in_flight to /v3/deployments

* This currently just creates initial set of deployment instances to max_in_flight, and then waits for all instances to be ready before batching another round of max_in_flight instances. A follow up PR can improve this behaviour so that as soon as a new instance is ready, we spin down an old one and create another new process.
* Canary deployments *always* start with a single single. max_in_flight is honoured post continuation for the rest of the instances.

Closes #3878

---------
Co-authored-by: Evan Farrar <[email protected]>
Co-authored-by: Sam Gunaratne <[email protected]>

Author: evanfarrar

app/actions/deployment_create.rb

@@ -41,23 +41,28 @@ def create(app:, user_audit_info:, message:)
             )
           end
 
+          desired_instances = desired_instances(app.oldest_web_process, previous_deployment)
+
           deployment = DeploymentModel.create(
             app: app,
             state: starting_state(message),
             status_value: DeploymentModel::ACTIVE_STATUS_VALUE,
             status_reason: DeploymentModel::DEPLOYING_STATUS_REASON,
             droplet: target_state.droplet,
             previous_droplet: previous_droplet,
-            original_web_process_instance_count: desired_instances(app.oldest_web_process, previous_deployment),
+            original_web_process_instance_count: desired_instances,
             revision_guid: revision&.guid,
             revision_version: revision&.version,
-            strategy: message.strategy
+            strategy: message.strategy,
+            max_in_flight: message.max_in_flight
           )
           MetadataUpdate.update(deployment, message)
 
           supersede_deployment(previous_deployment)
 
-          process = create_deployment_process(app, deployment.guid, revision)
+          process_instances = starting_process_instances(message, desired_instances)
+
+          process = create_deployment_process(app, deployment.guid, revision, process_instances)
           # Need to transition from STOPPED to STARTED to engage the ProcessObserver to desire the LRP.
           # It'd be better to do this via Diego::Runner.new(process, config).start,
           # but it is nontrivial to get that working in test.
@@ -75,8 +80,8 @@ def create(app:, user_audit_info:, message:)
         raise error
       end
 
-      def create_deployment_process(app, deployment_guid, revision)
-        process = clone_existing_web_process(app, revision)
+      def create_deployment_process(app, deployment_guid, revision, process_instances)
+        process = clone_existing_web_process(app, revision, process_instances)
 
         DeploymentProcessModel.create(
           deployment_guid: deployment_guid,
@@ -87,7 +92,7 @@ def create_deployment_process(app, deployment_guid, revision)
         process
       end
 
-      def clone_existing_web_process(app, revision)
+      def clone_existing_web_process(app, revision, process_instances)
         web_process = app.newest_web_process
         command = if revision
                     revision.commands_by_process_type[ProcessTypes::WEB]
@@ -99,7 +104,7 @@ def clone_existing_web_process(app, revision)
           app: app,
           type: ProcessTypes::WEB,
           state: ProcessModel::STOPPED,
-          instances: 1,
+          instances: process_instances,
           command: command,
           memory: web_process.memory,
           file_descriptors: web_process.file_descriptors,
@@ -170,7 +175,8 @@ def deployment_for_stopped_app(app, message, previous_deployment, previous_dropl
           original_web_process_instance_count: desired_instances(app.oldest_web_process, previous_deployment),
           revision_guid: revision&.guid,
           revision_version: revision&.version,
-          strategy: DeploymentModel::ROLLING_STRATEGY
+          strategy: DeploymentModel::ROLLING_STRATEGY,
+          max_in_flight: message.options ? message.options[:max_in_flight] : 1
         )
 
         MetadataUpdate.update(deployment, message)
@@ -211,6 +217,14 @@ def starting_state(message)
         end
       end
 
+      def starting_process_instances(message, desired_instances)
+        if message.strategy == DeploymentModel::CANARY_STRATEGY
+          1
+        else
+          [message.max_in_flight, desired_instances].min
+        end
+      end
+
       def log_rollback_event(app_guid, user_id, revision_id, strategy)
         TelemetryLogger.v3_emit(
           'rolled-back-app',

app/messages/deployment_create_message.rb

@@ -7,6 +7,7 @@ class DeploymentCreateMessage < MetadataBaseMessage
       droplet
       revision
       strategy
+      options
     ]
 
     validates_with NoAdditionalKeysValidator
@@ -15,6 +16,12 @@ class DeploymentCreateMessage < MetadataBaseMessage
               allow_nil: true
     validate :mutually_exclusive_droplet_sources
 
+    validates :options,
+              allow_nil: true,
+              hash: true
+
+    validate :validate_max_in_flight
+
     def app_guid
       relationships&.dig(:app, :data, :guid)
     end
@@ -27,12 +34,26 @@ def revision_guid
       revision&.dig(:guid)
     end
 
+    def max_in_flight
+      options&.dig(:max_in_flight) || 1
+    end
+
     private
 
     def mutually_exclusive_droplet_sources
       return unless revision.present? && droplet.present?
 
       errors.add(:droplet, "Cannot set both fields 'droplet' and 'revision'")
     end
+
+    def validate_max_in_flight
+      return unless options.present? && options.is_a?(Hash) && options[:max_in_flight]
+
+      max_in_flight = options[:max_in_flight]
+
+      return unless !max_in_flight.is_a?(Integer) || max_in_flight < 1
+
+      errors.add(:max_in_flight, 'must be an integer greater than 0')
+    end
   end
 end

app/presenters/v3/deployment_presenter.rb

@@ -19,6 +19,9 @@ def to_hash
           }
         },
         strategy: deployment.strategy,
+        options: {
+          max_in_flight: deployment.max_in_flight
+        },
         droplet: {
           guid: deployment.droplet_guid
         },

db/migrations/20240709231741_add_max_in_flight_to_deployment.rb

@@ -0,0 +1,7 @@
+Sequel.migration do
+  change do
+    alter_table(:deployments) do
+      add_column :max_in_flight, Integer, null: false, default: 1
+    end
+  end
+end

docs/v3/source/includes/api_resources/_deployments.erb

@@ -10,6 +10,9 @@
     }
   },
   "strategy": "canary",
+  "options" : {
+    "max_in_flight": 3,
+  },
   "droplet": {
     "guid": "44ccfa61-dbcf-4a0d-82fe-f668e9d2a962"
   },
@@ -80,6 +83,9 @@
         "reason": "DEPLOYED"
       },
       "strategy": "rolling",
+      "options" : {
+        "max_in_flight": 1,
+      },
       "droplet": {
         "guid": "44ccfa61-dbcf-4a0d-82fe-f668e9d2a962"
       },

docs/v3/source/includes/resources/deployments/_create.md.erb

@@ -77,6 +77,7 @@ Name | Type | Description | Default
 **droplet**<sup>[1]</sup> | _object_ | The droplet to deploy for the app; this will update the app's [current droplet](#get-current-droplet-association-for-an-app) to this droplet | The app's [current droplet](#get-current-droplet-association-for-an-app)
 **revision**<sup>[1]</sup> | _object_ | The [revision](#revisions) whose droplet to deploy for the app; this will update the app's [current droplet](#get-current-droplet-association-for-an-app) to this droplet |
 **strategy** | _string_ | The strategy to use for the deployment | `rolling`
+**options.max_in_flight** | _integer_ | The maximum number of new instances to deploy simultaneously | 1
 **metadata.labels** | [_label object_](#labels) | Labels applied to the deployment
 **metadata.annotations**  | [_annotation object_](#annotations) | Annotations applied to the deployment
 

docs/v3/source/includes/resources/deployments/_header.md

@@ -13,6 +13,7 @@ Deployments are different than the traditional method of pushing app updates whi
 Deployment strategies supported:
 
 * [Rolling deployments](https://docs.cloudfoundry.org/devguide/deploy-apps/rolling-deploy.html) allows for
-applications to be deployed without incurring downtime by gradually rolling out instances.
+applications to be deployed without incurring downtime by gradually rolling out instances. Max-in-flight can be configured
+to specify how many instances are rolled out simultaneously.
 
 * Canary deployments deploy a single instance and pause for user evaluation. If the canary instance is deemed successful, the deployment can be resumed via the [continue action](#continue-a-deployment). The deployment then continues like a rolling deployment. This feature is experimental and is subject to change.

docs/v3/source/includes/resources/deployments/_object.md.erb

@@ -17,6 +17,7 @@ Name | Type | Description
 **status.details.last_successful_healthcheck** | _[timestamp](#timestamps)_ | Timestamp of the last successful healthcheck
 **status.details.last_status_change** | _[timestamp](#timestamps)_ | Timestamp of last change to status.value or status.reason
 **strategy** | _string_ | Strategy used for the deployment; supported strategies are `rolling` and `canary` (experimental)
+**options.max_in_flight** | _integer_ | The maximum number of new instances to deploy simultaneously
 **droplet.guid** | _string_ | The droplet guid that the deployment is transitioning the app to
 **previous_droplet.guid** | _string_ | The app's [current droplet guid](#get-current-droplet-association-for-an-app) before the deployment was created
 **new_processes** | _array_ | List of processes created as part of the deployment

lib/cloud_controller/deployment_updater/updater.rb

@@ -110,7 +110,7 @@ def scale_deployment
           end
 
           scale_down_oldest_web_process_with_instances
-          deploying_web_process.update(instances: deploying_web_process.instances + 1)
+          deploying_web_process.update(instances: [deploying_web_process.instances + deployment.max_in_flight, deployment.original_web_process_instance_count].min)
         end
       end
 
@@ -161,12 +161,12 @@ def scale_canceled_web_processes_to_zero
       def scale_down_oldest_web_process_with_instances
         process = oldest_web_process_with_instances
 
-        if process.instances == 1 && is_interim_process?(process)
+        if process.instances <= deployment.max_in_flight && is_interim_process?(process)
           process.destroy
           return
         end
 
-        process.update(instances: process.instances - 1)
+        process.update(instances: [(process.instances - deployment.max_in_flight), 0].max)
       end
 
       def finalize_deployment