Add Canary steps (#4219)

Adds Canary steps

* Storing json in the DB as the overhead of a new canary_steps table seemed to have little benefit. We can always migrate in the future if required.

Resolves #4172
---------

Co-authored-by: Seth Boyles <[email protected]>

Author: Samze

app/actions/deployment_continue.rb

@@ -12,11 +12,21 @@ def continue(deployment:, user_audit_info:)
           reject_invalid_state!(deployment) unless deployment.continuable?
 
           record_audit_event(deployment, user_audit_info)
-          deployment.update(
-            state: DeploymentModel::DEPLOYING_STATE,
-            status_value: DeploymentModel::ACTIVE_STATUS_VALUE,
-            status_reason: DeploymentModel::DEPLOYING_STATUS_REASON
-          )
+
+          if deployment.canary_steps && deployment.canary_current_step < deployment.canary_steps.length
+            deployment.update(
+              state: DeploymentModel::PREPAUSED_STATE,
+              status_value: DeploymentModel::ACTIVE_STATUS_VALUE,
+              status_reason: DeploymentModel::DEPLOYING_STATUS_REASON,
+              canary_current_step: deployment.canary_current_step + 1
+            )
+          else
+            deployment.update(
+              state: DeploymentModel::DEPLOYING_STATE,
+              status_value: DeploymentModel::ACTIVE_STATUS_VALUE,
+              status_reason: DeploymentModel::DEPLOYING_STATUS_REASON
+            )
+          end
         end
       end
 

app/actions/deployment_create.rb

@@ -54,13 +54,15 @@ def create(app:, user_audit_info:, message:)
             revision_guid: revision&.guid,
             revision_version: revision&.version,
             strategy: message.strategy,
-            max_in_flight: message.max_in_flight
+            max_in_flight: message.max_in_flight,
+            canary_steps: message.options&.dig(:canary, :steps)
           )
+
           MetadataUpdate.update(deployment, message)
 
           supersede_deployment(previous_deployment)
 
-          process_instances = starting_process_instances(message, desired_instances)
+          process_instances = starting_process_instances(deployment, desired_instances)
 
           process = create_deployment_process(app, deployment.guid, revision, process_instances)
           # Need to transition from STOPPED to STARTED to engage the ProcessObserver to desire the LRP.
@@ -176,7 +178,8 @@ def deployment_for_stopped_app(app, message, previous_deployment, previous_dropl
           revision_guid: revision&.guid,
           revision_version: revision&.version,
           strategy: message.strategy,
-          max_in_flight: message.max_in_flight
+          max_in_flight: message.max_in_flight,
+          canary_steps: message.options&.dig(:canary, :steps)
         )
 
         MetadataUpdate.update(deployment, message)
@@ -217,12 +220,14 @@ def starting_state(message)
         end
       end
 
-      def starting_process_instances(message, desired_instances)
-        if message.strategy == DeploymentModel::CANARY_STRATEGY
-          1
-        else
-          [message.max_in_flight, desired_instances].min
-        end
+      def starting_process_instances(deployment, desired_instances)
+        starting_process_count = if deployment.strategy == DeploymentModel::CANARY_STRATEGY
+                                   deployment.canary_step[:canary]
+                                 else
+                                   desired_instances
+                                 end
+
+        [deployment.max_in_flight, starting_process_count].min
       end
 
       def log_rollback_event(app_guid, user_id, revision_id, strategy)

app/messages/deployment_create_message.rb

@@ -10,17 +10,22 @@ class DeploymentCreateMessage < MetadataBaseMessage
       options
     ]
 
+    ALLOWED_OPTION_KEYS = %i[
+      canary
+      max_in_flight
+    ].freeze
+
+    ALLOWED_STEP_KEYS = [
+      :instance_weight
+    ].freeze
+
     validates_with NoAdditionalKeysValidator
     validates :strategy,
               inclusion: { in: %w[rolling canary], message: "'%<value>s' is not a supported deployment strategy" },
               allow_nil: true
     validate :mutually_exclusive_droplet_sources
 
-    validates :options,
-              allow_nil: true,
-              hash: true
-
-    validate :validate_max_in_flight
+    validate :validate_options
 
     def app_guid
       relationships&.dig(:app, :data, :guid)
@@ -46,14 +51,76 @@ def mutually_exclusive_droplet_sources
       errors.add(:droplet, "Cannot set both fields 'droplet' and 'revision'")
     end
 
-    def validate_max_in_flight
-      return unless options.present? && options.is_a?(Hash) && options[:max_in_flight]
+    def validate_options
+      return if options.blank?
 
+      unless options.is_a?(Hash)
+        errors.add(:options, 'must be an object')
+        return
+      end
+
+      disallowed_keys = options.keys - ALLOWED_OPTION_KEYS
+      errors.add(:options, "has unsupported key(s): #{disallowed_keys.join(', ')}") if disallowed_keys.present?
+
+      validate_max_in_flight if options[:max_in_flight]
+      validate_canary if options[:canary]
+    end
+
+    def validate_max_in_flight
       max_in_flight = options[:max_in_flight]
 
       return unless !max_in_flight.is_a?(Integer) || max_in_flight < 1
 
       errors.add(:max_in_flight, 'must be an integer greater than 0')
     end
+
+    def validate_canary
+      canary_options = options[:canary]
+      unless canary_options.is_a?(Hash)
+        errors.add(:'options.canary', 'must be an object')
+        return
+      end
+
+      if canary_options && strategy != 'canary'
+        errors.add(:'options.canary', 'are only valid for Canary deployments')
+        return
+      end
+
+      validate_steps if options[:canary][:steps]
+    end
+
+    def validate_steps
+      steps = options[:canary][:steps]
+      if !steps.is_a?(Array) || steps.any? { |step| !step.is_a?(Hash) }
+        errors.add(:'options.canary.steps', 'must be an array of objects')
+        return
+      end
+
+      steps.each do |step|
+        disallowed_keys = step.keys - ALLOWED_STEP_KEYS
+
+        errors.add(:'options.canary.steps', "has unsupported key(s): #{disallowed_keys.join(', ')}") if disallowed_keys.present?
+      end
+
+      validate_step_instance_weights
+    end
+
+    def validate_step_instance_weights
+      steps = options[:canary][:steps]
+
+      errors.add(:'options.canary.steps', 'missing key: "instance_weight"') if steps.any? { |step| !step.key?(:instance_weight) }
+
+      if steps.any? { |step| !step[:instance_weight].is_a?(Integer) }
+        errors.add(:'options.canary.steps.instance_weight', 'must be an Integer between 1-100 (inclusive)')
+        return
+      end
+
+      errors.add(:'options.canary.steps.instance_weight', 'must be an Integer between 1-100 (inclusive)') if steps.any? { |step| (1..100).exclude?(step[:instance_weight]) }
+
+      weights = steps.pluck(:instance_weight)
+      return unless weights.sort != weights
+
+      errors.add(:'options.canary.steps.instance_weight', 'must be sorted in ascending order')
+    end
   end
 end

app/models/runtime/deployment_model.rb

@@ -1,5 +1,7 @@
 module VCAP::CloudController
   class DeploymentModel < Sequel::Model(:deployments)
+    plugin :serialization
+
     DEPLOYMENT_STATES = [
       DEPLOYING_STATE = 'DEPLOYING'.freeze,
       PREPAUSED_STATE = 'PREPAUSED'.freeze,
@@ -76,6 +78,8 @@ class DeploymentModel < Sequel::Model(:deployments)
     add_association_dependencies labels: :destroy
     add_association_dependencies annotations: :destroy
 
+    serialize_attributes :json, :canary_steps
+
     dataset_module do
       def deploying_count
         where(state: DeploymentModel::PROGRESSING_STATES).count
@@ -87,6 +91,16 @@ def before_update
       set_status_updated_at
     end
 
+    def before_create
+      self.canary_current_step = 1 if strategy == DeploymentModel::CANARY_STRATEGY
+
+      # unless canary_steps.nil?
+      #   # ensure that canary steps are in the correct format for serialization
+      #   self.canary_steps = canary_steps.map(&:stringify_keys)
+      # end
+      super
+    end
+
     def deploying?
       DeploymentModel::PROGRESSING_STATES.include?(state)
     end
@@ -99,6 +113,36 @@ def continuable?
       state == DeploymentModel::PAUSED_STATE
     end
 
+    def current_canary_instance_target
+      canary_step[:canary]
+    end
+
+    def canary_total_instances
+      canary_step[:canary] + canary_step[:original]
+    end
+
+    def canary_step
+      raise 'canary_step is only valid for canary deloyments' unless strategy == CANARY_STRATEGY
+
+      current_step = canary_current_step || 1
+      canary_step_plan[current_step - 1]
+    end
+
+    def canary_step_plan
+      raise 'canary_step_plan is only valid for canary deloyments' unless strategy == CANARY_STRATEGY
+
+      return [{ canary: 1, original: original_web_process_instance_count }] if canary_steps.nil?
+
+      canary_steps.map do |step|
+        weight = step['instance_weight']
+        target_canary = (original_web_process_instance_count * (weight.to_f / 100)).round.to_i
+        target_canary = 1 if target_canary.zero?
+        target_original = original_web_process_instance_count - target_canary + 1
+        target_original = 0 if weight == 100
+        { canary: target_canary, original: target_original }
+      end
+    end
+
     private
 
     def set_status_updated_at

app/presenters/v3/deployment_presenter.rb

@@ -10,18 +10,9 @@ def to_hash
         guid: deployment.guid,
         created_at: deployment.created_at,
         updated_at: deployment.updated_at,
-        status: {
-          value: deployment.status_value,
-          reason: deployment.status_reason,
-          details: {
-            last_successful_healthcheck: deployment.last_healthy_at,
-            last_status_change: deployment.status_updated_at
-          }
-        },
+        status: status(deployment),
         strategy: deployment.strategy,
-        options: {
-          max_in_flight: deployment.max_in_flight
-        },
+        options: options(deployment),
         droplet: {
           guid: deployment.droplet_guid
         },
@@ -64,6 +55,42 @@ def new_processes
       end
     end
 
+    def options(deployment)
+      options = {
+        max_in_flight: deployment.max_in_flight
+      }
+
+      if deployment.strategy == VCAP::CloudController::DeploymentModel::CANARY_STRATEGY && deployment.canary_steps
+        options[:canary] = {
+          steps: deployment.canary_steps
+        }
+      end
+
+      options
+    end
+
+    def status(deployment)
+      status = {
+        value: deployment.status_value,
+        reason: deployment.status_reason,
+        details: {
+          last_successful_healthcheck: deployment.last_healthy_at,
+          last_status_change: deployment.status_updated_at
+        }
+      }
+
+      if deployment.strategy == VCAP::CloudController::DeploymentModel::CANARY_STRATEGY
+        status[:canary] = {
+          steps: {
+            current: deployment.canary_current_step,
+            total: deployment.canary_steps&.length || 1
+          }
+        }
+      end
+
+      status
+    end
+
     def build_links
       {
         self: {

db/migrations/20250130172009_add_canary_steps_to_deployments.rb

@@ -0,0 +1,14 @@
+Sequel.migration do
+  up do
+    alter_table(:deployments) do
+      add_column :canary_steps, String, size: 4096
+      add_column :canary_current_step, :integer
+    end
+  end
+  down do
+    alter_table(:deployments) do
+      drop_column :canary_steps
+      drop_column :canary_current_step
+    end
+  end
+end

docs/v3/source/includes/api_resources/_deployments.erb

@@ -8,11 +8,27 @@
     "details": {
       "last_successful_healthcheck": "2018-04-25T22:42:10Z",
       "last_status_change": "2018-04-25T22:42:10Z"
+    },
+    "canary": {
+      "steps": {
+        "current": 1,
+        "total": 2
+      }
     }
   },
   "strategy": "canary",
   "options" : {
     "max_in_flight": 3,
+    "canary": {
+      "steps": [
+        {
+          "instance_weight": 10
+        },
+        {
+          "instance_weight": 20
+        }
+      ]
+    }
   },
   "droplet": {
     "guid": "44ccfa61-dbcf-4a0d-82fe-f668e9d2a962"

docs/v3/source/includes/resources/deployments/_create.md.erb

@@ -78,6 +78,7 @@ Name | Type | Description | Default
 **revision**<sup>[1]</sup> | _object_ | The [revision](#revisions) whose droplet to deploy for the app; this will update the app's [current droplet](#get-current-droplet-association-for-an-app) to this droplet |
 **strategy** | _string_ | The strategy to use for the deployment | `rolling`
 **options.max_in_flight** | _integer_ | The maximum number of new instances to deploy simultaneously | 1
+**options.canary.steps** | _array of [canary step objects](#canary-steps-object)_ | An array of canary steps to use for the deployment
 **metadata.labels** | [_label object_](#labels) | Labels applied to the deployment
 **metadata.annotations**  | [_annotation object_](#annotations) | Annotations applied to the deployment
 

docs/v3/source/includes/resources/deployments/_object.md.erb

@@ -15,9 +15,12 @@ Name | Type | Description
 **status.value** | _string_ | The current status of the deployment; valid values are `ACTIVE` (meaning in progress) and `FINALIZED` (meaning finished, either successfully or not)
 **status.reason** | _string_ | The reason for the status of the deployment;<br>following list represents valid values:<br>1. If **status.value** is `ACTIVE`<br>- `DEPLOYING`<br>- `PAUSED` (only valid for canary deployments) <br>- `CANCELING`<br>2. If **status.value** is `FINALIZED`<br>- `DEPLOYED`<br>- `CANCELED`<br>- `SUPERSEDED` (another deployment created for app before completion)<br>
 **status.details.last_successful_healthcheck** | _[timestamp](#timestamps)_ | Timestamp of the last successful healthcheck
-**status.details.last_status_change** | _[timestamp](#timestamps)_ | Timestamp of last change to status.value or status.reason
+**status.details.last_status_change** | _[timestamp](#timestamps)_ | Timestamp of last change to status.value or status.reason**status.details.last_status_change** | _[timestamp](#timestamps)_ | Timestamp of last change to status.value or status.reason
+**status.canary.steps.current** | _integer_ | The current canary step. Only available for deployments with strategy 'canary'. (experimental)
+**status.canary.steps.total** | _integer_ |  The total number of canary steps. Only available for deployments with strategy 'canary'. (experimental)
 **strategy** | _string_ | Strategy used for the deployment; supported strategies are `rolling` and `canary` (experimental)
 **options.max_in_flight** | _integer_ | The maximum number of new instances to deploy simultaneously
+**options.canary.steps** | _array of [canary step objects](#canary-steps-object)_ | Canary steps to use for the deployment. Only available for deployments with strategy 'canary'. (experimental)
 **droplet.guid** | _string_ | The droplet guid that the deployment is transitioning the app to
 **previous_droplet.guid** | _string_ | The app's [current droplet guid](#get-current-droplet-association-for-an-app) before the deployment was created
 **new_processes** | _array_ | List of processes created as part of the deployment
@@ -26,3 +29,8 @@ Name | Type | Description
 **metadata.labels** | [_label object_](#labels) | Labels applied to the deployment
 **metadata.annotations**  | [_annotation object_](#annotations) | Annotations applied to the deployment
 **links** | [_links object_](#links) | Links to related resources
+
+
+#### Canary steps object
+
+**instance_weight** | _integer_ | The percentage of instances to be deployed as part of the canary process in this step (experimental)