Add Canary deployments (#3892)

* Introduce Canary deployments as an experimental feature.

Closes #3837

Co-authored-by: Sam Gunaratne <[email protected]>
Co-authored-by: Seth Boyles <[email protected]>

Author: Samze

app/actions/deployment_continue.rb

@@ -0,0 +1,42 @@
+module VCAP::CloudController
+  class DeploymentContinue
+    class Error < StandardError
+    end
+    class InvalidStatus < Error
+    end
+
+    class << self
+      def continue(deployment:, user_audit_info:)
+        deployment.db.transaction do
+          deployment.lock!
+          reject_invalid_state!(deployment) unless deployment.continuable?
+
+          record_audit_event(deployment, user_audit_info)
+          deployment.update(
+            state: DeploymentModel::DEPLOYING_STATE,
+            status_value: DeploymentModel::ACTIVE_STATUS_VALUE,
+            status_reason: DeploymentModel::DEPLOYING_STATUS_REASON
+          )
+        end
+      end
+
+      private
+
+      def reject_invalid_state!(deployment)
+        raise InvalidStatus.new("Cannot continue a deployment with status: #{deployment.status_value} and reason: #{deployment.status_reason}")
+      end
+
+      def record_audit_event(deployment, user_audit_info)
+        app = deployment.app
+        Repositories::DeploymentEventRepository.record_continue(
+          deployment,
+          deployment.droplet,
+          user_audit_info,
+          app.name,
+          app.space_guid,
+          app.space.organization_guid
+        )
+      end
+    end
+  end
+end

app/actions/deployment_create.rb

@@ -13,14 +13,16 @@ def create(app:, user_audit_info:, message:)
         DeploymentModel.db.transaction do
           app.lock!
 
+          message.strategy ||= DeploymentModel::ROLLING_STRATEGY
+
           target_state = DeploymentTargetState.new(app, message)
 
           previous_droplet = app.droplet
           target_state.apply_to_app(app, user_audit_info)
 
           if target_state.rollback_target_revision
             revision = RevisionResolver.rollback_app_revision(app, target_state.rollback_target_revision, user_audit_info)
-            log_rollback_event(app.guid, user_audit_info.user_guid, target_state.rollback_target_revision.guid)
+            log_rollback_event(app.guid, user_audit_info.user_guid, target_state.rollback_target_revision.guid, message.strategy)
           else
             revision = RevisionResolver.update_app_revision(app, user_audit_info)
           end
@@ -41,15 +43,15 @@ def create(app:, user_audit_info:, message:)
 
           deployment = DeploymentModel.create(
             app: app,
-            state: DeploymentModel::DEPLOYING_STATE,
+            state: starting_state(message),
             status_value: DeploymentModel::ACTIVE_STATUS_VALUE,
             status_reason: DeploymentModel::DEPLOYING_STATUS_REASON,
             droplet: target_state.droplet,
             previous_droplet: previous_droplet,
             original_web_process_instance_count: desired_instances(app.oldest_web_process, previous_deployment),
             revision_guid: revision&.guid,
             revision_version: revision&.version,
-            strategy: DeploymentModel::ROLLING_STRATEGY
+            strategy: message.strategy
           )
           MetadataUpdate.update(deployment, message)
 
@@ -201,15 +203,23 @@ def supersede_deployment(previous_deployment)
         )
       end
 
-      def log_rollback_event(app_guid, user_id, revision_id)
+      def starting_state(message)
+        if message.strategy == DeploymentModel::CANARY_STRATEGY
+          DeploymentModel::PREPAUSED_STATE
+        else
+          DeploymentModel::DEPLOYING_STATE
+        end
+      end
+
+      def log_rollback_event(app_guid, user_id, revision_id, strategy)
         TelemetryLogger.v3_emit(
           'rolled-back-app',
           {
             'app-id' => app_guid,
             'user-id' => user_id,
             'revision-id' => revision_id
           },
-          { 'strategy' => 'rolling' }
+          { 'strategy' => strategy }
         )
       end
     end

app/controllers/v3/deployments_controller.rb

@@ -6,6 +6,7 @@
 require 'actions/deployment_create'
 require 'actions/deployment_update'
 require 'actions/deployment_cancel'
+require 'actions/deployment_continue'
 require 'cloud_controller/telemetry_logger'
 
 class DeploymentsController < ApplicationController
@@ -57,7 +58,7 @@ def create
           'app-id' => app.guid,
           'user-id' => current_user.guid
         },
-        { 'strategy' => 'rolling' }
+        { 'strategy' => deployment.strategy }
       )
     rescue DeploymentCreate::Error => e
       unprocessable!(e.message)
@@ -97,6 +98,22 @@ def cancel
     head :ok
   end
 
+  def continue
+    deployment = DeploymentModel.find(guid: hashed_params[:guid])
+
+    resource_not_found!(:deployment) unless deployment && permission_queryer.can_manage_apps_in_active_space?(deployment.app.space.id) &&
+                                            permission_queryer.is_space_active?(deployment.app.space.id)
+
+    begin
+      DeploymentContinue.continue(deployment:, user_audit_info:)
+      logger.info("Continued deployment #{deployment.guid} for app #{deployment.app_guid}")
+    rescue DeploymentContinue::Error => e
+      unprocessable!(e.message)
+    end
+
+    head :ok
+  end
+
   private
 
   def deployments_not_enabled!

app/jobs/runtime/prune_completed_deployments.rb

@@ -25,7 +25,7 @@ def perform
                                   select(:id)
 
             deployments_to_delete = deployments_dataset.
-                                    exclude(state: [DeploymentModel::DEPLOYING_STATE, DeploymentModel::CANCELING_STATE]).
+                                    exclude(state: DeploymentModel::ACTIVE_STATES).
                                     exclude(id: deployments_to_keep)
 
             delete_count = DeploymentDelete.delete(deployments_to_delete)

app/messages/deployment_create_message.rb

@@ -11,7 +11,7 @@ class DeploymentCreateMessage < MetadataBaseMessage
 
     validates_with NoAdditionalKeysValidator
     validates :strategy,
-              inclusion: { in: %w[rolling], message: "'%<value>s' is not a supported deployment strategy" },
+              inclusion: { in: %w[rolling canary], message: "'%<value>s' is not a supported deployment strategy" },
               allow_nil: true
     validate :mutually_exclusive_droplet_sources
 

app/models/runtime/app_model.rb

@@ -132,7 +132,7 @@ def stopped?
     end
 
     def deploying?
-      deployments_dataset.where(state: DeploymentModel::DEPLOYING_STATE).any?
+      deployments_dataset.where(state: DeploymentModel::PROGRESSING_STATES).any?
     end
 
     def self.user_visibility_filter(user)

app/models/runtime/deployment_model.rb

@@ -2,6 +2,8 @@ module VCAP::CloudController
   class DeploymentModel < Sequel::Model(:deployments)
     DEPLOYMENT_STATES = [
       DEPLOYING_STATE = 'DEPLOYING'.freeze,
+      PREPAUSED_STATE = 'PREPAUSED'.freeze,
+      PAUSED_STATE = 'PAUSED'.freeze,
       DEPLOYED_STATE = 'DEPLOYED'.freeze,
       CANCELING_STATE = 'CANCELING'.freeze,
       CANCELED_STATE = 'CANCELED'.freeze
@@ -13,15 +15,28 @@ class DeploymentModel < Sequel::Model(:deployments)
     ].freeze
 
     STATUS_REASONS = [
-      DEPLOYED_STATUS_REASON = 'DEPLOYED'.freeze,
       DEPLOYING_STATUS_REASON = 'DEPLOYING'.freeze,
+      PAUSED_STATUS_REASON = 'PAUSED'.freeze,
+      DEPLOYED_STATUS_REASON = 'DEPLOYED'.freeze,
       CANCELED_STATUS_REASON = 'CANCELED'.freeze,
       CANCELING_STATUS_REASON = 'CANCELING'.freeze,
       SUPERSEDED_STATUS_REASON = 'SUPERSEDED'.freeze
     ].freeze
 
     DEPLOYMENT_STRATEGIES = [
-      ROLLING_STRATEGY = 'rolling'.freeze
+      ROLLING_STRATEGY = 'rolling'.freeze,
+      CANARY_STRATEGY = 'canary'.freeze
+    ].freeze
+
+    PROGRESSING_STATES = [
+      DEPLOYING_STATE,
+      PREPAUSED_STATE,
+      PAUSED_STATE
+    ].freeze
+
+    ACTIVE_STATES = [
+      *PROGRESSING_STATES,
+      CANCELING_STATE
     ].freeze
 
     many_to_one :app,
@@ -63,7 +78,7 @@ class DeploymentModel < Sequel::Model(:deployments)
 
     dataset_module do
       def deploying_count
-        where(state: DeploymentModel::DEPLOYING_STATE).count
+        where(state: DeploymentModel::PROGRESSING_STATES).count
       end
     end
 
@@ -73,13 +88,15 @@ def before_update
     end
 
     def deploying?
-      state == DEPLOYING_STATE
+      DeploymentModel::PROGRESSING_STATES.include?(state)
     end
 
     def cancelable?
-      valid_states_for_cancel = [DeploymentModel::DEPLOYING_STATE,
-                                 DeploymentModel::CANCELING_STATE]
-      valid_states_for_cancel.include?(state)
+      DeploymentModel::ACTIVE_STATES.include?(state)
+    end
+
+    def continuable?
+      state == DeploymentModel::PAUSED_STATE
     end
 
     private

app/presenters/v3/deployment_presenter.rb

@@ -76,6 +76,13 @@ def build_links
             method: 'POST'
           }
         end
+      end.tap do |links|
+        if deployment.continuable?
+          links[:continue] = {
+            href: url_builder.build_url(path: "/v3/deployments/#{deployment.guid}/actions/continue"),
+            method: 'POST'
+          }
+        end
       end
     end
   end

app/repositories/deployment_event_repository.rb

@@ -11,7 +11,8 @@ def self.record_create(deployment, droplet, user_audit_info, v3_app_name, space_
           droplet_guid: droplet&.guid,
           type: type,
           revision_guid: deployment.revision_guid,
-          request: params
+          request: params,
+          strategy: deployment.strategy
         }
 
         Event.create(
@@ -53,6 +54,30 @@ def self.record_cancel(deployment, droplet, user_audit_info, v3_app_name, space_
           organization_guid: org_guid
         )
       end
+
+      def self.record_continue(deployment, droplet, user_audit_info, v3_app_name, space_guid, org_guid)
+        VCAP::AppLogEmitter.emit(deployment.app_guid, "Continuing deployment for app with guid #{deployment.app_guid}")
+
+        metadata = {
+          deployment_guid: deployment.guid,
+          droplet_guid: droplet&.guid
+        }
+
+        Event.create(
+          type: EventTypes::APP_DEPLOYMENT_CONTINUE,
+          actor: user_audit_info.user_guid,
+          actor_type: 'user',
+          actor_name: user_audit_info.user_email,
+          actor_username: user_audit_info.user_name,
+          actee: deployment.app_guid,
+          actee_type: 'app',
+          actee_name: v3_app_name,
+          timestamp: Sequel::CURRENT_TIMESTAMP,
+          metadata: metadata,
+          space_guid: space_guid,
+          organization_guid: org_guid
+        )
+      end
     end
   end
 end

app/repositories/event_types.rb

@@ -47,6 +47,7 @@ class EventTypesError < StandardError
         APP_REVISION_ENV_VARS_SHOW = 'audit.app.revision.environment_variables.show'.freeze,
         APP_DEPLOYMENT_CANCEL = 'audit.app.deployment.cancel'.freeze,
         APP_DEPLOYMENT_CREATE = 'audit.app.deployment.create'.freeze,
+        APP_DEPLOYMENT_CONTINUE = 'audit.app.deployment.continue'.freeze,
         APP_COPY_BITS = 'audit.app.copy-bits'.freeze,
         APP_UPLOAD_BITS = 'audit.app.upload-bits'.freeze,
         APP_APPLY_MANIFEST = 'audit.app.apply_manifest'.freeze,