adding process monitoring to blue-green deployment

Author: SylwekLewan-dev

operation/push.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
 	"time"
 
 	"gopkg.in/yaml.v3"
@@ -22,6 +23,13 @@ const (
 	StrategyRolling
 )
 
+const (
+	AppDeployedRunningCheckIntervalSecondsDefault = 5
+	AppDeployedRunningTimeoutMinutesDefault       = 5
+	EnvironmentVariablesConvertBaseDecimal        = 10
+	EnvironmentVariablesConvertBitsSize           = 32
+)
+
 // AppPushOperation can be used to push buildpack apps
 type AppPushOperation struct {
 	orgName   string
@@ -96,42 +104,103 @@ func (p *AppPushOperation) pushBlueGreenApp(ctx context.Context, space *resource
 		return originalApp, err
 	}
 
-	tempAppName := originalApp.Name + "-venerable"
+	originalAppName := originalApp.Name
+	tempAppName := originalAppName + "-venerable"
 
 	// Check if temporary app name already exists if yes gracefully delete the app and continue
-	tempApp, err := p.findApp(ctx, tempAppName, space)
-	if err == nil {
-		err = p.gracefulDeletion(ctx, tempApp)
-		if err != nil {
-			return nil, err
+	tempApp, tempFindError := p.findApp(ctx, tempAppName, space)
+	if tempFindError == nil {
+		deleteTmpErr := p.gracefulDeletion(ctx, tempApp)
+		if deleteTmpErr != nil {
+			return nil, fmt.Errorf("failed to delete temp app due to: %w", deleteTmpErr)
 		}
 	}
 
 	// Update the existing app's name
-	_, err = p.client.Applications.Update(ctx, originalApp.GUID, &resource.AppUpdate{
+	_, updateErr := p.client.Applications.Update(ctx, originalApp.GUID, &resource.AppUpdate{
 		Name: tempAppName,
 	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to update app name failed with: %w", err)
-	}
+	if updateErr != nil {
+		return nil, fmt.Errorf("failed to update: %s, failed with: %w", tempAppName, updateErr)
+	}
+
+	newApp, pushError := p.pushApp(ctx, space, manifest, zipFile)
+	if pushError != nil {
+		// If push fails delete new application and change back original app name
+		invalidNewApp, findAppErr := p.findApp(ctx, originalAppName, space)
+		if findAppErr == nil && invalidNewApp != nil {
+			deleteErr := p.gracefulDeletion(ctx, invalidNewApp)
+			if deleteErr != nil {
+				return nil, fmt.Errorf("failed to deploy the new application and failed to revert it.\nNow the old application is running under name %s and is attached to state. \nThe new application under name %s is detached from state, and failed to start correctly: %w", tempAppName, originalApp.Name, deleteErr)
+			}
+		}
 
-	// Apply the manifest
-	newApp, err := p.pushApp(ctx, space, manifest, zipFile)
-	if err != nil {
-		// If push fails change back original app name
-		_, err = p.client.Applications.Update(ctx, originalApp.GUID, &resource.AppUpdate{
+		_, updateErr := p.client.Applications.Update(ctx, originalApp.GUID, &resource.AppUpdate{
 			Name: originalApp.Name,
 		})
-		if err != nil {
-			return nil, fmt.Errorf("failed to update app name back to original name: failed with %w", err)
+		if updateErr != nil {
+			return nil, fmt.Errorf("failed to push new version of application %s.\nDuring rollback also failed to update old app name from %s back to original name: %s. The old version of the application is currently running as: %s. The operation failed due with: %w",
+				originalApp.Name, tempAppName, originalApp.Name, tempAppName, updateErr)
 		}
-		return nil, fmt.Errorf("blue green deployment failed with: %w", err)
+		return nil, fmt.Errorf("blue-green deployment for application %s failed with: %w", originalApp.Name, pushError)
 	}
+
 	if newApp.State == "STARTED" {
-		err = p.gracefulDeletion(ctx, originalApp)
-		return newApp, err
+
+		healthCheckErr := p.waitForAppHealthy(ctx, newApp)
+		// If health check fails, delete new app and change back original app name
+		if healthCheckErr != nil {
+			originalName := newApp.Name
+			deleteErr := p.gracefulDeletion(ctx, newApp)
+			if deleteErr != nil {
+				return nil, fmt.Errorf("failed to delete new app after health check failure: %w", deleteErr)
+			}
+			_, revertUpdateErr := p.client.Applications.Update(ctx, originalApp.GUID, &resource.AppUpdate{
+				Name: originalName,
+			})
+
+			if revertUpdateErr != nil {
+				return nil, fmt.Errorf("failed to update app name back to original name: failed with: %w", revertUpdateErr)
+			}
+
+			return nil, fmt.Errorf("new application failed health check with: %w", healthCheckErr)
+		}
+		cleanUpOldErr := p.gracefulDeletion(ctx, originalApp)
+		return newApp, cleanUpOldErr
 	}
-	return newApp, fmt.Errorf("failed to verify application start: %w", err)
+	return newApp, fmt.Errorf("failed to verify application start: %w", pushError)
+}
+
+func getAppDeployedRunningPollingOptions(ctx context.Context) (uint, uint) {
+	rawTimeout := ctx.Value("app_deployed_running_timeout")
+	processedTimeout := uint(AppDeployedRunningTimeoutMinutesDefault)
+
+	rawCheckInterval := ctx.Value("app_deployed_running_check_interval")
+	processedCheckInterval := uint(AppDeployedRunningCheckIntervalSecondsDefault)
+
+	if rawTimeout != nil {
+		t, ok := rawTimeout.(uint)
+		if ok == true {
+			processedTimeout = t
+		}
+	} else if envTimeout := os.Getenv("CF_APP_DEPLOYED_RUNNING_TIMEOUT"); envTimeout != "" {
+		if t, err := strconv.ParseUint(envTimeout, EnvironmentVariablesConvertBaseDecimal, EnvironmentVariablesConvertBitsSize); err == nil {
+			processedTimeout = uint(t)
+		}
+	}
+
+	if rawCheckInterval != nil {
+		ci, ok := rawCheckInterval.(uint)
+		if ok {
+			processedCheckInterval = ci
+		}
+	} else if envInterval := os.Getenv("CF_APP_DEPLOYED_RUNNING_CHECK_INTERVAL"); envInterval != "" {
+		if ci, err := strconv.ParseUint(envInterval, EnvironmentVariablesConvertBaseDecimal, EnvironmentVariablesConvertBitsSize); err == nil {
+			processedCheckInterval = uint(ci)
+		}
+	}
+
+	return processedTimeout, processedCheckInterval
 }
 
 func (p *AppPushOperation) pushRollingApp(ctx context.Context, space *resource.Space, manifest *AppManifest, zipFile io.Reader) (*resource.App, error) {
@@ -494,3 +563,25 @@ func (p *AppPushOperation) findSpace(ctx context.Context, orgGUID string) (*reso
 	}
 	return space, nil
 }
+
+func (p *AppPushOperation) waitForAppHealthy(ctx context.Context, app *resource.App) error {
+	timeout, checkInterval := getAppDeployedRunningPollingOptions(ctx)
+	pollOptions := client.NewPollingOptions()
+	pollOptions.Timeout = time.Duration(timeout) * time.Minute
+	pollOptions.CheckInterval = time.Duration(checkInterval) * time.Second
+	appPollErr := client.PollForStateOrTimeout(func() (string, string, error) {
+		for {
+			procData, err := p.client.Processes.GetStatsForApp(ctx, app.GUID, "web")
+			if err != nil {
+				return "FAILED", "Failed to get processes stats for application", err
+			}
+			for _, proc := range procData.Stats {
+				if proc.State != "RUNNING" {
+					return proc.State, "One or more instances are not running", nil
+				}
+			}
+			return "RUNNING", "", nil
+		}
+	}, "RUNNING", pollOptions)
+	return appPollErr
+}