feat(upgrade): add verification release is deployable (#3136)

* feat(upgrade): add verification release is deployable

* chore: tests and cleaning up code

* Apply suggestions from code review

Co-authored-by: Ethan Mosbaugh <[email protected]>

* chore: address feedback and fix airgap required version setup

* Update pkg-new/validation/upgradable.go

Co-authored-by: Ethan Mosbaugh <[email protected]>

* chore: more comments and tests

* chore: feedback

* fix: client param naming

* fix: client tests

* fix: also validate kuberntes downgrades

---------

Co-authored-by: Ethan Mosbaugh <[email protected]>

Author: JGAntunes

cmd/installer/cli/upgrade.go

@@ -16,6 +16,8 @@ import (
 	"github.com/replicatedhq/embedded-cluster/cmd/installer/goods"
 	"github.com/replicatedhq/embedded-cluster/cmd/installer/kotscli"
 	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
+	"github.com/replicatedhq/embedded-cluster/pkg-new/replicatedapi"
+	"github.com/replicatedhq/embedded-cluster/pkg-new/validation"
 	"github.com/replicatedhq/embedded-cluster/pkg/airgap"
 	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
 	"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
@@ -59,6 +61,8 @@ type upgradeConfig struct {
 	managerPort          int
 	requiresInfraUpgrade bool
 	kotsadmNamespace     string
+	currentAppVersion    *kotscli.AppVersionInfo
+	replicatedAPIClient  replicatedapi.Client
 }
 
 // UpgradeCmd returns a cobra command for upgrading the embedded cluster application.
@@ -110,7 +114,7 @@ func UpgradeCmd(ctx context.Context, appSlug, appTitle string) *cobra.Command {
 			if err := preRunUpgrade(ctx, flags, &upgradeConfig, existingRC, kcli, appSlug); err != nil {
 				return err
 			}
-			if err := verifyAndPromptUpgrade(ctx, flags, upgradeConfig, prompts.New()); err != nil {
+			if err := verifyAndPromptUpgrade(ctx, flags, upgradeConfig, prompts.New(), kcli); err != nil {
 				return err
 			}
 
@@ -254,8 +258,8 @@ func preRunUpgrade(ctx context.Context, flags UpgradeCmdFlags, upgradeConfig *up
 	}
 	upgradeConfig.license = l
 
-	// sync the license if a license is provided and we are not in airgap mode
-	if upgradeConfig.license != nil && flags.airgapBundle == "" {
+	// sync the license and initialize the replicated api client if we are not in airgap mode
+	if flags.airgapBundle == "" {
 		replicatedAPI, err := newReplicatedAPIClient(upgradeConfig.license, upgradeConfig.clusterID)
 		if err != nil {
 			return fmt.Errorf("failed to create replicated API client: %w", err)
@@ -267,6 +271,7 @@ func preRunUpgrade(ctx context.Context, flags UpgradeCmdFlags, upgradeConfig *up
 		}
 		upgradeConfig.license = updatedLicense
 		upgradeConfig.licenseBytes = licenseBytes
+		upgradeConfig.replicatedAPIClient = replicatedAPI
 	}
 
 	// Continue using "kotsadm" namespace if it exists for backwards compatibility, otherwise use the appSlug
@@ -337,10 +342,17 @@ func preRunUpgrade(ctx context.Context, flags UpgradeCmdFlags, upgradeConfig *up
 	}
 	upgradeConfig.requiresInfraUpgrade = requiresInfraUpgrade
 
+	// Get current app version for deployability validation
+	currentAppVersion, err := kotscli.GetCurrentAppVersion(appSlug, upgradeConfig.kotsadmNamespace)
+	if err != nil {
+		return fmt.Errorf("failed to get current app version: %w", err)
+	}
+	upgradeConfig.currentAppVersion = currentAppVersion
+
 	return nil
 }
 
-func verifyAndPromptUpgrade(ctx context.Context, flags UpgradeCmdFlags, upgradeConfig upgradeConfig, prompt prompts.Prompt) error {
+func verifyAndPromptUpgrade(ctx context.Context, flags UpgradeCmdFlags, upgradeConfig upgradeConfig, prompt prompts.Prompt, kcli client.Client) error {
 	isAirgap := flags.airgapBundle != ""
 
 	err := verifyChannelRelease("upgrade", isAirgap, flags.assumeYes)
@@ -355,6 +367,16 @@ func verifyAndPromptUpgrade(ctx context.Context, flags UpgradeCmdFlags, upgradeC
 		}
 	}
 
+	// Validate release upgradable
+	if err := validateIsReleaseUpgradable(ctx, upgradeConfig, kcli, isAirgap); err != nil {
+		var ve *validation.ValidationError
+		if errors.As(err, &ve) {
+			// This is a validation error that prevents the upgrade from proceeding, expose the error directly
+			return ve
+		}
+		return fmt.Errorf("upgrade validation execution failed: %w", err)
+	}
+
 	if !isAirgap {
 		if err := maybePromptForAppUpdate(ctx, prompt, upgradeConfig.license, flags.assumeYes); err != nil {
 			if errors.As(err, &ErrorNothingElseToAdd{}) {
@@ -554,3 +576,67 @@ func checkRequiresInfraUpgrade(ctx context.Context) (bool, error) {
 
 	return !bytes.Equal(currentJSON, targetJSON), nil
 }
+
+// validateIsReleaseUpgradable validates that the target release can be safely deployed
+func validateIsReleaseUpgradable(ctx context.Context, upgradeConfig upgradeConfig, kcli client.Client, isAirgap bool) error {
+	// Get current installation for version information
+	currentInstallation, err := kubeutils.GetLatestInstallation(ctx, kcli)
+	if err != nil {
+		return fmt.Errorf("get current installation: %w", err)
+	}
+
+	// Get target release data
+	releaseData := release.GetReleaseData()
+	if releaseData == nil {
+		return fmt.Errorf("release data not found")
+	}
+
+	// Get channel release info
+	channelRelease := releaseData.ChannelRelease
+	if channelRelease == nil {
+		return fmt.Errorf("channel release not found in release data")
+	}
+
+	// Get current and target EC/K8s versions
+	var currentECVersion string
+	if currentInstallation.Spec.Config != nil {
+		currentECVersion = currentInstallation.Spec.Config.Version
+	}
+
+	targetECVersion := versions.Version
+
+	// Build validation options
+	opts := validation.UpgradableOptions{
+		CurrentECVersion: currentECVersion,
+		TargetECVersion:  targetECVersion,
+		License:          upgradeConfig.license,
+	}
+
+	// Add current app version info if available
+	if upgradeConfig.currentAppVersion != nil {
+		opts.CurrentAppVersion = upgradeConfig.currentAppVersion.VersionLabel
+		opts.CurrentAppSequence = upgradeConfig.currentAppVersion.ChannelSequence
+	}
+
+	// Add target app version info
+	opts.TargetAppVersion = channelRelease.VersionLabel
+	opts.TargetAppSequence = channelRelease.ChannelSequence
+
+	// Extract the required releases depending on if it's airgap or online
+	if isAirgap {
+		if err := opts.WithAirgapRequiredReleases(upgradeConfig.airgapMetadata); err != nil {
+			return fmt.Errorf("failed to extract required releases from airgap metadata: %w", err)
+		}
+	} else {
+		if err := opts.WithOnlineRequiredReleases(ctx, upgradeConfig.replicatedAPIClient); err != nil {
+			return fmt.Errorf("failed to extract required releases from replicated API's pending release call: %w", err)
+		}
+	}
+
+	// Perform validation
+	if err := validation.ValidateIsReleaseUpgradable(ctx, opts); err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/installer/kotscli/kotscli.go

@@ -3,6 +3,7 @@ package kotscli
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -437,6 +438,65 @@ func createLicenseFile(license []byte) (string, error) {
 	return licenseFile.Name(), nil
 }
 
+// AppVersionInfo holds information about a deployed app version
+type AppVersionInfo struct {
+	VersionLabel    string `json:"versionLabel"`
+	ChannelSequence int64  `json:"channelSequence"`
+	Sequence        int64  `json:"sequence"`
+	Status          string `json:"status"`
+}
+
+// GetCurrentAppVersion retrieves the currently deployed app version and sequence
+func GetCurrentAppVersion(appSlug string, namespace string) (*AppVersionInfo, error) {
+	kotsBinPath, err := goods.InternalBinary("kubectl-kots")
+	if err != nil {
+		return nil, fmt.Errorf("materialize kubectl-kots binary: %w", err)
+	}
+	defer os.Remove(kotsBinPath)
+
+	// Build command arguments: kots get versions <appSlug> -n <namespace> -o json
+	args := []string{
+		"get", "versions",
+		appSlug,
+		"-n", namespace,
+		"-o", "json",
+	}
+
+	// Execute the command and capture output
+	var outputBuffer bytes.Buffer
+	runCommandOpts := helpers.RunCommandOptions{
+		Stdout: &outputBuffer,
+	}
+
+	if err := helpers.RunCommandWithOptions(runCommandOpts, kotsBinPath, args...); err != nil {
+		return nil, fmt.Errorf("get versions from kots: %w", err)
+	}
+
+	// Parse JSON output
+	var versions []AppVersionInfo
+	if err := json.Unmarshal(outputBuffer.Bytes(), &versions); err != nil {
+		return nil, fmt.Errorf("unmarshal versions output: %w", err)
+	}
+
+	version, err := getLastDeployedAppVersion(versions)
+	if err != nil {
+		return nil, fmt.Errorf("no deployed version found for app %s", appSlug)
+	}
+	return version, nil
+}
+
+// getLastDeployedAppVersion finds the last deployed version from a slice of versions
+func getLastDeployedAppVersion(versions []AppVersionInfo) (*AppVersionInfo, error) {
+	// Find the last deployed version. This can be either successful or failed deploys.
+	for _, v := range versions {
+		if v.Status == "deployed" || v.Status == "failed" {
+			return &v, nil
+		}
+	}
+
+	return nil, fmt.Errorf("no deployed version found")
+}
+
 // GetConfigValuesOptions holds options for getting config values
 type GetConfigValuesOptions struct {
 	AppSlug               string