ci: Updates to support the migration on the CLI.

Author: eamonnmoloney

scripts/camunda-core/pkg/scenarios/scenarios.go

@@ -34,6 +34,10 @@ type DeploymentConfig struct {
 	QA        bool // Enable QA configuration (test users, etc.)
 	ImageTags bool // Enable image tag overrides from env vars
 	Upgrade   bool // Enable upgrade flow configuration
+
+	// Migration support (for helm chart schema changes)
+	ChartVersion string // Chart version (e.g., "13.0.0") - used to determine if migrator is needed
+	Flow         string // Deployment flow: install, upgrade-patch, upgrade-minor
 }
 
 // Validate checks that required fields are set and feature constraints are satisfied.
@@ -75,7 +79,7 @@ func (c *DeploymentConfig) Validate() error {
 }
 
 // ResolvePaths returns the ordered list of values files based on the configuration.
-// Files are returned in order: base -> base modifiers -> identity -> persistence -> platform -> features
+// Files are returned in order: base -> base modifiers -> identity -> persistence -> platform -> features -> migrator -> image-tags
 func (c *DeploymentConfig) ResolvePaths(scenariosDir string) ([]string, error) {
 	valuesDir := filepath.Join(scenariosDir, ValuesDir)
 
@@ -142,7 +146,18 @@ func (c *DeploymentConfig) ResolvePaths(scenariosDir string) ([]string, error) {
 		}
 	}
 
-	// Layer 7: Image tags (applied last, needs env var processing)
+	// Layer 7: Migrator (for chart version 13.x minor upgrades)
+	// The migrator handles schema migrations like global.security.initialization -> orchestration.security.initialization
+	// Include when: chart version starts with "13" AND flow is not "upgrade-patch" (i.e., it's a minor upgrade or install)
+	if c.needsMigrator() {
+		// Migrator file is at chart-full-setup/values-enable-migrator.yaml (one level up from values/)
+		migratorPath := filepath.Join(scenariosDir, "values-enable-migrator.yaml")
+		if _, err := os.Stat(migratorPath); err == nil {
+			files = append(files, migratorPath)
+		}
+	}
+
+	// Layer 8: Image tags (applied last, needs env var processing)
 	// Note: ImageTags file path is returned but the caller is responsible
 	// for processing environment variable substitution
 	if c.ImageTags {
@@ -155,6 +170,23 @@ func (c *DeploymentConfig) ResolvePaths(scenariosDir string) ([]string, error) {
 	return files, nil
 }
 
+// needsMigrator returns true if the migrator values file should be included.
+// The migrator is needed for chart version 13.x when the flow is not a patch upgrade.
+func (c *DeploymentConfig) needsMigrator() bool {
+	// Only needed for chart version 13.x
+	if !strings.HasPrefix(c.ChartVersion, "13") {
+		return false
+	}
+
+	// Skip for patch upgrades (they don't need schema migration)
+	if c.Flow == "upgrade-patch" {
+		return false
+	}
+
+	// Include for install, upgrade-minor, or any other flow on chart 13.x
+	return true
+}
+
 // MapScenarioToConfig converts a legacy scenario name to a DeploymentConfig.
 // This provides backward compatibility with the old --scenario flag.
 func MapScenarioToConfig(scenario string) *DeploymentConfig {

scripts/camunda-deployer/pkg/deployer/prepare.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"scripts/camunda-core/pkg/logging"
+	"scripts/camunda-core/pkg/scenarios"
 	"sort"
 	"strings"
 )
@@ -26,11 +27,17 @@ var CommonValuesFiles = []string{
 // If commonFiles is provided (non-nil), those files are used directly as the common base layer.
 // If commonFiles is nil, the function will attempt to discover common files from ../common/ relative to scenarioDir.
 func BuildValuesList(scenarioDir string, scenarios []string, auth string, includeEnterprise, includeDigest bool, userValues []string, commonFiles []string) ([]string, error) {
+	return BuildValuesListWithConfig(scenarioDir, scenarios, auth, includeEnterprise, includeDigest, userValues, commonFiles, nil)
+}
+
+// BuildValuesListWithConfig is like BuildValuesList but accepts a DeploymentConfig
+// for layered values resolution. This allows passing ChartVersion and Flow for migrator detection.
+func BuildValuesListWithConfig(scenarioDir string, scenarioNames []string, auth string, includeEnterprise, includeDigest bool, userValues []string, commonFiles []string, config *scenarios.DeploymentConfig) ([]string, error) {
 	var files []string
 
 	logging.Logger.Debug().
 		Str("scenarioDir", scenarioDir).
-		Strs("scenarios", scenarios).
+		Strs("scenarios", scenarioNames).
 		Str("auth", auth).
 		Bool("includeEnterprise", includeEnterprise).
 		Bool("includeDigest", includeDigest).
@@ -65,7 +72,7 @@ func BuildValuesList(scenarioDir string, scenarios []string, auth string, includ
 		logging.Logger.Debug().
 			Str("auth", auth).
 			Msg("📋 [BuildValuesList] resolving auth scenario")
-		authFiles, err := ResolveScenarioFiles(scenarioDir, []string{auth})
+		authFiles, err := ResolveScenarioFilesWithConfig(scenarioDir, []string{auth}, config)
 		if err != nil {
 			logging.Logger.Debug().
 				Err(err).
@@ -81,13 +88,13 @@ func BuildValuesList(scenarioDir string, scenarios []string, auth string, includ
 
 	// Add main scenario values
 	logging.Logger.Debug().
-		Strs("scenarios", scenarios).
+		Strs("scenarios", scenarioNames).
 		Msg("📋 [BuildValuesList] resolving main scenario(s)")
-	scenarioFiles, err := ResolveScenarioFiles(scenarioDir, scenarios)
+	scenarioFiles, err := ResolveScenarioFilesWithConfig(scenarioDir, scenarioNames, config)
 	if err != nil {
 		logging.Logger.Debug().
 			Err(err).
-			Strs("scenarios", scenarios).
+			Strs("scenarios", scenarioNames).
 			Msg("❌ [BuildValuesList] failed to resolve scenario files")
 		return nil, err
 	}
@@ -216,4 +223,4 @@ func overlayIfExists(scenarioDir, fileName string) string {
 		return p
 	}
 	return ""
-}
+}

scripts/camunda-deployer/pkg/deployer/values.go

@@ -17,13 +17,20 @@ type ScenarioMeta struct {
 // ResolveScenarioFiles resolves scenario names to their values file paths.
 // It supports both layered values (values/ directory) and legacy single-file approach.
 func ResolveScenarioFiles(scenarioDir string, scenarioNames []string) ([]string, error) {
+	return ResolveScenarioFilesWithConfig(scenarioDir, scenarioNames, nil)
+}
+
+// ResolveScenarioFilesWithConfig resolves scenario names to their values file paths,
+// using the provided deployment config for layered values resolution.
+// This allows passing ChartVersion and Flow for migrator detection.
+func ResolveScenarioFilesWithConfig(scenarioDir string, scenarioNames []string, config *scenarios.DeploymentConfig) ([]string, error) {
 	if len(scenarioNames) == 0 {
 		return nil, nil
 	}
 
 	// Check if this scenario directory uses layered values
 	if scenarios.HasLayeredValues(scenarioDir) {
-		return resolveLayeredScenarioFiles(scenarioDir, scenarioNames)
+		return resolveLayeredScenarioFiles(scenarioDir, scenarioNames, config)
 	}
 
 	// Fall back to legacy single-file approach
@@ -32,7 +39,7 @@ func ResolveScenarioFiles(scenarioDir string, scenarioNames []string) ([]string,
 
 // resolveLayeredScenarioFiles resolves scenarios using the layered values structure.
 // For layered values, we combine all layer files for each scenario.
-func resolveLayeredScenarioFiles(scenarioDir string, scenarioNames []string) ([]string, error) {
+func resolveLayeredScenarioFiles(scenarioDir string, scenarioNames []string, baseConfig *scenarios.DeploymentConfig) ([]string, error) {
 	var allFiles []string
 	var missing []string
 
@@ -42,7 +49,14 @@ func resolveLayeredScenarioFiles(scenarioDir string, scenarioNames []string) ([]
 			continue
 		}
 
-		files, err := scenarios.ResolveLayeredPaths(scenarioDir, s, nil)
+		// Start with the provided config or derive from scenario name
+		var config *scenarios.DeploymentConfig
+		if baseConfig != nil {
+			// Use the provided config (which includes ChartVersion and Flow)
+			config = baseConfig
+		}
+
+		files, err := scenarios.ResolveLayeredPaths(scenarioDir, s, config)
 		if err != nil {
 			missing = append(missing, s)
 			continue
@@ -91,14 +105,3 @@ func resolveLegacyScenarioFiles(scenarioDir string, scenarioNames []string) ([]s
 
 	return files, nil
 }
-
-// ResolveScenarioFilesWithConfig resolves scenario files using an explicit layered config.
-// This allows callers to specify exactly which layers to use, bypassing auto-detection.
-func ResolveScenarioFilesWithConfig(scenarioDir string, config *scenarios.LayeredConfig) ([]string, error) {
-	if config == nil {
-		return nil, fmt.Errorf("layered config is required")
-	}
-
-	// Use an empty scenario name since config is explicit
-	return scenarios.ResolveLayeredPaths(scenarioDir, "", config)
-}

scripts/deploy-camunda/deploy/deploy.go

@@ -433,6 +433,10 @@ func buildDeploymentConfigFromFlags(flags *config.RuntimeFlags, scenarioName str
 		config.Upgrade = true
 	}
 
+	// Set chart version and flow for migrator detection
+	config.ChartVersion = flags.ChartVersion
+	config.Flow = flags.Flow
+
 	return config
 }
 
@@ -980,6 +984,9 @@ func prepareScenarioValues(scenarioCtx *ScenarioContext, flags *config.RuntimeFl
 		}
 	}
 
+	// Build deployment config for layered values resolution (includes ChartVersion and Flow for migrator)
+	deployConfig := buildDeploymentConfigFromFlags(flags, scenarioCtx.ScenarioName)
+
 	// Build values files list
 	logging.Logger.Debug().
 		Str("scenario", scenarioCtx.ScenarioName).