acc: add EnvVaryOutput (#3438)

## Changes
Add new flag to acceptance test: EnvVaryOutput. This allows recording
environment specific output in a single test.

## Why
In some cases the output of a command is too different between terraform
and direct (e.g. everything where an error happens). It is still
desirable to have it a single test where common things (configs,
recorded API requests) can be compared. Current workaround is to have
parent directory with config and common script and 2 tests that copy the
config and the script. That involves too much overhead when writing a
test. This feature allows having single test but record different
outputs as follows:

`$CLI bundle deploy &> out.$DATABRICKS_CLI_DEPLOYMENT.txt`

The test runner will compare only the variant applicable to current
subtest.

## Tests
New selftest. Existing tests that could benefit are converted to use
this.

Author: denik

acceptance/acceptance_test.go

@@ -299,7 +299,11 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 			if testdiff.OverwriteMode && len(expanded) > 1 {
 				// All variants of the test are producing the same output,
 				// there is no need to run the concurrently when updating.
-				expanded = expanded[0:1]
+				// Exception: if EnvVaryOutput is configured with multiple values, we must
+				// run all variants to record variant-specific outputs.
+				if config.EnvVaryOutput == nil || len(config.EnvMatrix[*config.EnvVaryOutput]) <= 1 {
+					expanded = expanded[0:1]
+				}
 			}
 
 			if len(expanded) == 1 {
@@ -635,8 +639,14 @@ func runTest(t *testing.T,
 
 	printedRepls := false
 
+	pathFilter := preparePathFilter(config, customEnv)
+
 	// Compare expected outputs
 	for relPath := range outputs {
+		if shouldSkip(pathFilter, relPath) {
+			continue
+		}
+
 		skipRepls := false
 		if relPath == internal.MaterializedConfigFile {
 			skipRepls = true
@@ -1336,3 +1346,53 @@ func loadUserReplacements(t *testing.T, repls *testdiff.ReplacementsContext, tmp
 		repls.SetWithOrder(old, "["+repl+"]", -100)
 	}
 }
+
+type pathFilter struct {
+	// contains substrings from the variants other than current.
+	// E.g. if EnvVaryOutput is DATABRICKS_CLI_DEPLOYMENT and current test running DATABRICKS_CLI_DEPLOYMENT="terraform" then
+	// notSelected contains ".direct-exp." meaning if filename contains that (e.g. out.deploy.direct-exp.txt) then we ignore it here.
+	notSelected []string
+}
+
+// preparePathFilter builds filter based on EnvVaryOutput and current variant env.
+func preparePathFilter(config internal.TestConfig, customEnv []string) pathFilter {
+	if config.EnvVaryOutput == nil {
+		return pathFilter{}
+	}
+	key := *config.EnvVaryOutput
+	vals := config.EnvMatrix[key]
+	if len(vals) <= 1 {
+		return pathFilter{}
+	}
+	selected := ""
+	for _, kv := range customEnv {
+		items := strings.SplitN(kv, "=", 2)
+		if len(items) == 2 && items[0] == key {
+			selected = items[1]
+			break
+		}
+	}
+	if selected == "" {
+		return pathFilter{}
+	}
+	var others []string
+	for _, v := range vals {
+		if v == selected {
+			continue
+		}
+		others = append(others, "."+v+".")
+	}
+	return pathFilter{
+		notSelected: others,
+	}
+}
+
+// shouldSkip returns true if the given file belongs to a non-selected variant.
+func shouldSkip(filter pathFilter, relPath string) bool {
+	for _, infix := range filter.notSelected {
+		if strings.Contains(relPath, infix) {
+			return true
+		}
+	}
+	return false
+}

…tance/bundle/debug/direct/databricks.yml acceptance/bundle/debug/databricks.ymlacceptance/bundle/debug/direct/databricks.yml renamed to acceptance/bundle/debug/databricks.yml acceptance/bundle/debug/direct/databricks.yml renamed to acceptance/bundle/debug/databricks.yml

@@ -1,5 +1,3 @@
-
->>> [CLI] bundle validate --debug
 10:07:59 Info: start pid=12345 version=[DEV_VERSION] args="[CLI], bundle, validate, --debug"
 10:07:59 Debug: Found bundle root at [TEST_TMP_DIR] (file [TEST_TMP_DIR]/databricks.yml) pid=12345
 10:07:59 Info: Phase: load pid=12345

acceptance/bundle/debug/direct/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]