pipelines and bundle run: specify refresh and full-refresh together (#3300)

## Changes
For pipelines run and bundle run command flags:
- Allows to specify list of tables for refresh and full-refresh at the
same time.
- however, other flags are still mutually exclusive, and can not be
specified with refresh or full-refresh.

## Why
Matching OSS spark implementation and allows for more flexible running
of tables

## Tests
Unit tests included.
Follow up with acceptance tests:
#3301

---------

Co-authored-by: Jeffery Cheng <[email protected]>

Author: alyssa-db

NEXT_CHANGELOG.md

@@ -10,5 +10,6 @@
 * Fixed auth login ignoring DATABRICKS_CONFIG_FILE environmental variable when saving profile ([#3266](https://github.com/databricks/cli/pull/3266))
 
 ### Bundles
+* Modified run flag validation to allow `--refresh` and `--full-refresh` flags to be used together ([#3300](https://github.com/databricks/cli/pull/3300))
 
 ### API Changes

bundle/run/pipeline_options.go

@@ -37,25 +37,24 @@ func (o *PipelineOptions) Define(fs *flag.FlagSet) {
 
 // Validate returns if the combination of options is valid.
 func (o *PipelineOptions) Validate(pipeline *resources.Pipeline) error {
-	var set []string
+	var mutuallyExclusiveFlags []string
+
 	if o.RefreshAll {
-		set = append(set, "--refresh-all")
-	}
-	if len(o.Refresh) > 0 {
-		set = append(set, "--refresh")
+		mutuallyExclusiveFlags = append(mutuallyExclusiveFlags, "--refresh-all")
 	}
 	if o.FullRefreshAll {
-		set = append(set, "--full-refresh-all")
-	}
-	if len(o.FullRefresh) > 0 {
-		set = append(set, "--full-refresh")
+		mutuallyExclusiveFlags = append(mutuallyExclusiveFlags, "--full-refresh-all")
 	}
 	if o.ValidateOnly {
-		set = append(set, "--validate-only")
+		mutuallyExclusiveFlags = append(mutuallyExclusiveFlags, "--validate-only")
 	}
-	if len(set) > 1 {
-		return fmt.Errorf("pipeline run arguments are mutually exclusive (got %s)", strings.Join(set, ", "))
+
+	if len(mutuallyExclusiveFlags) > 1 {
+		return fmt.Errorf("%s pipeline run flags are mutually exclusive", strings.Join(mutuallyExclusiveFlags, ", "))
+	} else if len(mutuallyExclusiveFlags) == 1 && (len(o.Refresh) > 0 || len(o.FullRefresh) > 0) {
+		return fmt.Errorf("cannot use --refresh or --full-refresh together with %s, these flags are mutually exclusive", strings.Join(mutuallyExclusiveFlags, ""))
 	}
+
 	return nil
 }
 

bundle/run/pipeline_options_test.go

@@ -67,12 +67,20 @@ func TestPipelineOptionsValidateSuccessWithSingleOption(t *testing.T) {
 	}
 }
 
+func TestPipelineOptionsValidateSuccessWithRefreshAndFullRefresh(t *testing.T) {
+	fs, opts := setupPipelineOptions(t)
+	err := fs.Parse([]string{`--refresh=arg1,arg2`, `--full-refresh=arg3,arg4`})
+	require.NoError(t, err)
+	err = opts.Validate(nil)
+	assert.NoError(t, err)
+	assert.Equal(t, []string{"arg1", "arg2"}, opts.Refresh)
+	assert.Equal(t, []string{"arg3", "arg4"}, opts.FullRefresh)
+}
+
 func TestPipelineOptionsValidateFailureWithMultipleOptions(t *testing.T) {
 	args := []string{
 		`--refresh-all`,
-		`--refresh=arg1,arg2,arg3`,
 		`--full-refresh-all`,
-		`--full-refresh=arg1,arg2,arg3`,
 		`--validate-only`,
 	}
 	for i := range args {
@@ -84,7 +92,29 @@ func TestPipelineOptionsValidateFailureWithMultipleOptions(t *testing.T) {
 			err := fs.Parse([]string{args[i], args[j]})
 			require.NoError(t, err)
 			err = opts.Validate(nil)
-			assert.ErrorContains(t, err, "pipeline run arguments are mutually exclusive")
+			assert.ErrorContains(t, err, "pipeline run flags are mutually exclusive")
+		}
+	}
+}
+
+func TestPipelineOptionsValidateFailureWithRefreshAndMutuallyExclusiveFlags(t *testing.T) {
+	args := []string{
+		`--refresh=arg1`,
+		`--full-refresh=arg1`,
+	}
+	mutuallyExclusiveArgs := []string{
+		`--refresh-all`,
+		`--full-refresh-all`,
+		`--validate-only`,
+	}
+
+	for _, refreshArg := range args {
+		for _, exclusiveArg := range mutuallyExclusiveArgs {
+			fs, opts := setupPipelineOptions(t)
+			err := fs.Parse([]string{refreshArg, exclusiveArg})
+			require.NoError(t, err)
+			err = opts.Validate(nil)
+			assert.ErrorContains(t, err, "cannot use --refresh or --full-refresh together with")
 		}
 	}
 }