Fix oss-source pipeline errors when using the Databricks CLI (#3889)

## Changes
Removes validation that incorrectly rejected bundles using the
`definitions` field in `databricks.yml`.

## Why
PR #3797 inadvertently applied validation to reject OSS Spark
Declarative Pipelines to all Databricks CLI bundles. This error was only
meant for the pipelines CLI. Customers commonly use the `definitions`
field for YAML anchors (e.g., to reuse cluster configurations), and this
validation blocked their deployments with the error:

```
Error: databricks.yml seems to be formatted for open-source Spark Declarative Pipelines.
Pipelines CLI currently only supports Lakeflow Declarative Pipelines development.
```

The validation has been removed entirely. The Pipelines CLI is not yet
in active use, and the pipelines team can add a better context-aware
check later that only applies when running as the Pipelines CLI.

## Tests
- New acceptance test:
`acceptance/bundle/validate/definitions_yaml_anchors/` verifies bundles
with `definitions` field validate successfully
- Existing test: `acceptance/pipelines/deploy/oss-spark-error/` ensures
the error continues to work for the pipelines CLI.

Author: shreyas-goenka

acceptance/bundle/validate/definitions_yaml_anchors/databricks.yml

@@ -0,0 +1,20 @@
+# This test verifies that bundles can use the 'definitions' field for YAML anchors
+# without triggering the OSS Spark Pipelines validation error. This was a regression
+# patched in https://github.com/databricks/cli/pull/3889
+
+definitions:
+  common_cluster_settings: &common_cluster_settings
+    spark_version: "13.3.x-scala2.12"
+    node_type_id: "i3.xlarge"
+    num_workers: 2
+
+resources:
+  jobs:
+    my_job:
+      name: "test job with anchors"
+      tasks:
+        - task_key: "main"
+          new_cluster:
+            <<: *common_cluster_settings
+          notebook_task:
+            notebook_path: "/some/notebook"

acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml

@@ -0,0 +1,9 @@
+
+>>> [CLI] bundle validate
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Validation OK!

acceptance/bundle/validate/definitions_yaml_anchors/output.txt

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/validate/definitions_yaml_anchors/script

@@ -2,8 +2,6 @@ package phases
 
 import (
 	"context"
-	"errors"
-	"path/filepath"
 
 	"github.com/databricks/cli/bundle/config/mutator/resourcemutator"
 
@@ -18,9 +16,7 @@ import (
 	"github.com/databricks/cli/bundle/permissions"
 	"github.com/databricks/cli/bundle/scripts"
 	"github.com/databricks/cli/bundle/trampoline"
-	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/log"
-	"github.com/databricks/cli/libs/logdiag"
 )
 
 // The initialize phase fills in defaults and connects to the workspace.
@@ -29,11 +25,6 @@ import (
 func Initialize(ctx context.Context, b *bundle.Bundle) {
 	log.Info(ctx, "Phase: initialize")
 
-	rejectDefinitions(ctx, b)
-	if logdiag.HasError(ctx) {
-		return
-	}
-
 	bundle.ApplySeqContext(ctx, b,
 		// Reads (dynamic): resource.*.*
 		// Checks that none of resources.<type>.<key> is nil. Raises error otherwise.
@@ -206,17 +197,3 @@ func Initialize(ctx context.Context, b *bundle.Bundle) {
 		scripts.Execute(config.ScriptPostInit),
 	)
 }
-
-func rejectDefinitions(ctx context.Context, b *bundle.Bundle) {
-	if b.Config.Definitions != nil {
-		v := dyn.GetValue(b.Config.Value(), "definitions")
-		loc := v.Locations()
-		filename := "input yaml"
-		if len(loc) > 0 {
-			filename = filepath.ToSlash(loc[0].File)
-		}
-		logdiag.LogError(ctx, errors.New(filename+` seems to be formatted for open-source Spark Declarative Pipelines.
-Pipelines CLI currently only supports Lakeflow Declarative Pipelines development.
-To see an example of a supported pipelines template, create a new Pipelines CLI project with "pipelines init".`))
-	}
-}

bundle/phases/initialize.go

@@ -2,6 +2,8 @@ package utils
 
 import (
 	"context"
+	"errors"
+	"path/filepath"
 	"time"
 
 	"github.com/databricks/cli/bundle"
@@ -11,6 +13,7 @@ import (
 	"github.com/databricks/cli/bundle/statemgmt"
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/logdiag"
 	"github.com/databricks/cli/libs/sync"
 	"github.com/databricks/cli/libs/telemetry/protos"
@@ -56,6 +59,9 @@ type ProcessOptions struct {
 	Validate     bool
 	Build        bool
 	Deploy       bool
+
+	// Indicate whether the bundle operation originates from the pipelines CLI
+	IsPipelinesCLI bool
 }
 
 func ProcessBundle(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle, error) {
@@ -166,6 +172,14 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle,
 		if logdiag.HasError(ctx) {
 			return b, isDirectEngine, root.ErrAlreadyPrinted
 		}
+
+		// Pipeline CLI only validation.
+		if opts.IsPipelinesCLI {
+			rejectDefinitions(ctx, b)
+			if logdiag.HasError(ctx) {
+				return b, isDirectEngine, root.ErrAlreadyPrinted
+			}
+		}
 	}
 
 	if opts.Validate {
@@ -210,3 +224,17 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle,
 
 	return b, isDirectEngine, nil
 }
+
+func rejectDefinitions(ctx context.Context, b *bundle.Bundle) {
+	if b.Config.Definitions != nil {
+		v := dyn.GetValue(b.Config.Value(), "definitions")
+		loc := v.Locations()
+		filename := "input yaml"
+		if len(loc) > 0 {
+			filename = filepath.ToSlash(loc[0].File)
+		}
+		logdiag.LogError(ctx, errors.New(filename+` seems to be formatted for open-source Spark Declarative Pipelines.
+Pipelines CLI currently only supports Lakeflow Declarative Pipelines development.
+To see an example of a supported pipelines template, create a new Pipelines CLI project with "pipelines init".`))
+	}
+}

cmd/bundle/utils/process.go

@@ -44,11 +44,12 @@ func deployCommand() *cobra.Command {
 					b.Config.Bundle.Deployment.FailOnActiveRuns = failOnActiveRuns
 				}
 			},
-			Verbose:      verbose,
-			AlwaysPull:   true,
-			FastValidate: true,
-			Build:        true,
-			Deploy:       true,
+			Verbose:        verbose,
+			AlwaysPull:     true,
+			FastValidate:   true,
+			Build:          true,
+			Deploy:         true,
+			IsPipelinesCLI: true,
 		})
 		if err != nil {
 			return err