pipelines deploy: oss spark pipelines YAML warning message (#3312)

## Changes
When deploying any project with an OSS formatted YAML file, ie one that
begins with the definitions: field intead of resources:, display an
error message explaining that Pipelines CLI does not support OSS files.
forces deploy to fail.

## Tests
Unit tests checking error message.
Acceptance tests in follow-up:
#3313

Author: alyssa-db

cmd/pipelines/deploy.go

@@ -42,10 +42,20 @@ func deployCommand() *cobra.Command {
 		ctx := logdiag.InitContext(cmd.Context())
 		cmd.SetContext(ctx)
 
+		// Enable collection of diagnostics to check for OSS template warning in ConfigureBundleWithVariables
+		logdiag.SetCollect(ctx, true)
+
 		b := utils.ConfigureBundleWithVariables(cmd)
 		if b == nil || logdiag.HasError(ctx) {
 			return root.ErrAlreadyPrinted
 		}
+		logdiag.SetCollect(ctx, false)
+
+		diags := logdiag.FlushCollected(ctx)
+		// Prevent deploying open-source Spark Declarative Pipelines YAML files with the Pipelines CLI.
+		if err := checkForOSSTemplateWarning(ctx, diags); err != nil {
+			return err
+		}
 
 		bundle.ApplyFuncContext(ctx, b, func(context.Context, *bundle.Bundle) {
 			b.Config.Bundle.Deployment.Lock.Force = forceLock

cmd/pipelines/deploy_test.go

@@ -0,0 +1,23 @@
+package pipelines
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFormatOSSTemplateWarningMessage(t *testing.T) {
+	d := diag.Diagnostic{
+		Severity: diag.Warning,
+		Summary:  "unknown field: definitions",
+		Paths:    []dyn.Path{dyn.EmptyPath},
+		Locations: []dyn.Location{
+			{File: "test-pipeline.yml"},
+		},
+	}
+
+	message := formatOSSTemplateWarningMessage(d)
+	assert.Contains(t, message, "test-pipeline.yml seems to be formatted for open-source Spark Declarative Pipelines.")
+}

cmd/pipelines/utils.go

@@ -4,12 +4,16 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/databricks/cli/bundle"
 	configresources "github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/bundle/resources"
 	"github.com/databricks/cli/bundle/run"
 	"github.com/databricks/cli/libs/cmdio"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/logdiag"
 )
 
 // Copied from cmd/bundle/run.go
@@ -88,3 +92,37 @@ func keyToRunner(b *bundle.Bundle, arg string) (run.Runner, error) {
 
 	return runner, nil
 }
+
+// formatOSSTemplateWarningMessage formats the warning message for OSS template pipeline YAML files.
+func formatOSSTemplateWarningMessage(d diag.Diagnostic) string {
+	fileName := "A pipeline YAML file"
+	if len(d.Locations) > 0 && d.Locations[0].File != "" {
+		fileName = d.Locations[0].File
+	}
+
+	return fileName + ` seems to be formatted for open-source Spark Declarative Pipelines.
+Pipelines CLI currently only supports Lakeflow Declarative Pipelines development.
+To see an example of a supported pipelines template, create a new Pipelines CLI project with "pipelines init".`
+}
+
+// checkForOSSTemplateWarning checks for a warning in the logs that suggests a pipelines YAML file
+// is formatted for open-source Spark Declarative Pipelines and returns an error if found.
+// For the Spark Declarative Pipelines template, see: https://github.com/apache/spark/blob/master/python/pyspark/pipelines/init_cli.py
+// Logs all collected diagnostics to expose them.
+func checkForOSSTemplateWarning(ctx context.Context, diags diag.Diagnostics) error {
+	var ossWarning *diag.Diagnostic
+
+	for _, d := range diags {
+		// The "definitions" field in the root is expected in OSS Spark Declarative Pipelines templates but is not supported in the Pipelines CLI.
+		if d.Severity == diag.Warning && strings.Contains(d.Summary, "unknown field: definitions") && len(d.Paths) == 1 && d.Paths[0].Equal(dyn.EmptyPath) {
+			ossWarning = &d
+		}
+		logdiag.LogDiag(ctx, d)
+	}
+
+	if ossWarning != nil {
+		return errors.New(formatOSSTemplateWarningMessage(*ossWarning))
+	}
+
+	return nil
+}