Add pydabs template (#3806)

kanterov · web-flow · commit 43bea6270d0a · 2025-10-23T11:01:04.000Z
## Changes
Add the pydabs template as a specialized version of the `default-python`
template.

The template is hidden until we move "python" outside of "experimental".

## Why
We want to have a separate template for the full Python version of the
default-python template.

## Tests
Existing acceptance tests and unit tests for additional templating
features
diff --git a/acceptance/bundle/templates/pydabs/check-consistency/output.txt b/acceptance/bundle/templates/pydabs/check-consistency/output.txt
@@ -1,6 +1,6 @@
 == Collecting resources into ./enable_pydabs=yes
 
->>> [CLI] bundle init default-python --config-file ./input.json
+>>> [CLI] bundle init pydabs --config-file ./input.json
 
 Welcome to the default Python template for Databricks Asset Bundles!
 Workspace to use (auto-detected, edit in 'my_pydabs/databricks.yml'): [DATABRICKS_URL]
@@ -17,7 +17,7 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde
 >>> [CLI] bundle validate --output json
 == Collecting resources into ./enable_pydabs=no
 
->>> [CLI] bundle init default-python --config-file ./input.json
+>>> [CLI] bundle init pydabs --config-file ./input.json
 
 Welcome to the default Python template for Databricks Asset Bundles!
 Workspace to use (auto-detected, edit in 'my_pydabs/databricks.yml'): [DATABRICKS_URL]
diff --git a/acceptance/bundle/templates/pydabs/check-consistency/script b/acceptance/bundle/templates/pydabs/check-consistency/script
@@ -3,7 +3,7 @@ for enable_pydabs in yes no; do
 
     enable_pydabs=$enable_pydabs envsubst < input.json.tmpl > input.json
 
-    trace $CLI bundle init default-python --config-file ./input.json
+    trace $CLI bundle init pydabs --config-file ./input.json
 
     (
         cd my_pydabs
diff --git a/acceptance/bundle/templates/pydabs/check-formatting/input.json.tmpl b/acceptance/bundle/templates/pydabs/check-formatting/input.json.tmpl
@@ -3,6 +3,5 @@
   "include_notebook": "$INCLUDE_NOTEBOOK",
   "include_dlt": "$INCLUDE_DLT",
   "include_python": "$INCLUDE_PYTHON",
-  "enable_pydabs": "yes",
   "serverless": "$SERVERLESS"
 }
diff --git a/acceptance/bundle/templates/pydabs/check-formatting/output.txt b/acceptance/bundle/templates/pydabs/check-formatting/output.txt
@@ -1,5 +1,5 @@
 
->>> [CLI] bundle init default-python --config-file ./input.json
+>>> [CLI] bundle init pydabs --config-file ./input.json
 
 Welcome to the default Python template for Databricks Asset Bundles!
 Workspace to use (auto-detected, edit in 'my_pydabs/databricks.yml'): [DATABRICKS_URL]
diff --git a/acceptance/bundle/templates/pydabs/check-formatting/script b/acceptance/bundle/templates/pydabs/check-formatting/script
@@ -1,6 +1,6 @@
 envsubst < input.json.tmpl > input.json
 
-trace $CLI bundle init default-python --config-file ./input.json
+trace $CLI bundle init pydabs --config-file ./input.json
 
 # only keep relevant files for snapshots
 trace find my_pydabs -mindepth 1 ! -name 'pyproject.toml' ! -regex '.*/resources.*' -delete
diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/input.json b/acceptance/bundle/templates/pydabs/deploy-classic/input.json
@@ -3,6 +3,5 @@
   "include_notebook": "yes",
   "include_dlt": "yes",
   "include_python": "yes",
-  "serverless": "no",
-  "enable_pydabs": "yes"
+  "serverless": "no"
 }
diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/output.txt b/acceptance/bundle/templates/pydabs/deploy-classic/output.txt
@@ -1,5 +1,5 @@
 
->>> [CLI] bundle init default-python --config-file ./input.json
+>>> [CLI] bundle init pydabs --config-file ./input.json
 
 Welcome to the default Python template for Databricks Asset Bundles!
 Workspace to use (auto-detected, edit in 'my_pydabs/databricks.yml'): [DATABRICKS_URL]
diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/script b/acceptance/bundle/templates/pydabs/deploy-classic/script
@@ -1,4 +1,4 @@
-trace $CLI bundle init default-python --config-file ./input.json
+trace $CLI bundle init pydabs --config-file ./input.json
 
 (
     cd my_pydabs
diff --git a/acceptance/bundle/templates/pydabs/init-classic/input.json b/acceptance/bundle/templates/pydabs/init-classic/input.json
@@ -3,6 +3,5 @@
   "include_notebook": "yes",
   "include_dlt": "yes",
   "include_python": "yes",
-  "serverless": "no",
-  "enable_pydabs": "yes"
+  "serverless": "no"
 }
diff --git a/acceptance/bundle/templates/pydabs/init-classic/output.txt b/acceptance/bundle/templates/pydabs/init-classic/output.txt
@@ -1,5 +1,5 @@
 
->>> [CLI] bundle init default-python --config-file ./input.json --output-dir output
+>>> [CLI] bundle init pydabs --config-file ./input.json --output-dir output
 
 Welcome to the default Python template for Databricks Asset Bundles!
 Workspace to use (auto-detected, edit in 'my_pydabs/databricks.yml'): [DATABRICKS_URL]
diff --git a/acceptance/bundle/templates/pydabs/init-classic/script b/acceptance/bundle/templates/pydabs/init-classic/script
@@ -1,4 +1,4 @@
-trace $CLI bundle init default-python --config-file ./input.json --output-dir output
+trace $CLI bundle init pydabs --config-file ./input.json --output-dir output
 
 # only keep relevant files for snapshots
 trace find output/my_pydabs -mindepth 1 ! -name 'pyproject.toml' ! -regex '.*/resources.*' -delete
diff --git a/libs/template/reader.go b/libs/template/reader.go
@@ -72,6 +72,33 @@ func (r *builtinReader) LoadSchemaAndTemplateFS(ctx context.Context) (*jsonschem
 
 func (r *builtinReader) Cleanup(ctx context.Context) {}
 
+// overridingReader wraps another Reader and overrides their properties
+type overridingReader struct {
+	underlying               Reader
+	propertyDefaultOverrides map[string]any
+}
+
+func (r *overridingReader) LoadSchemaAndTemplateFS(ctx context.Context) (*jsonschema.Schema, fs.FS, error) {
+	schema, schemaFS, err := r.underlying.LoadSchemaAndTemplateFS(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for propName, newDefault := range r.propertyDefaultOverrides {
+		if propSchema, ok := schema.Properties[propName]; ok {
+			propSchema.Default = newDefault
+		} else {
+			return nil, nil, fmt.Errorf("cannot override default for non-existing property %s", propName)
+		}
+	}
+
+	return schema, schemaFS, nil
+}
+
+func (r *overridingReader) Cleanup(ctx context.Context) {
+	r.underlying.Cleanup(ctx)
+}
+
 // gitReader reads a template from a git repository.
 type gitReader struct {
 	gitUrl string
diff --git a/libs/template/reader_test.go b/libs/template/reader_test.go
@@ -6,6 +6,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/databricks/cli/libs/jsonschema"
+
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/stretchr/testify/assert"
@@ -215,3 +217,39 @@ func TestGitReaderWithTemplateDir(t *testing.T) {
 	// Cleanup
 	r.Cleanup(ctx)
 }
+
+func TestOverridingReader_LoadSchemaAndTemplateFS(t *testing.T) {
+	tmpDir := t.TempDir()
+	content := `{
+		"welcome_message": "test",
+		"properties": {
+			"enable_feature_x": {
+				"type": "string",
+				"default": "no"
+			}
+		}
+	}`
+
+	testutil.WriteFile(t, filepath.Join(tmpDir, "databricks_template_schema.json"), content)
+	ctx := context.Background()
+
+	r := overridingReader{
+		underlying: &localReader{path: tmpDir},
+		propertyDefaultOverrides: map[string]any{
+			"enable_feature_x": "yes",
+		},
+	}
+
+	schema, fsys, err := r.LoadSchemaAndTemplateFS(ctx)
+
+	require.NoError(t, err)
+	require.NotNil(t, schema)
+	require.NotNil(t, fsys)
+
+	assert.Equal(t, map[string]*jsonschema.Schema{
+		"enable_feature_x": {
+			Type:    "string",
+			Default: "yes",
+		},
+	}, schema.Properties)
+}
diff --git a/libs/template/template.go b/libs/template/template.go
@@ -31,7 +31,7 @@ const (
 	CLIPipelines              TemplateName = "cli-pipelines"
 	DbtSql                    TemplateName = "dbt-sql"
 	MlopsStacks               TemplateName = "mlops-stacks"
-	DefaultPydabs             TemplateName = "default-pydabs"
+	Pydabs                    TemplateName = "pydabs"
 	Custom                    TemplateName = "custom"
 	ExperimentalJobsAsCode    TemplateName = "experimental-jobs-as-code"
 	Default                   TemplateName = "default"
@@ -85,11 +85,19 @@ var databricksTemplates = []Template{
 		Writer:      &writerWithFullTelemetry{defaultWriter: defaultWriter{name: MlopsStacks}},
 	},
 	{
-		name:        DefaultPydabs,
+		name:        Pydabs,
 		hidden:      true,
-		description: "The default PyDABs template",
-		Reader:      &gitReader{gitUrl: "https://databricks.github.io/workflows-authoring-toolkit/pydabs-template.git", cloneFunc: git.Clone},
-		Writer:      &writerWithFullTelemetry{defaultWriter: defaultWriter{name: DefaultPydabs}},
+		description: "A variant of 'default-python' template that defines resources in Python instead of YAML",
+		Reader: &overridingReader{
+			underlying: &builtinReader{name: string(DefaultPython)},
+			propertyDefaultOverrides: map[string]any{
+				// In 'default-python' template 'enable_pydabs' prompt is hidden with default of 'no'.
+				// We want a separate template called 'pydabs' to reuse 'default-python' but have
+				// the 'enable_pydabs' property enabled by default that switches from YAML to Python.
+				"enable_pydabs": "yes",
+			},
+		},
+		Writer: &writerWithFullTelemetry{defaultWriter: defaultWriter{name: Pydabs}},
 	},
 	{
 		name:        ExperimentalJobsAsCode,
diff --git a/libs/template/template_test.go b/libs/template/template_test.go
@@ -56,7 +56,7 @@ func TestTemplateGetDatabricksTemplate(t *testing.T) {
 		DefaultSql,
 		DbtSql,
 		MlopsStacks,
-		DefaultPydabs,
+		Pydabs,
 	}
 
 	for _, name := range names {

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@ for enable_pydabs in yes no; do`
`3`	`3`
`4`	`4`	`enable_pydabs=$enable_pydabs envsubst < input.json.tmpl > input.json`
`5`	`5`
`6`		`- trace $CLI bundle init default-python --config-file ./input.json`
	`6`	`+ trace $CLI bundle init pydabs --config-file ./input.json`
`7`	`7`
`8`	`8`	`(`
`9`	`9`	`cd my_pydabs`
Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,5 @@`
`3`	`3`	`"include_notebook": "$INCLUDE_NOTEBOOK",`
`4`	`4`	`"include_dlt": "$INCLUDE_DLT",`
`5`	`5`	`"include_python": "$INCLUDE_PYTHON",`
`6`		`- "enable_pydabs": "yes",`
`7`	`6`	`"serverless": "$SERVERLESS"`
`8`	`7`	`}`
Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,5 @@`
`3`	`3`	`"include_notebook": "yes",`
`4`	`4`	`"include_dlt": "yes",`
`5`	`5`	`"include_python": "yes",`
`6`		`- "serverless": "no",`
`7`		`- "enable_pydabs": "yes"`
	`6`	`+ "serverless": "no"`
`8`	`7`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-trace $CLI bundle init default-python --config-file ./input.json`
	`1`	`+trace $CLI bundle init pydabs --config-file ./input.json`
`2`	`2`
`3`	`3`	`(`
`4`	`4`	`cd my_pydabs`