Prepare a new "generic" template, use "src layout" for Lakeflow template

NEXT_CHANGELOG.md

@@ -9,6 +9,8 @@
 ### Dependency updates
 
 ### Bundles
+* Updated the internal lakeflow-pipelines template to use an "src" layout ([#3671](https://github.com/databricks/cli/pull/3671)).
+* Added support for a "template_dir" option in the databricks_template_schema.json format. ([#3671](https://github.com/databricks/cli/pull/3671)).
 
 ### API Changes
 * Added `databricks account account-groups-v2` command group.

acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "my_lakeflow_pipelines"
+version = "0.0.1"
+authors = [{ name = "[USERNAME]" }]
+requires-python = ">=3.10,<=3.13"
+dependencies = [
+    # Any dependencies for jobs and pipelines in this project can be added here
+    # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
+]
+
+[dependency-groups]
+dev = [
+    "pytest",
+    "databricks-dlt",
+    "databricks-connect>=15.4,<15.5",
+]
+
+[project.scripts]
+main = "my_lakeflow_pipelines.main:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.black]
+line-length = 125

acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/lakeflow_pipelines_etl.pipeline.yml

@@ -0,0 +1,21 @@
+# The main pipeline for my_lakeflow_pipelines
+
+resources:
+  pipelines:
+    lakeflow_pipelines_etl:
+      name: lakeflow_pipelines_etl
+      ## Catalog is required for serverless compute
+      catalog: ${var.catalog}
+      schema: ${var.schema}
+      serverless: true
+      root_path: "."
+
+      libraries:
+        - glob:
+            include: ../src/lakeflow_pipelines_etl/transformations/**
+
+      environment:
+        dependencies:
+          # We include every dependency defined by pyproject.toml by defining an editable environment
+          # that points to the folder where pyproject.toml is deployed.
+          - --editable ${workspace.file_path}

acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/lakeflow_pipelines_etl/lakeflow_pipelines_job.job.yml → acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml

@@ -1,9 +1,9 @@
-# The job that triggers lakeflow_pipelines_etl.
+# A sample job for my_lakeflow_pipelines.
 
 resources:
   jobs:
-    lakeflow_pipelines_job:
-      name: lakeflow_pipelines_job
+    sample_job:
+      name: sample_job
 
       trigger:
         # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger
@@ -15,7 +15,18 @@ resources:
       #  on_failure:
       #    - your_email@example.com
 
+      parameters:
+        - name: catalog
+          default: ${var.catalog}
+        - name: schema
+          default: ${var.schema}
+
       tasks:
         - task_key: refresh_pipeline
           pipeline_task:
             pipeline_id: ${resources.pipelines.lakeflow_pipelines_etl.id}
+
+      environments:
+        - environment_key: default
+          spec:
+            client: "2"

acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines/main.py

@@ -0,0 +1,22 @@
+import argparse
+from databricks.sdk.runtime import spark
+from my_lakeflow_pipelines import taxis
+
+
+def main():
+    # Process command-line arguments
+    parser = argparse.ArgumentParser(description="Databricks job with catalog and schema parameters")
+    parser.add_argument("--catalog", required=True)
+    parser.add_argument("--schema", required=True)
+    args = parser.parse_args()
+
+    # Set the default catalog and schema
+    spark.sql(f"USE CATALOG {args.catalog}")
+    spark.sql(f"USE SCHEMA {args.schema}")
+
+    # Example: just find all taxis from a sample catalog
+    taxis.find_all_taxis().show(5)
+
+
+if __name__ == "__main__":
+    main()

acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "my_lakeflow_pipelines"
+version = "0.0.1"
+authors = [{ name = "[USERNAME]" }]
+requires-python = ">=3.10,<=3.13"
+dependencies = [
+    # Any dependencies for jobs and pipelines in this project can be added here
+    # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
+]
+
+[dependency-groups]
+dev = [
+    "pytest",
+    "databricks-dlt",
+    "databricks-connect>=15.4,<15.5",
+]
+
+[project.scripts]
+main = "my_lakeflow_pipelines.main:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.black]
+line-length = 125

acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/lakeflow_pipelines_etl.pipeline.yml

@@ -0,0 +1,21 @@
+# The main pipeline for my_lakeflow_pipelines
+
+resources:
+  pipelines:
+    lakeflow_pipelines_etl:
+      name: lakeflow_pipelines_etl
+      ## Catalog is required for serverless compute
+      catalog: ${var.catalog}
+      schema: ${var.schema}
+      serverless: true
+      root_path: "."
+
+      libraries:
+        - glob:
+            include: ../src/lakeflow_pipelines_etl/transformations/**
+
+      environment:
+        dependencies:
+          # We include every dependency defined by pyproject.toml by defining an editable environment
+          # that points to the folder where pyproject.toml is deployed.
+          - --editable ${workspace.file_path}

acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/lakeflow_pipelines_etl/lakeflow_pipelines_job.job.yml → acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml

@@ -1,9 +1,9 @@
-# The job that triggers lakeflow_pipelines_etl.
+# A sample job for my_lakeflow_pipelines.
 
 resources:
   jobs:
-    lakeflow_pipelines_job:
-      name: lakeflow_pipelines_job
+    sample_job:
+      name: sample_job
 
       trigger:
         # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger
@@ -15,7 +15,18 @@ resources:
       #  on_failure:
       #    - your_email@example.com
 
+      parameters:
+        - name: catalog
+          default: ${var.catalog}
+        - name: schema
+          default: ${var.schema}
+
       tasks:
         - task_key: refresh_pipeline
           pipeline_task:
             pipeline_id: ${resources.pipelines.lakeflow_pipelines_etl.id}
+
+      environments:
+        - environment_key: default
+          spec:
+            client: "2"

acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines/main.py

@@ -0,0 +1,22 @@
+import argparse
+from databricks.sdk.runtime import spark
+from my_lakeflow_pipelines import taxis
+
+
+def main():
+    # Process command-line arguments
+    parser = argparse.ArgumentParser(description="Databricks job with catalog and schema parameters")
+    parser.add_argument("--catalog", required=True)
+    parser.add_argument("--schema", required=True)
+    args = parser.parse_args()
+
+    # Set the default catalog and schema
+    spark.sql(f"USE CATALOG {args.catalog}")
+    spark.sql(f"USE SCHEMA {args.schema}")
+
+    # Example: just find all taxis from a sample catalog
+    taxis.find_all_taxis().show(5)
+
+
+if __name__ == "__main__":
+    main()

acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/src/my_lakeflow_pipelines/taxis.py

@@ -0,0 +1,7 @@
+from databricks.sdk.runtime import spark
+from pyspark.sql import DataFrame
+
+
+def find_all_taxis() -> DataFrame:
+    """Find all taxi data."""
+    return spark.read.table("samples.nyctaxi.trips")

libs/jsonschema/extension.go

@@ -35,6 +35,12 @@ type Extension struct {
 	// compatible with the current CLI version.
 	Version *int `json:"version,omitempty"`
 
+	// TemplateDir specifies the directory containing the template files to use.
+	// If not specified, the template files are expected to be in the same directory
+	// as the schema file. This allows schema files to reference template files
+	// from a different directory (e.g., "../default").
+	TemplateDir string `json:"template_dir,omitempty"`
+
 	// Preview indicates launch stage (e.g. PREVIEW).
 	//
 	// This field indicates whether the associated field is part of a private preview feature.

libs/template/builtin_test.go

@@ -27,7 +27,7 @@ func TestBuiltin(t *testing.T) {
 	// Verify the filesystems work for each template
 	_, err = fs.Stat(templates["dbt-sql"].FS, `template/{{.project_name}}/dbt_project.yml.tmpl`)
 	assert.NoError(t, err)
-	_, err = fs.Stat(templates["default-python"].FS, `template/{{.project_name}}/tests/main_test.py.tmpl`)
+	_, err = fs.Stat(templates["default"].FS, `template/{{.project_name}}/tests/sample_taxis_test.py.tmpl`)
 	assert.NoError(t, err)
 	_, err = fs.Stat(templates["default-sql"].FS, `template/{{.project_name}}/src/orders_daily.sql.tmpl`)
 	assert.NoError(t, err)

libs/template/config.go

@@ -34,6 +34,10 @@ func newConfig(ctx context.Context, templateFS fs.FS, schemaPath string) (*confi
 	if err != nil {
 		return nil, err
 	}
+	return newConfigFromSchema(ctx, schema)
+}
+
+func newConfigFromSchema(ctx context.Context, schema *jsonschema.Schema) (*config, error) {
 	if err := validateSchema(schema); err != nil {
 		return nil, err
 	}