Update templates to use serverless environment version 4 and matching Python version (#3897)

lennartkats-db · claude · web-flow · commit e54dfe5500c2 · 2025-11-14T21:38:08.000Z
## Changes

This updates templates to serverless environment version 4 and
associated Python versions

Some considerations:
* Client version 4, client version 5 (upcoming), DBR 16, DBR 17 all use
Python 3.12
* DB Connect 15 supports all DBR versions above and supports Python 3.12
* DB Connect 15 also supports DBR 15, which uses Python 3.11 at runtime,
but that discrepancy already exists and seems okay
* Pinning Python to a very narrow range in pyproject.toml means users
always need to use uv to install the right version, which may not be
great for usability. so I haven't change that approach either

## Why
* Environment version 4 provides better startup performance and new
features

## Tests
* Standard acceptance testing

---------

Co-authored-by: Claude &lt;noreply@anthropic.com&gt;
diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md
@@ -9,6 +9,7 @@
 ### Dependency updates
 
 ### Bundles
+* Update templates to use serverless environment version 4 and matching Python version ([#3897](https://github.com/databricks/cli/pull/3897))
 * Add a language prompt to the `default-minimal` template ([#3918](https://github.com/databricks/cli/pull/3918))
 * Add `default-scala` template for Scala projects with SBT build configuration and example code ([#3906](https://github.com/databricks/cli/pull/3906))
 
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml
@@ -29,6 +29,6 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "4"
             dependencies:
               - dbt-databricks>=1.8.0,<2.0.0
diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff
@@ -51,7 +51,7 @@
 -      environments:
 -        - environment_key: default
 -          spec:
--            environment_version: "2"
+-            environment_version: "4"
 -            dependencies:
 -              # By default we just include the .whl file generated for the my_default_python package.
 -              # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml
@@ -2,7 +2,7 @@
 name = "my_default_python"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<=3.13"
+requires-python = ">=3.10,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb
@@ -72,7 +72,7 @@
     "dependencies": [
      "--editable .."
     ],
-    "environment_version": "2"
+    "environment_version": "4"
    },
    "language": "python",
    "notebookMetadata": {
@@ -141,7 +141,7 @@
   },
   "language_info": {
    "name": "python",
-   "version": "3.11.4"
+   "version": "3.12"
   }
  },
  "nbformat": 4,
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml
@@ -2,7 +2,7 @@
 name = "my_default_python"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<=3.13"
+requires-python = ">=3.10,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml
@@ -46,7 +46,7 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "4"
             dependencies:
               # By default we just include the .whl file generated for the my_default_python package.
               # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb
@@ -72,7 +72,7 @@
     "dependencies": [
      "--editable .."
     ],
-    "environment_version": "2"
+    "environment_version": "4"
    },
    "language": "python",
    "notebookMetadata": {
@@ -141,7 +141,7 @@
   },
   "language_info": {
    "name": "python",
-   "version": "3.11.4"
+   "version": "3.12"
   }
  },
  "nbformat": 4,
diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml
@@ -2,7 +2,7 @@
 name = "my_lakeflow_pipelines"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<=3.13"
+requires-python = ">=3.10,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml
@@ -29,4 +29,4 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "4"
diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml
@@ -29,4 +29,4 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "4"
diff --git a/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml b/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml
@@ -2,7 +2,7 @@
 name = "my_pydabs"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<=3.13"
+requires-python = ">=3.10,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
diff --git a/libs/template/templates/dbt-sql/library/versions.tmpl b/libs/template/templates/dbt-sql/library/versions.tmpl
@@ -5,3 +5,10 @@
 {{define "latest_lts_db_connect_version_spec" -}}
   >=15.4,<15.5
 {{- end}}
+
+{{/* The serverless environment version to use.
+   * See https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/
+   */}}
+{{define "serverless_environment_version" -}}
+  4
+{{- end}}
diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl
@@ -60,7 +60,7 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "{{template "serverless_environment_version"}}"
             dependencies:
               - dbt-databricks>=1.8.0,<2.0.0
       {{- end }}
diff --git a/libs/template/templates/default/library/versions.tmpl b/libs/template/templates/default/library/versions.tmpl
@@ -1,4 +1,5 @@
 {{/* The latest LTS DBR version; this should be updated a few months after each LTS.
+   * See https://docs.databricks.com/release-notes/runtime/index.html for available versions.
    */}}
 {{define "latest_lts_dbr_version" -}}
   16.4.x-scala2.12
@@ -11,16 +12,39 @@
    * only connect to compute of that same version and higher. If the version is
    * deemed too old, customers can update the version themselves after initializing
    * the template.
+   *
+   * See https://docs.databricks.com/dev-tools/databricks-connect/python/index.html#requirements
+   * for DB Connect release notes and version compatibility.
    */}}
 {{define "conservative_db_connect_version_spec" -}}
   >=15.4,<15.5
 {{- end}}
 
-{{/* DB Connect 15 only supports versions up to 3.13 because of a limitation in
-   * pyarrow: https://arrow.apache.org/docs/python/install.html#python-compatibility
+{{/* The serverless environment version to use.
+   * See https://docs.databricks.com/release-notes/runtime/index.html for available versions.
+   */}}
+{{define "serverless_environment_version" -}}
+  4
+{{- end}}
+
+{{/* Python version specification for the template.
+   * This needs to be compatible with the version of DB Connect and the serverless_environment_version.
+   *
+   * Serverless environment version 4 (and upcoming v5), DBR 16, and DBR 17 all use Python 3.12.
+   * DB Connect 15 supports all of these and is compatible with Python 3.10-3.12.
+   *
+   * We use >=3.10,<3.13 rather than pinning to 3.12 specifically because overly narrow pinning
+   * means users always need to use uv or similar tools to install the exact version. A broader
+   * range improves usability while maintaining compatibility with both DB Connect and serverless.
+   */}}
+{{define "python_version_spec" -}}
+  >=3.10,<3.13
+{{- end}}
+
+{{/* The default Python version used in notebook metadata.
    */}}
-{{define "conservative_db_connect_python_version_spec" -}}
-  >=3.10,<=3.13
+{{define "default_python_version" -}}
+  3.12
 {{- end}}
 
 {{define "latest_databricks_bundles_version" -}}0.277.0{{- end}}
diff --git a/libs/template/templates/default/template/{{.project_name}}/pyproject.toml.tmpl b/libs/template/templates/default/template/{{.project_name}}/pyproject.toml.tmpl
@@ -2,7 +2,7 @@
 name = "{{.project_name}}"
 version = "0.0.1"
 authors = [{ name = "{{user_name}}" }]
-requires-python = "{{template "conservative_db_connect_python_version_spec"}}"
+requires-python = "{{template "python_version_spec"}}"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies
diff --git a/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl b/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl
@@ -94,7 +94,7 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "{{template "serverless_environment_version"}}"
   {{- if $python_package}}
             dependencies:
               # By default we just include the .whl file generated for the {{.project_name}} package.
diff --git a/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.py.tmpl b/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.py.tmpl
@@ -116,7 +116,7 @@ sample_job = Job.from_dict(
             {
                 "environment_key": "default",
                 "spec": {
-                    "environment_version": "2",
+                    "environment_version": "{{template "serverless_environment_version"}}",
   {{- if $python_package}}
                     "dependencies": [
                         # By default we just include the .whl file generated for the {{.project_name}} package.
diff --git a/libs/template/templates/default/template/{{.project_name}}/src/sample_notebook.ipynb.tmpl b/libs/template/templates/default/template/{{.project_name}}/src/sample_notebook.ipynb.tmpl
@@ -77,7 +77,7 @@
      {{- /* Make sure that this file always gets any library dependencies (when using serverless)*/}}
      "--editable .."
     ],
-    "environment_version": "2"
+    "environment_version": "{{template "serverless_environment_version"}}"
    },
    "language": "python",
    "notebookMetadata": {
@@ -146,7 +146,7 @@
   },
   "language_info": {
    "name": "python",
-   "version": "3.11.4"
+   "version": "{{template "default_python_version"}}"
   }
  },
  "nbformat": 4,
