Update templates to use serverless environment version 4 and matching Python version

NEXT_CHANGELOG.md

@@ -10,5 +10,6 @@
 
 ### Bundles
 * Add validation that served_models and served_entities are not used at the same time. Add client side translation logic. ([#3880](https://github.com/databricks/cli/pull/3880))
+* Update templates to use serverless environment version 4 and matching Python version ([#3897](https://github.com/databricks/cli/pull/3897))
 
 ### API Changes

acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml

@@ -29,6 +29,6 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "4"
             dependencies:
               - dbt-databricks>=1.8.0,<2.0.0

acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff

@@ -51,7 +51,7 @@
 -      environments:
 -        - environment_key: default
 -          spec:
--            environment_version: "2"
+-            environment_version: "4"
 -            dependencies:
 -              # By default we just include the .whl file generated for the my_default_python package.
 -              # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html

acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml

@@ -2,7 +2,7 @@
 name = "my_default_python"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<=3.13"
+requires-python = ">=3.10,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies

acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb

@@ -72,7 +72,7 @@
     "dependencies": [
      "--editable .."
     ],
-    "environment_version": "2"
+    "environment_version": "4"
    },
    "language": "python",
    "notebookMetadata": {
@@ -141,7 +141,7 @@
   },
   "language_info": {
    "name": "python",
-   "version": "3.11.4"
+   "version": "3.12"
   }
  },
  "nbformat": 4,

acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml

@@ -2,7 +2,7 @@
 name = "my_default_python"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<=3.13"
+requires-python = ">=3.10,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies

acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml

@@ -46,7 +46,7 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "4"
             dependencies:
               # By default we just include the .whl file generated for the my_default_python package.
               # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html

acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb

@@ -72,7 +72,7 @@
     "dependencies": [
      "--editable .."
     ],
-    "environment_version": "2"
+    "environment_version": "4"
    },
    "language": "python",
    "notebookMetadata": {
@@ -141,7 +141,7 @@
   },
   "language_info": {
    "name": "python",
-   "version": "3.11.4"
+   "version": "3.12"
   }
  },
  "nbformat": 4,

acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml

@@ -2,7 +2,7 @@
 name = "my_lakeflow_pipelines"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<=3.13"
+requires-python = ">=3.10,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies

acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml

@@ -29,4 +29,4 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "4"

acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/pyproject.toml

@@ -2,7 +2,7 @@
 name = "my_lakeflow_pipelines"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<=3.13"
+requires-python = ">=3.10,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies

acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml

@@ -29,4 +29,4 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "4"

acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml

@@ -2,7 +2,7 @@
 name = "my_pydabs"
 version = "0.0.1"
 authors = [{ name = "[USERNAME]" }]
-requires-python = ">=3.10,<=3.13"
+requires-python = ">=3.10,<3.13"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies

libs/template/templates/dbt-sql/library/versions.tmpl

@@ -5,3 +5,10 @@
 {{define "latest_lts_db_connect_version_spec" -}}
   >=15.4,<15.5
 {{- end}}
+
+{{/* The serverless environment version to use.
+   * See https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/
+   */}}
+{{define "serverless_environment_version" -}}
+  4
+{{- end}}

libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl

@@ -60,7 +60,7 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "{{template "serverless_environment_version"}}"
             dependencies:
               - dbt-databricks>=1.8.0,<2.0.0
       {{- end }}

libs/template/templates/default/library/versions.tmpl

@@ -1,4 +1,5 @@
 {{/* The latest LTS DBR version; this should be updated a few months after each LTS.
+   * See https://docs.databricks.com/release-notes/runtime/index.html for available versions.
    */}}
 {{define "latest_lts_dbr_version" -}}
   16.4.x-scala2.12
@@ -11,16 +12,32 @@
    * only connect to compute of that same version and higher. If the version is
    * deemed too old, customers can update the version themselves after initializing
    * the template.
+   *
+   * See https://docs.databricks.com/dev-tools/databricks-connect/python/index.html#requirements
+   * for DB Connect release notes and version compatibility.
    */}}
 {{define "conservative_db_connect_version_spec" -}}
   >=15.4,<15.5
 {{- end}}
 
-{{/* DB Connect 15 only supports versions up to 3.13 because of a limitation in
-   * pyarrow: https://arrow.apache.org/docs/python/install.html#python-compatibility
+{{define "latest_databricks_bundles_version" -}}0.276.0{{- end}}
+
+{{/* The serverless environment version to use.
+   * See https://docs.databricks.com/release-notes/runtime/index.html for available versions.
    */}}
-{{define "conservative_db_connect_python_version_spec" -}}
-  >=3.10,<=3.13
+{{define "serverless_environment_version" -}}
+  4
 {{- end}}
 
-{{define "latest_databricks_bundles_version" -}}0.276.0{{- end}}
+{{/* Python version specification for the template.
+   * This needs to be compatible with the version of DB Connect and the serverless_environment_version.
+   */}}
+{{define "python_version_spec" -}}
+  >=3.10,<3.13
+{{- end}}
+
+{{/* The default Python version used in notebook metadata.
+   */}}
+{{define "default_python_version" -}}
+  3.12
+{{- end}}

libs/template/templates/default/template/{{.project_name}}/pyproject.toml.tmpl

@@ -2,7 +2,7 @@
 name = "{{.project_name}}"
 version = "0.0.1"
 authors = [{ name = "{{user_name}}" }]
-requires-python = "{{template "conservative_db_connect_python_version_spec"}}"
+requires-python = "{{template "python_version_spec"}}"
 dependencies = [
     # Any dependencies for jobs and pipelines in this project can be added here
     # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies

libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl

@@ -94,7 +94,7 @@ resources:
       environments:
         - environment_key: default
           spec:
-            environment_version: "2"
+            environment_version: "{{template "serverless_environment_version"}}"
   {{- if $python_package}}
             dependencies:
               # By default we just include the .whl file generated for the {{.project_name}} package.

libs/template/templates/default/template/{{.project_name}}/resources/sample_job.py.tmpl

@@ -116,7 +116,7 @@ sample_job = Job.from_dict(
             {
                 "environment_key": "default",
                 "spec": {
-                    "environment_version": "2",
+                    "environment_version": "{{template "serverless_environment_version"}}",
   {{- if $python_package}}
                     "dependencies": [
                         # By default we just include the .whl file generated for the {{.project_name}} package.

libs/template/templates/default/template/{{.project_name}}/src/sample_notebook.ipynb.tmpl

@@ -77,7 +77,7 @@
      {{- /* Make sure that this file always gets any library dependencies (when using serverless)*/}}
      "--editable .."
     ],
-    "environment_version": "2"
+    "environment_version": "{{template "serverless_environment_version"}}"
    },
    "language": "python",
    "notebookMetadata": {
@@ -146,7 +146,7 @@
   },
   "language_info": {
    "name": "python",
-   "version": "3.11.4"
+   "version": "{{template "default_python_version"}}"
   }
  },
  "nbformat": 4,