diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 88f45aac96..6a6341af0d 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,5 +10,6 @@ ### Bundles * Add validation that served_models and served_entities are not used at the same time. Add client side translation logic. ([#3880](https://github.com/databricks/cli/pull/3880)) +* Update templates to use serverless environment version 4 and matching Python version ([#3897](https://github.com/databricks/cli/pull/3897)) ### API Changes diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml index ea834cd9de..d01c17860a 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml @@ -29,6 +29,6 @@ resources: environments: - environment_key: default spec: - environment_version: "2" + environment_version: "4" dependencies: - dbt-databricks>=1.8.0,<2.0.0 diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index f4929ebb1b..22c5698c64 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -51,7 +51,7 @@ - environments: - - environment_key: default - spec: -- environment_version: "2" +- environment_version: "4" - dependencies: - # By default we just include the .whl file generated for the my_default_python package. - # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml index 12114d057f..ffa9337ef4 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml @@ -2,7 +2,7 @@ name = "my_default_python" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<=3.13" +requires-python = ">=3.10,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb index 226cb2d064..d5e4804cb8 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb @@ -72,7 +72,7 @@ "dependencies": [ "--editable .." ], - "environment_version": "2" + "environment_version": "4" }, "language": "python", "notebookMetadata": { @@ -141,7 +141,7 @@ }, "language_info": { "name": "python", - "version": "3.11.4" + "version": "3.12" } }, "nbformat": 4, diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml index 12114d057f..ffa9337ef4 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml @@ -2,7 +2,7 @@ name = "my_default_python" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<=3.13" +requires-python = ">=3.10,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml index 7fd0530bd4..c64690910f 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml @@ -46,7 +46,7 @@ resources: environments: - environment_key: default spec: - environment_version: "2" + environment_version: "4" dependencies: # By default we just include the .whl file generated for the my_default_python package. # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb index 226cb2d064..d5e4804cb8 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb @@ -72,7 +72,7 @@ "dependencies": [ "--editable .." ], - "environment_version": "2" + "environment_version": "4" }, "language": "python", "notebookMetadata": { @@ -141,7 +141,7 @@ }, "language_info": { "name": "python", - "version": "3.11.4" + "version": "3.12" } }, "nbformat": 4, diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml index c5cf343c0b..82c73daa9c 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml @@ -2,7 +2,7 @@ name = "my_lakeflow_pipelines" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<=3.13" +requires-python = ">=3.10,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml index a72aa871eb..9880071565 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml @@ -29,4 +29,4 @@ resources: environments: - environment_key: default spec: - environment_version: "2" + environment_version: "4" diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/pyproject.toml b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/pyproject.toml index c5cf343c0b..82c73daa9c 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/pyproject.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/pyproject.toml @@ -2,7 +2,7 @@ name = "my_lakeflow_pipelines" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<=3.13" +requires-python = ">=3.10,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml index a72aa871eb..9880071565 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml @@ -29,4 +29,4 @@ resources: environments: - environment_key: default spec: - environment_version: "2" + environment_version: "4" diff --git a/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml b/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml index 1dba700078..070a22c735 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml +++ b/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml @@ -2,7 +2,7 @@ name = "my_pydabs" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<=3.13" +requires-python = ">=3.10,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies diff --git a/libs/template/templates/dbt-sql/library/versions.tmpl b/libs/template/templates/dbt-sql/library/versions.tmpl index 7d0c88e7df..cb393461f3 100644 --- a/libs/template/templates/dbt-sql/library/versions.tmpl +++ b/libs/template/templates/dbt-sql/library/versions.tmpl @@ -5,3 +5,10 @@ {{define "latest_lts_db_connect_version_spec" -}} >=15.4,<15.5 {{- end}} + +{{/* The serverless environment version to use. + * See https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/ + */}} +{{define "serverless_environment_version" -}} + 4 +{{- end}} diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index 7e2c48616c..fede1892c7 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -60,7 +60,7 @@ resources: environments: - environment_key: default spec: - environment_version: "2" + environment_version: "{{template "serverless_environment_version"}}" dependencies: - dbt-databricks>=1.8.0,<2.0.0 {{- end }} diff --git a/libs/template/templates/default/library/versions.tmpl b/libs/template/templates/default/library/versions.tmpl index 63891ea9e3..44690bf156 100644 --- a/libs/template/templates/default/library/versions.tmpl +++ b/libs/template/templates/default/library/versions.tmpl @@ -1,4 +1,5 @@ {{/* The latest LTS DBR version; this should be updated a few months after each LTS. + * See https://docs.databricks.com/release-notes/runtime/index.html for available versions. */}} {{define "latest_lts_dbr_version" -}} 16.4.x-scala2.12 @@ -11,16 +12,39 @@ * only connect to compute of that same version and higher. If the version is * deemed too old, customers can update the version themselves after initializing * the template. + * + * See https://docs.databricks.com/dev-tools/databricks-connect/python/index.html#requirements + * for DB Connect release notes and version compatibility. */}} {{define "conservative_db_connect_version_spec" -}} >=15.4,<15.5 {{- end}} -{{/* DB Connect 15 only supports versions up to 3.13 because of a limitation in - * pyarrow: https://arrow.apache.org/docs/python/install.html#python-compatibility +{{define "latest_databricks_bundles_version" -}}0.276.0{{- end}} + +{{/* The serverless environment version to use. + * See https://docs.databricks.com/release-notes/runtime/index.html for available versions. */}} -{{define "conservative_db_connect_python_version_spec" -}} - >=3.10,<=3.13 +{{define "serverless_environment_version" -}} + 4 {{- end}} -{{define "latest_databricks_bundles_version" -}}0.276.0{{- end}} +{{/* Python version specification for the template. + * This needs to be compatible with the version of DB Connect and the serverless_environment_version. + * + * Serverless environment version 4 (and upcoming v5), DBR 16, and DBR 17 all use Python 3.12. + * DB Connect 15 supports all of these and is compatible with Python 3.10-3.12. + * + * We use >=3.10,<3.13 rather than pinning to 3.12 specifically because overly narrow pinning + * means users always need to use uv or similar tools to install the exact version. A broader + * range improves usability while maintaining compatibility with both DB Connect and serverless. + */}} +{{define "python_version_spec" -}} + >=3.10,<3.13 +{{- end}} + +{{/* The default Python version used in notebook metadata. + */}} +{{define "default_python_version" -}} + 3.12 +{{- end}} diff --git a/libs/template/templates/default/template/{{.project_name}}/pyproject.toml.tmpl b/libs/template/templates/default/template/{{.project_name}}/pyproject.toml.tmpl index bf8c74c61c..d6c8318c21 100644 --- a/libs/template/templates/default/template/{{.project_name}}/pyproject.toml.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/pyproject.toml.tmpl @@ -2,7 +2,7 @@ name = "{{.project_name}}" version = "0.0.1" authors = [{ name = "{{user_name}}" }] -requires-python = "{{template "conservative_db_connect_python_version_spec"}}" +requires-python = "{{template "python_version_spec"}}" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies diff --git a/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl b/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl index 00c921ef44..2a80e04e87 100644 --- a/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.job.yml.tmpl @@ -94,7 +94,7 @@ resources: environments: - environment_key: default spec: - environment_version: "2" + environment_version: "{{template "serverless_environment_version"}}" {{- if $python_package}} dependencies: # By default we just include the .whl file generated for the {{.project_name}} package. diff --git a/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.py.tmpl b/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.py.tmpl index 5a64325b80..ef80b97494 100644 --- a/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.py.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/resources/sample_job.py.tmpl @@ -116,7 +116,7 @@ sample_job = Job.from_dict( { "environment_key": "default", "spec": { - "environment_version": "2", + "environment_version": "{{template "serverless_environment_version"}}", {{- if $python_package}} "dependencies": [ # By default we just include the .whl file generated for the {{.project_name}} package. diff --git a/libs/template/templates/default/template/{{.project_name}}/src/sample_notebook.ipynb.tmpl b/libs/template/templates/default/template/{{.project_name}}/src/sample_notebook.ipynb.tmpl index 23d2003538..63d00ba51a 100644 --- a/libs/template/templates/default/template/{{.project_name}}/src/sample_notebook.ipynb.tmpl +++ b/libs/template/templates/default/template/{{.project_name}}/src/sample_notebook.ipynb.tmpl @@ -77,7 +77,7 @@ {{- /* Make sure that this file always gets any library dependencies (when using serverless)*/}} "--editable .." ], - "environment_version": "2" + "environment_version": "{{template "serverless_environment_version"}}" }, "language": "python", "notebookMetadata": { @@ -146,7 +146,7 @@ }, "language_info": { "name": "python", - "version": "3.11.4" + "version": "{{template "default_python_version"}}" } }, "nbformat": 4,