Add serverless support to dbt-sql template

This mirrors the approach in default-python.

Author: pietern

acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml

@@ -15,6 +15,7 @@ resources:
 
       tasks:
         - task_key: dbt
+          environment_key: default
           dbt_task:
             project_directory: ../
             # The default schema, catalog, etc. are defined in ../dbt_profiles/profiles.yml
@@ -25,17 +26,12 @@ resources:
               - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"'
               - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"'
 
-          libraries:
-            - pypi:
-                package: dbt-databricks>=1.8.0,<2.0.0
+      environments:
+        - environment_key: default
 
-          new_cluster:
-            spark_version: 15.4.x-scala2.12
-            node_type_id: [NODE_TYPE_ID]
-            data_security_mode: SINGLE_USER
-            num_workers: 0
-            spark_conf:
-              spark.master: "local[*, 4]"
-              spark.databricks.cluster.profile: singleNode
-            custom_tags:
-              ResourceClass: SingleNode
+          # Full documentation of this spec can be found at:
+          # https://docs.databricks.com/api/workspace/jobs/create#environments-spec
+          spec:
+            environment_version: "2"
+            dependencies:
+              - dbt-databricks>=1.8.0,<2.0.0

acceptance/bundle/templates/telemetry/dbt-sql/out.requests.txt

@@ -28,7 +28,7 @@
     "uploadTime": [UNIX_TIME_MILLIS],
     "items": [],
     "protoLogs": [
-      "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[CMD-EXEC-ID]\",\"version\":\"[DEV_VERSION]\",\"command\":\"bundle_init\",\"operating_system\":\"[OS]\",\"execution_time_ms\":\"SMALL_INT\",\"exit_code\":0},\"bundle_init_event\":{\"bundle_uuid\":\"[BUNDLE-UUID]\",\"template_name\":\"dbt-sql\",\"template_enum_args\":[{\"key\":\"personal_schemas\",\"value\":\"yes, use a schema based on the current user name during development\"}]}}}}"
+      "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[CMD-EXEC-ID]\",\"version\":\"[DEV_VERSION]\",\"command\":\"bundle_init\",\"operating_system\":\"[OS]\",\"execution_time_ms\":\"SMALL_INT\",\"exit_code\":0},\"bundle_init_event\":{\"bundle_uuid\":\"[BUNDLE-UUID]\",\"template_name\":\"dbt-sql\",\"template_enum_args\":[{\"key\":\"personal_schemas\",\"value\":\"yes, use a schema based on the current user name during development\"},{\"key\":\"serverless\",\"value\":\"yes\"}]}}}}"
     ]
   }
 }

acceptance/bundle/templates/telemetry/dbt-sql/output.txt

@@ -29,6 +29,10 @@ Refer to the README.md file for full "getting started" guide and production setu
           {
             "key": "personal_schemas",
             "value": "yes, use a schema based on the current user name during development"
+          },
+          {
+            "key": "serverless",
+            "value": "yes"
           }
         ]
       }

libs/template/templates/dbt-sql/databricks_template_schema.json

@@ -47,6 +47,13 @@
             "pattern_match_failure_message": "Invalid schema name.",
             "description": "\nPlease provide an initial schema during development.\ndefault_schema",
             "order": 5
+        },
+        "serverless": {
+            "type": "string",
+            "default": "yes",
+            "enum": ["yes", "no"],
+            "description": "Use serverless compute",
+            "order": 6
         }
     },
     "success_message": "\n📊 Your new project has been created in the '{{.project_name}}' directory!\nIf you already have dbt installed, just type 'cd {{.project_name}}; dbt init' to get started.\nRefer to the README.md file for full \"getting started\" guide and production setup instructions.\n"

libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl

@@ -1,3 +1,5 @@
+{{$with_serverless := (eq .serverless "yes") -}}
+
 resources:
   jobs:
     {{.project_name}}_job:
@@ -15,6 +17,9 @@ resources:
 
       tasks:
         - task_key: dbt
+          {{- if $with_serverless }}
+          environment_key: default
+          {{- end }}
           dbt_task:
             project_directory: ../
             # The default schema, catalog, etc. are defined in ../dbt_profiles/profiles.yml
@@ -32,6 +37,8 @@ resources:
               - 'dbt run --target=${bundle.target}'
 {{- end}}
 
+          {{- if not $with_serverless }}
+
           libraries:
             - pypi:
                 package: dbt-databricks>=1.8.0,<2.0.0
@@ -46,3 +53,17 @@ resources:
               spark.databricks.cluster.profile: singleNode
             custom_tags:
               ResourceClass: SingleNode
+          {{- end }}
+
+      {{- if $with_serverless }}
+
+      environments:
+        - environment_key: default
+
+          # Full documentation of this spec can be found at:
+          # https://docs.databricks.com/api/workspace/jobs/create#environments-spec
+          spec:
+            environment_version: "2"
+            dependencies:
+              - dbt-databricks>=1.8.0,<2.0.0
+      {{- end }}