feat: unity cluster; mount with secrets output; mount with default cluster

Author: owlleg6

README.md

@@ -45,6 +45,7 @@ resource "databricks_cluster" "this" {
   cluster_name  = "shared autoscaling"
   spark_version = var.spark_version
 
+  data_security_mode      = var.data_security_mode
   node_type_id            = var.node_type
   autotermination_minutes = var.autotermination_minutes
 
@@ -63,5 +64,9 @@ resource "databricks_cluster" "this" {
     ignore_changes = [
       state
     ]
+    precondition {
+      condition     = var.data_security_mode == "USER_ISOLATION" ? contains(["11.3.x-scala2.12", "12.0.x-scala2.12"], var.spark_version) : true
+      error_message = "When USER_ISOLATION is selected, please set spark version to be either one of these values: '11.3.x-scala2.12', '12.0.x-scala2.12'"
+    }
   }
 }

main.tf

@@ -1,19 +1,21 @@
+locals {
+  secret_scope_name = var.use_local_secret_scope ? databricks_secret_scope.this[0].name : "main"
+  mount_secret_name = var.use_local_secret_scope ? databricks_secret.this[var.sp_key_secret_name].config_reference : "{{secrets/${local.secret_scope_name}/${data.azurerm_key_vault_secret.sp_key.name}}}"
+}
+
 resource "databricks_mount" "adls" {
   for_each = var.mountpoints
 
-  cluster_id = databricks_cluster.this.id
-  name       = each.key
-  uri        = "abfss://${each.value["container_name"]}@${each.value["storage_account_name"]}.dfs.core.windows.net/${each.value["root_path"]}"
+  name = each.key
+  uri  = "abfss://${each.value["container_name"]}@${each.value["storage_account_name"]}.dfs.core.windows.net/${each.value["root_path"]}"
   extra_configs = {
     "fs.azure.account.auth.type" : "OAuth",
     "fs.azure.account.oauth.provider.type" : "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
     "fs.azure.account.oauth2.client.id" : data.azurerm_key_vault_secret.sp_client_id.value,
-    "fs.azure.account.oauth2.client.secret" : "{{secrets/${local.secret_scope_name}/${data.azurerm_key_vault_secret.sp_key.name}}}",
-    "fs.azure.account.oauth2.client.secret" : "{{secrets/${local.secret_scope_name}/${local.mount_secret_name}}}",
+    "fs.azure.account.oauth2.client.secret" : local.mount_secret_name,
     "fs.azure.account.oauth2.client.endpoint" : "https://login.microsoftonline.com/${data.azurerm_key_vault_secret.tenant_id.value}/oauth2/token",
     "fs.azure.createRemoteFileSystemDuringInitialization" : "false",
     "spark.databricks.sqldw.jdbc.service.principal.client.id" : data.azurerm_key_vault_secret.sp_client_id.value,
-    "spark.databricks.sqldw.jdbc.service.principal.client.secret" : "{{secrets/${local.secret_scope_name}/${data.azurerm_key_vault_secret.sp_key.name}}}",
-    "spark.databricks.sqldw.jdbc.service.principal.client.secret" : "{{secrets/${local.secret_scope_name}/${local.mount_secret_name}}}",
+    "spark.databricks.sqldw.jdbc.service.principal.client.secret" : local.mount_secret_name,
   }
 }

mount.tf

@@ -1,7 +1,3 @@
-locals {
-  secret_scope_name = var.use_local_secret_scope ? databricks_secret_scope.this[0].name : "main"
-  mount_secret_name = var.use_local_secret_scope ? databricks_secret.this[var.sp_key_secret_name].key : data.azurerm_key_vault_secret.sp_key.name
-}
 resource "databricks_secret_scope" "this" {
   count = var.use_local_secret_scope ? 1 : 0
 

secrets.tf

@@ -101,10 +101,20 @@ variable "permissions" {
   ]
 }
 
+variable "data_security_mode" {
+  type        = string
+  description = "Security features of the cluster"
+  default     = "USER_ISOLATION"
+  validation {
+    condition     = contains(["SINGLE_USER", "USER_ISOLATION", "NONE"], var.data_security_mode)
+    error_message = "Catalog Access mode must be either 'SINGLE_USER', 'USER_ISOLATION' or 'NONE' value"
+  }
+}
+
 variable "spark_version" {
   type        = string
   description = "Runtime version"
-  default     = "9.1.x-scala2.12"
+  default     = "11.3.x-scala2.12"
 }
 
 variable "node_type" {

variables.tf

@@ -8,7 +8,7 @@ terraform {
     }
     databricks = {
       source  = "databricks/databricks"
-      version = ">=1.4.0"
+      version = ">=1.8.0"
     }
   }
 }