Skip to content

Commit d425441

Browse files
owlleg6owlleg6
authored
Merge pull request #5 from data-platform-hq/unity_catalog
feat: unity catalog
2 parents 180f550 + 8738ea8 commit d425441

File tree

7 files changed

+294
-60
lines changed

7 files changed

+294
-60
lines changed

‎README.md‎

Lines changed: 60 additions & 36 deletions
Large diffs are not rendered by default.

‎iam.tf‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -101,18 +101,3 @@ resource "databricks_permissions" "sql_endpoint" {
101101

102102
depends_on = [databricks_group.this]
103103
}
104-
105-
resource "databricks_permissions" "token" {
106-
authorization = "tokens"
107-
108-
dynamic "access_control" {
109-
for_each = { for entry in flatten([for resource, permissions in var.iam_permissions : [for permission, groups in permissions : [for group in groups : {
110-
resource = resource, permission = permission, group = group
111-
} if resource == "token"]]]) : "${entry.resource}.${entry.permission}.${entry.group}" => entry }
112-
content {
113-
group_name = access_control.value.group
114-
permission_level = access_control.value.permission
115-
}
116-
}
117-
depends_on = [databricks_group.this]
118-
}

‎main.tf‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
locals {
22
ip_rules = var.ip_rules == null ? null : values(var.ip_rules)
3+
suffix = length(var.suffix) == 0 ? "" : "-${var.suffix}"
34
}
45

56
resource "databricks_workspace_conf" "this" {
@@ -23,7 +24,7 @@ resource "databricks_ip_access_list" "this" {
2324
resource "databricks_sql_endpoint" "this" {
2425
for_each = var.sql_endpoint
2526

26-
name = "${each.key}-${var.project}-${var.env}"
27+
name = "${each.key}-${var.project}-${var.env}${local.suffix}"
2728
cluster_size = lookup(each.value, "cluster_size", var.default_values_sql_endpoint["cluster_size"])
2829
min_num_clusters = lookup(each.value, "min_num_clusters", var.default_values_sql_endpoint["min_num_clusters"])
2930
max_num_clusters = lookup(each.value, "max_num_clusters", var.default_values_sql_endpoint["max_num_clusters"])

‎outputs.tf‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,8 @@ output "sql_endpoint_data_source_id" {
77
value = [for n in databricks_sql_endpoint.this : n.data_source_id]
88
description = "ID of the data source for this endpoint"
99
}
10+
11+
output "metastore_id" {
12+
value = var.create_metastore ? databricks_metastore.this[0].id : ""
13+
description = "Unity Catalog Metastore Id"
14+
}

‎unity.tf‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
resource "azurerm_storage_data_lake_gen2_filesystem" "this" {
2+
count = var.create_metastore ? 1 : 0
3+
4+
name = "meta-${var.project}-${var.env}"
5+
storage_account_id = var.storage_account_id
6+
7+
lifecycle {
8+
precondition {
9+
condition = alltrue([
10+
for variable in [var.storage_account_id, var.access_connector_id, var.storage_account_name] : false if length(variable) == 0
11+
])
12+
error_message = "To create Metastore in a Region it is required to provide proper values for these variables: access_connector_id, storage_account_id, storage_account_name"
13+
}
14+
}
15+
}
16+
17+
resource "databricks_metastore" "this" {
18+
count = var.create_metastore ? 1 : 0
19+
20+
name = "meta-${var.project}-${var.env}-${var.location}${local.suffix}"
21+
storage_root = format("abfss://%s@%s.dfs.core.windows.net/", azurerm_storage_data_lake_gen2_filesystem.this[0].name, var.storage_account_name)
22+
force_destroy = true
23+
}
24+
25+
resource "databricks_grants" "metastore" {
26+
for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : {
27+
for k, v in var.metastore_grants : k => v
28+
if v != null
29+
}
30+
31+
metastore = length(var.external_metastore_id) == 0 ? databricks_metastore.this[0].id : var.external_metastore_id
32+
grant {
33+
principal = each.key
34+
privileges = each.value
35+
}
36+
}
37+
38+
resource "databricks_metastore_data_access" "this" {
39+
count = var.create_metastore ? 1 : 0
40+
41+
metastore_id = databricks_metastore.this[0].id
42+
name = "data-access-${var.project}-${var.env}-${var.location}${local.suffix}"
43+
azure_managed_identity {
44+
access_connector_id = var.access_connector_id
45+
}
46+
is_default = true
47+
}
48+
49+
resource "databricks_metastore_assignment" "this" {
50+
count = !var.create_metastore && length(var.external_metastore_id) == 0 ? 0 : 1
51+
52+
workspace_id = var.workspace_id
53+
metastore_id = length(var.external_metastore_id) == 0 ? databricks_metastore.this[0].id : var.external_metastore_id
54+
default_catalog_name = "hive_metastore"
55+
}
56+
57+
# Catalog
58+
resource "databricks_catalog" "this" {
59+
for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : var.catalog
60+
61+
metastore_id = length(var.external_metastore_id) == 0 ? databricks_metastore.this[0].id : var.external_metastore_id
62+
name = each.key
63+
comment = lookup(each.value, "catalog_comment", "default comment")
64+
properties = merge(lookup(each.value, "catalog_properties", {}), { env = var.env })
65+
force_destroy = true
66+
67+
depends_on = [databricks_metastore_assignment.this[0]]
68+
}
69+
70+
# Catalog grants
71+
resource "databricks_grants" "catalog" {
72+
for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : {
73+
for name, params in var.catalog : name => params.catalog_grants
74+
if params.catalog_grants != null
75+
}
76+
77+
catalog = databricks_catalog.this[each.key].name
78+
dynamic "grant" {
79+
for_each = each.value
80+
content {
81+
principal = grant.key
82+
privileges = grant.value
83+
}
84+
}
85+
}
86+
87+
# Schema
88+
locals {
89+
schema = flatten([
90+
for catalog, params in var.catalog : [
91+
for schema in params.schema_name : {
92+
catalog = catalog,
93+
schema = schema,
94+
comment = lookup(params, "schema_comment", "default comment"),
95+
properties = lookup(params, "schema_properties", {})
96+
}
97+
] if params.schema_name != null
98+
])
99+
}
100+
101+
resource "databricks_schema" "this" {
102+
for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : {
103+
for entry in local.schema : "${entry.catalog}.${entry.schema}" => entry
104+
}
105+
106+
catalog_name = databricks_catalog.this[each.value.catalog].name
107+
name = each.value.schema
108+
comment = each.value.comment
109+
properties = merge(each.value.properties, { env = var.env })
110+
force_destroy = true
111+
}
112+
113+
# Schema grants
114+
locals {
115+
schema_grants = flatten([
116+
for catalog, params in var.catalog : [for schema in params.schema_name : [for principal in flatten(keys(params.schema_grants)) : {
117+
catalog = catalog,
118+
schema = schema,
119+
principal = principal,
120+
permission = flatten(values(params.schema_grants)),
121+
}]] if params.schema_grants != null
122+
])
123+
}
124+
125+
resource "databricks_grants" "schema" {
126+
for_each = !var.create_metastore && length(var.external_metastore_id) == 0 ? {} : {
127+
for entry in local.schema_grants : "${entry.catalog}.${entry.schema}.${entry.principal}" => entry
128+
}
129+
130+
schema = databricks_schema.this["${each.value.catalog}.${each.value.schema}"].id
131+
grant {
132+
principal = each.value.principal
133+
privileges = each.value.permission
134+
}
135+
}

‎variables.tf‎

Lines changed: 88 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,21 @@
1+
variable "project" {
2+
type = string
3+
description = "Project name"
4+
}
5+
16
variable "env" {
27
type = string
38
description = "Environment name"
49
}
510

6-
variable "project" {
11+
variable "location" {
712
type = string
8-
description = "Project name"
13+
description = "Azure location"
14+
}
15+
16+
variable "workspace_id" {
17+
type = string
18+
description = "Id of Azure Databricks workspace"
919
}
1020

1121
variable "user_object_ids" {
@@ -19,7 +29,7 @@ variable "workspace_admins" {
1929
user = list(string)
2030
service_principal = list(string)
2131
})
22-
description = "Provide users or service principals to grant them Admin permissions."
32+
description = "Provide users or service principals to grant them Admin permissions in Workspace."
2333
default = {
2434
user = null
2535
service_principal = null
@@ -54,10 +64,6 @@ variable "iam_permissions" {
5464
"CAN_USE" = ["default"]
5565
"CAN_MANAGE" = []
5666
}
57-
"token" = {
58-
"CAN_USE" = ["default"]
59-
"CAN_MANAGE" = []
60-
}
6167
}
6268
}
6369

@@ -68,7 +74,14 @@ variable "ip_rules" {
6874
}
6975

7076
variable "sql_endpoint" {
71-
type = map(map(string))
77+
type = map(object({
78+
cluster_size = string
79+
min_num_clusters = optional(number)
80+
max_num_clusters = optional(number)
81+
auto_stop_mins = optional(string)
82+
enable_photon = optional(bool)
83+
enable_serverless_compute = optional(bool)
84+
}))
7285
description = "Map of SQL Endoints to be deployed in Databricks Workspace"
7386
default = {}
7487
}
@@ -92,3 +105,70 @@ variable "default_values_sql_endpoint" {
92105
enable_serverless_compute = false
93106
}
94107
}
108+
109+
variable "create_metastore" {
110+
type = bool
111+
description = "Boolean flag for Unity Catalog Metastore current in this environment. One Metastore per region"
112+
default = false
113+
}
114+
115+
variable "access_connector_id" {
116+
type = string
117+
description = "Databricks Access Connector Id that lets you to connect managed identities to an Azure Databricks account. Provides an ability to access Unity Catalog with assigned identity"
118+
default = ""
119+
}
120+
121+
variable "storage_account_id" {
122+
type = string
123+
description = "Storage Account Id where Unity Catalog Metastore would be provisioned"
124+
default = ""
125+
}
126+
variable "storage_account_name" {
127+
type = string
128+
description = "Storage Account Name where Unity Catalog Metastore would be provisioned"
129+
default = ""
130+
}
131+
132+
variable "catalog" {
133+
type = map(object({
134+
catalog_grants = optional(map(list(string)))
135+
catalog_comment = optional(string)
136+
catalog_properties = optional(map(string))
137+
schema_name = optional(list(string))
138+
schema_grants = optional(map(list(string)))
139+
schema_comment = optional(string)
140+
schema_properties = optional(map(string))
141+
}))
142+
description = "Map of catalog name and its parameters"
143+
default = {}
144+
}
145+
146+
variable "suffix" {
147+
type = string
148+
description = "Optional suffix that would be added to the end of resources names."
149+
default = ""
150+
}
151+
152+
variable "external_metastore_id" {
153+
type = string
154+
description = "Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore"
155+
default = ""
156+
validation {
157+
condition = length(var.external_metastore_id) == 36 || length(var.external_metastore_id) == 0
158+
error_message = "UUID has to be either in nnnnnnnn-nnnn-nnnn-nnnn-nnnnnnnnnnnn format or empty string"
159+
}
160+
}
161+
162+
variable "metastore_grants" {
163+
type = map(list(string))
164+
description = "Permissions to give on metastore to group"
165+
default = {}
166+
validation {
167+
condition = values(var.metastore_grants) != null ? alltrue([
168+
for item in toset(flatten([for group, params in var.metastore_grants : params if params != null])) : contains([
169+
"CREATE_CATALOG", "CREATE_EXTERNAL_LOCATION", "CREATE_SHARE", "CREATE_RECIPIENT", "CREATE_PROVIDER"
170+
], item)
171+
]) : true
172+
error_message = "Metastore permission validation. The only possible values for permissions are: CREATE_CATALOG, CREATE_EXTERNAL_LOCATION, CREATE_SHARE, CREATE_RECIPIENT, CREATE_PROVIDER"
173+
}
174+
}

‎versions.tf‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ terraform {
22
required_version = ">=1.0.0"
33

44
required_providers {
5+
azurerm = {
6+
source = "hashicorp/azurerm"
7+
version = ">=3.40.0"
8+
}
59
databricks = {
610
source = "databricks/databricks"
711
version = ">=1.9.0"

0 commit comments

Comments
 (0)