Skip to content

Commit f9a7f45

Browse files
owlleg6owlleg6
authored
Merge pull request #1 from data-platform-hq/feat/synapse_analytics
feat: synapse analytics
2 parents a45f8c0 + 9ce4f2d commit f9a7f45

File tree

7 files changed

+913
-2
lines changed

7 files changed

+913
-2
lines changed

‎README.md‎

Lines changed: 100 additions & 2 deletions
Large diffs are not rendered by default.

‎locals.tf‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
data "http" "client_ip" {
2+
count = var.allow_own_ip ? 1 : 0
3+
url = "https://ipv4.icanhazip.com"
4+
}
5+
6+
locals {
7+
# private endpoint role assignments
8+
pe_role_assignments = { for ra in flatten([
9+
for pe_k, pe_v in var.private_endpoints : [
10+
for rk, rv in pe_v.role_assignments : {
11+
private_endpoint_key = pe_k
12+
ra_key = rk
13+
role_assignment = rv
14+
}
15+
]
16+
]) : "${ra.private_endpoint_key}-${ra.ra_key}" => ra }
17+
# Private endpsoint application security group associations
18+
private_endpoint_application_security_group_associations = { for assoc in flatten([
19+
for pe_k, pe_v in var.private_endpoints : [
20+
for asg_k, asg_v in pe_v.application_security_group_associations : {
21+
asg_key = asg_k
22+
pe_key = pe_k
23+
asg_resource_id = asg_v
24+
}
25+
]
26+
]) : "${assoc.pe_key}-${assoc.asg_key}" => assoc }
27+
role_definition_resource_substring = "/providers/Microsoft.Authorization/roleDefinitions"
28+
}

‎main.privateendpoint.tf‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
resource "azurerm_synapse_private_link_hub" "this" {
2+
count = length(var.private_endpoints) > 0 ? 1 : 0
3+
4+
name = var.private_link_hub_name != null ? var.private_link_hub_name : lower(join("", split("-", "plh-${var.name}")))
5+
resource_group_name = var.resource_group_name
6+
location = var.location
7+
}
8+
9+
resource "azurerm_private_endpoint" "this" {
10+
for_each = { for k, v in var.private_endpoints : k => v if var.private_endpoints_manage_dns_zone_group }
11+
12+
location = each.value.location != null ? each.value.location : var.location
13+
name = each.value.name != null ? each.value.name : "pe-${var.name}"
14+
resource_group_name = each.value.resource_group_name != null ? each.value.resource_group_name : var.resource_group_name
15+
subnet_id = each.value.subnet_resource_id
16+
custom_network_interface_name = each.value.network_interface_name
17+
tags = each.value.tags
18+
19+
private_service_connection {
20+
is_manual_connection = false
21+
name = each.value.private_service_connection_name != null ? each.value.private_service_connection_name : "pse-${var.name}"
22+
private_connection_resource_id = each.value.subresource_name == "Web" ? azurerm_synapse_private_link_hub.this[0].id : azurerm_synapse_workspace.this.id
23+
subresource_names = [each.value.subresource_name]
24+
}
25+
dynamic "ip_configuration" {
26+
for_each = each.value.ip_configurations
27+
28+
content {
29+
name = ip_configuration.value.name
30+
private_ip_address = ip_configuration.value.private_ip_address
31+
member_name = each.value.subresource_name
32+
subresource_name = each.value.subresource_name
33+
}
34+
}
35+
dynamic "private_dns_zone_group" {
36+
for_each = length(each.value.private_dns_zone_resource_ids) > 0 ? ["this"] : []
37+
38+
content {
39+
name = each.value.private_dns_zone_group_name
40+
private_dns_zone_ids = each.value.private_dns_zone_resource_ids
41+
}
42+
}
43+
}
44+
45+
# The PE resource when we are managing **not** the private_dns_zone_group block:
46+
resource "azurerm_private_endpoint" "this_unmanaged_dns_zone_groups" {
47+
for_each = { for k, v in var.private_endpoints : k => v if !var.private_endpoints_manage_dns_zone_group }
48+
49+
location = each.value.location != null ? each.value.location : var.location
50+
name = each.value.name != null ? each.value.name : "pe-${var.name}"
51+
resource_group_name = each.value.resource_group_name != null ? each.value.resource_group_name : var.resource_group_name
52+
subnet_id = each.value.subnet_resource_id
53+
custom_network_interface_name = each.value.network_interface_name
54+
tags = each.value.tags
55+
56+
private_service_connection {
57+
is_manual_connection = false
58+
name = each.value.private_service_connection_name != null ? each.value.private_service_connection_name : "pse-${var.name}"
59+
private_connection_resource_id = each.value.subresource_name == "Web" ? azurerm_synapse_private_link_hub.this[0].id : azurerm_synapse_workspace.this.id
60+
subresource_names = [each.value.subresource_name]
61+
}
62+
dynamic "ip_configuration" {
63+
for_each = each.value.ip_configurations
64+
65+
content {
66+
name = ip_configuration.value.name
67+
private_ip_address = ip_configuration.value.private_ip_address
68+
member_name = each.value.subresource_name
69+
subresource_name = each.value.subresource_name
70+
}
71+
}
72+
73+
lifecycle {
74+
ignore_changes = [private_dns_zone_group]
75+
}
76+
}
77+
78+
resource "azurerm_private_endpoint_application_security_group_association" "this" {
79+
for_each = local.private_endpoint_application_security_group_associations
80+
81+
application_security_group_id = each.value.asg_resource_id
82+
private_endpoint_id = var.private_endpoints_manage_dns_zone_group ? azurerm_private_endpoint.this[each.value.pe_key].id : azurerm_private_endpoint.this_unmanaged_dns_zone_groups[each.value.pe_key].id
83+
}
84+
85+
resource "azurerm_role_assignment" "private_endpoint" {
86+
for_each = local.pe_role_assignments
87+
88+
principal_id = each.value.role_assignment.principal_id
89+
scope = try(azurerm_private_endpoint.this[each.value.private_endpoint_key].id, azurerm_private_endpoint.this_unmanaged_dns_zone_groups[each.value.private_endpoint_key].id)
90+
condition = each.value.role_assignment.condition
91+
condition_version = each.value.role_assignment.condition_version
92+
delegated_managed_identity_resource_id = each.value.role_assignment.delegated_managed_identity_resource_id
93+
principal_type = each.value.role_assignment.principal_type
94+
role_definition_id = strcontains(lower(each.value.role_assignment.role_definition_id_or_name), lower(local.role_definition_resource_substring)) ? each.value.role_assignment.role_definition_id_or_name : null
95+
role_definition_name = strcontains(lower(each.value.role_assignment.role_definition_id_or_name), lower(local.role_definition_resource_substring)) ? null : each.value.role_assignment.role_definition_id_or_name
96+
skip_service_principal_aad_check = each.value.role_assignment.skip_service_principal_aad_check
97+
}
98+
99+
############################### Private Links ###############################
100+
resource "azurerm_synapse_managed_private_endpoint" "private_link" {
101+
for_each = var.private_links
102+
name = each.value.name != null ? each.value.name : "pl-${var.name}"
103+
synapse_workspace_id = azurerm_synapse_workspace.this.id
104+
target_resource_id = each.value.target_resource_id
105+
subresource_name = each.value.subresource_name
106+
107+
depends_on = [
108+
azurerm_synapse_firewall_rule.this,
109+
azurerm_synapse_firewall_rule.azureservices,
110+
azurerm_synapse_firewall_rule.client_ip
111+
]
112+
}

‎main.tf‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
resource "random_password" "sql_password" {
2+
count = !var.azuread_authentication_only && var.auth_sql_administrator_password == null ? 1 : 0
3+
length = 16
4+
special = true
5+
override_special = "!#$%&*()-_=+[]{}<>:?"
6+
}
7+
8+
resource "azurerm_synapse_workspace" "this" {
9+
name = var.name
10+
resource_group_name = var.resource_group_name
11+
managed_resource_group_name = var.managed_resource_group_name != "" ? var.managed_resource_group_name : null
12+
location = var.location
13+
storage_data_lake_gen2_filesystem_id = var.storage_data_lake_gen2_filesystem_id
14+
sql_administrator_login = !var.azuread_authentication_only ? coalesce(var.auth_sql_administrator, "sqladminuser") : null
15+
sql_administrator_login_password = !var.azuread_authentication_only ? coalesce(var.auth_sql_administrator_password, random_password.sql_password[0].result) : null
16+
azuread_authentication_only = var.azuread_authentication_only
17+
purview_id = var.purview_id
18+
managed_virtual_network_enabled = var.managed_virtual_network_enabled
19+
20+
dynamic "github_repo" {
21+
for_each = var.github != null ? [var.github] : []
22+
content {
23+
account_name = github_repo.value.account_name
24+
repository_name = github_repo.value.repository_name
25+
branch_name = github_repo.value.branch_name
26+
root_folder = github_repo.value.root_folder
27+
git_url = lookup(github_repo.value, "git_url", null)
28+
last_commit_id = lookup(github_repo.value, "last_commit_id", null)
29+
}
30+
}
31+
32+
dynamic "azure_devops_repo" {
33+
for_each = var.azure_devops_repo != null ? [var.azure_devops_repo] : []
34+
content {
35+
account_name = azure_devops_repo.value.account_name
36+
project_name = azure_devops_repo.value.project_name
37+
repository_name = azure_devops_repo.value.repository_name
38+
branch_name = azure_devops_repo.value.branch_name
39+
root_folder = azure_devops_repo.value.root_folder
40+
last_commit_id = lookup(azure_devops_repo.value, "last_commit_id", null)
41+
tenant_id = azure_devops_repo.value.tenant_id
42+
}
43+
}
44+
45+
dynamic "identity" {
46+
for_each = var.identity_type != null ? [1] : []
47+
content {
48+
type = var.identity_type
49+
identity_ids = var.identity_type == "UserAssigned" ? var.identity_ids : []
50+
}
51+
}
52+
53+
lifecycle {
54+
ignore_changes = [
55+
github_repo[0].last_commit_id,
56+
azure_devops_repo[0].last_commit_id,
57+
sql_administrator_login]
58+
}
59+
60+
tags = var.tags
61+
}
62+
63+
############################### Access Control ###############################
64+
# To storage account
65+
resource "azurerm_role_assignment" "storage_blob_contributor" {
66+
count = var.add_storage_contributor_role ? length(azurerm_synapse_workspace.this.identity) : 0
67+
scope = var.storage_account_id
68+
role_definition_name = "Storage Blob Data Contributor"
69+
principal_id = azurerm_synapse_workspace.this.identity[count.index].principal_id
70+
}
71+
72+
resource "azurerm_role_assignment" "this" {
73+
count = length(var.azure_role_assignments)
74+
scope = azurerm_synapse_workspace.this.id
75+
role_definition_name = var.azure_role_assignments[count.index].role_name
76+
principal_id = var.azure_role_assignments[count.index].principal_id
77+
}
78+
79+
resource "azurerm_synapse_role_assignment" "this" {
80+
count = length(var.synapse_role_assignments)
81+
synapse_workspace_id = azurerm_synapse_workspace.this.id
82+
role_name = var.synapse_role_assignments[count.index].role_name
83+
principal_id = var.synapse_role_assignments[count.index].principal_id
84+
principal_type = lookup(var.synapse_role_assignments[count.index], "principal_type", null)
85+
86+
depends_on = [
87+
time_sleep.wait_for_firewall_operations
88+
]
89+
}
90+
91+
############################### Firewall Rules ###############################
92+
resource "azurerm_synapse_firewall_rule" "this" {
93+
count = length(var.firewall_rules)
94+
synapse_workspace_id = azurerm_synapse_workspace.this.id
95+
name = var.firewall_rules[count.index].name
96+
start_ip_address = var.firewall_rules[count.index].start_ip_address
97+
end_ip_address = var.firewall_rules[count.index].end_ip_address
98+
}
99+
100+
resource "azurerm_synapse_firewall_rule" "azureservices" {
101+
count = var.allow_azure_services_access ? 1 : 0
102+
synapse_workspace_id = azurerm_synapse_workspace.this.id
103+
name = "AllowAllWindowsAzureIps"
104+
start_ip_address = "0.0.0.0"
105+
end_ip_address = "0.0.0.0"
106+
}
107+
108+
resource "azurerm_synapse_firewall_rule" "client_ip" {
109+
count = var.allow_own_ip ? 1 : 0
110+
synapse_workspace_id = azurerm_synapse_workspace.this.id
111+
name = "ClientIp"
112+
start_ip_address = chomp(data.http.client_ip[0].response_body)
113+
end_ip_address = chomp(data.http.client_ip[0].response_body)
114+
}
115+
116+
resource "time_sleep" "wait_for_firewall_operations" {
117+
count = var.allow_own_ip ? 1 : 0
118+
119+
create_duration = var.wait_for_firewall_operations.create
120+
destroy_duration = var.wait_for_firewall_operations.destroy
121+
122+
depends_on = [
123+
azurerm_synapse_firewall_rule.client_ip
124+
]
125+
}
126+
127+
############################### Linked Services ###############################
128+
resource "azurerm_synapse_linked_service" "this" {
129+
for_each = var.linked_services
130+
name = each.key
131+
description = each.value["description"]
132+
synapse_workspace_id = azurerm_synapse_workspace.this.id
133+
type = each.value["type"]
134+
type_properties_json = each.value["type_properties_json"]
135+
additional_properties = each.value["additional_properties"]
136+
annotations = each.value["annotations"]
137+
parameters = each.value["parameters"]
138+
dynamic "integration_runtime" {
139+
for_each = each.value["integration_runtime_name"] != null ? [1] : []
140+
content {
141+
name = each.value["integration_runtime_name"]
142+
parameters = each.value["integration_runtime_parameters"]
143+
}
144+
}
145+
146+
depends_on = [
147+
time_sleep.wait_for_firewall_operations
148+
]
149+
150+
}
151+
152+
############################### Spark Pools ###############################
153+
resource "azurerm_synapse_spark_pool" "this" {
154+
for_each = var.spark_pools
155+
name = each.key
156+
synapse_workspace_id = azurerm_synapse_workspace.this.id
157+
node_size_family = each.value["node_size_family"]
158+
node_size = each.value["node_size"]
159+
node_count = each.value["node_count"]
160+
cache_size = each.value["cache_size"]
161+
compute_isolation_enabled = each.value["compute_isolation_enabled"]
162+
dynamic_executor_allocation_enabled = each.value["dynamic_executor_allocation_enabled"]
163+
min_executors = each.value["min_executors"]
164+
max_executors = each.value["max_executors"]
165+
session_level_packages_enabled = each.value["session_level_packages_enabled"]
166+
spark_log_folder = each.value["spark_log_folder"]
167+
spark_events_folder = each.value["spark_events_folder"]
168+
spark_version = each.value["spark_version"]
169+
170+
dynamic "auto_scale" {
171+
for_each = each.value["autoscale_max_node_count"] != null ? [1] : []
172+
content {
173+
max_node_count = each.value["autoscale_max_node_count"]
174+
min_node_count = each.value["autoscale_min_node_count"]
175+
}
176+
}
177+
178+
dynamic "auto_pause" {
179+
for_each = each.value["autopause_delay_in_minutes"] != null ? [1] : []
180+
content {
181+
delay_in_minutes = each.value["autopause_delay_in_minutes"]
182+
}
183+
}
184+
185+
dynamic "library_requirement" {
186+
for_each = each.value["requirements_content"] != null ? [1] : []
187+
content {
188+
content = each.value["requirements_content"]
189+
filename = each.value["requirements_filename"]
190+
}
191+
}
192+
193+
dynamic "spark_config" {
194+
for_each = each.value["spark_config_content"] != null ? [1] : []
195+
content {
196+
content = each.value["spark_config_content"]
197+
filename = each.value["spark_config_filename"]
198+
}
199+
}
200+
201+
tags = var.tags
202+
203+
depends_on = [
204+
azurerm_synapse_firewall_rule.this,
205+
azurerm_synapse_firewall_rule.azureservices,
206+
azurerm_synapse_firewall_rule.client_ip,
207+
]
208+
}
209+
210+
############################### Integration Runtimes ###############################
211+
resource "azurerm_synapse_integration_runtime_azure" "this" {
212+
for_each = var.azure_integration_runtimes
213+
name = each.key
214+
synapse_workspace_id = azurerm_synapse_workspace.this.id
215+
description = each.value["description"]
216+
location = each.value["location"]
217+
compute_type = each.value["compute_type"]
218+
core_count = each.value["core_count"]
219+
time_to_live_min = each.value["time_to_live_min"]
220+
}
221+
222+
resource "azurerm_synapse_integration_runtime_self_hosted" "this" {
223+
for_each = var.self_hosted_integration_runtimes
224+
name = each.key
225+
synapse_workspace_id = azurerm_synapse_workspace.this.id
226+
description = each.value["description"]
227+
}
228+
229+
############################## SQL Pools ###############################
230+
resource "azurerm_synapse_sql_pool" "this" {
231+
for_each = var.sql_pools
232+
name = each.key
233+
synapse_workspace_id = azurerm_synapse_workspace.this.id
234+
sku_name = each.value["sku_name"]
235+
create_mode = each.value["create_mode"]
236+
collation = each.value["collation"]
237+
data_encrypted = each.value["data_encrypted"]
238+
recovery_database_id = each.value["recovery_database_id"]
239+
geo_backup_policy_enabled = each.value["geo_backup_policy_enabled"]
240+
storage_account_type = each.value["storage_account_type"]
241+
dynamic "restore" {
242+
for_each = each.value["restore_source_database_id"] != null ? [1] : []
243+
content {
244+
source_database_id = each.value["restore_source_database_id"]
245+
point_in_time = each.value["restore_point_in_time"]
246+
}
247+
}
248+
tags = var.tags
249+
}

0 commit comments

Comments
 (0)