Azure terraform AiO deployment

terraform/all_in_one_azure/README.md

@@ -0,0 +1,22 @@
+This AiO will create Everything: 
+
+The first file (main.tf) will deal with required azure resources: Resource Group + Two Storage accounts (one for the cluster logs, another for the overwatch database), configure diagnostic settings, create databricks workspace, secret scope, keyvault, put proper permission and roles, access policies in the keyvault, create the Event Hub Namespace, the event hub and it's authorization rules
+
+The second file (main_databricks) will upload the overwatch notebook, create the secret scope, add a few entries in the keyvault, generate thew required PAT token, create a dummy job and create and schedule the overwatch job with the required libraries from maven put as a dependencies.
+
+USAGE:
+* Do a terraform init to download the azure and databricks providers
+* Fill the Variables below in a new file called terraform.tfvars file with your own values
+* run az login (to be able to create the secret scope, as service principal is not allowed)
+* run terraform plan / terraform apply
+* run the dummy job and then the overwatch job is scheduled to run every day (you can change the schedule in the variables.tf file, using the cron expression)
+
+
+The terraform.tfvars file should contain the following vars defined:
+
+tenant_id=
+subscription_id= 
+service_principal_id_mount=
+user_id= 
+overwatch_spn=
+overwatch_spn_pass=

terraform/all_in_one_azure/main.tf

@@ -0,0 +1,216 @@
+// Albert Nogués. 23/05/2022. Automate Overwatch Depoloyment
+// This file does all the overwatch prerequisites deployment
+
+data "azurerm_storage_account" "owsa" {
+  name                = azurerm_storage_account.owsa.name
+  resource_group_name = azurerm_resource_group.rg.name
+}
+
+data "azurerm_storage_account" "logsa" {
+  name                = azurerm_storage_account.logsa.name
+  resource_group_name = azurerm_resource_group.rg.name
+}
+
+data "azurerm_databricks_workspace" "adb" {
+  name                = azurerm_databricks_workspace.adb.name
+  resource_group_name = azurerm_resource_group.rg.name
+}
+
+resource "random_string" "strapp" {
+  length  = 5
+  lower = true
+  upper = false
+  special = false
+}
+
+resource "azurerm_resource_group" "rg" {
+  name = join("", [var.resource_group_name,random_string.strapp.result])
+  location  = var.resource_group_location
+
+  tags = {
+    environment = "Overwatch"
+  }
+}
+
+resource "azurerm_storage_account" "owsa" {
+  name                     = join("", [var.overwatch_storage_account_name,random_string.strapp.result])
+  resource_group_name      = azurerm_resource_group.rg.name
+  location                 = azurerm_resource_group.rg.location
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+  is_hns_enabled           = true
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  tags = {
+    environment = "Overwatch"
+    purpose="Overwatch Delta Database"    
+  }
+}
+
+resource "azurerm_storage_account" "logsa" {
+  name                     = join("", [var.logs_storage_account_name,random_string.strapp.result])
+  resource_group_name      = azurerm_resource_group.rg.name
+  location                 = azurerm_resource_group.rg.location
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+  is_hns_enabled           = true
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  tags = {
+    environment = "Overwatch"
+    purpose="Overwatch Cluster Logs Storage"
+  }
+}
+
+resource "azurerm_role_assignment" "data-contributor-role"{
+  scope = azurerm_storage_account.owsa.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id = var.service_principal_id_mount
+}
+
+resource "azurerm_role_assignment" "data-contributor-role-log"{
+  scope = azurerm_storage_account.logsa.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id = var.service_principal_id_mount
+}
+
+resource "azurerm_storage_data_lake_gen2_filesystem" "overwatch" {
+  name               = "overwatch"
+  storage_account_id = azurerm_storage_account.owsa.id
+}
+
+resource "azurerm_storage_data_lake_gen2_filesystem" "logs" {
+  name               = "logs"
+  storage_account_id = azurerm_storage_account.logsa.id
+}
+
+resource "azurerm_key_vault" "kv" {
+  name                = join("", [var.key_vault_name,random_string.strapp.result])
+  location            = azurerm_resource_group.rg.location
+  resource_group_name = azurerm_resource_group.rg.name
+  tenant_id           = var.tenant_id
+  sku_name            = "standard"
+
+  purge_protection_enabled = false
+}
+
+
+resource "azurerm_key_vault_access_policy" "storage" {
+  key_vault_id = azurerm_key_vault.kv.id
+  tenant_id    = var.tenant_id
+  object_id    = azurerm_storage_account.owsa.identity[0].principal_id
+
+  key_permissions    = ["Get", "Create", "List", "Restore", "Recover", "UnwrapKey", "WrapKey", "Purge", "Encrypt", "Decrypt", "Sign", "Verify"]
+  secret_permissions = ["Get", "List", "Set"]
+}
+
+resource "azurerm_key_vault_access_policy" "tfuser" {
+  key_vault_id = azurerm_key_vault.kv.id
+  tenant_id    = var.tenant_id
+  object_id    = var.user_id
+
+  key_permissions    = ["Get", "Create", "List", "Restore", "Recover", "UnwrapKey", "WrapKey", "Purge", "Encrypt", "Decrypt", "Sign", "Verify"]
+  secret_permissions = ["Get", "List", "Set", "Delete", "Purge", "Recover"]
+}
+
+resource "azurerm_databricks_workspace" "adb" {
+  name                = var.workspace_name
+  resource_group_name = azurerm_resource_group.rg.name
+  location            = azurerm_resource_group.rg.location
+  sku                 = "premium"
+
+  tags = {
+    Environment = "Overwatch"
+  }
+}
+
+
+//EvenHub Part
+resource "azurerm_eventhub_namespace" "ehn" {
+  name                = join("", [var.evenhub_namespace_name,random_string.strapp.result])
+  location            = azurerm_resource_group.rg.location
+  resource_group_name = azurerm_resource_group.rg.name
+  sku                 = "Basic"
+  capacity            = 1
+
+  tags = {
+    environment = "Overwatch"
+  }
+}
+
+data "azurerm_eventhub_namespace" "ehn" {
+  name = azurerm_eventhub_namespace.ehn.name
+  resource_group_name = azurerm_resource_group.rg.name
+}
+
+resource "azurerm_eventhub" "eh" {
+  name                = var.evenhub_name
+  namespace_name      = azurerm_eventhub_namespace.ehn.name
+  resource_group_name = azurerm_resource_group.rg.name
+  partition_count     = 2
+  message_retention   = 1
+}
+
+resource "azurerm_eventhub_authorization_rule" "ehar" {
+  name                = "overwatch"
+  namespace_name      = azurerm_eventhub_namespace.ehn.name
+  eventhub_name       = azurerm_eventhub.eh.name
+  resource_group_name = azurerm_resource_group.rg.name
+  listen              = true
+  send                = true
+  manage              = true
+}
+
+data "azurerm_eventhub_authorization_rule" "ehar" {
+  name = azurerm_eventhub_authorization_rule.ehar.name
+  resource_group_name = azurerm_resource_group.rg.name
+  namespace_name      = azurerm_eventhub_namespace.ehn.name
+  eventhub_name       = azurerm_eventhub.eh.name
+}
+
+
+resource "azurerm_eventhub_namespace_authorization_rule" "ehnar" {
+  name                = "overwatch"
+  namespace_name      = azurerm_eventhub_namespace.ehn.name
+  resource_group_name = azurerm_resource_group.rg.name
+  listen              = true
+  send                = true
+  manage              = true
+}
+
+data "azurerm_monitor_diagnostic_categories" "cat"{
+  resource_id = azurerm_databricks_workspace.adb.id
+}
+
+resource "azurerm_monitor_diagnostic_setting" "ovwdgs" {
+  name               = "OverwatchDGS"
+  target_resource_id = azurerm_databricks_workspace.adb.id
+  eventhub_name = azurerm_eventhub.eh.name
+  eventhub_authorization_rule_id = azurerm_eventhub_namespace_authorization_rule.ehnar.id
+
+  dynamic "log" {
+    iterator = log_category
+    for_each = data.azurerm_monitor_diagnostic_categories.cat.logs
+    content {
+      enabled = true
+      category = log_category.value
+      retention_policy {
+        enabled = false
+      }
+    }
+  }
+
+  log{
+    category = "clusters"
+    enabled = true
+    retention_policy {
+      enabled = false
+    }
+  }
+}