Update UC Azure guide to use MI (#1543)

Author: nkvuong

docs/guides/unity-catalog-azure.md

@@ -4,7 +4,7 @@ page_title: "Unity Catalog set up on Azure"
 
 # Deploying pre-requisite resources and enabling Unity Catalog (Azure Preview)
 
--> **Public Preview** This feature is in [Public Preview](https://docs.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog). Contact your Databricks representative to request access. 
+-> **Public Preview** This feature is in [Public Preview](https://docs.microsoft.com/en-us/azure/databricks/data-governance/unity-catalog). Contact your Databricks representative to request access.
 
 Databricks Unity Catalog brings fine-grained governance and security to Lakehouse data using a familiar, open interface. You can use Terraform to deploy the underlying cloud resources and Unity Catalog objects automatically, using a programmatic approach.
 
@@ -15,6 +15,7 @@ This guide uses the following variables in configurations:
 This guide is provided as-is and you can use this guide as the basis for your custom Terraform module.
 
 To get started with Unity Catalog, this guide takes you throw the following high-level steps:
+
 - [Deploying pre-requisite resources and enabling Unity Catalog (Azure Preview)](#deploying-pre-requisite-resources-and-enabling-unity-catalog-azure-preview)
   - [Provider initialization](#provider-initialization)
   - [Configure Azure objects](#configure-azure-objects)
@@ -61,22 +62,20 @@ data "azurerm_databricks_workspace" "this" {
 ```hcl
 terraform {
   required_providers {
-    azurerm = {
-      source  = "hashicorp/azurerm"
-      version = "~>2.99.0"
+    azapi = {
+      source = "azure/azapi"
     }
-    azuread = {
-      source  = "hashicorp/azuread"
-      version = "~>2.19.0"
+    azurerm = {
+      source = "hashicorp/azurerm"
     }
     databricks = {
       source = "databricks/databricks"
     }
   }
 }
 
-provider "azuread" {
-  tenant_id = local.tenant_id
+provider "azapi" {
+  subscription_id = local.subscription_id
 }
 
 provider "azurerm" {
@@ -90,22 +89,24 @@ provider "databricks" {
 ```
 
 ## Configure Azure objects
+
 The first step is to create the required Azure objects:
+
 - An Azure storage account, which is the default storage location for managed tables in Unity Catalog. Please use a dedicated account for each metastore.
-- An AAD service principal that provides Unity Catalog permissions to access and manage data in the bucket.
+- A Databricks Access Connector that provides Unity Catalog permissions to access and manage data in the storage account.
 
 ```hcl
-resource "azuread_application" "unity_catalog" {
-  display_name = "${local.prefix}-root-sp"
-}
-
-resource "azuread_application_password" "unity_catalog" {
-  application_object_id = azuread_application.unity_catalog.object_id
-}
-
-resource "azuread_service_principal" "unity_catalog" {
-  application_id               = azuread_application.unity_catalog.application_id
-  app_role_assignment_required = false
+resource "azapi_resource" "access_connector" {
+  type      = "Microsoft.Databricks/accessConnectors@2022-04-01-preview"
+  name      = "${local.prefix}-databricks-mi"
+  location  = data.azurerm_resource_group.this.location
+  parent_id = data.azurerm_resource_group.this.id
+  identity {
+    type = "SystemAssigned"
+  }
+  body = jsonencode({
+    properties = {}
+  })
 }
 
 resource "azurerm_storage_account" "unity_catalog" {
@@ -127,7 +128,7 @@ resource "azurerm_storage_container" "unity_catalog" {
 resource "azurerm_role_assignment" "example" {
   scope                = azurerm_storage_account.unity_catalog.id
   role_definition_name = "Storage Blob Data Contributor"
-  principal_id         = azuread_service_principal.unity_catalog.object_id
+  principal_id         = azapi_resource.access_connector.identity[0].principal_id
 }
 ```
 
@@ -147,10 +148,8 @@ resource "databricks_metastore" "this" {
 resource "databricks_metastore_data_access" "first" {
   metastore_id = databricks_metastore.this.id
   name         = "the-keys"
-  azure_service_principal {
-    directory_id   = local.tenant_id
-    application_id = azuread_application.unity_catalog.application_id
-    client_secret  = azuread_application_password.unity_catalog.value
+  azure_managed_identity {
+    access_connector_id = azapi_resource.access_connector.id
   }
 
   is_default = true
@@ -165,7 +164,7 @@ resource "databricks_metastore_assignment" "this" {
 
 ## Create Unity Catalog objects in the metastore
 
-Each metastore exposes a 3-level namespace (catalog-schema-table) by which data can be organized. 
+Each metastore exposes a 3-level namespace (catalog-schema-table) by which data can be organized.
 
 ```hcl
 resource "databricks_catalog" "sandbox" {
@@ -211,23 +210,24 @@ resource "databricks_grants" "things" {
 ## Configure external tables and credentials
 
 To work with external tables, Unity Catalog introduces two new objects to access and work with external cloud storage:
-- [databricks_storage_credential](../resources/storage_credential.md) represent authentication methods to access cloud storage (e.g. an IAM role for Amazon S3 or a service principal for Azure Storage). Storage credentials are access-controlled to determine which users can use the credential.
-- [databricks_external_location](../resources/external_location.md) are objects that combine a cloud storage path with a Storage Credential that can be used to access the location. 
+
+- [databricks_storage_credential](../resources/storage_credential.md) represent authentication methods to access cloud storage (e.g. an IAM role for Amazon S3 or a managed identity for Azure Storage). Storage credentials are access-controlled to determine which users can use the credential.
+- [databricks_external_location](../resources/external_location.md) are objects that combine a cloud storage path with a Storage Credential that can be used to access the location.
 
 First, create the required objects in Azure.
 
 ```hcl
-resource "azuread_application" "ext_cred" {
-  display_name = "${local.prefix}-cred"
-}
-
-resource "azuread_application_password" "ext_cred" {
-  application_object_id = azuread_application.ext_cred.object_id
-}
-
-resource "azuread_service_principal" "ext_cred" {
-  application_id               = azuread_application.ext_cred.application_id
-  app_role_assignment_required = false
+resource "azapi_resource" "ext_access_connector" {
+  type      = "Microsoft.Databricks/accessConnectors@2022-04-01-preview"
+  name      = "ext-databricks-mi"
+  location  = data.azurerm_resource_group.this.location
+  parent_id = data.azurerm_resource_group.this.id
+  identity {
+    type = "SystemAssigned"
+  }
+  body = jsonencode({
+    properties = {}
+  })
 }
 
 resource "azurerm_storage_account" "ext_storage" {
@@ -249,19 +249,16 @@ resource "azurerm_storage_container" "ext_storage" {
 resource "azurerm_role_assignment" "ext_storage" {
   scope                = azurerm_storage_account.ext_storage.id
   role_definition_name = "Storage Blob Data Contributor"
-  principal_id         = azuread_service_principal.ext_cred.object_id
-}
+  principal_id         = azapi_resource.ext_access_connector.identity[0].principal_id
 ```
 
 Then create the [databricks_storage_credential](../resources/storage_credential.md) and [databricks_external_location](../resources/external_location.md) in Unity Catalog.
 
 ```hcl
 resource "databricks_storage_credential" "external" {
-  name = azuread_application.ext_cred.display_name
-  azure_service_principal {
-    directory_id   = local.tenant_id
-    application_id = azuread_application.ext_cred.application_id
-    client_secret  = azuread_application_password.ext_cred.value
+  name = azapi_resource.ext_access_connector.name
+  azure_managed_identity {
+    access_connector_id = azapi_resource.ext_access_connector.id
   }
   comment = "Managed by TF"
   depends_on = [

docs/resources/storage_credential.md

@@ -35,67 +35,31 @@ resource "databricks_grants" "external_creds" {
 For Azure
 
 ```hcl
-resource "azurerm_resource_group_template_deployment" "access_connector" {
-  name                = "databricks-access-connectors"
-  resource_group_name = "vn-sandbox"
-  deployment_mode     = "Incremental"
-  parameters_content = jsonencode({
-    "connectorName" = {
-      value = "vn-databricks-mi"
-    }
-    "accessConnectorRegion" = {
-      value = "uksouth"
-    }
-    "enableSystemAssignedIdentity" = {
-      value = true
-    }
-  })
-  template_content = <<TEMPLATE
-{
-   "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
-   "contentVersion": "1.0.0.0",
-   "parameters": {
-      "connectorName": {
-         "defaultValue": "testConnector",
-         "type": "String",
-         "metadata": {
-               "description": "The name of the Azure Databricks Access Connector to create."
-         }
-      },
-      "accessConnectorRegion": {
-         "defaultValue": "[resourceGroup().location]",
-         "type": "String",
-         "metadata": {
-               "description": "Location for the access connector resource."
-         }
-      },
-      "enableSystemAssignedIdentity": {
-         "defaultValue": true,
-         "type": "Bool",
-         "metadata": {
-               "description": "Whether the system assigned managed identity is enabled"
-         }
-      }
-   },
-   "resources": [
-      {
-         "type": "Microsoft.Databricks/accessConnectors",
-         "apiVersion": "2022-04-01-preview",
-         "name": "[parameters('connectorName')]",
-         "location": "[parameters('accessConnectorRegion')]",
-         "identity": {
-               "type": "[if(parameters('enableSystemAssignedIdentity'), 'SystemAssigned', 'None')]"
-         }
-      }
-   ]
+data "azurerm_resource_group" "this" {
+  name     = "example-rg"
 }
-TEMPLATE
+
+resource "azapi_resource" "access_connector" {
+  type      = "Microsoft.Databricks/accessConnectors@2022-04-01-preview"
+  name      = "example-databricks-mi"
+  location  = data.azurerm_resource_group.this.location
+  parent_id = data.azurerm_resource_group.this.id
+  tags = {
+    tagName1 = "tagValue1"
+    tagName2 = "tagValue2"
+  }
+  identity {
+    type = "SystemAssigned"
+  }
+  body = jsonencode({
+    properties = {}
+  })
 }
 
 resource "databricks_storage_credential" "external_mi" {
   name = "mi_credential"
   azure_managed_identity {
-    access_connector_id = "${split("/Microsoft.Resources", azurerm_resource_group_template_deployment.access_connector.id)[0]}/Microsoft.Databricks/accessConnectors/${jsondecode(azurerm_resource_group_template_deployment.access_connector.parameters_content).connectorName.value}"
+    access_connector_id = azapi_resource.access_connector.id
   }
   comment = "Managed identity credential managed by TF"
 }