Added azure_managed_identity block to databricks_storage_credential and databricks_metastore_data_access resources (#1354)

Author: nkvuong

catalog/resource_metastore_data_access.go

@@ -27,12 +27,17 @@ type AzureServicePrincipal struct {
 	ClientSecret  string `json:"client_secret"`
 }
 
+type AzureManagedIdentity struct {
+	AccessConnectorID string `json:"access_connector_id"`
+}
+
 type DataAccessConfiguration struct {
 	ID                string                 `json:"id,omitempty" tf:"computed"`
 	Name              string                 `json:"name"`
 	ConfigurationType string                 `json:"configuration_type,omitempty" tf:"computed"`
 	Aws               *AwsIamRole            `json:"aws_iam_role,omitempty" tf:"group:access"`
 	Azure             *AzureServicePrincipal `json:"azure_service_principal,omitempty" tf:"group:access"`
+	AzMI              *AzureManagedIdentity  `json:"azure_managed_identity,omitempty" tf:"group:access"`
 }
 
 func (a DataAccessConfigurationsAPI) Create(metastoreID string, dac *DataAccessConfiguration) error {
@@ -65,9 +70,10 @@ func ResourceDataAccessConfiguration() *schema.Resource {
 				Type:     schema.TypeBool,
 				Optional: true,
 			}
-			alof := []string{"aws_iam_role", "azure_service_principal"}
+			alof := []string{"aws_iam_role", "azure_service_principal", "azure_managed_identity"}
 			m["aws_iam_role"].AtLeastOneOf = alof
 			m["azure_service_principal"].AtLeastOneOf = alof
+			m["azure_managed_identity"].AtLeastOneOf = alof
 			return m
 		})
 	p := common.NewPairID("metastore_id", "id")

catalog/resource_metastore_data_access_test.go

@@ -64,3 +64,57 @@ func TestCreateDac(t *testing.T) {
 		`,
 	}.ApplyNoError(t)
 }
+
+func TestCreateDacWithAzMI(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "POST",
+				Resource: "/api/2.0/unity-catalog/metastores/abc/data-access-configurations",
+				ExpectedRequest: DataAccessConfiguration{
+					Name: "bcd",
+					AzMI: &AzureManagedIdentity{
+						AccessConnectorID: "def",
+					},
+				},
+				Response: DataAccessConfiguration{
+					ID: "efg",
+				},
+			},
+			{
+				Method:   "PATCH",
+				Resource: "/api/2.0/unity-catalog/metastores/abc",
+				ExpectedRequest: map[string]interface{}{
+					"default_data_access_config_id": "efg",
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/unity-catalog/metastores/abc/data-access-configurations/efg",
+				Response: DataAccessConfiguration{
+					Name: "bcd",
+					AzMI: &AzureManagedIdentity{
+						AccessConnectorID: "def",
+					},
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/unity-catalog/metastores/abc",
+				Response: MetastoreInfo{
+					DefaultDacID: "efg",
+				},
+			},
+		},
+		Create:   true,
+		Resource: ResourceDataAccessConfiguration(),
+		HCL: `
+		metastore_id = "abc"
+		name = "bcd"
+		is_default = true
+		azure_managed_identity {
+			access_connector_id = "def"
+		}
+		`,
+	}.ApplyNoError(t)
+}

catalog/resource_storage_credentials.go

@@ -22,6 +22,7 @@ type StorageCredentialInfo struct {
 	Comment     string                 `json:"comment,omitempty"`
 	Aws         *AwsIamRole            `json:"aws_iam_role,omitempty" tf:"group:access"`
 	Azure       *AzureServicePrincipal `json:"azure_service_principal,omitempty" tf:"group:access"`
+	AzMI        *AzureManagedIdentity  `json:"azure_managed_identity,omitempty" tf:"group:access"`
 	MetastoreID string                 `json:"metastore_id,omitempty" tf:"computed"`
 }
 
@@ -41,9 +42,10 @@ func (a StorageCredentialsAPI) delete(id string) error {
 func ResourceStorageCredential() *schema.Resource {
 	s := common.StructToSchema(StorageCredentialInfo{},
 		func(m map[string]*schema.Schema) map[string]*schema.Schema {
-			alof := []string{"aws_iam_role", "azure_service_principal"}
+			alof := []string{"aws_iam_role", "azure_service_principal", "azure_managed_identity"}
 			m["aws_iam_role"].AtLeastOneOf = alof
 			m["azure_service_principal"].AtLeastOneOf = alof
+			m["azure_managed_identity"].AtLeastOneOf = alof
 			return m
 		})
 	update := updateFunctionFactory("/unity-catalog/storage-credentials", []string{"owner", "comment", "aws_iam_role", "azure_service_principal"})

catalog/resource_storage_credentials_test.go

@@ -137,3 +137,44 @@ func TestUpdateStorageCredentials(t *testing.T) {
 		`,
 	}.ApplyNoError(t)
 }
+
+func TestCreateStorageCredentialWithAzMI(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "POST",
+				Resource: "/api/2.0/unity-catalog/storage-credentials",
+				ExpectedRequest: StorageCredentialInfo{
+					Name: "a",
+					AzMI: &AzureManagedIdentity{
+						AccessConnectorID: "def",
+					},
+					Comment: "c",
+				},
+				Response: StorageCredentialInfo{
+					Name: "a",
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/unity-catalog/storage-credentials/a",
+				Response: StorageCredentialInfo{
+					Name: "a",
+					AzMI: &AzureManagedIdentity{
+						AccessConnectorID: "def",
+					},
+					MetastoreID: "d",
+				},
+			},
+		},
+		Resource: ResourceStorageCredential(),
+		Create:   true,
+		HCL: `
+		name = "a"
+		azure_managed_identity {
+			access_connector_id = "def"
+		}
+		comment = "c"
+		`,
+	}.ApplyNoError(t)
+}

docs/resources/metastore_data_access.md

@@ -29,7 +29,7 @@ resource "databricks_metastore_data_access" "this" {
 }
 ```
 
-For Azure
+For Azure using service principal as credential
 
 ```hcl
 resource "databricks_metastore" "this" {
@@ -43,7 +43,7 @@ resource "databricks_metastore" "this" {
 
 resource "databricks_metastore_data_access" "this" {
   metastore_id = databricks_metastore.this.id
-  name         = aws_iam_role.metastore_data_access.name
+  name         = "sp_dac"
   azure_service_principal {
     directory_id   = var.tenant_id
     application_id = azuread_application.unity_catalog.application_id
@@ -53,6 +53,28 @@ resource "databricks_metastore_data_access" "this" {
 }
 ```
 
+For Azure using managed identity as credential (Private Preview)
+
+```hcl
+resource "databricks_metastore" "this" {
+  name = "primary"
+  storage_root = format("abfss://%s@%s.dfs.core.windows.net/",
+    azurerm_storage_account.unity_catalog.name,
+  azurerm_storage_container.unity_catalog.name)
+  owner         = "uc admins"
+  force_destroy = true
+}
+
+resource "databricks_metastore_data_access" "this" {
+  metastore_id = databricks_metastore.this.id
+  name         = "mi_dac"
+  azure_managed_identity {
+    access_connector_id   = var.access_connector_id
+  }
+  is_default = true
+}
+```
+
 ## Argument Reference
 
 The following arguments are required:
@@ -68,6 +90,9 @@ The following arguments are required:
 * `application_id` - The application ID of the application registration within the referenced AAD tenant
 * `client_secret` - The client secret generated for the above app ID in AAD. **This field is redacted on output**
 
+`azure_managed_identity` optional configuration block for using managed identity as credential details for Azure:
+* `access_connector_id` - The Resource ID of the Azure Databricks Access Connector resource, of the form `/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-name/providers/Microsoft.Databricks/accessConnectors/connector-name`
+
 ## Import
 
 -> **Note** Importing this resource is not currently supported.

docs/resources/storage_credential.md

@@ -34,14 +34,22 @@ resource "databricks_grants" "external_creds" {
 For Azure
 
 ```hcl
-resource "databricks_storage_credential" "external" {
+resource "databricks_storage_credential" "external_sp" {
   name = azuread_application.ext_cred.display_name
   azure_service_principal {
     directory_id   = var.tenant_id
     application_id = azuread_application.ext_cred.application_id
     client_secret  = azuread_application_password.ext_cred.value
   }
-  comment = "Managed by TF"
+  comment = "SP credential managed by TF"
+}
+
+resource "databricks_storage_credential" "external_mi" {
+  name = "mi_credential"
+  azure_managed_identity {
+    access_connector_id   = var.access_connector_id
+  }
+  comment = "Managed identity credential managed by TF"
 }
 
 resource "databricks_grants" "external_creds" {
@@ -62,11 +70,13 @@ The following arguments are required:
 `aws_iam_role` optional configuration block for credential details for AWS:
 * `role_arn` - The Amazon Resource Name (ARN) of the AWS IAM role for S3 data access, of the form `arn:aws:iam::1234567890:role/MyRole-AJJHDSKSDF`
 
-`azure_service_principal` optional configuration block for credential details for Azure:
+`azure_service_principal` optional configuration block to use service principal as credential details for Azure:
 * `directory_id` - The directory ID corresponding to the Azure Active Directory (AAD) tenant of the application
 * `application_id` - The application ID of the application registration within the referenced AAD tenant
 * `client_secret` - The client secret generated for the above app ID in AAD. **This field is redacted on output**
-* `owner` - (Optional) Username/groupname/sp application_id storage credential owner.
+
+`azure_managed_identity` optional configuration block for using managed identity as credential details for Azure:
+* `access_connector_id` - The Resource ID of the Azure Databricks Access Connector resource, of the form `/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-name/providers/Microsoft.Databricks/accessConnectors/connector-name`
 
 ## Import