Add delegation support for databricks_storage_credential (#2973)

* refactor storage credential to support delegation

* update guide

* doc

Author: nkvuong

catalog/resource_metastore_data_access.go

@@ -12,10 +12,6 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 )
 
-type AwsIamRole struct {
-	RoleARN string `json:"role_arn"`
-}
-
 type GcpServiceAccountKey struct {
 	Email        string `json:"email"`
 	PrivateKeyId string `json:"private_key_id"`
@@ -41,6 +37,8 @@ func adjustDataAccessSchema(m map[string]*schema.Schema) map[string]*schema.Sche
 	// suppress changes for private_key
 	m["gcp_service_account_key"].DiffSuppressFunc = SuppressGcpSAKeyDiff
 
+	common.MustSchemaPath(m, "aws_iam_role", "external_id").Computed = true
+	common.MustSchemaPath(m, "aws_iam_role", "unity_catalog_iam_arn").Computed = true
 	common.MustSchemaPath(m, "azure_managed_identity", "credential_id").Computed = true
 	common.MustSchemaPath(m, "databricks_gcp_service_account", "email").Computed = true
 	common.MustSchemaPath(m, "databricks_gcp_service_account", "credential_id").Computed = true
@@ -55,10 +53,6 @@ func adjustDataAccessSchema(m map[string]*schema.Schema) map[string]*schema.Sche
 
 var dacSchema = common.StructToSchema(StorageCredentialInfo{},
 	func(m map[string]*schema.Schema) map[string]*schema.Schema {
-		m["metastore_id"] = &schema.Schema{
-			Type:     schema.TypeString,
-			Required: true,
-		}
 		m["is_default"] = &schema.Schema{
 			// having more than one default DAC per metastore will lead
 			// to Terraform re-assigning default_data_access_config_id

catalog/resource_storage_credential.go

@@ -10,16 +10,16 @@ import (
 )
 
 type StorageCredentialInfo struct {
-	Name        string                                       `json:"name" tf:"force_new"`
-	Owner       string                                       `json:"owner,omitempty" tf:"computed"`
-	Comment     string                                       `json:"comment,omitempty"`
-	Aws         AwsIamRole                                   `json:"aws_iam_role,omitempty" tf:"group:access"`
-	Azure       *catalog.AzureServicePrincipal               `json:"azure_service_principal,omitempty" tf:"group:access"`
-	AzMI        *catalog.AzureManagedIdentity                `json:"azure_managed_identity,omitempty" tf:"group:access"`
-	GcpSAKey    *GcpServiceAccountKey                        `json:"gcp_service_account_key,omitempty" tf:"group:access"`
-	DBGcpSA     *catalog.DatabricksGcpServiceAccountResponse `json:"databricks_gcp_service_account,omitempty" tf:"computed"`
-	MetastoreID string                                       `json:"metastore_id,omitempty" tf:"computed"`
-	ReadOnly    bool                                         `json:"read_only,omitempty"`
+	Name                        string                                       `json:"name" tf:"force_new"`
+	Owner                       string                                       `json:"owner,omitempty" tf:"computed"`
+	Comment                     string                                       `json:"comment,omitempty"`
+	Aws                         *catalog.AwsIamRole                          `json:"aws_iam_role,omitempty" tf:"group:access"`
+	Azure                       *catalog.AzureServicePrincipal               `json:"azure_service_principal,omitempty" tf:"group:access"`
+	AzMI                        *catalog.AzureManagedIdentity                `json:"azure_managed_identity,omitempty" tf:"group:access"`
+	GcpSAKey                    *GcpServiceAccountKey                        `json:"gcp_service_account_key,omitempty" tf:"group:access"`
+	DatabricksGcpServiceAccount *catalog.DatabricksGcpServiceAccountResponse `json:"databricks_gcp_service_account,omitempty" tf:"computed"`
+	MetastoreID                 string                                       `json:"metastore_id,omitempty" tf:"computed"`
+	ReadOnly                    bool                                         `json:"read_only,omitempty"`
 }
 
 func removeGcpSaField(originalSchema map[string]*schema.Schema) map[string]*schema.Schema {

catalog/resource_storage_credential_test.go

@@ -34,7 +34,8 @@ func TestCreateStorageCredentials(t *testing.T) {
 				Response: catalog.StorageCredentialInfo{
 					Name: "a",
 					AwsIamRole: &catalog.AwsIamRole{
-						RoleArn: "def",
+						RoleArn:    "def",
+						ExternalId: "123",
 					},
 					MetastoreId: "d",
 				},
@@ -49,7 +50,11 @@ func TestCreateStorageCredentials(t *testing.T) {
 		}
 		comment = "c"
 		`,
-	}.ApplyNoError(t)
+	}.ApplyAndExpectData(t, map[string]any{
+		"aws_iam_role.0.external_id": "123",
+		"aws_iam_role.0.role_arn":    "def",
+		"name":                       "a",
+	})
 }
 
 func TestCreateStorageCredentialWithOwner(t *testing.T) {

docs/guides/unity-catalog.md

@@ -79,7 +79,6 @@ variable "databricks_client_id" {}
 variable "databricks_client_secret" {}
 variable "databricks_account_id" {}
 variable "databricks_workspace_url" {}
-variable "aws_account_id" {}
 
 variable "tags" {
   default = {}
@@ -196,7 +195,31 @@ Unity Catalog introduces two new objects to access and work with external cloud
 - [databricks_storage_credential](../resources/storage_credential.md) represent authentication methods to access cloud storage (e.g. an IAM role for Amazon S3 or a service principal for Azure Storage). Storage credentials are access-controlled to determine which users can use the credential.
 - [databricks_external_location](../resources/external_location.md) are objects that combine a cloud storage path with a Storage Credential that can be used to access the location.
 
-First, create the required objects in AWS.
+First, we need to create the storage credential in Databricks before creating the IAM role in AWS. This is because the external ID of the Databricks storage credential is required in the IAM role trust policy.
+
+```hcl
+data "aws_caller_identity" "current" {}
+
+resource "databricks_storage_credential" "external" {
+  provider = databricks.workspace
+  name     = "${local.prefix}-external-access"
+  aws_iam_role {
+    role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${local.prefix}-uc-access" //cannot reference aws_iam_role directly, as it will create circular dependency
+  }
+  comment = "Managed by TF"
+}
+
+resource "databricks_grants" "external_creds" {
+  provider           = databricks.workspace
+  storage_credential = databricks_storage_credential.external.id
+  grant {
+    principal  = "Data Engineers"
+    privileges = ["CREATE_TABLE"]
+  }
+}
+```
+
+Then we can create the required objects in AWS
 
 ```hcl
 resource "aws_s3_bucket" "external" {
@@ -222,6 +245,36 @@ resource "aws_s3_bucket_public_access_block" "external" {
   depends_on         = [aws_s3_bucket.external]
 }
 
+data "aws_iam_policy_document" "passrole_for_uc" {
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+    principals {
+      identifiers = [databricks_storage_credential.external.aws_iam_role.unity_catalog_iam_arn]
+      type        = "AWS"
+    }
+    condition {
+      test     = "StringEquals"
+      variable = "sts:ExternalId"
+      values   = [databricks_storage_credential.external.aws_iam_role.external_id]
+    }
+  }
+  statement {
+    sid     = "ExplicitSelfRoleAssumption"
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+    condition {
+      test     = "ArnLike"
+      variable = "aws:PrincipalArn"
+      values   = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${local.prefix}-uc-access"]
+    }
+  }
+}
+
 resource "aws_iam_policy" "external_data_access" {
   // Terraform's "jsonencode" function converts a
   // Terraform expression's result to valid JSON syntax.
@@ -262,26 +315,9 @@ resource "aws_iam_role" "external_data_access" {
 }
 ```
 
-Then create the [databricks_storage_credential](../resources/storage_credential.md) and [databricks_external_location](../resources/external_location.md) in Unity Catalog.
+Then we can create the [databricks_external_location](../resources/external_location.md) in Unity Catalog.
 
 ```hcl
-resource "databricks_storage_credential" "external" {
-  provider = databricks.workspace
-  name     = aws_iam_role.external_data_access.name
-  aws_iam_role {
-    role_arn = aws_iam_role.external_data_access.arn
-  }
-  comment = "Managed by TF"
-}
-
-resource "databricks_grants" "external_creds" {
-  provider           = databricks.workspace
-  storage_credential = databricks_storage_credential.external.id
-  grant {
-    principal  = "Data Engineers"
-    privileges = ["CREATE_TABLE"]
-  }
-}
 
 resource "databricks_external_location" "some" {
   provider        = databricks.workspace

docs/resources/grants.md

@@ -29,7 +29,7 @@ Unlike the [SQL specification](https://docs.databricks.com/sql/language-manual/s
 
 ## Metastore grants
 
-You can grant `CREATE_CATALOG`, `CREATE_CONNECTION`, `CREATE_EXTERNAL_LOCATION`, `CREATE_PROVIDER`, `CREATE_RECIPIENT`, `CREATE_SHARE`, `MANAGE_ALLOWLIST`, `SET_SHARE_PERMISSION`, `USE_MARKETPLACE_ASSETS`, `USE_CONNECTION`, `USE_PROVIDER`, `USE_RECIPIENT` and `USE_SHARE` privileges to [databricks_metastore](metastore.md) id specified in `metastore` attribute.
+You can grant `CREATE_CATALOG`, `CREATE_CONNECTION`, `CREATE_EXTERNAL_LOCATION`, `CREATE_PROVIDER`, `CREATE_RECIPIENT`, `CREATE_SHARE`, `CREATE_STORAGE_CREDENTIAL`, `MANAGE_ALLOWLIST`, `SET_SHARE_PERMISSION`, `USE_MARKETPLACE_ASSETS`, `USE_CONNECTION`, `USE_PROVIDER`, `USE_RECIPIENT` and `USE_SHARE` privileges to [databricks_metastore](metastore.md) id specified in `metastore` attribute.
 
 ```hcl
 resource "databricks_grants" "sandbox" {

docs/resources/metastore_data_access.md

@@ -3,7 +3,7 @@ subcategory: "Unity Catalog"
 ---
 # databricks_metastore_data_access (Resource)
 
-Optionally, each [databricks_metastore](docs/resources/metastore.md) can have root storage credential defined as `databricks_metastore_data_access`. This will be used by Unity Catalog to access data in the root storage location if defined.
+Optionally, each [databricks_metastore](docs/resources/metastore.md) can have a default [databricks_storage_credential](storage_credential.md) defined as `databricks_metastore_data_access`. This will be used by Unity Catalog to access data in the root storage location if defined.
 
 ## Example Usage
 
@@ -53,31 +53,9 @@ resource "databricks_metastore_data_access" "this" {
 
 ## Argument Reference
 
-The following arguments are required:
+The arguments are the same as of [databricks_storage_credential](storage_credential.md). Additionally
 
-* `name` - Name of Data Access Configuration, which must be unique within the [databricks_metastore](metastore.md). Change forces creation of a new resource.
-* `metastore_id` - Unique identifier of the parent Metastore
-* `owner` - (Optional) Username/groupname/sp application_id of the data access configuration owner.
-* `force_destroy` - (Optional) Delete the data access configuration regardless of its dependencies.
-
-`aws_iam_role` optional configuration block for credential details for AWS:
-
-* `role_arn` - The Amazon Resource Name (ARN) of the AWS IAM role for S3 data access, of the form `arn:aws:iam::1234567890:role/MyRole-AJJHDSKSDF`
-
-`azure_managed_identity` optional configuration block for using managed identity as credential details for Azure (Recommended):
-
-* `access_connector_id` - The Resource ID of the Azure Databricks Access Connector resource, of the form `/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-name/providers/Microsoft.Databricks/accessConnectors/connector-name`.
-* `managed_identity_id` - (Optional) The Resource ID of the Azure User Assigned Managed Identity associated with Azure Databricks Access Connector, of the form `/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-name/providers/Microsoft.ManagedIdentity/userAssignedIdentities/user-managed-identity-name`.
-
-`databricks_gcp_service_account` optional configuration block for creating a Databricks-managed GCP Service Account:
-
-* `email` (output only) - The email of the GCP service account created, to be granted access to relevant buckets.
-
-`azure_service_principal` optional configuration block for credential details for Azure (Legacy):
-
-* `directory_id` - The directory ID corresponding to the Azure Active Directory (AAD) tenant of the application
-* `application_id` - The application ID of the application registration within the referenced AAD tenant
-* `client_secret` - The client secret generated for the above app ID in AAD. **This field is redacted on output**
+* `is_default` -  whether to set this credential as the default for the metastore. In practice, this should always be true.
 
 ## Attribute Reference
 

docs/resources/storage_credential.md

@@ -74,11 +74,14 @@ The following arguments are required:
 - `name` - Name of Storage Credentials, which must be unique within the [databricks_metastore](metastore.md). Change forces creation of a new resource.
 - `metastore_id` - (Required for account-level) Unique identifier of the parent Metastore
 - `owner` - (Optional) Username/groupname/sp application_id of the storage credential owner.
+- `read_only` - (Optional) Indicates whether the storage credential is only usable for read operations.
 - `force_destroy` - (Optional) Delete storage credential regardless of its dependencies.
 
 `aws_iam_role` optional configuration block for credential details for AWS:
 
 - `role_arn` - The Amazon Resource Name (ARN) of the AWS IAM role for S3 data access, of the form `arn:aws:iam::1234567890:role/MyRole-AJJHDSKSDF`
+- `external_id` (output only) - The external ID used in role assumption to prevent confused deputy problem.
+- `unity_catalog_iam_arn` (output only) - The Amazon Resource Name (ARN) of the AWS IAM user managed by Databricks. This is the identity that is going to assume the AWS IAM role.
 
 `azure_managed_identity` optional configuration block for using managed identity as credential details for Azure (recommended over service principal):
 
@@ -90,8 +93,6 @@ The following arguments are required:
 
 - `email` (output only) - The email of the GCP service account created, to be granted access to relevant buckets.
 
-- `read_only` - (Optional) Indicates whether the storage credential is only usable for read operations.
-
 `azure_service_principal` optional configuration block to use service principal as credential details for Azure (Legacy):
 
 - `directory_id` - The directory ID corresponding to the Azure Active Directory (AAD) tenant of the application