Skip to content

Commit 2dd4f7c

Browse files
840840
authored
Add databricks_aws_unity_catalog_policy data source (#2483)
* WIP Add `data_aws_unity_catalog_policy` data source * Create `aws_unity_catalog_policy` data with option to provide KMS key * Add tests for `databricks_aws_unity_catalog_policy` * Add documentation for `databricks_aws_unity_catalog_policy` * Refactor `databricks_aws_unity_catalog_policy`, remove redundant test, made ID derive from parameters * Use evaluated conditional to pass role * Revert "Use evaluated conditional to pass role" This reverts commit 272d548. * Use `common.Resource` * Update docs * Correct indentiation for tests * Remove redundant statements in policy * Improve tests
1 parent 622d36f commit 2dd4f7c

File tree

4 files changed

+344
-40
lines changed

4 files changed

+344
-40
lines changed

‎aws/data_aws_unity_catalog_policy.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package aws
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"regexp"
8+
9+
"github.com/databricks/terraform-provider-databricks/common"
10+
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
11+
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
12+
)
13+
14+
func generateReadContext(ctx context.Context, d *schema.ResourceData, m *common.DatabricksClient) error {
15+
bucket := d.Get("bucket_name").(string)
16+
awsAccountId := d.Get("aws_account_id").(string)
17+
roleName := d.Get("role_name").(string)
18+
policy := awsIamPolicy{
19+
Version: "2012-10-17",
20+
Statements: []*awsIamPolicyStatement{
21+
{
22+
Effect: "Allow",
23+
Actions: []string{
24+
"s3:GetObject",
25+
"s3:PutObject",
26+
"s3:DeleteObject",
27+
"s3:ListBucket",
28+
"s3:GetBucketLocation",
29+
},
30+
Resources: []string{
31+
fmt.Sprintf("arn:aws:s3:::%s/*", bucket),
32+
fmt.Sprintf("arn:aws:s3:::%s", bucket),
33+
},
34+
},
35+
{
36+
Effect: "Allow",
37+
Actions: []string{
38+
"sts:AssumeRole",
39+
},
40+
Resources: []string{
41+
fmt.Sprintf("arn:aws:iam::%s:role/%s", awsAccountId, roleName),
42+
},
43+
},
44+
},
45+
}
46+
if kmsKey, ok := d.GetOk("kms_name"); ok {
47+
policy.Statements = append(policy.Statements, &awsIamPolicyStatement{
48+
Effect: "Allow",
49+
Actions: []string{
50+
"kms:Decrypt",
51+
"kms:Encrypt",
52+
"kms:GenerateDataKey*",
53+
},
54+
Resources: []string{
55+
fmt.Sprintf("arn:aws:kms:%s", kmsKey),
56+
},
57+
})
58+
}
59+
policyJSON, err := json.MarshalIndent(policy, "", " ")
60+
if err != nil {
61+
return err
62+
}
63+
d.SetId(fmt.Sprintf("%s-%s-%s", bucket, awsAccountId, roleName))
64+
err = d.Set("json", string(policyJSON))
65+
if err != nil {
66+
return err
67+
}
68+
return nil
69+
}
70+
71+
func validateSchema() map[string]*schema.Schema {
72+
return map[string]*schema.Schema{
73+
"kms_name": {
74+
Type: schema.TypeString,
75+
Optional: true,
76+
ValidateFunc: validation.StringMatch(
77+
regexp.MustCompile(`^[0-9a-zA-Z/_-]+$`),
78+
"must contain only alphanumeric, hyphens, forward slashes, and underscores characters"),
79+
},
80+
"bucket_name": {
81+
Type: schema.TypeString,
82+
Required: true,
83+
ValidateFunc: validation.StringMatch(
84+
regexp.MustCompile(`^[0-9a-zA-Z_-]+$`),
85+
"must contain only alphanumeric, underscore, and hyphen characters"),
86+
},
87+
"role_name": {
88+
Type: schema.TypeString,
89+
Required: true,
90+
},
91+
"aws_account_id": {
92+
Type: schema.TypeString,
93+
Required: true,
94+
},
95+
"json": {
96+
Type: schema.TypeString,
97+
Computed: true,
98+
},
99+
}
100+
}
101+
102+
func DataAwsUnityCatalogPolicy() common.Resource {
103+
return common.Resource{
104+
Read: generateReadContext,
105+
Schema: validateSchema(),
106+
}
107+
}

‎aws/data_aws_unity_catalog_policy_test.go‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package aws
2+
3+
import (
4+
"encoding/json"
5+
"testing"
6+
7+
"github.com/databricks/terraform-provider-databricks/qa"
8+
"github.com/stretchr/testify/assert"
9+
)
10+
11+
func TestDataAwsUnityCatalogPolicy(t *testing.T) {
12+
d, err := qa.ResourceFixture{
13+
Read: true,
14+
Resource: DataAwsUnityCatalogPolicy(),
15+
NonWritable: true,
16+
ID: ".",
17+
HCL: `
18+
aws_account_id = "123456789098"
19+
bucket_name = "databricks-bucket"
20+
role_name = "databricks-role"
21+
kms_name = "databricks-kms"
22+
`,
23+
}.Apply(t)
24+
assert.NoError(t, err)
25+
j := d.Get("json").(string)
26+
p := `{
27+
"Version": "2012-10-17",
28+
"Statement": [
29+
{
30+
"Effect": "Allow",
31+
"Action": [
32+
"s3:GetObject",
33+
"s3:PutObject",
34+
"s3:DeleteObject",
35+
"s3:ListBucket",
36+
"s3:GetBucketLocation"
37+
],
38+
"Resource": [
39+
"arn:aws:s3:::databricks-bucket/*",
40+
"arn:aws:s3:::databricks-bucket"
41+
]
42+
},
43+
{
44+
"Effect": "Allow",
45+
"Action": [
46+
"sts:AssumeRole"
47+
],
48+
"Resource": [
49+
"arn:aws:iam::123456789098:role/databricks-role"
50+
]
51+
},
52+
{
53+
"Effect": "Allow",
54+
"Action": [
55+
"kms:Decrypt",
56+
"kms:Encrypt",
57+
"kms:GenerateDataKey*"
58+
],
59+
"Resource": [
60+
"arn:aws:kms:databricks-kms"
61+
]
62+
}
63+
]
64+
}`
65+
compareJSON(t, j, p)
66+
}
67+
68+
func TestDataAwsUnityCatalogPolicyWithoutKMS(t *testing.T) {
69+
d, err := qa.ResourceFixture{
70+
Read: true,
71+
Resource: DataAwsUnityCatalogPolicy(),
72+
NonWritable: true,
73+
ID: ".",
74+
HCL: `
75+
aws_account_id = "123456789098"
76+
bucket_name = "databricks-bucket"
77+
role_name = "databricks-role"
78+
`,
79+
}.Apply(t)
80+
assert.NoError(t, err)
81+
j := d.Get("json").(string)
82+
p := `{
83+
"Version": "2012-10-17",
84+
"Statement": [
85+
{
86+
"Effect": "Allow",
87+
"Action": [
88+
"s3:GetObject",
89+
"s3:PutObject",
90+
"s3:DeleteObject",
91+
"s3:ListBucket",
92+
"s3:GetBucketLocation"
93+
],
94+
"Resource": [
95+
"arn:aws:s3:::databricks-bucket/*",
96+
"arn:aws:s3:::databricks-bucket"
97+
]
98+
},
99+
{
100+
"Effect": "Allow",
101+
"Action": [
102+
"sts:AssumeRole"
103+
],
104+
"Resource": [
105+
"arn:aws:iam::123456789098:role/databricks-role"
106+
]
107+
}
108+
]
109+
}`
110+
compareJSON(t, j, p)
111+
}
112+
113+
func compareJSON(t *testing.T, json1 string, json2 string) {
114+
var i1 interface{}
115+
var i2 interface{}
116+
err := json.Unmarshal([]byte(json1), &i1)
117+
assert.NoError(t, err, "error while unmarshalling")
118+
err = json.Unmarshal([]byte(json2), &i2)
119+
assert.NoError(t, err, "error while unmarshalling")
120+
assert.Equal(t, i1, i2)
121+
}

‎docs/data-sources/aws_unity_catalog_policy.md‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
---
2+
subcategory: "Deployment"
3+
---
4+
# databricks_aws_unity_catalog_policy Data Source
5+
6+
-> **Note** This resource has an evolving API, which may change in future versions of the provider. Please always consult [latest documentation](https://docs.databricks.com/administration-guide/account-api/iam-role.html#language-Your%C2%A0VPC,%C2%A0default) in case of any questions.
7+
8+
This data source constructs necessary AWS Unity Catalog policy for you, which is based on [official documentation](https://docs.databricks.com/data-governance/unity-catalog/get-started.html#configure-a-storage-bucket-and-iam-role-in-aws).
9+
10+
## Example Usage
11+
12+
```hcl
13+
data "databricks_aws_unity_catalog_policy" "this" {
14+
aws_account_id = var.aws_account_id
15+
bucket_name = "databricks-bucket"
16+
role_name = "databricks-role"
17+
kms_name = "databricks-kms"
18+
}
19+
20+
data "aws_iam_policy_document" "passrole_for_uc" {
21+
statement {
22+
effect = "Allow"
23+
actions = ["sts:AssumeRole"]
24+
principals {
25+
identifiers = [
26+
"arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL" # Databricks Account ID
27+
]
28+
type = "AWS"
29+
}
30+
condition {
31+
test = "StringEquals"
32+
variable = "sts:ExternalId"
33+
values = [var.databricks_account_id]
34+
}
35+
}
36+
statement {
37+
sid = "ExplicitSelfRoleAssumption"
38+
effect = "Allow"
39+
actions = ["sts:AssumeRole"]
40+
principals {
41+
type = "AWS"
42+
identifiers = ["arn:aws:iam::${var.aws_account_id}:root"]
43+
}
44+
condition {
45+
test = "ArnLike"
46+
variable = "aws:PrincipalArn"
47+
values = ["arn:aws:iam::${var.aws_account_id}:role/${var.prefix}-uc-access"]
48+
}
49+
}
50+
}
51+
52+
resource "aws_iam_policy" "unity_metastore" {
53+
name = "${var.prefix}-unity-catalog-metastore-access-iam-policy"
54+
policy = data.databricks_aws_unity_catalog_policy.this.json
55+
}
56+
57+
resource "aws_iam_role" "metastore_data_access" {
58+
name = "${var.prefix}-uc-access"
59+
assume_role_policy = data.aws_iam_policy_document.passrole_for_uc.json
60+
managed_policy_arns = [aws_iam_policy.unity_metastore.arn]
61+
}
62+
```
63+
64+
## Argument Reference
65+
66+
* `aws_account_id` (Required) The Account ID of the current AWS account (not your Databricks account).
67+
* `bucket_name` (Required) The name of the S3 bucket used as root storage location for [managed tables](https://docs.databricks.com/data-governance/unity-catalog/index.html#managed-table) in Unity Catalog.
68+
* `role_name` (Required) The name of the AWS IAM role that you created in the previous step in the [official documentation](https://docs.databricks.com/data-governance/unity-catalog/get-started.html#configure-a-storage-bucket-and-iam-role-in-aws).
69+
* `kms_name` (Optional) If encryption is enabled, provide the name of the KMS key that encrypts the S3 bucket contents. If encryption is disabled, do not provide this argument.
70+
71+
## Attribute Reference
72+
73+
In addition to all arguments above, the following attributes are exported:
74+
75+
* `json` - AWS IAM Policy JSON document

‎provider/provider.go‎

Lines changed: 41 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -52,46 +52,47 @@ func init() {
5252
func DatabricksProvider() *schema.Provider {
5353
p := &schema.Provider{
5454
DataSourcesMap: map[string]*schema.Resource{ // must be in alphabetical order
55-
"databricks_aws_crossaccount_policy": aws.DataAwsCrossaccountPolicy().ToResource(),
56-
"databricks_aws_assume_role_policy": aws.DataAwsAssumeRolePolicy().ToResource(),
57-
"databricks_aws_bucket_policy": aws.DataAwsBucketPolicy().ToResource(),
58-
"databricks_cluster": clusters.DataSourceCluster().ToResource(),
59-
"databricks_clusters": clusters.DataSourceClusters().ToResource(),
60-
"databricks_cluster_policy": policies.DataSourceClusterPolicy().ToResource(),
61-
"databricks_catalogs": catalog.DataSourceCatalogs().ToResource(),
62-
"databricks_current_config": mws.DataSourceCurrentConfiguration().ToResource(),
63-
"databricks_current_metastore": catalog.DataSourceCurrentMetastore().ToResource(),
64-
"databricks_current_user": scim.DataSourceCurrentUser().ToResource(),
65-
"databricks_dbfs_file": storage.DataSourceDbfsFile().ToResource(),
66-
"databricks_dbfs_file_paths": storage.DataSourceDbfsFilePaths().ToResource(),
67-
"databricks_directory": workspace.DataSourceDirectory().ToResource(),
68-
"databricks_group": scim.DataSourceGroup().ToResource(),
69-
"databricks_instance_pool": pools.DataSourceInstancePool().ToResource(),
70-
"databricks_instance_profiles": aws.DataSourceInstanceProfiles().ToResource(),
71-
"databricks_jobs": jobs.DataSourceJobs().ToResource(),
72-
"databricks_job": jobs.DataSourceJob().ToResource(),
73-
"databricks_metastore": catalog.DataSourceMetastore().ToResource(),
74-
"databricks_metastores": catalog.DataSourceMetastores().ToResource(),
75-
"databricks_mlflow_model": mlflow.DataSourceModel().ToResource(),
76-
"databricks_mws_credentials": mws.DataSourceMwsCredentials().ToResource(),
77-
"databricks_mws_workspaces": mws.DataSourceMwsWorkspaces().ToResource(),
78-
"databricks_node_type": clusters.DataSourceNodeType().ToResource(),
79-
"databricks_notebook": workspace.DataSourceNotebook().ToResource(),
80-
"databricks_notebook_paths": workspace.DataSourceNotebookPaths().ToResource(),
81-
"databricks_pipelines": pipelines.DataSourcePipelines().ToResource(),
82-
"databricks_schemas": catalog.DataSourceSchemas().ToResource(),
83-
"databricks_service_principal": scim.DataSourceServicePrincipal().ToResource(),
84-
"databricks_service_principals": scim.DataSourceServicePrincipals().ToResource(),
85-
"databricks_share": catalog.DataSourceShare().ToResource(),
86-
"databricks_shares": catalog.DataSourceShares().ToResource(),
87-
"databricks_spark_version": clusters.DataSourceSparkVersion().ToResource(),
88-
"databricks_sql_warehouse": sql.DataSourceWarehouse().ToResource(),
89-
"databricks_sql_warehouses": sql.DataSourceWarehouses().ToResource(),
90-
"databricks_tables": catalog.DataSourceTables().ToResource(),
91-
"databricks_views": catalog.DataSourceViews().ToResource(),
92-
"databricks_volumes": catalog.DataSourceVolumes().ToResource(),
93-
"databricks_user": scim.DataSourceUser().ToResource(),
94-
"databricks_zones": clusters.DataSourceClusterZones().ToResource(),
55+
"databricks_aws_crossaccount_policy": aws.DataAwsCrossaccountPolicy().ToResource(),
56+
"databricks_aws_assume_role_policy": aws.DataAwsAssumeRolePolicy().ToResource(),
57+
"databricks_aws_bucket_policy": aws.DataAwsBucketPolicy().ToResource(),
58+
"databricks_aws_unity_catalog_policy": aws.DataAwsUnityCatalogPolicy().ToResource(),
59+
"databricks_cluster": clusters.DataSourceCluster().ToResource(),
60+
"databricks_clusters": clusters.DataSourceClusters().ToResource(),
61+
"databricks_cluster_policy": policies.DataSourceClusterPolicy().ToResource(),
62+
"databricks_catalogs": catalog.DataSourceCatalogs().ToResource(),
63+
"databricks_current_config": mws.DataSourceCurrentConfiguration().ToResource(),
64+
"databricks_current_metastore": catalog.DataSourceCurrentMetastore().ToResource(),
65+
"databricks_current_user": scim.DataSourceCurrentUser().ToResource(),
66+
"databricks_dbfs_file": storage.DataSourceDbfsFile().ToResource(),
67+
"databricks_dbfs_file_paths": storage.DataSourceDbfsFilePaths().ToResource(),
68+
"databricks_directory": workspace.DataSourceDirectory().ToResource(),
69+
"databricks_group": scim.DataSourceGroup().ToResource(),
70+
"databricks_instance_pool": pools.DataSourceInstancePool().ToResource(),
71+
"databricks_instance_profiles": aws.DataSourceInstanceProfiles().ToResource(),
72+
"databricks_jobs": jobs.DataSourceJobs().ToResource(),
73+
"databricks_job": jobs.DataSourceJob().ToResource(),
74+
"databricks_metastore": catalog.DataSourceMetastore().ToResource(),
75+
"databricks_metastores": catalog.DataSourceMetastores().ToResource(),
76+
"databricks_mlflow_model": mlflow.DataSourceModel().ToResource(),
77+
"databricks_mws_credentials": mws.DataSourceMwsCredentials().ToResource(),
78+
"databricks_mws_workspaces": mws.DataSourceMwsWorkspaces().ToResource(),
79+
"databricks_node_type": clusters.DataSourceNodeType().ToResource(),
80+
"databricks_notebook": workspace.DataSourceNotebook().ToResource(),
81+
"databricks_notebook_paths": workspace.DataSourceNotebookPaths().ToResource(),
82+
"databricks_pipelines": pipelines.DataSourcePipelines().ToResource(),
83+
"databricks_schemas": catalog.DataSourceSchemas().ToResource(),
84+
"databricks_service_principal": scim.DataSourceServicePrincipal().ToResource(),
85+
"databricks_service_principals": scim.DataSourceServicePrincipals().ToResource(),
86+
"databricks_share": catalog.DataSourceShare().ToResource(),
87+
"databricks_shares": catalog.DataSourceShares().ToResource(),
88+
"databricks_spark_version": clusters.DataSourceSparkVersion().ToResource(),
89+
"databricks_sql_warehouse": sql.DataSourceWarehouse().ToResource(),
90+
"databricks_sql_warehouses": sql.DataSourceWarehouses().ToResource(),
91+
"databricks_tables": catalog.DataSourceTables().ToResource(),
92+
"databricks_views": catalog.DataSourceViews().ToResource(),
93+
"databricks_volumes": catalog.DataSourceVolumes().ToResource(),
94+
"databricks_user": scim.DataSourceUser().ToResource(),
95+
"databricks_zones": clusters.DataSourceClusterZones().ToResource(),
9596
},
9697
ResourcesMap: map[string]*schema.Resource{ // must be in alphabetical order
9798
"databricks_access_control_rule_set": permissions.ResourceAccessControlRuleSet().ToResource(),

0 commit comments

Comments
 (0)