Skip to content

Commit 119a6c0

Browse files
borremosch-dbborremosch-db
authored
[Feature] Updated AWS UC storage credential to include permissions for file events (#4406)
## Changes [Databricks documentation for storage credentials](https://docs.databricks.com/en/connect/unity-catalog/cloud-storage/storage-credentials.html) contains instructions to add permissions for file events, but as of yet these are missing from the terraform provider. This PR adds them for AWS. PRs for Azure and GCP will follow soon ## Tests Updated test: `aws/data_aws_unity_catalog_policy_test.go` - [x] `make test` run locally - [ ] relevant change in `docs/` folder - [x] covered with integration tests in `internal/acceptance` - [ ] using Go SDK - [ ] using TF Plugin Framework
1 parent b8cf8fb commit 119a6c0

File tree

2 files changed

+280
-0
lines changed

2 files changed

+280
-0
lines changed

‎aws/data_aws_unity_catalog_policy.go‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,62 @@ func generateReadContext(ctx context.Context, d *schema.ResourceData, m *common.
6060
Resources: []string{kmsArn},
6161
})
6262
}
63+
policy.Statements = append(policy.Statements, &awsIamPolicyStatement{
64+
Sid: "ManagedFileEventsSetupStatement",
65+
Effect: "Allow",
66+
Actions: []string{
67+
"s3:GetBucketNotification",
68+
"s3:PutBucketNotification",
69+
"sns:ListSubscriptionsByTopic",
70+
"sns:GetTopicAttributes",
71+
"sns:SetTopicAttributes",
72+
"sns:CreateTopic",
73+
"sns:TagResource",
74+
"sns:Publish",
75+
"sns:Subscribe",
76+
"sqs:CreateQueue",
77+
"sqs:DeleteMessage",
78+
"sqs:ReceiveMessage",
79+
"sqs:SendMessage",
80+
"sqs:GetQueueUrl",
81+
"sqs:GetQueueAttributes",
82+
"sqs:SetQueueAttributes",
83+
"sqs:TagQueue",
84+
"sqs:ChangeMessageVisibility",
85+
"sqs:PurgeQueue",
86+
},
87+
Resources: []string{
88+
fmt.Sprintf("arn:%s:s3:::%s", awsPartition, bucket),
89+
fmt.Sprintf("arn:%s:sqs:*:%s:csms-*", awsPartition, awsAccountId),
90+
fmt.Sprintf("arn:%s:sns:*:%s:csms-*", awsPartition, awsAccountId),
91+
},
92+
},
93+
&awsIamPolicyStatement{
94+
Sid: "ManagedFileEventsListStatement",
95+
Effect: "Allow",
96+
Actions: []string{
97+
"sqs:ListQueues",
98+
"sqs:ListQueueTags",
99+
"sns:ListTopics",
100+
},
101+
Resources: []string{
102+
fmt.Sprintf("arn:%s:sqs:*:%s:csms-*", awsPartition, awsAccountId),
103+
fmt.Sprintf("arn:%s:sns:*:%s:csms-*", awsPartition, awsAccountId),
104+
},
105+
},
106+
&awsIamPolicyStatement{
107+
Sid: "ManagedFileEventsTeardownStatement",
108+
Effect: "Allow",
109+
Actions: []string{
110+
"sns:Unsubscribe",
111+
"sns:DeleteTopic",
112+
"sqs:DeleteQueue",
113+
},
114+
Resources: []string{
115+
fmt.Sprintf("arn:%s:sqs:*:%s:csms-*", awsPartition, awsAccountId),
116+
fmt.Sprintf("arn:%s:sns:*:%s:csms-*", awsPartition, awsAccountId),
117+
},
118+
})
63119
policyJSON, err := json.MarshalIndent(policy, "", " ")
64120
if err != nil {
65121
return err

‎aws/data_aws_unity_catalog_policy_test.go‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,62 @@ func TestDataAwsUnityCatalogPolicy(t *testing.T) {
5959
"Resource": [
6060
"arn:aws:kms:databricks-kms"
6161
]
62+
},
63+
{
64+
"Sid": "ManagedFileEventsSetupStatement",
65+
"Effect": "Allow",
66+
"Action": [
67+
"s3:GetBucketNotification",
68+
"s3:PutBucketNotification",
69+
"sns:ListSubscriptionsByTopic",
70+
"sns:GetTopicAttributes",
71+
"sns:SetTopicAttributes",
72+
"sns:CreateTopic",
73+
"sns:TagResource",
74+
"sns:Publish",
75+
"sns:Subscribe",
76+
"sqs:CreateQueue",
77+
"sqs:DeleteMessage",
78+
"sqs:ReceiveMessage",
79+
"sqs:SendMessage",
80+
"sqs:GetQueueUrl",
81+
"sqs:GetQueueAttributes",
82+
"sqs:SetQueueAttributes",
83+
"sqs:TagQueue",
84+
"sqs:ChangeMessageVisibility",
85+
"sqs:PurgeQueue"
86+
],
87+
"Resource": [
88+
"arn:aws:s3:::databricks-bucket",
89+
"arn:aws:sqs:*:123456789098:csms-*",
90+
"arn:aws:sns:*:123456789098:csms-*"
91+
]
92+
},
93+
{
94+
"Sid": "ManagedFileEventsListStatement",
95+
"Effect": "Allow",
96+
"Action": [
97+
"sqs:ListQueues",
98+
"sqs:ListQueueTags",
99+
"sns:ListTopics"
100+
],
101+
"Resource": [
102+
"arn:aws:sqs:*:123456789098:csms-*",
103+
"arn:aws:sns:*:123456789098:csms-*"
104+
]
105+
},
106+
{
107+
"Sid": "ManagedFileEventsTeardownStatement",
108+
"Effect": "Allow",
109+
"Action": [
110+
"sns:Unsubscribe",
111+
"sns:DeleteTopic",
112+
"sqs:DeleteQueue"
113+
],
114+
"Resource": [
115+
"arn:aws:sqs:*:123456789098:csms-*",
116+
"arn:aws:sns:*:123456789098:csms-*"
117+
]
62118
}
63119
]
64120
}`
@@ -116,6 +172,62 @@ func TestDataAwsUnityCatalogPolicyFullKms(t *testing.T) {
116172
"Resource": [
117173
"arn:aws:kms:us-west-2:111122223333:key/databricks-kms"
118174
]
175+
},
176+
{
177+
"Sid": "ManagedFileEventsSetupStatement",
178+
"Effect": "Allow",
179+
"Action": [
180+
"s3:GetBucketNotification",
181+
"s3:PutBucketNotification",
182+
"sns:ListSubscriptionsByTopic",
183+
"sns:GetTopicAttributes",
184+
"sns:SetTopicAttributes",
185+
"sns:CreateTopic",
186+
"sns:TagResource",
187+
"sns:Publish",
188+
"sns:Subscribe",
189+
"sqs:CreateQueue",
190+
"sqs:DeleteMessage",
191+
"sqs:ReceiveMessage",
192+
"sqs:SendMessage",
193+
"sqs:GetQueueUrl",
194+
"sqs:GetQueueAttributes",
195+
"sqs:SetQueueAttributes",
196+
"sqs:TagQueue",
197+
"sqs:ChangeMessageVisibility",
198+
"sqs:PurgeQueue"
199+
],
200+
"Resource": [
201+
"arn:aws:s3:::databricks-bucket",
202+
"arn:aws:sqs:*:123456789098:csms-*",
203+
"arn:aws:sns:*:123456789098:csms-*"
204+
]
205+
},
206+
{
207+
"Sid": "ManagedFileEventsListStatement",
208+
"Effect": "Allow",
209+
"Action": [
210+
"sqs:ListQueues",
211+
"sqs:ListQueueTags",
212+
"sns:ListTopics"
213+
],
214+
"Resource": [
215+
"arn:aws:sqs:*:123456789098:csms-*",
216+
"arn:aws:sns:*:123456789098:csms-*"
217+
]
218+
},
219+
{
220+
"Sid": "ManagedFileEventsTeardownStatement",
221+
"Effect": "Allow",
222+
"Action": [
223+
"sns:Unsubscribe",
224+
"sns:DeleteTopic",
225+
"sqs:DeleteQueue"
226+
],
227+
"Resource": [
228+
"arn:aws:sqs:*:123456789098:csms-*",
229+
"arn:aws:sns:*:123456789098:csms-*"
230+
]
119231
}
120232
]
121233
}`
@@ -161,6 +273,62 @@ func TestDataAwsUnityCatalogPolicyWithoutKMS(t *testing.T) {
161273
"Resource": [
162274
"arn:aws:iam::123456789098:role/databricks-role"
163275
]
276+
},
277+
{
278+
"Sid": "ManagedFileEventsSetupStatement",
279+
"Effect": "Allow",
280+
"Action": [
281+
"s3:GetBucketNotification",
282+
"s3:PutBucketNotification",
283+
"sns:ListSubscriptionsByTopic",
284+
"sns:GetTopicAttributes",
285+
"sns:SetTopicAttributes",
286+
"sns:CreateTopic",
287+
"sns:TagResource",
288+
"sns:Publish",
289+
"sns:Subscribe",
290+
"sqs:CreateQueue",
291+
"sqs:DeleteMessage",
292+
"sqs:ReceiveMessage",
293+
"sqs:SendMessage",
294+
"sqs:GetQueueUrl",
295+
"sqs:GetQueueAttributes",
296+
"sqs:SetQueueAttributes",
297+
"sqs:TagQueue",
298+
"sqs:ChangeMessageVisibility",
299+
"sqs:PurgeQueue"
300+
],
301+
"Resource": [
302+
"arn:aws:s3:::databricks-bucket",
303+
"arn:aws:sqs:*:123456789098:csms-*",
304+
"arn:aws:sns:*:123456789098:csms-*"
305+
]
306+
},
307+
{
308+
"Sid": "ManagedFileEventsListStatement",
309+
"Effect": "Allow",
310+
"Action": [
311+
"sqs:ListQueues",
312+
"sqs:ListQueueTags",
313+
"sns:ListTopics"
314+
],
315+
"Resource": [
316+
"arn:aws:sqs:*:123456789098:csms-*",
317+
"arn:aws:sns:*:123456789098:csms-*"
318+
]
319+
},
320+
{
321+
"Sid": "ManagedFileEventsTeardownStatement",
322+
"Effect": "Allow",
323+
"Action": [
324+
"sns:Unsubscribe",
325+
"sns:DeleteTopic",
326+
"sqs:DeleteQueue"
327+
],
328+
"Resource": [
329+
"arn:aws:sqs:*:123456789098:csms-*",
330+
"arn:aws:sns:*:123456789098:csms-*"
331+
]
164332
}
165333
]
166334
}`
@@ -219,6 +387,62 @@ func TestDataAwsUnityCatalogPolicyPartionGov(t *testing.T) {
219387
"Resource": [
220388
"arn:aws-us-gov:kms:databricks-kms"
221389
]
390+
},
391+
{
392+
"Sid": "ManagedFileEventsSetupStatement",
393+
"Effect": "Allow",
394+
"Action": [
395+
"s3:GetBucketNotification",
396+
"s3:PutBucketNotification",
397+
"sns:ListSubscriptionsByTopic",
398+
"sns:GetTopicAttributes",
399+
"sns:SetTopicAttributes",
400+
"sns:CreateTopic",
401+
"sns:TagResource",
402+
"sns:Publish",
403+
"sns:Subscribe",
404+
"sqs:CreateQueue",
405+
"sqs:DeleteMessage",
406+
"sqs:ReceiveMessage",
407+
"sqs:SendMessage",
408+
"sqs:GetQueueUrl",
409+
"sqs:GetQueueAttributes",
410+
"sqs:SetQueueAttributes",
411+
"sqs:TagQueue",
412+
"sqs:ChangeMessageVisibility",
413+
"sqs:PurgeQueue"
414+
],
415+
"Resource": [
416+
"arn:aws-us-gov:s3:::databricks-bucket",
417+
"arn:aws-us-gov:sqs:*:123456789098:csms-*",
418+
"arn:aws-us-gov:sns:*:123456789098:csms-*"
419+
]
420+
},
421+
{
422+
"Sid": "ManagedFileEventsListStatement",
423+
"Effect": "Allow",
424+
"Action": [
425+
"sqs:ListQueues",
426+
"sqs:ListQueueTags",
427+
"sns:ListTopics"
428+
],
429+
"Resource": [
430+
"arn:aws-us-gov:sqs:*:123456789098:csms-*",
431+
"arn:aws-us-gov:sns:*:123456789098:csms-*"
432+
]
433+
},
434+
{
435+
"Sid": "ManagedFileEventsTeardownStatement",
436+
"Effect": "Allow",
437+
"Action": [
438+
"sns:Unsubscribe",
439+
"sns:DeleteTopic",
440+
"sqs:DeleteQueue"
441+
],
442+
"Resource": [
443+
"arn:aws-us-gov:sqs:*:123456789098:csms-*",
444+
"arn:aws-us-gov:sns:*:123456789098:csms-*"
445+
]
222446
}
223447
]
224448
}`

0 commit comments

Comments
 (0)