Skip to content

Commit ae10bfe

Browse files
Tian-2017Tian-2017
authored
feat/DL-165: Create a role for Glue catalog metadata ingestion used by the ECS container (#2626)
* feat/DL-165 create a role for the glue metadata ingestion * allow reading all database, but only write to metastore * change the file name to meet the naming convention
1 parent 08a4810 commit ae10bfe

File tree

2 files changed

+206
-2
lines changed

2 files changed

+206
-2
lines changed

‎terraform/core/47-mwaa.tf‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,28 @@ resource "aws_iam_role_policy" "mwaa_assume_role_policy" {
130130
})
131131
}
132132

133+
# To allow the MWAA execution role to pass the cross-department glue metadata role to ECS tasks
134+
resource "aws_iam_role_policy" "mwaa_pass_cross_dept_glue_metadata_role" {
135+
name = "mwaa_pass_cross_dept_glue_metadata_role"
136+
role = aws_iam_role.mwaa_role.id
137+
138+
policy = jsonencode({
139+
Version = "2012-10-17"
140+
Statement = [
141+
{
142+
Effect = "Allow"
143+
Action = "iam:PassRole"
144+
Resource = aws_iam_role.cross_department_glue_metadata_role.arn
145+
Condition = {
146+
StringLike = {
147+
"iam:PassedToService" = "ecs-tasks.amazonaws.com"
148+
}
149+
}
150+
}
151+
]
152+
})
153+
}
154+
133155

134156
# Security group for MWAA - self-referencing and allowing all traffic out
135157
# This is recommended in the doc, Matt recommended at current stage.
@@ -266,5 +288,3 @@ resource "aws_secretsmanager_secret_version" "mwaa_alloy_api_connection" {
266288
ignore_changes = [secret_string]
267289
}
268290
}
269-
270-

‎terraform/core/49-aws-ecs-iam.tf‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
# ==============================================================================
2+
# Cross-Department Glue Metadata Role and Permissions:
3+
# This role is used by ECS tasks to collect metadata from AWS Glue Catalog and S3
4+
# and store it in the metastore database.
5+
# ==============================================================================
6+
7+
# IAM Role for Cross-Department Glue Metadata
8+
resource "aws_iam_role" "cross_department_glue_metadata_role" {
9+
name = "${local.identifier_prefix}-cross-department-glue-metadata-role"
10+
11+
assume_role_policy = jsonencode({
12+
Version = "2012-10-17"
13+
Statement = [
14+
{
15+
Effect = "Allow"
16+
Principal = {
17+
AWS = aws_iam_role.mwaa_role.arn
18+
}
19+
Action = "sts:AssumeRole"
20+
}
21+
]
22+
})
23+
24+
tags = module.tags.values
25+
}
26+
27+
# Glue Permissions Policy
28+
data "aws_iam_policy_document" "cross_dept_glue_metadata_glue_permissions" {
29+
statement {
30+
effect = "Allow"
31+
actions = [
32+
"glue:GetDatabase",
33+
"glue:GetDatabases",
34+
"glue:GetTable",
35+
"glue:GetTables",
36+
"glue:GetPartitions"
37+
]
38+
resources = ["*"]
39+
}
40+
statement {
41+
effect = "Allow"
42+
actions = [
43+
"glue:UpdateTable"
44+
]
45+
resources = [
46+
"arn:aws:glue:${var.aws_deploy_region}:${var.aws_deploy_account_id}:catalog",
47+
"arn:aws:glue:${var.aws_deploy_region}:${var.aws_deploy_account_id}:database/metastore",
48+
"arn:aws:glue:${var.aws_deploy_region}:${var.aws_deploy_account_id}:table/metastore/*"
49+
]
50+
}
51+
}
52+
53+
resource "aws_iam_policy" "cross_dept_glue_metadata_glue_permissions" {
54+
name = "${local.identifier_prefix}-cross-dept-glue-metadata-glue"
55+
policy = data.aws_iam_policy_document.cross_dept_glue_metadata_glue_permissions.json
56+
tags = module.tags.values
57+
}
58+
59+
resource "aws_iam_role_policy_attachment" "cross_dept_glue_metadata_glue_attach" {
60+
role = aws_iam_role.cross_department_glue_metadata_role.name
61+
policy_arn = aws_iam_policy.cross_dept_glue_metadata_glue_permissions.arn
62+
}
63+
64+
# Athena Permissions Policy
65+
data "aws_iam_policy_document" "cross_dept_glue_metadata_athena_permissions" {
66+
statement {
67+
effect = "Allow"
68+
actions = [
69+
"athena:StartQueryExecution",
70+
"athena:GetQueryExecution",
71+
"athena:GetQueryResults",
72+
"athena:ListTableMetadata"
73+
]
74+
resources = ["*"]
75+
}
76+
}
77+
78+
resource "aws_iam_policy" "cross_dept_glue_metadata_athena_permissions" {
79+
name = "${local.identifier_prefix}-cross-dept-glue-metadata-athena"
80+
policy = data.aws_iam_policy_document.cross_dept_glue_metadata_athena_permissions.json
81+
tags = module.tags.values
82+
}
83+
84+
resource "aws_iam_role_policy_attachment" "cross_dept_glue_metadata_athena_attach" {
85+
role = aws_iam_role.cross_department_glue_metadata_role.name
86+
policy_arn = aws_iam_policy.cross_dept_glue_metadata_athena_permissions.arn
87+
}
88+
89+
# S3 Permissions Policy
90+
data "aws_iam_policy_document" "cross_dept_glue_metadata_s3_permissions" {
91+
statement {
92+
effect = "Allow"
93+
actions = [
94+
"s3:GetObject",
95+
"s3:GetBucketLocation",
96+
"s3:ListBucket"
97+
]
98+
resources = ["*"]
99+
}
100+
101+
# Write access only to raw zone and Athena storage (data-and-insight prefix)
102+
statement {
103+
effect = "Allow"
104+
actions = [
105+
"s3:PutObject",
106+
"s3:DeleteObject"
107+
]
108+
resources = [
109+
"${module.raw_zone.bucket_arn}/data-and-insight/dataplatform_metadata/catalog_metadata/*",
110+
"${module.athena_storage.bucket_arn}/data-and-insight/temp/*"
111+
]
112+
}
113+
}
114+
115+
resource "aws_iam_policy" "cross_dept_glue_metadata_s3_permissions" {
116+
name = "${local.identifier_prefix}-cross-dept-glue-metadata-s3"
117+
policy = data.aws_iam_policy_document.cross_dept_glue_metadata_s3_permissions.json
118+
tags = module.tags.values
119+
}
120+
121+
resource "aws_iam_role_policy_attachment" "cross_dept_glue_metadata_s3_attach" {
122+
role = aws_iam_role.cross_department_glue_metadata_role.name
123+
policy_arn = aws_iam_policy.cross_dept_glue_metadata_s3_permissions.arn
124+
}
125+
126+
# KMS Permissions Policy
127+
data "aws_iam_policy_document" "cross_dept_glue_metadata_kms_permissions" {
128+
statement {
129+
effect = "Allow"
130+
actions = [
131+
"kms:Decrypt",
132+
"kms:DescribeKey"
133+
]
134+
resources = ["*"]
135+
}
136+
137+
# Encrypt/GenerateDataKey only for raw zone and Athena storage (for writing)
138+
statement {
139+
effect = "Allow"
140+
actions = [
141+
"kms:Encrypt",
142+
"kms:GenerateDataKey"
143+
]
144+
resources = [
145+
module.raw_zone.kms_key_arn,
146+
module.athena_storage.kms_key_arn
147+
]
148+
}
149+
}
150+
151+
resource "aws_iam_policy" "cross_dept_glue_metadata_kms_permissions" {
152+
name = "${local.identifier_prefix}-cross-dept-glue-metadata-kms"
153+
policy = data.aws_iam_policy_document.cross_dept_glue_metadata_kms_permissions.json
154+
tags = module.tags.values
155+
}
156+
157+
resource "aws_iam_role_policy_attachment" "cross_dept_glue_metadata_kms_attach" {
158+
role = aws_iam_role.cross_department_glue_metadata_role.name
159+
policy_arn = aws_iam_policy.cross_dept_glue_metadata_kms_permissions.arn
160+
}
161+
162+
163+
data "aws_iam_policy_document" "cross_dept_glue_metadata_secrets_permissions" {
164+
statement {
165+
effect = "Allow"
166+
actions = [
167+
"secretsmanager:GetSecretValue"
168+
]
169+
resources = [
170+
"arn:aws:secretsmanager:${var.aws_deploy_region}:${var.aws_deploy_account_id}:secret:airflow/variables/*"
171+
]
172+
}
173+
}
174+
175+
resource "aws_iam_policy" "cross_dept_glue_metadata_secrets_permissions" {
176+
name = "${local.identifier_prefix}-cross-dept-glue-metadata-secrets"
177+
policy = data.aws_iam_policy_document.cross_dept_glue_metadata_secrets_permissions.json
178+
tags = module.tags.values
179+
}
180+
181+
resource "aws_iam_role_policy_attachment" "cross_dept_glue_metadata_secrets_attach" {
182+
role = aws_iam_role.cross_department_glue_metadata_role.name
183+
policy_arn = aws_iam_policy.cross_dept_glue_metadata_secrets_permissions.arn
184+
}

0 commit comments

Comments
 (0)