Skip to content

Commit c8c0a32

Browse files
Tian-2017Tian-2017
authored
Feature/DL-114-Create-bucket,-grant-permission,-create-database-for-manual-uploads (#2560)
* create "parking_user_uploads_db" * create bucket * add to compliance * permission to access own departmental prefix * add exclude to both * allow put and delete in the new bucket * remove exclude from second section
1 parent 939c435 commit c8c0a32

File tree

7 files changed

+80
-7
lines changed

7 files changed

+80
-7
lines changed

‎terraform/compliance/s3.feature‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ Feature: S3
1212
@exclude_module.file_sync_destination_nec.aws_s3_bucket.bucket
1313
@exclude_module.file_sync_destination_nec.aws_s3_bucket.log_bucket
1414
@exclude_module.arcus_data_storage.aws_s3_bucket.bucket
15+
@exclude_module.user_uploads.aws_s3_bucket.bucket
1516

1617
# This rule is in place for legacy buckets created with the deprecated block within the aws_s3_bucket resource
1718
Scenario: Data must be encrypted at rest for buckets created using server_side_encryption_configuration property within bucket resource

‎terraform/core/05-departments.tf‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ module "department_housing_repairs" {
3939
google_group_admin_display_name = local.google_group_admin_display_name
4040
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
4141
mwaa_key_arn = aws_kms_key.mwaa_key.arn
42+
user_uploads_bucket = module.user_uploads
4243
}
4344

4445
module "department_parking" {
@@ -73,12 +74,14 @@ module "department_parking" {
7374
departmental_airflow_user = true
7475
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
7576
mwaa_key_arn = aws_kms_key.mwaa_key.arn
77+
user_uploads_bucket = module.user_uploads
7678
additional_glue_database_access = {
7779
read_only = [
7880
"${local.identifier_prefix}-liberator-raw-zone",
7981
"${local.identifier_prefix}-liberator-refined-zone",
8082
"${local.identifier_prefix}-liberator-trusted-zone",
8183
"parking-ringgo-sftp-raw-zone",
84+
"parking_user_uploads_db",
8285
]
8386
read_write = []
8487
}
@@ -114,6 +117,7 @@ module "department_finance" {
114117
google_group_admin_display_name = local.google_group_admin_display_name
115118
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
116119
mwaa_key_arn = aws_kms_key.mwaa_key.arn
120+
user_uploads_bucket = module.user_uploads
117121
}
118122

119123
module "department_data_and_insight" {
@@ -148,6 +152,7 @@ module "department_data_and_insight" {
148152
departmental_airflow_user = true
149153
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
150154
mwaa_key_arn = aws_kms_key.mwaa_key.arn
155+
user_uploads_bucket = module.user_uploads
151156
cloudtrail_bucket = module.cloudtrail_storage
152157
additional_glue_database_access = {
153158
read_only = []
@@ -193,6 +198,7 @@ module "department_env_enforcement" {
193198
google_group_admin_display_name = local.google_group_admin_display_name
194199
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
195200
mwaa_key_arn = aws_kms_key.mwaa_key.arn
201+
user_uploads_bucket = module.user_uploads
196202
}
197203

198204
module "department_planning" {
@@ -226,6 +232,7 @@ module "department_planning" {
226232
google_group_display_name = "[email protected]"
227233
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
228234
mwaa_key_arn = aws_kms_key.mwaa_key.arn
235+
user_uploads_bucket = module.user_uploads
229236
additional_glue_database_access = {
230237
read_only = ["${local.identifier_prefix}-tascomi*"]
231238
read_write = []
@@ -263,6 +270,7 @@ module "department_unrestricted" {
263270
departmental_airflow_user = true
264271
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
265272
mwaa_key_arn = aws_kms_key.mwaa_key.arn
273+
user_uploads_bucket = module.user_uploads
266274
}
267275

268276
module "department_sandbox" {
@@ -296,6 +304,7 @@ module "department_sandbox" {
296304
google_group_display_name = "[email protected]"
297305
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
298306
mwaa_key_arn = aws_kms_key.mwaa_key.arn
307+
user_uploads_bucket = module.user_uploads
299308
}
300309

301310
module "department_benefits_and_housing_needs" {
@@ -330,6 +339,7 @@ module "department_benefits_and_housing_needs" {
330339
departmental_airflow_user = true
331340
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
332341
mwaa_key_arn = aws_kms_key.mwaa_key.arn
342+
user_uploads_bucket = module.user_uploads
333343
additional_glue_database_access = {
334344
read_only = ["hben_raw_zone"]
335345
read_write = []
@@ -368,6 +378,7 @@ module "department_revenues" {
368378
departmental_airflow_user = true
369379
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
370380
mwaa_key_arn = aws_kms_key.mwaa_key.arn
381+
user_uploads_bucket = module.user_uploads
371382
additional_glue_database_access = {
372383
read_only = [
373384
"nndr_raw_zone",
@@ -409,6 +420,7 @@ module "department_environmental_services" {
409420
departmental_airflow_user = true
410421
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
411422
mwaa_key_arn = aws_kms_key.mwaa_key.arn
423+
user_uploads_bucket = module.user_uploads
412424
}
413425

414426
module "department_housing" {
@@ -443,6 +455,7 @@ module "department_housing" {
443455
departmental_airflow_user = true
444456
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
445457
mwaa_key_arn = aws_kms_key.mwaa_key.arn
458+
user_uploads_bucket = module.user_uploads
446459
additional_s3_access = [
447460
{
448461
bucket_arn = module.housing_nec_migration_storage.bucket_arn
@@ -500,6 +513,7 @@ module "department_children_and_education" {
500513
google_group_display_name = "saml-aws-data-platform-collaborator-children-and-family-services@hackney.gov.uk"
501514
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
502515
mwaa_key_arn = aws_kms_key.mwaa_key.arn
516+
user_uploads_bucket = module.user_uploads
503517
}
504518

505519
module "department_customer_services" {
@@ -533,6 +547,7 @@ module "department_customer_services" {
533547
google_group_display_name = "saml-aws-data-platform-collaborator-customer-services@hackney.gov.uk"
534548
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
535549
mwaa_key_arn = aws_kms_key.mwaa_key.arn
550+
user_uploads_bucket = module.user_uploads
536551
}
537552

538553
module "department_hr_and_od" {
@@ -566,6 +581,7 @@ module "department_hr_and_od" {
566581
google_group_display_name = "[email protected]"
567582
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
568583
mwaa_key_arn = aws_kms_key.mwaa_key.arn
584+
user_uploads_bucket = module.user_uploads
569585
}
570586

571587
module "department_streetscene" {
@@ -600,6 +616,7 @@ module "department_streetscene" {
600616
departmental_airflow_user = true
601617
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
602618
mwaa_key_arn = aws_kms_key.mwaa_key.arn
619+
user_uploads_bucket = module.user_uploads
603620
}
604621

605622
module "department_children_family_services" {
@@ -634,6 +651,7 @@ module "department_children_family_services" {
634651
departmental_airflow_user = true
635652
mwaa_etl_scripts_bucket_arn = aws_s3_bucket.mwaa_etl_scripts_bucket.arn
636653
mwaa_key_arn = aws_kms_key.mwaa_key.arn
654+
user_uploads_bucket = module.user_uploads
637655
additional_glue_database_access = {
638656
read_only = ["child_edu_refined", "hackney_casemanagement_live", "hackney_synergy_live"]
639657
read_write = []

‎terraform/core/10-aws-s3-utility-buckets.tf‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,22 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "addresses_api_rds
148148
provider = aws.aws_api_account
149149
}
150150

151+
#===============================================================================
152+
# User Uploads Bucket
153+
#===============================================================================
154+
155+
module "user_uploads" {
156+
source = "../modules/s3-bucket"
157+
158+
tags = module.tags.values
159+
project = var.project
160+
environment = var.environment
161+
identifier_prefix = local.identifier_prefix
162+
bucket_name = "user-uploads"
163+
bucket_identifier = "user-uploads"
164+
include_backup_policy_tags = false
165+
}
166+
151167
#===============================================================================
152168
# MWAA Buckets
153169
#===============================================================================

‎terraform/etl/61-aws-glue-catalog-database.tf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,3 +85,11 @@ resource "aws_glue_catalog_database" "arcus_archive" {
8585
prevent_destroy = true
8686
}
8787
}
88+
89+
resource "aws_glue_catalog_database" "parking_user_uploads" {
90+
name = "parking_user_uploads_db"
91+
92+
lifecycle {
93+
prevent_destroy = true
94+
}
95+
}

‎terraform/modules/department/01-inputs-required.tf‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,3 +140,12 @@ variable "mwaa_etl_scripts_bucket_arn" {
140140
variable "mwaa_key_arn" {
141141
type = string
142142
}
143+
144+
variable "user_uploads_bucket" {
145+
description = "User uploads S3 bucket"
146+
type = object({
147+
bucket_id = string
148+
bucket_arn = string
149+
kms_key_arn = string
150+
})
151+
}

‎terraform/modules/department/02-inputs-optional.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,13 +72,13 @@ variable "additional_glue_database_access" {
7272
description = <<EOF
7373
Additional Glue database access to grant to the department.
7474
Databases are grouped by permission level.
75-
75+
7676
Example:
7777
additional_glue_database_access = {
7878
read_only = ["database1", "database2*"]
7979
read_write = ["database3"]
8080
}
81-
81+
8282
Permission levels:
8383
- "read_only": Grants Get* and BatchGet* permissions (for reading databases, tables, partitions)
8484
- "read_write": Grants Get*, BatchGet*, Create*, Update*, Delete*, BatchCreate*, BatchUpdate*, BatchDelete* permissions

‎terraform/modules/department/50-aws-iam-policies.tf‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@ data "aws_iam_policy_document" "read_only_s3_department_access" {
6060
var.trusted_zone_bucket.kms_key_arn,
6161
var.athena_storage_bucket.kms_key_arn,
6262
var.glue_scripts_bucket.kms_key_arn,
63-
var.spark_ui_output_storage_bucket.kms_key_arn
63+
var.spark_ui_output_storage_bucket.kms_key_arn,
64+
var.user_uploads_bucket.kms_key_arn
6465
]
6566
}
6667

@@ -91,7 +92,6 @@ data "aws_iam_policy_document" "read_only_s3_department_access" {
9192
"s3:List*",
9293
]
9394
resources = [
94-
9595
var.athena_storage_bucket.bucket_arn,
9696
"${var.athena_storage_bucket.bucket_arn}/${local.department_identifier}/*",
9797

@@ -120,7 +120,10 @@ data "aws_iam_policy_document" "read_only_s3_department_access" {
120120
"${var.trusted_zone_bucket.bucket_arn}/unrestricted/*",
121121

122122
var.spark_ui_output_storage_bucket.bucket_arn,
123-
"${var.spark_ui_output_storage_bucket.bucket_arn}/${local.department_identifier}/*"
123+
"${var.spark_ui_output_storage_bucket.bucket_arn}/${local.department_identifier}/*",
124+
125+
var.user_uploads_bucket.bucket_arn,
126+
"${var.user_uploads_bucket.bucket_arn}/${local.department_identifier}/*"
124127
]
125128
}
126129

@@ -158,6 +161,18 @@ data "aws_iam_policy_document" "read_only_s3_department_access" {
158161
"${var.landing_zone_bucket.bucket_arn}/${local.department_identifier}/manual/*",
159162
]
160163
}
164+
165+
statement {
166+
sid = "S3WriteToUserUploads"
167+
effect = "Allow"
168+
actions = [
169+
"s3:Put*",
170+
"s3:Delete*"
171+
]
172+
resources = [
173+
"${var.user_uploads_bucket.bucket_arn}/${local.department_identifier}/*",
174+
]
175+
}
161176
}
162177

163178
resource "aws_iam_policy" "read_only_s3_access" {
@@ -276,7 +291,8 @@ data "aws_iam_policy_document" "s3_department_access" {
276291
var.glue_scripts_bucket.kms_key_arn,
277292
var.spark_ui_output_storage_bucket.kms_key_arn,
278293
var.glue_temp_storage_bucket.kms_key_arn,
279-
var.mwaa_key_arn
294+
var.mwaa_key_arn,
295+
var.user_uploads_bucket.kms_key_arn
280296
]
281297
}
282298

@@ -344,7 +360,10 @@ data "aws_iam_policy_document" "s3_department_access" {
344360
var.mwaa_etl_scripts_bucket_arn,
345361
"${var.mwaa_etl_scripts_bucket_arn}/${replace(local.department_identifier, "-", "_")}/*",
346362
"${var.mwaa_etl_scripts_bucket_arn}/unrestricted/*",
347-
"${var.mwaa_etl_scripts_bucket_arn}/shared/*"
363+
"${var.mwaa_etl_scripts_bucket_arn}/shared/*",
364+
365+
var.user_uploads_bucket.bucket_arn,
366+
"${var.user_uploads_bucket.bucket_arn}/${local.department_identifier}/*"
348367
]
349368
}
350369

@@ -401,6 +420,8 @@ data "aws_iam_policy_document" "s3_department_access" {
401420

402421
"${var.trusted_zone_bucket.bucket_arn}/${local.department_identifier}/*",
403422
"${var.trusted_zone_bucket.bucket_arn}/quality-metrics/department=${local.department_identifier}/*",
423+
424+
"${var.user_uploads_bucket.bucket_arn}/${local.department_identifier}/*"
404425
]
405426
}
406427

0 commit comments

Comments
 (0)