Skip to content

Commit 3e901cc

Browse files
mmg-nhsemmg-nhse
authored
feat(athena): FDOS-614 Add Athena stack with DynamoDB and RDS connectors (#833)
* feat(athena): FDOS-614 Add Athena stack with DynamoDB and RDS connectors * feat(athena): FDOS-614 Add serverlessrepo permissions for SAR deployments * feat(athena): FDOS-614 Update IAM policy for enhanced permissions and resource specificity * feat(athena): FDOS-614 Add KMS encryption for Athena resources and update IAM policies
1 parent 019a83e commit 3e901cc

File tree

21 files changed

+518
-15
lines changed

21 files changed

+518
-15
lines changed

‎.github/actions/derive-workspace/action.yaml‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,25 @@ runs:
1212
- name: "Derive workspace"
1313
id: "derive-workspace"
1414
shell: bash
15+
env:
16+
GITHUB_REF_TYPE: ${{ github.ref_type }}
17+
GITHUB_REF_NAME: ${{ github.ref_name }}
18+
GITHUB_EVENT_NAME: ${{ github.event_name }}
19+
GITHUB_HEAD_REF: ${{ github.head_ref }}
20+
GITHUB_EVENT_REF: ${{ github.event.ref }}
1521
run: |
1622
# Determine if we're on a tag
1723
if git describe --exact-match --tags HEAD 2>/dev/null; then
1824
export TRIGGER=tag
1925
export TRIGGER_REFERENCE=$(git describe --exact-match --tags HEAD)
2026
else
21-
export TRIGGER=${{ github.ref_type }}
22-
export TRIGGER_REFERENCE=${{ github.ref_name }}
27+
export TRIGGER="$GITHUB_REF_TYPE"
28+
export TRIGGER_REFERENCE="$GITHUB_REF_NAME"
2329
fi
2430
25-
export TRIGGER_ACTION=${{ github.event_name }}
26-
export TRIGGER_HEAD_REFERENCE=${{ github.head_ref }}
27-
export TRIGGER_EVENT_REF=${{ github.event.ref}}
31+
export TRIGGER_ACTION="$GITHUB_EVENT_NAME"
32+
export TRIGGER_HEAD_REFERENCE="$GITHUB_HEAD_REF"
33+
export TRIGGER_EVENT_REF="$GITHUB_EVENT_REF"
2834
export COMMIT_HASH=$(git rev-parse --short $GITHUB_SHA)
2935
. scripts/workflow/derive-workspace.sh
3036
echo "Workspace Name: ${WORKSPACE}"

‎.github/workflows/infrastructure-cleardown.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ jobs:
7272
release_tag: ${{ inputs.release_tag }}
7373
download_appconfig_artifact: true
7474
appconfig_artifact_name: "appconfig_feature_flags"
75-
stacks: "['database', 'crud_apis', 'data_migration', 'read_only_viewer', 'opensearch', 'etl_ods', 'dos_search', 'is_performance', 'ui']"
75+
stacks: "['database', 'crud_apis', 'data_migration', 'read_only_viewer', 'opensearch', 'etl_ods', 'dos_search', 'is_performance', 'ui', 'athena']"
7676
secrets:
7777
ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }}
7878
MGMT_ACCOUNT_ID: ${{ secrets.MGMT_ACCOUNT_ID }}

‎.github/workflows/pipeline-build-release.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ jobs:
149149
- deploy-toggle-infrastructure
150150
uses: ./.github/workflows/deploy-application-infrastructure.yaml
151151
with:
152-
stacks: "['database', 'crud_apis', 'data_migration', 'read_only_viewer', 'opensearch', 'etl_ods', 'dos_search', 'is_performance', 'ui']"
152+
stacks: "['database', 'crud_apis', 'data_migration', 'read_only_viewer', 'opensearch', 'etl_ods', 'dos_search', 'is_performance', 'ui', 'athena']"
153153
environment: ${{ needs.metadata.outputs.environment }}
154154
workspace: "rc"
155155
release_tag: ${{ needs.tag-release.outputs.release_tag }}

‎.github/workflows/pipeline-deploy-application.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ jobs:
4848
with:
4949
environment: ${{ needs.metadata.outputs.environment }}
5050
workspace: ${{ needs.metadata.outputs.workspace }}
51-
stacks: "['database', 'crud_apis', 'data_migration', 'read_only_viewer', 'opensearch', 'etl_ods', 'dos_search', 'is_performance', 'ui']"
51+
stacks: "['database', 'crud_apis', 'data_migration', 'read_only_viewer', 'opensearch', 'etl_ods', 'dos_search', 'is_performance', 'ui', 'athena']"
5252
type: app
5353
build_timestamp: ${{ needs.metadata.outputs.build_timestamp }}
5454
skip_dirs: "services/dos-ui,services/read-only-viewer"
@@ -191,7 +191,7 @@ jobs:
191191
workspace: ${{ needs.metadata.outputs.workspace }}
192192
ref: ${{ inputs.ref }}
193193
workflow_timeout: 30
194-
stacks: "['database', 'crud_apis', 'data_migration', 'read_only_viewer', 'opensearch', 'etl_ods', 'dos_search', 'is_performance', 'ui']"
194+
stacks: "['database', 'crud_apis', 'data_migration', 'read_only_viewer', 'opensearch', 'etl_ods', 'dos_search', 'is_performance', 'ui', 'athena']"
195195
secrets:
196196
ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }}
197197
MGMT_ACCOUNT_ID: ${{ secrets.MGMT_ACCOUNT_ID }}

‎.github/workflows/pipeline-deploy-release.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ jobs:
2929
steps:
3030
- name: "Validate release tag"
3131
id: validate
32+
env:
33+
RELEASE_TAG: ${{ inputs.release_tag }}
3234
run: |
33-
RELEASE_TAG="${{ inputs.release_tag }}"
3435
if [[ ! $RELEASE_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
3536
echo "Invalid release tag format: $RELEASE_TAG"
3637
exit 1

‎infrastructure/common/common-variables.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,3 +196,9 @@ variable "read_only_viewer_stack_enabled" {
196196
type = bool
197197
default = true
198198
}
199+
200+
variable "athena_stack_enabled" {
201+
description = "Enable or disable the Amazon athena stack"
202+
type = bool
203+
default = true
204+
}

‎infrastructure/common/locals.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ locals {
5454
s3 = "alias/${local.project_prefix}-s3-kms"
5555
rds = "alias/${local.project_prefix}-rds-kms"
5656
opensearch = "alias/${local.project_prefix}-opensearch-kms"
57+
athena = "alias/${local.project_prefix}-athena-kms"
5758
firehose = "alias/${local.project_prefix}-firehose-kms"
5859
}
5960

‎infrastructure/stacks/account_policies/policies/rw_compute.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55
"Sid": "ComputeFullAccess",
66
"Action": [
77
"ec2:*",
8+
"ecr:*",
89
"elasticloadbalancing:*",
910
"autoscaling:*",
1011
"kms:ListAliases",
1112
"lambda:*",
13+
"serverlessrepo:*",
1214
"states:DescribeStateMachine",
1315
"states:ListStateMachines",
1416
"tag:GetResources",
@@ -45,4 +47,4 @@
4547
}
4648
}
4749
]
48-
}
50+
}

‎infrastructure/stacks/account_wide/kms.tf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,14 @@ module "s3_encryption_key" {
138138
]
139139
}
140140

141+
module "athena_encryption_key" {
142+
source = "../../modules/kms"
143+
alias_name = local.kms_aliases.athena
144+
account_id = data.aws_caller_identity.current.account_id
145+
aws_service_name = "athena.amazonaws.com"
146+
description = "Encryption key for Athena workgroups in ${var.environment} environment"
147+
}
148+
141149
module "firehose_encryption_key" {
142150
source = "../../modules/kms"
143151
alias_name = local.kms_aliases.firehose

‎infrastructure/stacks/athena/athena.tf‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
resource "aws_glue_catalog_database" "athena_glue_catalog_database" {
2+
count = local.stack_enabled == 1 && local.is_primary_environment ? 1 : 0
3+
4+
name = "${local.resource_prefix}-glue-db"
5+
}
6+
7+
resource "aws_athena_workgroup" "athena_workgroup" {
8+
count = local.stack_enabled == 1 && local.is_primary_environment ? 1 : 0
9+
10+
name = "${local.resource_prefix}-workgroup"
11+
configuration {
12+
result_configuration {
13+
output_location = "s3://${module.athena_output_bucket[0].s3_bucket_id}/results/"
14+
encryption_configuration {
15+
encryption_option = "SSE_KMS"
16+
kms_key_arn = data.aws_kms_key.athena_kms_key[0].arn
17+
}
18+
}
19+
}
20+
}
21+
22+
# Athena Federated Query (DynamoDB Connector)
23+
# Use this approach if you require real-time querying of DynamoDB tables via Athena.
24+
25+
resource "aws_serverlessapplicationrepository_cloudformation_stack" "dynamodb_connector" {
26+
count = local.stack_enabled == 1 && local.is_primary_environment ? 1 : 0
27+
28+
name = "${local.resource_prefix}-dynamodb-connector"
29+
application_id = var.athena_dynamodb_connector_app_id
30+
31+
capabilities = [
32+
"CAPABILITY_IAM",
33+
"CAPABILITY_RESOURCE_POLICY",
34+
]
35+
36+
parameters = {
37+
AthenaCatalogName = "${local.resource_prefix}-dynamodb-connector"
38+
SpillBucket = module.athena_spill_bucket[0].s3_bucket_id
39+
LambdaRole = aws_iam_role.athena_dynamodb_role[0].arn
40+
}
41+
42+
depends_on = [
43+
module.athena_spill_bucket,
44+
aws_iam_role.athena_dynamodb_role,
45+
aws_iam_role_policy.athena_dynamodb_policy
46+
]
47+
}
48+
49+
resource "aws_athena_data_catalog" "athena_data_catalog_dynamodb" {
50+
count = local.stack_enabled == 1 && local.is_primary_environment ? 1 : 0
51+
52+
name = "${local.resource_prefix}-dynamodb-connector"
53+
description = "Athena DynamoDB data catalog"
54+
type = "LAMBDA"
55+
56+
parameters = {
57+
"function" = data.aws_lambda_function.dynamodb_lambda_connector[0].arn
58+
}
59+
60+
depends_on = [
61+
aws_serverlessapplicationrepository_cloudformation_stack.dynamodb_connector
62+
]
63+
}
64+
65+
# Athena Federated Query (RDS Connector)
66+
# Use this approach if you require real-time querying of RDS databases via Athena.
67+
68+
resource "aws_serverlessapplicationrepository_cloudformation_stack" "rds_connector" {
69+
count = local.stack_enabled == 1 && local.is_primary_environment ? 1 : 0
70+
71+
name = "${local.resource_prefix}-rds-connector"
72+
application_id = var.athena_postgres_connector_app_id
73+
74+
capabilities = [
75+
"CAPABILITY_IAM",
76+
"CAPABILITY_AUTO_EXPAND",
77+
"CAPABILITY_RESOURCE_POLICY",
78+
]
79+
80+
parameters = {
81+
SpillBucket = module.athena_spill_bucket[0].s3_bucket_id
82+
DefaultConnectionString = "postgres://jdbc:postgresql://${local.rds_secret.host}:${local.rds_secret.port}/${local.rds_secret.dbname}"
83+
SecretNamePrefix = "/${var.project}/${var.environment}/target-rds"
84+
LambdaFunctionName = "${local.resource_prefix}-rds-connector"
85+
SecurityGroupIds = aws_security_group.rds_connector_sg[0].id
86+
SubnetIds = join(",", data.aws_subnets.private_subnets.ids)
87+
}
88+
89+
depends_on = [
90+
module.athena_spill_bucket
91+
]
92+
}
93+
94+
resource "aws_athena_data_catalog" "athena_data_catalog_rds" {
95+
count = local.stack_enabled == 1 && local.is_primary_environment ? 1 : 0
96+
97+
name = "${local.resource_prefix}-rds-connector"
98+
description = "Athena RDS data catalog"
99+
type = "LAMBDA"
100+
101+
parameters = {
102+
"function" = data.aws_lambda_function.rds_lambda_connector[0].arn
103+
}
104+
105+
depends_on = [
106+
aws_serverlessapplicationrepository_cloudformation_stack.rds_connector
107+
]
108+
}

0 commit comments

Comments
 (0)