Skip to content

Commit bdc0a14

Browse files
mattdean3-nhsmattdean3-nhs
authored
Merge pull request #937 from NHSDigital/feature/made14-NRL-1386-prod-dashboards-infra
NRL-1386 Add infra and config for prod dashboards
2 parents f0be629 + 02eff29 commit bdc0a14

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+758
-211
lines changed

‎terraform/account-wide-infrastructure/README.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -134,9 +134,11 @@ If deploying the EC2 set up to a new environment, these steps need to be followe
134134
aws ssm start-session --target <AMI> --document-name AWS-StartPortForwardingSession --parameters "localPortNumber=13389,portNumber=3389"
135135
```
136136
137-
2. Install Athena ODBC driver and Power BI personal on premises gateway
138-
3. Configure ODBC driver to connect to relevant Athena instance and log in to the gateway using NHS email
139-
4. Log into power bi and test the refresh on the relevant data sources
137+
2. Install Athena ODBC driver and Power BI standard on premises gateway
138+
3. Configure ODBC driver to connect to relevant Athena instance
139+
4. Log in to the gateway using NHS email, name the cluster to nhsd-nrlf-{env}--reporting-gw
140+
5. Log on to power bi, navigate to Manage Connections and Gateways in settings and set up Athena connector with authentication method: Anonymous and privacy level: Private
141+
6. Set dataset to point to this gateway, define schedule as needed
140142
141143
## Tear down account wide resources
142144

‎terraform/account-wide-infrastructure/dev/athena.tf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
module "dev-athena" {
2+
count = var.enable_reporting ? 1 : 0
23
source = "../modules/athena"
34
name_prefix = "nhsd-nrlf--dev"
45
target_bucket_name = module.dev-glue.target_bucket_name
6+
glue_database = module.dev-glue.glue_database
57
}

‎terraform/account-wide-infrastructure/dev/ec2.tf‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
module "vpc" {
2+
count = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
23
source = "../modules/vpc"
34
vpc_cidr_block = var.vpc_cidr_block
45
enable_dns_hostnames = var.enable_dns_hostnames
@@ -8,16 +9,17 @@ module "vpc" {
89
name_prefix = "nhsd-nrlf--dev"
910
}
1011

11-
module "powerbi_gw_instance_v2" {
12-
source = "../modules/ec2"
13-
use_custom_ami = true
14-
instance_type = var.instance_type
15-
name_prefix = "nhsd-nrlf--dev-powerbi-gw-v2"
12+
module "powerbi_gw_instance" {
13+
count = var.enable_reporting && var.enable_powerbi_auto_push ? 1 : 0
14+
source = "../modules/powerbi-gw-ec2"
15+
use_custom_ami = var.use_powerbi_gw_custom_ami
16+
instance_type = var.powerbi_gw_instance_type
17+
name_prefix = "nhsd-nrlf--dev-powerbi-gw"
1618
target_bucket_arn = module.dev-glue.target_bucket_arn
1719
glue_kms_key_arn = module.dev-glue.aws_kms_key_arn
18-
athena_kms_key_arn = module.dev-athena.kms_key_arn
19-
athena_bucket_arn = module.dev-athena.bucket_arn
20+
athena_kms_key_arn = module.dev-athena[0].kms_key_arn
21+
athena_bucket_arn = module.dev-athena[0].bucket_arn
2022

21-
subnet_id = module.vpc.private_subnet_id
22-
security_groups = [module.vpc.powerbi_gw_security_group_id]
23+
subnet_id = module.vpc[0].private_subnet_id
24+
security_groups = [module.vpc[0].powerbi_gw_security_group_id]
2325
}

‎terraform/account-wide-infrastructure/dev/glue.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
module "dev-glue" {
2+
is_enabled = var.enable_reporting
23
source = "../modules/glue"
34
name_prefix = "nhsd-nrlf--dev"
45
python_version = 3

‎terraform/account-wide-infrastructure/dev/vars.tf‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,12 @@ variable "devsandbox_api_domain_name" {
1414
default = "dev-sandbox.api.record-locator.dev.national.nhs.uk"
1515
}
1616

17+
variable "enable_reporting" {
18+
type = bool
19+
description = "Enable account-wide reporting processes in the dev account"
20+
default = true
21+
}
22+
1723
variable "aws_azs" {
1824
type = string
1925
description = "AWS Availability Zones"
@@ -44,14 +50,20 @@ variable "vpc_private_subnets_cidr_block" {
4450
default = "10.0.1.0/24"
4551
}
4652

47-
variable "instance_type" {
53+
variable "enable_powerbi_auto_push" {
54+
type = bool
55+
description = "Enable automatic pushing of info into PowerBI"
56+
default = true
57+
}
58+
59+
variable "powerbi_gw_instance_type" {
4860
type = string
49-
description = "Type for EC2 Instance"
61+
description = "Type for PowerBI GW EC2 Instance"
5062
default = "t2.micro"
5163
}
5264

53-
variable "use_custom_ami" {
65+
variable "use_powerbi_gw_custom_ami" {
5466
type = bool
55-
description = "Use custom image"
56-
default = false
67+
description = "Use custom image for PowerBI GW instance"
68+
default = true
5769
}

‎terraform/account-wide-infrastructure/modules/athena/athena.tf‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,17 @@ resource "aws_athena_workgroup" "athena" {
1616
}
1717

1818
}
19+
20+
resource "aws_athena_named_query" "rep_consumer" {
21+
name = "rep_consumer"
22+
workgroup = aws_athena_workgroup.athena.id
23+
database = var.glue_database
24+
query = file("${path.module}/sql/rep_consumer.sql")
25+
}
26+
27+
resource "aws_athena_named_query" "rep_producer" {
28+
name = "rep_producer"
29+
workgroup = aws_athena_workgroup.athena.id
30+
database = var.glue_database
31+
query = file("${path.module}/sql/rep_producer.sql")
32+
}

‎terraform/account-wide-infrastructure/modules/athena/sql/rep_consumer.sql‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
CREATE OR REPLACE VIEW "rep_consumer" AS
2+
WITH
3+
cc AS (
4+
SELECT
5+
time
6+
, event_timestamp
7+
, date
8+
, host
9+
, event_log_reference
10+
, event_level
11+
, event_location
12+
, event_message
13+
, event_service
14+
, event_function_request_id
15+
, event_correlation_id
16+
, event_xray_trace_id
17+
, event_pointer_types
18+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
19+
FROM
20+
consumer_countdocumentreference
21+
)
22+
, cr AS (
23+
SELECT
24+
time
25+
, event_timestamp
26+
, date
27+
, host
28+
, event_log_reference
29+
, event_level
30+
, event_location
31+
, event_message
32+
, event_service
33+
, event_function_request_id
34+
, event_correlation_id
35+
, event_xray_trace_id
36+
, event_pointer_types
37+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
38+
FROM
39+
consumer_readdocumentreference
40+
)
41+
, cs AS (
42+
SELECT
43+
time
44+
, event_timestamp
45+
, date
46+
, host
47+
, event_log_reference
48+
, event_level
49+
, event_location
50+
, event_message
51+
, event_service
52+
, event_function_request_id
53+
, event_correlation_id
54+
, event_xray_trace_id
55+
, event_pointer_types
56+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
57+
FROM
58+
consumer_searchdocumentreference
59+
)
60+
, csp AS (
61+
SELECT
62+
time
63+
, event_timestamp
64+
, date
65+
, host
66+
, event_log_reference
67+
, event_level
68+
, event_location
69+
, event_message
70+
, event_service
71+
, event_function_request_id
72+
, event_correlation_id
73+
, event_xray_trace_id
74+
, event_pointer_types
75+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
76+
FROM
77+
consumer_searchpostdocumentreference
78+
)
79+
, base AS (
80+
SELECT *
81+
FROM
82+
cc
83+
UNION SELECT *
84+
FROM
85+
cr
86+
UNION SELECT *
87+
FROM
88+
cs
89+
UNION SELECT *
90+
FROM
91+
csp
92+
)
93+
, ods_codes AS (
94+
SELECT DISTINCT
95+
user_ods
96+
, event_xray_trace_id
97+
FROM
98+
base
99+
WHERE (user_ods IS NOT NULL)
100+
)
101+
SELECT
102+
time
103+
, event_timestamp
104+
, date
105+
, host
106+
, event_log_reference
107+
, event_level
108+
, event_location
109+
, event_message
110+
, event_service
111+
, event_function_request_id
112+
, b.event_correlation_id
113+
, b.event_xray_trace_id
114+
, event_pointer_types
115+
, oc.user_ods
116+
FROM
117+
(base b
118+
LEFT JOIN ods_codes oc ON (b.event_xray_trace_id = oc.event_xray_trace_id))

0 commit comments

Comments
 (0)