Skip to content

Commit e72cc3a

Browse files
mattdean3-nhsmattdean3-nhs
committed
Merge remote-tracking branch 'origin/feature/jale13-NRL-1479-temp-changes' into feature/made14-NRL-1386-prod-dashboards-infra
2 parents 53acc81 + 9cf8f54 commit e72cc3a

File tree

12 files changed

+402
-71
lines changed

12 files changed

+402
-71
lines changed

‎terraform/account-wide-infrastructure/dev/athena.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@ module "dev-athena" {
22
source = "../modules/athena"
33
name_prefix = "nhsd-nrlf--dev"
44
target_bucket_name = module.dev-glue.target_bucket_name
5+
glue_database = module.dev-glue.glue_database
56
}

‎terraform/account-wide-infrastructure/modules/athena/athena.tf‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,17 @@ resource "aws_athena_workgroup" "athena" {
1616
}
1717

1818
}
19+
20+
resource "aws_athena_named_query" "rep_consumer" {
21+
name = "rep_consumer"
22+
workgroup = aws_athena_workgroup.athena.id
23+
database = var.glue_database
24+
query = file("${path.module}/sql/rep_consumer.sql")
25+
}
26+
27+
resource "aws_athena_named_query" "rep_producer" {
28+
name = "rep_producer"
29+
workgroup = aws_athena_workgroup.athena.id
30+
database = var.glue_database
31+
query = file("${path.module}/sql/rep_producer.sql")
32+
}

‎terraform/account-wide-infrastructure/modules/athena/sql/rep_consumer.sql‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
CREATE OR REPLACE VIEW "rep_consumer" AS
2+
WITH
3+
cc AS (
4+
SELECT
5+
time
6+
, event_timestamp
7+
, date
8+
, host
9+
, event_log_reference
10+
, event_level
11+
, event_location
12+
, event_message
13+
, event_service
14+
, event_function_request_id
15+
, event_correlation_id
16+
, event_xray_trace_id
17+
, event_pointer_types
18+
, '' event_custodian
19+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
20+
FROM
21+
consumer_countdocumentreference
22+
)
23+
, cr AS (
24+
SELECT
25+
time
26+
, event_timestamp
27+
, date
28+
, host
29+
, event_log_reference
30+
, event_level
31+
, event_location
32+
, event_message
33+
, event_service
34+
, event_function_request_id
35+
, event_correlation_id
36+
, event_xray_trace_id
37+
, event_pointer_types
38+
, event_custodian
39+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
40+
FROM
41+
consumer_readdocumentreference
42+
)
43+
, cs AS (
44+
SELECT
45+
time
46+
, event_timestamp
47+
, date
48+
, host
49+
, event_log_reference
50+
, event_level
51+
, event_location
52+
, event_message
53+
, event_service
54+
, event_function_request_id
55+
, event_correlation_id
56+
, event_xray_trace_id
57+
, event_pointer_types
58+
, event_custodian
59+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
60+
FROM
61+
consumer_searchdocumentreference
62+
)
63+
, csp AS (
64+
SELECT
65+
time
66+
, event_timestamp
67+
, date
68+
, host
69+
, event_log_reference
70+
, event_level
71+
, event_location
72+
, event_message
73+
, event_service
74+
, event_function_request_id
75+
, event_correlation_id
76+
, event_xray_trace_id
77+
, event_pointer_types
78+
, event_custodian
79+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
80+
FROM
81+
consumer_searchpostdocumentreference
82+
)
83+
, base AS (
84+
SELECT *
85+
FROM
86+
cc
87+
UNION SELECT *
88+
FROM
89+
cr
90+
UNION SELECT *
91+
FROM
92+
cs
93+
UNION SELECT *
94+
FROM
95+
csp
96+
)
97+
, ods_codes AS (
98+
SELECT DISTINCT
99+
user_ods
100+
, event_xray_trace_id
101+
FROM
102+
base
103+
WHERE (user_ods IS NOT NULL)
104+
)
105+
SELECT
106+
time
107+
, event_timestamp
108+
, date
109+
, host
110+
, event_log_reference
111+
, event_level
112+
, event_location
113+
, event_message
114+
, event_service
115+
, event_function_request_id
116+
, b.event_correlation_id
117+
, b.event_xray_trace_id
118+
, event_pointer_types
119+
, COALESCE(COALESCE(event_custodian, LAG(event_custodian) IGNORE NULLS OVER (PARTITION BY b.event_xray_trace_id ORDER BY event_timestamp ASC)), COALESCE(event_custodian, LEAD(event_custodian) IGNORE NULLS OVER (PARTITION BY b.event_xray_trace_id ORDER BY event_timestamp ASC))) event_custodian
120+
, oc.user_ods
121+
FROM
122+
(base b
123+
LEFT JOIN ods_codes oc ON (b.event_xray_trace_id = oc.event_xray_trace_id))

‎terraform/account-wide-infrastructure/modules/athena/sql/rep_producer.sql‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
CREATE OR REPLACE VIEW "rep_producer" AS
2+
WITH
3+
pc AS (
4+
SELECT
5+
time
6+
, event_timestamp
7+
, date
8+
, host
9+
, event_log_reference
10+
, event_level
11+
, event_location
12+
, event_message
13+
, event_service
14+
, event_function_request_id
15+
, event_correlation_id
16+
, event_xray_trace_id
17+
, event_pointer_types
18+
, event_custodian
19+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
20+
FROM
21+
producer_createdocumentreference
22+
)
23+
, pd AS (
24+
SELECT
25+
time
26+
, event_timestamp
27+
, date
28+
, host
29+
, event_log_reference
30+
, event_level
31+
, event_location
32+
, event_message
33+
, event_service
34+
, event_function_request_id
35+
, event_correlation_id
36+
, event_xray_trace_id
37+
, event_pointer_types
38+
, event_custodian
39+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
40+
FROM
41+
producer_deletedocumentreference
42+
)
43+
, pr AS (
44+
SELECT
45+
time
46+
, event_timestamp
47+
, date
48+
, host
49+
, event_log_reference
50+
, event_level
51+
, event_location
52+
, event_message
53+
, event_service
54+
, event_function_request_id
55+
, event_correlation_id
56+
, event_xray_trace_id
57+
, event_pointer_types
58+
, event_custodian
59+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
60+
FROM
61+
producer_readdocumentreference
62+
)
63+
, ps AS (
64+
SELECT
65+
time
66+
, event_timestamp
67+
, date
68+
, host
69+
, event_log_reference
70+
, event_level
71+
, event_location
72+
, event_message
73+
, event_service
74+
, event_function_request_id
75+
, event_correlation_id
76+
, event_xray_trace_id
77+
, event_pointer_types
78+
, event_custodian
79+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
80+
FROM
81+
producer_searchdocumentreference
82+
)
83+
, psp AS (
84+
SELECT
85+
time
86+
, event_timestamp
87+
, date
88+
, host
89+
, event_log_reference
90+
, event_level
91+
, event_location
92+
, event_message
93+
, event_service
94+
, event_function_request_id
95+
, event_correlation_id
96+
, event_xray_trace_id
97+
, event_pointer_types
98+
, event_custodian
99+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
100+
FROM
101+
producer_searchpostdocumentreference
102+
)
103+
, pu AS (
104+
SELECT
105+
time
106+
, event_timestamp
107+
, date
108+
, host
109+
, event_log_reference
110+
, event_level
111+
, event_location
112+
, event_message
113+
, event_service
114+
, event_function_request_id
115+
, event_correlation_id
116+
, event_xray_trace_id
117+
, event_pointer_types
118+
, event_custodian
119+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
120+
FROM
121+
producer_updatedocumentreference
122+
)
123+
, pus AS (
124+
SELECT
125+
time
126+
, event_timestamp
127+
, date
128+
, host
129+
, event_log_reference
130+
, event_level
131+
, event_location
132+
, event_message
133+
, event_service
134+
, event_function_request_id
135+
, event_correlation_id
136+
, event_xray_trace_id
137+
, event_pointer_types
138+
, event_custodian
139+
, COALESCE("event_headers_nhsd-end-user-organisation-ods", event_metadata_ods_code) user_ods
140+
FROM
141+
producer_upsertdocumentreference
142+
)
143+
, base AS (
144+
SELECT *
145+
FROM
146+
pc
147+
UNION SELECT *
148+
FROM
149+
pd
150+
UNION SELECT *
151+
FROM
152+
pr
153+
UNION SELECT *
154+
FROM
155+
ps
156+
UNION SELECT *
157+
FROM
158+
psp
159+
UNION SELECT *
160+
FROM
161+
pu
162+
UNION SELECT *
163+
FROM
164+
pus
165+
)
166+
, ods_codes AS (
167+
SELECT DISTINCT
168+
user_ods
169+
, event_xray_trace_id
170+
FROM
171+
base
172+
WHERE (user_ods IS NOT NULL)
173+
)
174+
SELECT
175+
time
176+
, event_timestamp
177+
, date
178+
, host
179+
, event_log_reference
180+
, event_level
181+
, event_location
182+
, event_message
183+
, event_service
184+
, event_function_request_id
185+
, b.event_correlation_id
186+
, b.event_xray_trace_id
187+
, event_pointer_types
188+
, COALESCE(COALESCE(event_custodian, LAG(event_custodian) IGNORE NULLS OVER (PARTITION BY b.event_xray_trace_id ORDER BY event_timestamp ASC)), COALESCE(event_custodian, LEAD(event_custodian) IGNORE NULLS OVER (PARTITION BY b.event_xray_trace_id ORDER BY event_timestamp ASC))) event_custodian
189+
, oc.user_ods
190+
FROM
191+
(base b
192+
LEFT JOIN ods_codes oc ON (b.event_xray_trace_id = oc.event_xray_trace_id))

‎terraform/account-wide-infrastructure/modules/athena/vars.tf‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,3 @@
1-
variable "database" {
2-
description = "What the db will be called"
3-
default = "nrl_reporting"
4-
}
5-
61
variable "name_prefix" {
72
type = string
83
description = "The prefix to apply to all resources in the module."
@@ -11,3 +6,8 @@ variable "name_prefix" {
116
variable "target_bucket_name" {
127
type = string
138
}
9+
10+
variable "glue_database" {
11+
type = string
12+
description = "The Glue database in use"
13+
}

‎terraform/account-wide-infrastructure/modules/glue/iam.tf‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,8 @@ data "aws_iam_policy_document" "glue_service" {
9898
]
9999
effect = "Allow"
100100
resources = [
101-
"arn:aws:iam::*:role/AWSGlueServiceRole*"
101+
"arn:aws:iam::*:role/AWSGlueServiceRole*",
102+
aws_iam_role.glue_service_role.arn,
102103
]
103104
}
104105
}

‎terraform/account-wide-infrastructure/modules/glue/outputs.tf‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,7 @@ output "aws_kms_key_arn" {
2121
output "glue_crawler_name" {
2222
value = "s3//${aws_s3_bucket.source-data-bucket.id}/"
2323
}
24+
25+
output "glue_database" {
26+
value = aws_glue_catalog_database.log_database.name
27+
}

0 commit comments

Comments
 (0)