Skip to content

Commit 1cbbf55

Browse files
jacklearyjackleary
authored
Merge pull request #822 from NHSDigital/jale13-nrl-1273-update-schema
NRL-1273 update schema for countDocumentReference logs
2 parents 6c1d5e9 + c4e3136 commit 1cbbf55

File tree

1 file changed

+232
-0
lines changed

1 file changed

+232
-0
lines changed

‎terraform/account-wide-infrastructure/modules/glue/src/transformations.py‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from pyspark.sql.types import (
99
BooleanType,
1010
DoubleType,
11+
LongType,
1112
StringType,
1213
StructField,
1314
StructType,
@@ -44,19 +45,250 @@
4445
StructField("accept-encoding", StringType(), True),
4546
StructField("Authorization", StringType(), True),
4647
StructField("Host", StringType(), True),
48+
StructField(
49+
"NHSD-Client-RP-Details", StringType(), True
50+
),
4751
StructField(
4852
"NHSD-Connection-Metadata", StringType(), True
4953
),
5054
StructField("NHSD-Correlation-Id", StringType(), True),
5155
StructField("User-Agent", StringType(), True),
5256
StructField("X-Forwarded-For", StringType(), True),
5357
StructField("X-Request-Id", StringType(), True),
58+
StructField(
59+
"NHSD-End-User-Organisation-ODS", StringType(), True
60+
),
61+
StructField("NHSD-Request-ID", StringType(), True),
62+
StructField("X-Forwarded-Port", StringType(), True),
63+
StructField("X-Forwarded-Proto", StringType(), True),
5464
]
5565
),
5666
True,
5767
),
5868
StructField("log_reference", StringType(), True),
5969
StructField("xray_trace_id", StringType(), True),
70+
StructField(
71+
"config",
72+
StructType(
73+
[
74+
StructField("AWS_REGION", StringType(), True),
75+
StructField("PREFIX", StringType(), True),
76+
StructField("ENVIRONMENT", StringType(), True),
77+
StructField("SPLUNK_INDEX", StringType(), True),
78+
StructField("SOURCE", StringType(), True),
79+
StructField("AUTH_STORE", StringType(), True),
80+
StructField("TABLE_NAME", StringType(), True),
81+
]
82+
),
83+
True,
84+
),
85+
StructField(
86+
"metadata",
87+
StructType(
88+
[
89+
StructField("ods_code", StringType(), True),
90+
StructField("ods_code_extension", StringType(), True),
91+
StructField("nrl_app_id", StringType(), True),
92+
StructField("is_test_event", BooleanType(), True),
93+
StructField(
94+
"client_rp_details",
95+
StructType(
96+
[
97+
StructField(
98+
"developer_app_name", StringType(), True
99+
),
100+
StructField(
101+
"developer_app_id", StringType(), True
102+
),
103+
]
104+
),
105+
True,
106+
),
107+
]
108+
),
109+
True,
110+
),
111+
StructField(
112+
"params",
113+
StructType(
114+
[
115+
StructField("subject:identifier", StringType(), True),
116+
]
117+
),
118+
True,
119+
),
120+
StructField("model", StringType(), True),
121+
StructField(
122+
"parsed_params",
123+
StructType(
124+
[
125+
StructField("subject_identifier", StringType(), True),
126+
]
127+
),
128+
True,
129+
),
130+
StructField("table_name", StringType(), True),
131+
StructField("item_type", StringType(), True),
132+
StructField("original_kwargs_keys", StringType(), True),
133+
StructField("filtered_kwargs_keys", StringType(), True),
134+
StructField("nhs_number", StringType(), True),
135+
StructField(
136+
"query",
137+
StructType(
138+
[
139+
StructField("IndexName", StringType(), True),
140+
StructField(
141+
"KeyConditionExpression", StringType(), True
142+
),
143+
StructField(
144+
"ExpressionAttributeValues",
145+
StructType(
146+
[
147+
StructField(
148+
":patient_key", StringType(), True
149+
),
150+
StructField(
151+
":patient_sort", StringType(), True
152+
),
153+
]
154+
),
155+
True,
156+
),
157+
StructField("Select", StringType(), True),
158+
StructField(
159+
"ReturnConsumedCapacity", StringType(), True
160+
),
161+
]
162+
),
163+
True,
164+
),
165+
StructField("count", LongType(), True),
166+
StructField(
167+
"result",
168+
StructType(
169+
[
170+
StructField("Count", LongType(), True),
171+
StructField("ScannedCount", LongType(), True),
172+
StructField(
173+
"ConsumedCapacity",
174+
StructType(
175+
[
176+
StructField(
177+
"TableName", StringType(), True
178+
),
179+
StructField(
180+
"CapacityUnits", DoubleType(), True
181+
),
182+
StructField(
183+
"Table",
184+
StructType(
185+
[
186+
StructField(
187+
"CapacityUnits",
188+
DoubleType(),
189+
True,
190+
),
191+
]
192+
),
193+
True,
194+
),
195+
StructField(
196+
"GlobalSecondaryIndexes",
197+
StructType(
198+
[
199+
StructField(
200+
"patient_gsi",
201+
StructType(
202+
[
203+
StructField(
204+
"CapacityUnits",
205+
DoubleType(),
206+
True,
207+
),
208+
]
209+
),
210+
True,
211+
),
212+
]
213+
),
214+
True,
215+
),
216+
]
217+
),
218+
True,
219+
),
220+
StructField(
221+
"ResponseMetadata",
222+
StructType(
223+
[
224+
StructField(
225+
"RequestId", StringType(), True
226+
),
227+
StructField(
228+
"HTTPStatusCode", LongType(), True
229+
),
230+
StructField(
231+
"HTTPHeaders",
232+
StructType(
233+
[
234+
StructField(
235+
"server", StringType(), True
236+
),
237+
StructField(
238+
"date", StringType(), True
239+
),
240+
StructField(
241+
"content-type",
242+
StringType(),
243+
True,
244+
),
245+
StructField(
246+
"content-length",
247+
StringType(),
248+
True,
249+
),
250+
StructField(
251+
"connection",
252+
StringType(),
253+
True,
254+
),
255+
StructField(
256+
"x-amzn-requestid",
257+
StringType(),
258+
True,
259+
),
260+
StructField(
261+
"x-amz-crc32",
262+
StringType(),
263+
True,
264+
),
265+
]
266+
),
267+
True,
268+
),
269+
StructField(
270+
"RetryAttempts", LongType(), True
271+
),
272+
]
273+
),
274+
True,
275+
),
276+
]
277+
),
278+
True,
279+
),
280+
StructField("status_code", StringType(), True),
281+
StructField(
282+
"response",
283+
StructType(
284+
[
285+
StructField("statusCode", StringType(), True),
286+
StructField("body", StringType(), True),
287+
StructField("isBase64Encoded", BooleanType(), True),
288+
]
289+
),
290+
True,
291+
),
60292
]
61293
),
62294
True,

0 commit comments

Comments
 (0)