Skip to content

Commit 8ea181a

Browse files
jacklearyjackleary
committed
NRL-1273 update schema for countDocumentReference logs
1 parent c4e4698 commit 8ea181a

File tree

1 file changed

+234
-2
lines changed

1 file changed

+234
-2
lines changed

‎terraform/account-wide-infrastructure/modules/glue/src/transformations.py‎

Lines changed: 234 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
from pyspark.sql.functions import to_timestamp
22
from pyspark.sql.types import (
33
BooleanType,
4+
DoubleType,
5+
LongType,
46
StringType,
57
StructField,
68
StructType,
7-
TimestampType,
89
)
910

1011
logSchema = StructType(
1112
[
12-
StructField("time", TimestampType(), True),
13+
StructField("time", DoubleType(), True),
1314
StructField("index", StringType(), True),
1415
StructField("host", StringType(), True),
1516
StructField("source", StringType(), True),
@@ -38,19 +39,250 @@
3839
StructField("accept-encoding", StringType(), True),
3940
StructField("Authorization", StringType(), True),
4041
StructField("Host", StringType(), True),
42+
StructField(
43+
"NHSD-Client-RP-Details", StringType(), True
44+
),
4145
StructField(
4246
"NHSD-Connection-Metadata", StringType(), True
4347
),
4448
StructField("NHSD-Correlation-Id", StringType(), True),
4549
StructField("User-Agent", StringType(), True),
4650
StructField("X-Forwarded-For", StringType(), True),
4751
StructField("X-Request-Id", StringType(), True),
52+
StructField(
53+
"NHSD-End-User-Organisation-ODS", StringType(), True
54+
),
55+
StructField("NHSD-Request-ID", StringType(), True),
56+
StructField("X-Forwarded-Port", StringType(), True),
57+
StructField("X-Forwarded-Proto", StringType(), True),
4858
]
4959
),
5060
True,
5161
),
5262
StructField("log_reference", StringType(), True),
5363
StructField("xray_trace_id", StringType(), True),
64+
StructField(
65+
"config",
66+
StructType(
67+
[
68+
StructField("AWS_REGION", StringType(), True),
69+
StructField("PREFIX", StringType(), True),
70+
StructField("ENVIRONMENT", StringType(), True),
71+
StructField("SPLUNK_INDEX", StringType(), True),
72+
StructField("SOURCE", StringType(), True),
73+
StructField("AUTH_STORE", StringType(), True),
74+
StructField("TABLE_NAME", StringType(), True),
75+
]
76+
),
77+
True,
78+
),
79+
StructField(
80+
"metadata",
81+
StructType(
82+
[
83+
StructField("ods_code", StringType(), True),
84+
StructField("ods_code_extension", StringType(), True),
85+
StructField("nrl_app_id", StringType(), True),
86+
StructField("is_test_event", BooleanType(), True),
87+
StructField(
88+
"client_rp_details",
89+
StructType(
90+
[
91+
StructField(
92+
"developer_app_name", StringType(), True
93+
),
94+
StructField(
95+
"developer_app_id", StringType(), True
96+
),
97+
]
98+
),
99+
True,
100+
),
101+
]
102+
),
103+
True,
104+
),
105+
StructField(
106+
"params",
107+
StructType(
108+
[
109+
StructField("subject:identifier", StringType(), True),
110+
]
111+
),
112+
True,
113+
),
114+
StructField("model", StringType(), True),
115+
StructField(
116+
"parsed_params",
117+
StructType(
118+
[
119+
StructField("subject_identifier", StringType(), True),
120+
]
121+
),
122+
True,
123+
),
124+
StructField("table_name", StringType(), True),
125+
StructField("item_type", StringType(), True),
126+
StructField("original_kwargs_keys", StringType(), True),
127+
StructField("filtered_kwargs_keys", StringType(), True),
128+
StructField("nhs_number", StringType(), True),
129+
StructField(
130+
"query",
131+
StructType(
132+
[
133+
StructField("IndexName", StringType(), True),
134+
StructField(
135+
"KeyConditionExpression", StringType(), True
136+
),
137+
StructField(
138+
"ExpressionAttributeValues",
139+
StructType(
140+
[
141+
StructField(
142+
":patient_key", StringType(), True
143+
),
144+
StructField(
145+
":patient_sort", StringType(), True
146+
),
147+
]
148+
),
149+
True,
150+
),
151+
StructField("Select", StringType(), True),
152+
StructField(
153+
"ReturnConsumedCapacity", StringType(), True
154+
),
155+
]
156+
),
157+
True,
158+
),
159+
StructField("count", LongType(), True),
160+
StructField(
161+
"result",
162+
StructType(
163+
[
164+
StructField("Count", LongType(), True),
165+
StructField("ScannedCount", LongType(), True),
166+
StructField(
167+
"ConsumedCapacity",
168+
StructType(
169+
[
170+
StructField(
171+
"TableName", StringType(), True
172+
),
173+
StructField(
174+
"CapacityUnits", DoubleType(), True
175+
),
176+
StructField(
177+
"Table",
178+
StructType(
179+
[
180+
StructField(
181+
"CapacityUnits",
182+
DoubleType(),
183+
True,
184+
),
185+
]
186+
),
187+
True,
188+
),
189+
StructField(
190+
"GlobalSecondaryIndexes",
191+
StructType(
192+
[
193+
StructField(
194+
"patient_gsi",
195+
StructType(
196+
[
197+
StructField(
198+
"CapacityUnits",
199+
DoubleType(),
200+
True,
201+
),
202+
]
203+
),
204+
True,
205+
),
206+
]
207+
),
208+
True,
209+
),
210+
]
211+
),
212+
True,
213+
),
214+
StructField(
215+
"ResponseMetadata",
216+
StructType(
217+
[
218+
StructField(
219+
"RequestId", StringType(), True
220+
),
221+
StructField(
222+
"HTTPStatusCode", LongType(), True
223+
),
224+
StructField(
225+
"HTTPHeaders",
226+
StructType(
227+
[
228+
StructField(
229+
"server", StringType(), True
230+
),
231+
StructField(
232+
"date", StringType(), True
233+
),
234+
StructField(
235+
"content-type",
236+
StringType(),
237+
True,
238+
),
239+
StructField(
240+
"content-length",
241+
StringType(),
242+
True,
243+
),
244+
StructField(
245+
"connection",
246+
StringType(),
247+
True,
248+
),
249+
StructField(
250+
"x-amzn-requestid",
251+
StringType(),
252+
True,
253+
),
254+
StructField(
255+
"x-amz-crc32",
256+
StringType(),
257+
True,
258+
),
259+
]
260+
),
261+
True,
262+
),
263+
StructField(
264+
"RetryAttempts", LongType(), True
265+
),
266+
]
267+
),
268+
True,
269+
),
270+
]
271+
),
272+
True,
273+
),
274+
StructField("status_code", StringType(), True),
275+
StructField(
276+
"response",
277+
StructType(
278+
[
279+
StructField("statusCode", StringType(), True),
280+
StructField("body", StringType(), True),
281+
StructField("isBase64Encoded", BooleanType(), True),
282+
]
283+
),
284+
True,
285+
),
54286
]
55287
),
56288
True,

0 commit comments

Comments
 (0)