Skip to content

Commit c76192a

Browse files
steffengsteffeng
committed
Integrates fields added to CloudFront 12.12.2019
- new fields: https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html#BasicDistributionFileFormat - renames fields according to https://docs.aws.amazon.com/athena/latest/ug/cloudfront-logs.html
1 parent ce4808f commit c76192a

File tree

1 file changed

+113
-64
lines changed

1 file changed

+113
-64
lines changed

‎template.yaml

Lines changed: 113 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ Resources:
210210
Type: string
211211
- Name: bytes
212212
Type: bigint
213-
- Name: requestip
213+
- Name: request_ip
214214
Type: string
215215
- Name: method
216216
Type: string
@@ -222,38 +222,52 @@ Resources:
222222
Type: int
223223
- Name: referrer
224224
Type: string
225-
- Name: useragent
225+
- Name: user_agent
226226
Type: string
227-
- Name: querystring
227+
- Name: query_string
228228
Type: string
229229
- Name: cookie
230230
Type: string
231-
- Name: resulttype
231+
- Name: result_type
232232
Type: string
233-
- Name: requestid
233+
- Name: request_id
234234
Type: string
235-
- Name: hostheader
235+
- Name: host_header
236236
Type: string
237-
- Name: requestprotocol
237+
- Name: request_protocol
238238
Type: string
239-
- Name: requestbytes
239+
- Name: request_bytes
240240
Type: bigint
241-
- Name: timetaken
241+
- Name: time_taken
242242
Type: float
243-
- Name: xforwardedfor
243+
- Name: xforwarded_for
244244
Type: string
245-
- Name: sslprotocol
245+
- Name: ssl_protocol
246246
Type: string
247-
- Name: sslcipher
247+
- Name: ssl_cipher
248248
Type: string
249-
- Name: responseresulttype
249+
- Name: response_result_type
250250
Type: string
251-
- Name: httpversion
251+
- Name: http_version
252252
Type: string
253-
- Name: filestatus
253+
- Name: fle_status
254254
Type: string
255-
- Name: encryptedfields
255+
- Name: fle_encrypted_fields
256256
Type: int
257+
- Name: c_port
258+
Type: int
259+
- Name: time_to_first_byte
260+
Type: float
261+
- Name: x_edge_detailed_result_type
262+
Type: string
263+
- Name: sc_content_type
264+
Type: string
265+
- Name: sc_content_len
266+
Type: bigint
267+
- Name: sc_range_start
268+
Type: bigint
269+
- Name: sc_range_end
270+
Type: bigint
257271
InputFormat: org.apache.hadoop.mapred.TextInputFormat
258272
Location: !Sub "s3://${ResourcePrefix}-${AWS::AccountId}-cf-access-logs/${GzKeyPrefix}"
259273
SerdeInfo:
@@ -291,7 +305,7 @@ Resources:
291305
Type: string
292306
- Name: bytes
293307
Type: bigint
294-
- Name: requestip
308+
- Name: request_ip
295309
Type: string
296310
- Name: method
297311
Type: string
@@ -303,38 +317,52 @@ Resources:
303317
Type: int
304318
- Name: referrer
305319
Type: string
306-
- Name: useragent
320+
- Name: user_agent
307321
Type: string
308-
- Name: querystring
322+
- Name: query_string
309323
Type: string
310324
- Name: cookie
311325
Type: string
312-
- Name: resulttype
326+
- Name: result_type
313327
Type: string
314-
- Name: requestid
328+
- Name: request_id
315329
Type: string
316-
- Name: hostheader
330+
- Name: host_header
317331
Type: string
318-
- Name: requestprotocol
332+
- Name: request_protocol
319333
Type: string
320-
- Name: requestbytes
334+
- Name: request_bytes
321335
Type: bigint
322-
- Name: timetaken
336+
- Name: time_taken
323337
Type: float
324-
- Name: xforwardedfor
338+
- Name: xforwarded_for
325339
Type: string
326-
- Name: sslprotocol
340+
- Name: ssl_protocol
327341
Type: string
328-
- Name: sslcipher
342+
- Name: ssl_cipher
329343
Type: string
330-
- Name: responseresulttype
344+
- Name: response_result_type
331345
Type: string
332-
- Name: httpversion
346+
- Name: http_version
333347
Type: string
334-
- Name: filestatus
348+
- Name: fle_status
335349
Type: string
336-
- Name: encryptedfields
350+
- Name: fle_encrypted_fields
351+
Type: int
352+
- Name: c_port
337353
Type: int
354+
- Name: time_to_first_byte
355+
Type: float
356+
- Name: x_edge_detailed_result_type
357+
Type: string
358+
- Name: sc_content_type
359+
Type: string
360+
- Name: sc_content_len
361+
Type: bigint
362+
- Name: sc_range_start
363+
Type: bigint
364+
- Name: sc_range_end
365+
Type: bigint
338366
InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
339367
Location: !Sub "s3://${ResourcePrefix}-${AWS::AccountId}-cf-access-logs/${ParquetKeyPrefix}"
340368
SerdeInfo:
@@ -360,7 +388,7 @@ Resources:
360388
Type: string
361389
- Name: bytes
362390
Type: bigint
363-
- Name: requestip
391+
- Name: request_ip
364392
Type: string
365393
- Name: method
366394
Type: string
@@ -372,38 +400,52 @@ Resources:
372400
Type: int
373401
- Name: referrer
374402
Type: string
375-
- Name: useragent
403+
- Name: user_agent
376404
Type: string
377-
- Name: querystring
405+
- Name: query_string
378406
Type: string
379407
- Name: cookie
380408
Type: string
381-
- Name: resulttype
409+
- Name: result_type
382410
Type: string
383-
- Name: requestid
411+
- Name: request_id
384412
Type: string
385-
- Name: hostheader
413+
- Name: host_header
386414
Type: string
387-
- Name: requestprotocol
415+
- Name: request_protocol
388416
Type: string
389-
- Name: requestbytes
417+
- Name: request_bytes
390418
Type: bigint
391-
- Name: timetaken
419+
- Name: time_taken
392420
Type: float
393-
- Name: xforwardedfor
421+
- Name: xforwarded_for
394422
Type: string
395-
- Name: sslprotocol
423+
- Name: ssl_protocol
396424
Type: string
397-
- Name: sslcipher
425+
- Name: ssl_cipher
398426
Type: string
399-
- Name: responseresulttype
427+
- Name: response_result_type
400428
Type: string
401-
- Name: httpversion
429+
- Name: http_version
402430
Type: string
403-
- Name: filestatus
431+
- Name: fle_status
404432
Type: string
405-
- Name: encryptedfields
433+
- Name: fle_encrypted_fields
406434
Type: int
435+
- Name: c_port
436+
Type: int
437+
- Name: time_to_first_byte
438+
Type: float
439+
- Name: x_edge_detailed_result_type
440+
Type: string
441+
- Name: sc_content_type
442+
Type: string
443+
- Name: sc_content_len
444+
Type: bigint
445+
- Name: sc_range_start
446+
Type: bigint
447+
- Name: sc_range_end
448+
Type: bigint
407449
- Name: year
408450
Type: string
409451
- Name: month
@@ -431,28 +473,35 @@ Resources:
431473
{"name": "time", "type": "varchar"},
432474
{"name": "location", "type": "varchar"},
433475
{"name": "bytes", "type": "bigint"},
434-
{"name": "requestip", "type": "varchar"},
476+
{"name": "request_ip", "type": "varchar"},
435477
{"name": "method", "type": "varchar"},
436478
{"name": "host", "type": "varchar"},
437479
{"name": "uri", "type": "varchar"},
438480
{"name": "status", "type": "integer"},
439481
{"name": "referrer", "type": "varchar"},
440-
{"name": "useragent", "type": "varchar"},
441-
{"name": "querystring", "type": "varchar"},
482+
{"name": "user_agent", "type": "varchar"},
483+
{"name": "query_string", "type": "varchar"},
442484
{"name": "cookie", "type": "varchar"},
443-
{"name": "resulttype", "type": "varchar"},
444-
{"name": "requestid", "type": "varchar"},
445-
{"name": "hostheader", "type": "varchar"},
446-
{"name": "requestprotocol", "type": "varchar"},
447-
{"name": "requestbytes", "type": "bigint"},
448-
{"name": "timetaken", "type": "real"},
449-
{"name": "xforwardedfor", "type": "varchar"},
450-
{"name": "sslprotocol", "type": "varchar"},
451-
{"name": "sslcipher", "type": "varchar"},
452-
{"name": "responseresulttype", "type": "varchar"},
453-
{"name": "httpversion", "type": "varchar"},
454-
{"name": "filestatus", "type": "varchar"},
455-
{"name": "encryptedfields", "type": "integer"},
485+
{"name": "result_type", "type": "varchar"},
486+
{"name": "request_id", "type": "varchar"},
487+
{"name": "host_header", "type": "varchar"},
488+
{"name": "request_protocol", "type": "varchar"},
489+
{"name": "request_bytes", "type": "bigint"},
490+
{"name": "time_taken", "type": "real"},
491+
{"name": "xforwarded_for", "type": "varchar"},
492+
{"name": "ssl_protocol", "type": "varchar"},
493+
{"name": "ssl_cipher", "type": "varchar"},
494+
{"name": "response_result_type", "type": "varchar"},
495+
{"name": "http_version", "type": "varchar"},
496+
{"name": "fle_status", "type": "varchar"},
497+
{"name": "fle_encrypted_fields", "type": "integer"},
498+
{"name": "c_port", "type": "integer"},
499+
{"name": "time_to_first_byte", "type": "real"},
500+
{"name": "x_edge_detailed_result_type", "type": "varchar"},
501+
{"name": "sc_content_type", "type": "varchar"},
502+
{"name": "sc_content_len", "type": "bigint"},
503+
{"name": "sc_range_start", "type": "bigint"},
504+
{"name": "sc_range_end", "type": "bigint"},
456505
{"name": "year", "type": "varchar"},
457506
{"name": "month", "type": "varchar"},
458507
{"name": "day", "type": "varchar"},

0 commit comments

Comments
 (0)