Skip to content

Commit 3e9c5e9

Browse files
kaiyan-shengkaiyan-sheng
authored
[AWS] Support multiple forwarded IPs in cloudfront integration (elastic#4676)
1 parent d6fb9d0 commit 3e9c5e9

File tree

7 files changed

+132
-6
lines changed

7 files changed

+132
-6
lines changed

‎packages/aws/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "1.27.3"
3+
changes:
4+
- description: Support multiple forwarded IPs in cloudfront integration
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/4676
27
- version: "1.27.2"
38
changes:
49
- description: Update the pagination termination condition.

‎packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@
66
2019-12-13 22:37:02 SEA19-C2 900 89.160.20.112 GET d111111abcdef8.cloudfront.net / 502 - curl/7.55.1 - - Error kBkDzGnceVtWHqSCqBUqtA_cEs2T3tFUBbnBNkB9El_uVRhHgcZfcw== www.example.com http 387 0.103 - - - Error HTTP/1.1 - - 12644 0.103 OriginDnsError text/html 507 - -
77
2022-04-19 12:29:36 SEA19-C2 10157 81.2.69.143 POST d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36 source=global - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.238 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 4203 0.238 Miss application/json;charset=UTF-8 - - -
88
2022-04-19 12:29:36 SEA19-C2 10157 81.2.69.143 POST d111111abcdef8.cloudfront.net /getApplications 000 https://test.com/global Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36 source=global - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.238 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 4203 0.238 Miss application/json;charset=UTF-8 - - -
9+
2022-11-15 08:43:04 SEA19-C2 10157 81.2.69.143 GET d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(X11;%20Linux%20x86_64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20HeadlessChrome/100.0.4896.88%20Safari/537.36 - - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.093 81.2.69.142,216.160.83.56 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Miss HTTP/1.1 - - 33359 0.093 Miss application/javascript - - -

‎packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log-expected.json‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -876,6 +876,116 @@
876876
},
877877
"version": "100.0.4896.127"
878878
}
879+
},
880+
{
881+
"@timestamp": "2022-11-15T08:43:04.000Z",
882+
"aws": {
883+
"cloudfront": {
884+
"content_type": "application/javascript",
885+
"domain": "d111111abcdef8.cloudfront.net",
886+
"edge_detailed_result_type": "Miss",
887+
"edge_location": "SEA19-C2",
888+
"edge_response_result_type": "Miss",
889+
"edge_result_type": "Miss",
890+
"time_to_first_byte": 0.093
891+
}
892+
},
893+
"cloud": {
894+
"provider": "aws"
895+
},
896+
"destination": {
897+
"address": "test.com",
898+
"domain": "test.com"
899+
},
900+
"ecs": {
901+
"version": "8.0.0"
902+
},
903+
"event": {
904+
"category": "web",
905+
"id": "hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ==",
906+
"kind": "event",
907+
"original": "2022-11-15 08:43:04 SEA19-C2 10157 81.2.69.143 GET d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(X11;%20Linux%20x86_64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20HeadlessChrome/100.0.4896.88%20Safari/537.36 - - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.093 81.2.69.142,216.160.83.56 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Miss HTTP/1.1 - - 33359 0.093 Miss application/javascript - - -",
908+
"outcome": "success",
909+
"type": [
910+
"access"
911+
]
912+
},
913+
"http": {
914+
"request": {
915+
"bytes": 1057,
916+
"id": "hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ==",
917+
"method": "GET",
918+
"referrer": "https://test.com/global"
919+
},
920+
"response": {
921+
"bytes": 10157,
922+
"status_code": 200
923+
},
924+
"version": "1.1"
925+
},
926+
"network": {
927+
"forwarded_ip": [
928+
"81.2.69.142",
929+
"216.160.83.56"
930+
],
931+
"protocol": "https",
932+
"type": "ipv4"
933+
},
934+
"related": {
935+
"hosts": [
936+
"test.com",
937+
"d111111abcdef8.cloudfront.net"
938+
],
939+
"ip": [
940+
"81.2.69.142",
941+
"216.160.83.56",
942+
"81.2.69.143"
943+
]
944+
},
945+
"source": {
946+
"address": "81.2.69.143",
947+
"geo": {
948+
"city_name": "London",
949+
"continent_name": "Europe",
950+
"country_iso_code": "GB",
951+
"country_name": "United Kingdom",
952+
"location": {
953+
"lat": 51.5142,
954+
"lon": -0.0931
955+
},
956+
"region_iso_code": "GB-ENG",
957+
"region_name": "England"
958+
},
959+
"ip": "81.2.69.143",
960+
"port": 33359
961+
},
962+
"tags": [
963+
"preserve_original_event"
964+
],
965+
"tls": {
966+
"cipher": "ECDHE-RSA-AES128-GCM-SHA256",
967+
"version": "1.2",
968+
"version_protocol": "tls"
969+
},
970+
"url": {
971+
"domain": "test.com",
972+
"full": "https://test.com/getApplications",
973+
"path": "/getApplications",
974+
"registered_domain": "test.com",
975+
"scheme": "https",
976+
"top_level_domain": "com"
977+
},
978+
"user_agent": {
979+
"device": {
980+
"name": "Other"
981+
},
982+
"name": "HeadlessChrome",
983+
"original": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/100.0.4896.88 Safari/537.36",
984+
"os": {
985+
"name": "Linux"
986+
},
987+
"version": "100.0.4896"
988+
}
879989
}
880990
]
881991
}

‎packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,12 @@ processors:
3131
- grok:
3232
field: event.original
3333
patterns:
34-
- '%{TIMESTAMP:_tmp.time}\s%{EDGE_LOCATION:aws.cloudfront.edge_location}\s%{INT:http.response.bytes:long}\s%{IP:source.address}\s%{WORD:http.request.method}\s%{HOSTNAME:aws.cloudfront.domain}\s%{UNIXPATH:url.path}\s%{INT:http.response.status_code:long}\s(-|%{DATA:http.request.referrer})\s%{DATA:_tmp.user_agent}\s(-|%{DATA:url.query})\s(-|%{DATA:aws.cloudfront.cookies})\s%{WORD:aws.cloudfront.edge_result_type}\s%{DATA:http.request.id}\s%{HOSTNAME:destination.address}\s%{WORD:network.protocol}\s%{INT:http.request.bytes:long}\s%{NUMBER:_tmp.duration:float}\s(-|%{IP:network.forwarded_ip})\s(-|%{TLS:tls.version_protocol}v%{NUMBER:tls.version})\s(-|%{DATA:tls.cipher})\s%{WORD:aws.cloudfront.edge_response_result_type}\s%{DATA:_tmp.protocol}\s(-|%{WORD:aws.cloudfront.fle_status})\s(-|%{DATA:aws.cloudfront.fle_encrypted_fields})\s(-|%{POSINT:source.port:long})\s(-|%{NUMBER:aws.cloudfront.time_to_first_byte:float})\s(-|%{WORD:aws.cloudfront.edge_detailed_result_type})\s%{DATA:aws.cloudfront.content_type}\s(-|%{INT:http.response.body.bytes:long})\s(-|%{DATA:aws.cloudfront.range_start})\s(-|%{DATA:aws.cloudfront.range_end})'
34+
- '%{TIMESTAMP:_tmp.time}\s%{EDGE_LOCATION:aws.cloudfront.edge_location}\s%{INT:http.response.bytes:long}\s%{IP:source.address}\s%{WORD:http.request.method}\s%{HOSTNAME:aws.cloudfront.domain}\s%{UNIXPATH:url.path}\s%{INT:http.response.status_code:long}\s(-|%{DATA:http.request.referrer})\s%{DATA:_tmp.user_agent}\s(-|%{DATA:url.query})\s(-|%{DATA:aws.cloudfront.cookies})\s%{WORD:aws.cloudfront.edge_result_type}\s%{DATA:http.request.id}\s%{HOSTNAME:destination.address}\s%{WORD:network.protocol}\s%{INT:http.request.bytes:long}\s%{NUMBER:_tmp.duration:float}\s(-|%{FORWARDED_IPS:_tmp.forwarded_ip})\s(-|%{TLS:tls.version_protocol}v%{NUMBER:tls.version})\s(-|%{DATA:tls.cipher})\s%{WORD:aws.cloudfront.edge_response_result_type}\s%{DATA:_tmp.protocol}\s(-|%{WORD:aws.cloudfront.fle_status})\s(-|%{DATA:aws.cloudfront.fle_encrypted_fields})\s(-|%{POSINT:source.port:long})\s(-|%{NUMBER:aws.cloudfront.time_to_first_byte:float})\s(-|%{WORD:aws.cloudfront.edge_detailed_result_type})\s%{DATA:aws.cloudfront.content_type}\s(-|%{INT:http.response.body.bytes:long})\s(-|%{DATA:aws.cloudfront.range_start})\s(-|%{DATA:aws.cloudfront.range_end})'
3535
pattern_definitions:
3636
TIMESTAMP: '%{YEAR}-%{MONTHNUM}-%{MONTHDAY}%{SPACE}%{HOUR}:%{MINUTE}:%{SECOND}'
3737
TLS: '(TLS|SSL)'
3838
EDGE_LOCATION: '[A-Z]{3}\d+(-[A-Z]+\d+)?'
39+
FORWARDED_IPS: '(%{IP}?(,(\s*)%{IP})*)+'
3940
- gsub:
4041
field: _tmp.time
4142
pattern: \t
@@ -130,14 +131,20 @@ processors:
130131
field: source.as.organization_name
131132
target_field: source.as.organization.name
132133
ignore_missing: true
134+
- split:
135+
field: _tmp.forwarded_ip
136+
target_field: related.ip
137+
separator: ','
138+
ignore_missing: true
133139
- append:
134140
field: related.ip
135141
value: "{{source.ip}}"
136142
if: ctx.source?.ip != null
137-
- append:
138-
field: related.ip
139-
value: "{{network.forwarded_ip}}"
140-
if: ctx.network?.forwarded_ip != null
143+
- split:
144+
field: _tmp.forwarded_ip
145+
target_field: network.forwarded_ip
146+
separator: ','
147+
ignore_missing: true
141148
- append:
142149
field: related.hosts
143150
value: "{{destination.domain}}"

‎packages/aws/data_stream/cloudfront_logs/fields/ecs.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626
name: http.response.status_code
2727
- external: ecs
2828
name: http.version
29+
- external: ecs
30+
name: network.forwarded_ip
2931
- external: ecs
3032
name: network.protocol
3133
- external: ecs

‎packages/aws/docs/cloudfront.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ CloudFront standard logs provide detailed records about every request that’s m
105105
| http.response.bytes | Total size in bytes of the response (body and headers). | long |
106106
| http.response.status_code | HTTP response status code. | long |
107107
| http.version | HTTP version. | keyword |
108+
| network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip |
108109
| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword |
109110
| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword |
110111
| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword |

‎packages/aws/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
format_version: 1.0.0
22
name: aws
33
title: AWS
4-
version: 1.27.2
4+
version: 1.27.3
55
license: basic
66
description: Collect logs and metrics from Amazon Web Services with Elastic Agent.
77
type: integration

0 commit comments

Comments
 (0)