Skip to content

Commit a8f3f67

Browse files
ge0Ajaclaude
ge0Aja
and
claude
authored
fix(waf): serialize WAF message to JSON string so EXCLUDE_AT_MATCH can match (#1070)
WAF log transformation was leaving message as a Python dict, causing regex-based filtering (EXCLUDE_AT_MATCH/INCLUDE_AT_MATCH) to fail since str(dict) uses single quotes instead of JSON double quotes. Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 2cd93ec commit a8f3f67

10 files changed

+40
-138
lines changed

‎aws/logs_monitoring/steps/transformation.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ def parse_aws_waf_logs(event):
171171
non_terminating_rules
172172
)
173173

174-
event_copy["message"] = message
174+
event_copy["message"] = json.dumps(message)
175175
return event_copy
176176

177177

‎aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_headers.approved.json‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,4 @@
11
{
22
"ddsource": "waf",
3-
"message": {
4-
"httpRequest": {
5-
"headers": {
6-
"header1": "value1",
7-
"header2": "value2"
8-
}
9-
}
10-
}
3+
"message": "{\"httpRequest\": {\"headers\": {\"header1\": \"value1\", \"header2\": \"value2\"}}}"
114
}

‎aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_non_terminating_matching_rules.approved.json‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,4 @@
11
{
22
"ddsource": "waf",
3-
"message": {
4-
"nonTerminatingMatchingRules": {
5-
"nonterminating1": {
6-
"action": "COUNT"
7-
},
8-
"nonterminating2": {
9-
"action": "COUNT"
10-
}
11-
}
12-
}
3+
"message": "{\"nonTerminatingMatchingRules\": {\"nonterminating1\": {\"action\": \"COUNT\"}, \"nonterminating2\": {\"action\": \"COUNT\"}}}"
134
}

‎aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rate_based_rules.approved.json‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,4 @@
11
{
22
"ddsource": "waf",
3-
"message": {
4-
"rateBasedRuleList": {
5-
"no-rate-limit": {
6-
"limitKey": "IP",
7-
"limitValue": "195.154.122.189",
8-
"maxRateAllowed": 300,
9-
"rateBasedRuleId": "arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889"
10-
},
11-
"tf-rate-limit-5-min": {
12-
"limitKey": "IP",
13-
"limitValue": "195.154.122.189",
14-
"maxRateAllowed": 300,
15-
"rateBasedRuleId": "arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889"
16-
}
17-
}
18-
}
3+
"message": "{\"rateBasedRuleList\": {\"tf-rate-limit-5-min\": {\"limitValue\": \"195.154.122.189\", \"rateBasedRuleId\": \"arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889\", \"maxRateAllowed\": 300, \"limitKey\": \"IP\"}, \"no-rate-limit\": {\"limitValue\": \"195.154.122.189\", \"rateBasedRuleId\": \"arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889\", \"maxRateAllowed\": 300, \"limitKey\": \"IP\"}}}"
194
}

‎aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_three_rules_two_group_ids.approved.json‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,4 @@
11
{
22
"ddsource": "waf",
3-
"message": {
4-
"ruleGroupList": {
5-
"AWS#AWSManagedRulesSQLiRuleSet": {
6-
"terminatingRule": {
7-
"SQLi_QUERYARGUMENTS": {
8-
"action": "BLOCK"
9-
},
10-
"secondRULE": {
11-
"action": "BLOCK"
12-
}
13-
}
14-
},
15-
"A_DIFFERENT_ID": {
16-
"terminatingRule": {
17-
"thirdRULE": {
18-
"action": "BLOCK"
19-
}
20-
}
21-
}
22-
}
23-
}
3+
"message": "{\"ruleGroupList\": {\"AWS#AWSManagedRulesSQLiRuleSet\": {\"terminatingRule\": {\"SQLi_QUERYARGUMENTS\": {\"action\": \"BLOCK\"}, \"secondRULE\": {\"action\": \"BLOCK\"}}}, \"A_DIFFERENT_ID\": {\"terminatingRule\": {\"thirdRULE\": {\"action\": \"BLOCK\"}}}}}"
244
}

‎aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_two_rules_same_group_id.approved.json‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,4 @@
11
{
22
"ddsource": "waf",
3-
"message": {
4-
"ruleGroupList": {
5-
"AWS#AWSManagedRulesSQLiRuleSet": {
6-
"terminatingRule": {
7-
"SQLi_QUERYARGUMENTS": {
8-
"action": "BLOCK"
9-
},
10-
"secondRULE": {
11-
"action": "BLOCK"
12-
}
13-
}
14-
}
15-
}
16-
}
3+
"message": "{\"ruleGroupList\": {\"AWS#AWSManagedRulesSQLiRuleSet\": {\"terminatingRule\": {\"SQLi_QUERYARGUMENTS\": {\"action\": \"BLOCK\"}, \"secondRULE\": {\"action\": \"BLOCK\"}}}}}"
174
}

‎aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_with_excluded_and_nonterminating_rules.approved.json‎

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,4 @@
11
{
22
"ddsource": "waf",
3-
"message": {
4-
"ruleGroupList": {
5-
"AWS#AWSManagedRulesSQLiRuleSet": {
6-
"excludedRules": {
7-
"GenericRFI_BODY": {
8-
"exclusionType": "EXCLUDED_AS_COUNT"
9-
},
10-
"second_exclude": {
11-
"exclusionType": "EXCLUDED_AS_COUNT"
12-
}
13-
},
14-
"nonTerminatingMatchingRules": {
15-
"first_nonterminating": {
16-
"exclusionType": "REGULAR"
17-
},
18-
"second_nonterminating": {
19-
"exclusionType": "REGULAR"
20-
}
21-
},
22-
"terminatingRule": {
23-
"SQLi_QUERYARGUMENTS": {
24-
"action": "BLOCK"
25-
}
26-
}
27-
}
28-
}
29-
}
3+
"message": "{\"ruleGroupList\": {\"AWS#AWSManagedRulesSQLiRuleSet\": {\"terminatingRule\": {\"SQLi_QUERYARGUMENTS\": {\"action\": \"BLOCK\"}}, \"nonTerminatingMatchingRules\": {\"first_nonterminating\": {\"exclusionType\": \"REGULAR\"}, \"second_nonterminating\": {\"exclusionType\": \"REGULAR\"}}, \"excludedRules\": {\"GenericRFI_BODY\": {\"exclusionType\": \"EXCLUDED_AS_COUNT\"}, \"second_exclude\": {\"exclusionType\": \"EXCLUDED_AS_COUNT\"}}}}}"
304
}

‎aws/logs_monitoring/tests/approved_files/TestTransform.test_transform_mixed.approved.json‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,11 @@
11
[
22
{
33
"ddsource": "waf",
4-
"message": {
5-
"httpRequest": {
6-
"headers": {
7-
"header2": "value2"
8-
}
9-
},
10-
"request_id": "1"
11-
}
4+
"message": "{\"request_id\": \"1\", \"httpRequest\": {\"headers\": {\"header2\": \"value2\"}}}"
125
},
136
{
147
"ddsource": "waf",
15-
"message": {
16-
"httpRequest": {
17-
"headers": {
18-
"header1": "value1"
19-
}
20-
},
21-
"request_id": "2"
22-
}
8+
"message": "{\"request_id\": \"2\", \"httpRequest\": {\"headers\": {\"header1\": \"value1\"}}}"
239
},
2410
{
2511
"ddsource": "securityhub",

‎aws/logs_monitoring/tests/approved_files/TestTransform.test_transform_waf.approved.json‎

Lines changed: 3 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,14 @@
11
[
22
{
33
"ddsource": "waf",
4-
"message": {
5-
"httpRequest": {
6-
"headers": {
7-
"header1": "value1",
8-
"header2": "value2"
9-
}
10-
},
11-
"request_id": "1"
12-
}
4+
"message": "{\"httpRequest\": {\"headers\": {\"header1\": \"value1\", \"header2\": \"value2\"}}, \"request_id\": \"1\"}"
135
},
146
{
157
"ddsource": "waf",
16-
"message": {
17-
"httpRequest": {
18-
"headers": {
19-
"header1": "value1"
20-
}
21-
},
22-
"request_id": "2"
23-
}
8+
"message": "{\"httpRequest\": {\"headers\": {\"header1\": \"value1\"}}, \"request_id\": \"2\"}"
249
},
2510
{
2611
"ddsource": "waf",
27-
"message": {
28-
"httpRequest": {
29-
"headers": {
30-
"header3": "value3"
31-
}
32-
},
33-
"request_id": "3"
34-
}
12+
"message": "{\"httpRequest\": {\"headers\": {\"header3\": \"value3\"}}, \"request_id\": \"3\"}"
3513
}
3614
]

‎aws/logs_monitoring/tests/test_logs.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,34 @@ def test_no_filtering_rules(self):
112112
filtered_logs = filter_logs(DatadogMatcher(), self.example_logs)
113113
self.assertEqual(filtered_logs, self.example_logs)
114114

115+
def test_exclude_at_match_with_waf_logs(self):
116+
"""Test that EXCLUDE_AT_MATCH works on WAF logs after transformation serializes message back to JSON string"""
117+
import json
118+
119+
logs = [
120+
{"message": json.dumps({"action": "ALLOW", "httpRequest": {}})},
121+
{"message": json.dumps({"action": "BLOCK", "httpRequest": {}})},
122+
]
123+
filtered = filter_logs(
124+
DatadogMatcher(exclude_pattern='"action": "BLOCK"'), logs
125+
)
126+
self.assertEqual(len(filtered), 1)
127+
self.assertIn("ALLOW", filtered[0]["message"])
128+
129+
def test_include_at_match_with_waf_logs(self):
130+
"""Test that INCLUDE_AT_MATCH works on WAF logs after transformation serializes message back to JSON string"""
131+
import json
132+
133+
logs = [
134+
{"message": json.dumps({"action": "ALLOW", "httpRequest": {}})},
135+
{"message": json.dumps({"action": "BLOCK", "httpRequest": {}})},
136+
]
137+
filtered = filter_logs(
138+
DatadogMatcher(include_pattern='"action": "ALLOW"'), logs
139+
)
140+
self.assertEqual(len(filtered), 1)
141+
self.assertIn("ALLOW", filtered[0]["message"])
142+
115143

116144
def filter_logs(matcher, logs):
117145
filtered = []

0 commit comments

Comments
 (0)