Skip to content

Commit 9e65424

Browse files
jandro996jandro996
authored
Add support for waf_timeout tag in telemetry (#7696)
What Does This Do Add waf_timeout tag to dd.instrumentation_telemetry_data.appsec.waf.requests metric Motivation This will allow us to better understand WAF timeout behaviours, possibly helping us to: improve the default, and spot performance issues.
1 parent 962b539 commit 9e65424

File tree

5 files changed

+108
-18
lines changed

5 files changed

+108
-18
lines changed

‎dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFModule.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,7 @@ public void onDataAvailable(
429429
resultWithData = doRunPowerwaf(reqCtx, newData, ctxAndAddr, gwCtx);
430430
} catch (TimeoutPowerwafException tpe) {
431431
reqCtx.increaseTimeouts();
432+
WafMetricCollector.get().wafRequestTimeout();
432433
log.debug(LogCollector.EXCLUDE_TELEMETRY, "Timeout calling the WAF", tpe);
433434
if (gwCtx.isRasp) {
434435
WafMetricCollector.get().raspTimeout(gwCtx.raspRuleType);

‎dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/powerwaf/PowerWAFModuleSpecification.groovy‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import datadog.trace.api.ConfigDefaults
2424
import datadog.trace.api.internal.TraceSegment
2525
import datadog.appsec.api.blocking.BlockingContentType
2626
import datadog.trace.api.gateway.Flow
27+
import datadog.trace.api.telemetry.WafMetricCollector
2728
import datadog.trace.bootstrap.instrumentation.api.AgentSpan
2829
import datadog.trace.bootstrap.instrumentation.api.AgentTracer
2930
import datadog.trace.test.util.DDSpecification
@@ -44,6 +45,9 @@ class PowerWAFModuleSpecification extends DDSpecification {
4445
@Shared
4546
protected static final AgentTracer.TracerAPI ORIGINAL_TRACER = AgentTracer.get()
4647

48+
@Shared
49+
protected static final ORIGINAL_METRIC_COLLECTOR = WafMetricCollector.get()
50+
4751
private static final DataBundle ATTACK_BUNDLE = MapDataBundle.of(KnownAddresses.HEADERS_NO_COOKIES,
4852
new CaseInsensitiveMap<List<String>>(['user-agent': 'Arachni/v0']))
4953

@@ -70,6 +74,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
7074
}
7175

7276
void cleanup() {
77+
WafMetricCollector.INSTANCE = ORIGINAL_METRIC_COLLECTOR
7378
AgentTracer.forceRegister(ORIGINAL_TRACER)
7479
pwafAdditive?.close()
7580
release pwafModule
@@ -966,6 +971,9 @@ class PowerWAFModuleSpecification extends DDSpecification {
966971
TraceSegment segment = Mock()
967972
TraceSegmentPostProcessor pp = service.traceSegmentPostProcessors.last()
968973

974+
def mockWafMetricCollector = Mock(WafMetricCollector)
975+
WafMetricCollector.INSTANCE = mockWafMetricCollector
976+
969977
when:
970978
dataListener.onDataAvailable(flow, ctx, db, gwCtx)
971979

@@ -974,6 +982,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
974982
pwafAdditive = it[0].openAdditive() }
975983
assert !flow.blocking
976984
1 * ctx.increaseTimeouts()
985+
1 * mockWafMetricCollector.get().wafRequestTimeout()
977986

978987
when:
979988
pp.processTraceSegment(segment, ctx, [])

‎internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,16 @@
1212

1313
public class WafMetricCollector implements MetricCollector<WafMetricCollector.WafMetric> {
1414

15-
public static final WafMetricCollector INSTANCE = new WafMetricCollector();
15+
public static WafMetricCollector INSTANCE = new WafMetricCollector();
1616

1717
public static WafMetricCollector get() {
1818
return WafMetricCollector.INSTANCE;
1919
}
2020

21+
private WafMetricCollector() {
22+
// Prevent external instantiation
23+
}
24+
2125
private static final String NAMESPACE = "appsec";
2226

2327
private static final BlockingQueue<WafMetric> rawMetricsQueue =
@@ -29,11 +33,12 @@ public static WafMetricCollector get() {
2933
private static final AtomicRequestCounter wafRequestCounter = new AtomicRequestCounter();
3034
private static final AtomicRequestCounter wafTriggeredRequestCounter = new AtomicRequestCounter();
3135
private static final AtomicRequestCounter wafBlockedRequestCounter = new AtomicRequestCounter();
36+
private static final AtomicRequestCounter wafTimeoutRequestCounter = new AtomicRequestCounter();
3237
private static final AtomicLongArray raspRuleEvalCounter =
3338
new AtomicLongArray(RuleType.getNumValues());
3439
private static final AtomicLongArray raspRuleMatchCounter =
3540
new AtomicLongArray(RuleType.getNumValues());
36-
private static final AtomicLongArray respTimeoutCounter =
41+
private static final AtomicLongArray raspTimeoutCounter =
3742
new AtomicLongArray(RuleType.getNumValues());
3843
private static final AtomicRequestCounter missingUserIdCounter = new AtomicRequestCounter();
3944

@@ -78,6 +83,10 @@ public void wafRequestBlocked() {
7883
wafBlockedRequestCounter.increment();
7984
}
8085

86+
public void wafRequestTimeout() {
87+
wafTimeoutRequestCounter.increment();
88+
}
89+
8190
public void raspRuleEval(final RuleType ruleType) {
8291
raspRuleEvalCounter.incrementAndGet(ruleType.ordinal());
8392
}
@@ -87,7 +96,7 @@ public void raspRuleMatch(final RuleType ruleType) {
8796
}
8897

8998
public void raspTimeout(final RuleType ruleType) {
90-
respTimeoutCounter.incrementAndGet(ruleType.ordinal());
99+
raspTimeoutCounter.incrementAndGet(ruleType.ordinal());
91100
}
92101

93102
public void missingUserId() {
@@ -116,6 +125,7 @@ public void prepareMetrics() {
116125
WafMetricCollector.wafVersion,
117126
WafMetricCollector.rulesVersion,
118127
false,
128+
false,
119129
false))) {
120130
return;
121131
}
@@ -129,6 +139,7 @@ public void prepareMetrics() {
129139
WafMetricCollector.wafVersion,
130140
WafMetricCollector.rulesVersion,
131141
true,
142+
false,
132143
false))) {
133144
return;
134145
}
@@ -142,6 +153,21 @@ public void prepareMetrics() {
142153
WafMetricCollector.wafVersion,
143154
WafMetricCollector.rulesVersion,
144155
true,
156+
true,
157+
false))) {
158+
return;
159+
}
160+
}
161+
162+
// Timeout requests
163+
if (wafTimeoutRequestCounter.get() > 0) {
164+
if (!rawMetricsQueue.offer(
165+
new WafRequestsRawMetric(
166+
wafTimeoutRequestCounter.getAndReset(),
167+
WafMetricCollector.wafVersion,
168+
WafMetricCollector.rulesVersion,
169+
false,
170+
false,
145171
true))) {
146172
return;
147173
}
@@ -171,7 +197,7 @@ public void prepareMetrics() {
171197

172198
// RASP timeout per rule type
173199
for (RuleType ruleType : RuleType.values()) {
174-
long counter = respTimeoutCounter.getAndSet(ruleType.ordinal(), 0);
200+
long counter = raspTimeoutCounter.getAndSet(ruleType.ordinal(), 0);
175201
if (counter > 0) {
176202
if (!rawMetricsQueue.offer(
177203
new RaspTimeout(counter, ruleType, WafMetricCollector.wafVersion))) {
@@ -228,14 +254,16 @@ public WafRequestsRawMetric(
228254
final String wafVersion,
229255
final String rulesVersion,
230256
final boolean triggered,
231-
final boolean blocked) {
257+
final boolean blocked,
258+
final boolean wafTimeout) {
232259
super(
233260
"waf.requests",
234261
counter,
235262
"waf_version:" + wafVersion,
236263
"event_rules_version:" + rulesVersion,
237264
"rule_triggered:" + triggered,
238-
"request_blocked:" + blocked);
265+
"request_blocked:" + blocked,
266+
"waf_timeout:" + wafTimeout);
239267
}
240268
}
241269

‎internal-api/src/test/groovy/datadog/trace/api/telemetry/WafMetricCollectorTest.groovy‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,15 @@ class WafMetricCollectorTest extends DDSpecification {
2222
WafMetricCollector.get().wafRequest()
2323
WafMetricCollector.get().wafRequestTriggered()
2424
WafMetricCollector.get().wafRequestBlocked()
25+
WafMetricCollector.get().wafRequestTimeout()
2526
WafMetricCollector.get().raspRuleEval(RuleType.SQL_INJECTION)
2627
WafMetricCollector.get().raspRuleEval(RuleType.SQL_INJECTION)
2728
WafMetricCollector.get().raspRuleMatch(RuleType.SQL_INJECTION)
2829
WafMetricCollector.get().raspRuleEval(RuleType.SQL_INJECTION)
2930
WafMetricCollector.get().raspTimeout(RuleType.SQL_INJECTION)
3031

32+
33+
3134
WafMetricCollector.get().prepareMetrics()
3235

3336
then:
@@ -63,7 +66,8 @@ class WafMetricCollectorTest extends DDSpecification {
6366
'waf_version:waf_ver1',
6467
'event_rules_version:rules.3',
6568
'rule_triggered:false',
66-
'request_blocked:false'
69+
'request_blocked:false',
70+
'waf_timeout:false'
6771
].toSet()
6872

6973
def requestTriggeredMetric = (WafMetricCollector.WafRequestsRawMetric)metrics[4]
@@ -74,9 +78,11 @@ class WafMetricCollectorTest extends DDSpecification {
7478
'waf_version:waf_ver1',
7579
'event_rules_version:rules.3',
7680
'rule_triggered:true',
77-
'request_blocked:false'
81+
'request_blocked:false',
82+
'waf_timeout:false'
7883
].toSet()
7984

85+
8086
def requestBlockedMetric = (WafMetricCollector.WafRequestsRawMetric)metrics[5]
8187
requestBlockedMetric.namespace == 'appsec'
8288
requestBlockedMetric.metricName == 'waf.requests'
@@ -86,24 +92,38 @@ class WafMetricCollectorTest extends DDSpecification {
8692
'waf_version:waf_ver1',
8793
'event_rules_version:rules.3',
8894
'rule_triggered:true',
89-
'request_blocked:true'
95+
'request_blocked:true',
96+
'waf_timeout:false'
97+
].toSet()
98+
99+
def requestTimeoutMetric = (WafMetricCollector.WafRequestsRawMetric)metrics[6]
100+
requestTimeoutMetric.namespace == 'appsec'
101+
requestTimeoutMetric.metricName == 'waf.requests'
102+
requestTimeoutMetric.type == 'count'
103+
requestTimeoutMetric.value == 1
104+
requestTimeoutMetric.tags.toSet() == [
105+
'waf_version:waf_ver1',
106+
'event_rules_version:rules.3',
107+
'rule_triggered:false',
108+
'request_blocked:false',
109+
'waf_timeout:true'
90110
].toSet()
91111

92-
def raspRuleEvalSqli = (WafMetricCollector.RaspRuleEval)metrics[6]
112+
def raspRuleEvalSqli = (WafMetricCollector.RaspRuleEval)metrics[7]
93113
raspRuleEvalSqli.type == 'count'
94114
raspRuleEvalSqli.value == 3
95115
raspRuleEvalSqli.namespace == 'appsec'
96116
raspRuleEvalSqli.metricName == 'rasp.rule.eval'
97117
raspRuleEvalSqli.tags.toSet() == ['rule_type:sql_injection', 'waf_version:waf_ver1'].toSet()
98118

99-
def raspRuleMatch = (WafMetricCollector.RaspRuleMatch)metrics[7]
119+
def raspRuleMatch = (WafMetricCollector.RaspRuleMatch)metrics[8]
100120
raspRuleMatch.type == 'count'
101121
raspRuleMatch.value == 1
102122
raspRuleMatch.namespace == 'appsec'
103123
raspRuleMatch.metricName == 'rasp.rule.match'
104124
raspRuleMatch.tags.toSet() == ['rule_type:sql_injection', 'waf_version:waf_ver1'].toSet()
105125

106-
def raspTimeout = (WafMetricCollector.RaspTimeout)metrics[8]
126+
def raspTimeout = (WafMetricCollector.RaspTimeout)metrics[9]
107127
raspTimeout.type == 'count'
108128
raspTimeout.value == 1
109129
raspTimeout.namespace == 'appsec'

‎telemetry/src/test/groovy/datadog/telemetry/metric/WafMetricPeriodicActionSpecification.groovy‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
4848
WafMetricCollector.get().wafRequest()
4949
WafMetricCollector.get().wafRequestBlocked()
5050
WafMetricCollector.get().wafRequest()
51+
WafMetricCollector.get().wafRequestTimeout()
5152
WafMetricCollector.get().prepareMetrics()
5253
periodicAction.doIteration(telemetryService)
5354

@@ -64,7 +65,8 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
6465
'waf_version:0.0.0',
6566
'event_rules_version:rules_ver_1',
6667
'rule_triggered:false',
67-
'request_blocked:false'
68+
'request_blocked:false',
69+
'waf_timeout:false'
6870
]
6971
} )
7072
1 * telemetryService.addMetric( { Metric metric ->
@@ -75,7 +77,8 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
7577
'waf_version:0.0.0',
7678
'event_rules_version:rules_ver_1',
7779
'rule_triggered:true',
78-
'request_blocked:false'
80+
'request_blocked:false',
81+
'waf_timeout:false'
7982
]
8083
} )
8184
1 * telemetryService.addMetric( { Metric metric ->
@@ -86,7 +89,20 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
8689
'waf_version:0.0.0',
8790
'event_rules_version:rules_ver_1',
8891
'rule_triggered:true',
89-
'request_blocked:true'
92+
'request_blocked:true',
93+
'waf_timeout:false'
94+
]
95+
} )
96+
1 * telemetryService.addMetric( { Metric metric ->
97+
metric.namespace == 'appsec' &&
98+
metric.metric == 'waf.requests' &&
99+
metric.points[0][1] == 1 &&
100+
metric.tags == [
101+
'waf_version:0.0.0',
102+
'event_rules_version:rules_ver_1',
103+
'rule_triggered:false',
104+
'request_blocked:false',
105+
'waf_timeout:true'
90106
]
91107
} )
92108
0 * _._
@@ -96,6 +112,7 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
96112
WafMetricCollector.get().wafRequest()
97113
WafMetricCollector.get().wafRequestTriggered()
98114
WafMetricCollector.get().wafRequestBlocked()
115+
WafMetricCollector.get().wafRequestTimeout()
99116
WafMetricCollector.get().prepareMetrics()
100117
periodicAction.doIteration(telemetryService)
101118

@@ -112,7 +129,8 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
112129
'waf_version:0.0.0',
113130
'event_rules_version:rules_ver_2',
114131
'rule_triggered:false',
115-
'request_blocked:false'
132+
'request_blocked:false',
133+
'waf_timeout:false'
116134
]
117135
} )
118136
1 * telemetryService.addMetric( { Metric metric ->
@@ -123,7 +141,8 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
123141
'waf_version:0.0.0',
124142
'event_rules_version:rules_ver_2',
125143
'rule_triggered:true',
126-
'request_blocked:false'
144+
'request_blocked:false',
145+
'waf_timeout:false'
127146
]
128147
} )
129148
1 * telemetryService.addMetric( { Metric metric ->
@@ -134,7 +153,20 @@ class WafMetricPeriodicActionSpecification extends DDSpecification {
134153
'waf_version:0.0.0',
135154
'event_rules_version:rules_ver_2',
136155
'rule_triggered:true',
137-
'request_blocked:true'
156+
'request_blocked:true',
157+
'waf_timeout:false'
158+
]
159+
} )
160+
1 * telemetryService.addMetric( { Metric metric ->
161+
metric.namespace == 'appsec' &&
162+
metric.metric == 'waf.requests' &&
163+
metric.points[0][1] == 1 &&
164+
metric.tags == [
165+
'waf_version:0.0.0',
166+
'event_rules_version:rules_ver_2',
167+
'rule_triggered:false',
168+
'request_blocked:false',
169+
'waf_timeout:true'
138170
]
139171
} )
140172
0 * _._

0 commit comments

Comments
 (0)