[AAP] Added WAF metrics. Reorganized other metrics to match RFC (#7356)

## Summary of changes
Implemented some WAF missing metrics, as `WafTimeouts`
Reorganized existing metrics which were defined as Tags instead of
Metrics, matching the definition in the RFC

## Reason for change
WAF metrics
[RFC](https://docs.google.com/document/d/1D4hkC0jwwUyeo0hEQgyKP54kM1LZU98GL8MaP60tQrA/edit?tab=t.0)

## Implementation details

## Test coverage

## Other details
<!-- Fixes #{issue} -->


<!--  ⚠️ Note:

Where possible, please obtain 2 approvals prior to merging. Unless
CODEOWNERS specifies otherwise, for external teams it is typically best
to have one review from a team member, and one review from apm-dotnet.
Trivial changes do not require 2 reviews.

MergeQueue is NOT enabled in this repository. If you have write access
to the repo, the PR has 1-2 approvals (see above), and all of the
required checks have passed, you can use the Squash and Merge button to
merge the PR. If you don't have write access, or you need help, reach
out in the #apm-dotnet channel in Slack.
-->

Author: daniel-romano-DD

tracer/src/Datadog.Trace/AppSec/AppSecRequestContext.cs

@@ -6,6 +6,7 @@
 #nullable enable
 
 using System.Collections.Generic;
+using System.Threading;
 using Datadog.Trace.AppSec.Rasp;
 using Datadog.Trace.AppSec.Waf;
 using Datadog.Trace.Logging;
@@ -22,13 +23,14 @@ internal partial class AppSecRequestContext
     private const string AppsecKey = "appsec";
     private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor<AppSecRequestContext>();
     private readonly object _sync = new();
-    private readonly RaspTelemetryHelper? _raspTelemetryHelper = Security.Instance.RaspEnabled ? new RaspTelemetryHelper() : null;
+    private readonly RaspMetricsHelper? _raspMetricsHelper = Security.Instance.RaspEnabled ? new RaspMetricsHelper() : null;
     private readonly List<object> _wafSecurityEvents = new();
+    private int _wafTimeout = 0;
     private int? _wafError = null;
     private int? _wafRaspError = null;
     private Dictionary<string, List<Dictionary<string, object>>>? _raspStackTraces;
 
-    internal void CloseWebSpan(TraceTagCollection tags, Span span)
+    internal void CloseWebSpan(Span span)
     {
         lock (_sync)
         {
@@ -43,31 +45,42 @@ internal void CloseWebSpan(TraceTagCollection tags, Span span)
                 else
                 {
                     var triggers = JsonConvert.SerializeObject(_wafSecurityEvents);
-                    tags.SetTag(Tags.AppSecJson, "{\"triggers\":" + triggers + "}");
+                    span.Tags.SetTag(Tags.AppSecJson, "{\"triggers\":" + triggers + "}");
                 }
             }
 
-            if (_raspStackTraces?.Count > 0)
+            if (_wafTimeout > 0)
             {
-                span.SetMetaStruct(StackKey, MetaStructHelper.ObjectToByteArray(_raspStackTraces));
+                span.Tags.SetMetric(Metrics.WafTimeouts, _wafTimeout);
             }
 
             if (_wafError != null)
             {
-                tags.SetTag(Tags.WafError, _wafError.ToString());
+                span.Tags.SetMetric(Metrics.WafError, _wafError);
             }
 
             if (_wafRaspError != null)
             {
-                tags.SetTag(Tags.RaspWafError, _wafRaspError.ToString());
+                span.Tags.SetMetric(Metrics.RaspWafError, _wafRaspError);
+            }
+
+            if (_raspStackTraces?.Count > 0)
+            {
+                span.SetMetaStruct(StackKey, MetaStructHelper.ObjectToByteArray(_raspStackTraces));
             }
 
-            _raspTelemetryHelper?.GenerateRaspSpanMetricTags(span.Tags);
+            _raspMetricsHelper?.GenerateRaspSpanMetricTags(span.Tags);
         }
     }
 
-    internal void CheckWAFError(int code, bool isRasp)
+    internal void CheckWAFError(IResult result, bool isRasp)
     {
+        if (result.Timeout)
+        {
+            _wafTimeout++;
+        }
+
+        var code = (int)result.ReturnCode;
         int? existingValue = isRasp ? _wafRaspError : _wafError;
         if (code < 0 && (existingValue == null || existingValue < code))
         {
@@ -86,7 +99,7 @@ internal void AddRaspSpanMetrics(ulong duration, ulong durationWithBindings, boo
     {
         lock (_sync)
         {
-            _raspTelemetryHelper?.AddRaspSpanMetrics(duration, durationWithBindings, timeout);
+            _raspMetricsHelper?.AddRaspSpanMetrics(duration, durationWithBindings, timeout);
         }
     }
 

tracer/src/Datadog.Trace/AppSec/Coordinator/SecurityCoordinator.cs

@@ -99,7 +99,7 @@ private void SetErrorInformation(bool isRasp, IResult? result)
     {
         if (result is not null)
         {
-            _localRootSpan.Context.TraceContext.AppSecRequestContext.CheckWAFError((int)result.ReturnCode, isRasp);
+            _localRootSpan.Context.TraceContext.AppSecRequestContext.CheckWAFError(result, isRasp);
         }
     }
 

tracer/src/Datadog.Trace/AppSec/Rasp/RaspTelemetryHelper.cs renamed to tracer/src/Datadog.Trace/AppSec/Rasp/RaspMetricsHelper.cs

@@ -1,4 +1,4 @@
-// <copyright file="RaspTelemetryHelper.cs" company="Datadog">
+// <copyright file="RaspMetricsHelper.cs" company="Datadog">
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
@@ -8,7 +8,7 @@
 #nullable enable
 namespace Datadog.Trace.AppSec.Rasp;
 
-internal class RaspTelemetryHelper
+internal class RaspMetricsHelper
 {
     private ulong _raspWafDuration = 0;
     private ulong _raspWafAndBindingsDuration = 0;

tracer/src/Datadog.Trace/Metrics.cs

@@ -87,6 +87,21 @@ internal static class Metrics
         /// </summary>
         public const string AppSecWafAndBindingsDuration = "_dd.appsec.waf.duration_ext";
 
+        /// <summary>
+        /// The number of WAF timeouts during a request
+        /// </summary>
+        public const string WafTimeouts = "_dd.appsec.waf.timeouts";
+
+        /// <summary>
+        /// The most relevant WAF error code during a request
+        /// </summary>
+        public const string WafError = "_dd.appsec.waf.error";
+
+        /// <summary>
+        /// The most relevant WAF error code during a request when using RASP
+        /// </summary>
+        public const string RaspWafError = "_dd.appsec.rasp.error";
+
         /// <summary>
         /// Total cumulative waf duration for RASP calls across spans for one request
         /// </summary>

tracer/src/Datadog.Trace/Tags.cs

@@ -614,16 +614,6 @@ public static partial class Tags
         /// </summary>
         internal const string AppSecWafVersion = "_dd.appsec.waf.version";
 
-        /// <summary>
-        /// The most relevant WAF error code during a request
-        /// </summary>
-        public const string WafError = "_dd.appsec.waf.error";
-
-        /// <summary>
-        /// The most relevant WAF error code during a request when using RASP
-        /// </summary>
-        public const string RaspWafError = "_dd.appsec.rasp.error";
-
         /// <summary>
         ///  String-serialized JSON array, each item being a map containing:
         ///  Error(e) - the error string.

tracer/src/Datadog.Trace/TraceContext.cs

@@ -180,7 +180,7 @@ public void CloseSpan(Span span)
 
                     if (_appSecRequestContext is not null)
                     {
-                        _appSecRequestContext.CloseWebSpan(Tags, span);
+                        _appSecRequestContext.CloseWebSpan(span);
                         _appSecRequestContext.DisposeAdditiveContext();
                     }
                 }

tracer/test/Datadog.Trace.Security.Unit.Tests/AppSecContextTests.cs

@@ -4,6 +4,7 @@
 // </copyright>
 
 using System.Collections.Generic;
+using Datadog.Trace.AppSec.Waf;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.Tagging;
 using FluentAssertions;
@@ -13,23 +14,70 @@ namespace Datadog.Trace.Security.Unit.Tests;
 
 public class AppSecContextTests
 {
-    [InlineData(-2, -2, -1, -2, -1)]
-    [InlineData(2, -2, -1, null, -1)]
-    [InlineData(2, 2, 1, null, null)]
+    [InlineData(-2, -2, -1, 0, -2, -1)]
+    [InlineData(2, -2, -1, 1, null, -1)]
+    [InlineData(2, 2, 1, 2, null, null)]
     [Theory]
-    public void GivenAQuery_WhenWAFError_ThenSpanHasErrorTags(int raspErrorCode, int wafErrorCode, int wafErrorCode2, int? expectedRaspErrorCode, int? expectedWafErrorCode)
+    public void GivenAQuery_WhenWAFError_ThenSpanHasErrorTags(int raspErrorCode, int wafErrorCode, int wafErrorCode2, int wafTimeouts, int? expectedRaspErrorCode, int? expectedWafErrorCode)
     {
         var settings = TracerSettings.Create(new Dictionary<string, object>());
         var tracer = new Tracer(settings, null, null, null, null);
         var rootTestScope = (Scope)tracer.StartActive("test.trace");
 
         var appSecContext = rootTestScope.Span.Context.TraceContext.AppSecRequestContext;
-        appSecContext.CheckWAFError(raspErrorCode, true);
-        appSecContext.CheckWAFError(wafErrorCode, false);
-        appSecContext.CheckWAFError(wafErrorCode2, false);
-        TraceTagCollection tags = new();
-        appSecContext.CloseWebSpan(tags, rootTestScope.Span);
-        tags.GetTag(Tags.WafError).Should().Be(expectedWafErrorCode?.ToString());
-        tags.GetTag(Tags.RaspWafError).Should().Be(expectedRaspErrorCode?.ToString());
+        int timeouts = wafTimeouts;
+
+        appSecContext.CheckWAFError(new MockResult(raspErrorCode, timeouts-- > 0), true);
+        appSecContext.CheckWAFError(new MockResult(wafErrorCode, timeouts-- > 0), false);
+        appSecContext.CheckWAFError(new MockResult(wafErrorCode2, timeouts-- > 0), false);
+        appSecContext.CloseWebSpan(rootTestScope.Span);
+        rootTestScope.Span.GetMetric(Metrics.WafError).Should().Be(expectedWafErrorCode);
+        rootTestScope.Span.GetMetric(Metrics.RaspWafError).Should().Be(expectedRaspErrorCode);
+
+        if (wafTimeouts > 0)
+        {
+            rootTestScope.Span.GetMetric(Metrics.WafTimeouts).Should().Be(wafTimeouts);
+        }
+    }
+
+    private class MockResult : IResult
+    {
+        public MockResult(int returnCode, bool timeout)
+        {
+            ReturnCode = (WafReturnCode)returnCode;
+            Timeout = timeout;
+        }
+
+        public WafReturnCode ReturnCode { get; }
+
+        public bool Timeout { get; }
+
+        public bool ShouldBlock => throw new System.NotImplementedException();
+
+        public Dictionary<string, object> BlockInfo => throw new System.NotImplementedException();
+
+        public Dictionary<string, object> RedirectInfo => throw new System.NotImplementedException();
+
+        public Dictionary<string, object> SendStackInfo => throw new System.NotImplementedException();
+
+        public IReadOnlyCollection<object> Data => throw new System.NotImplementedException();
+
+        public Dictionary<string, object> Actions => throw new System.NotImplementedException();
+
+        public ulong AggregatedTotalRuntime => throw new System.NotImplementedException();
+
+        public ulong AggregatedTotalRuntimeWithBindings => throw new System.NotImplementedException();
+
+        public ulong AggregatedTotalRuntimeRasp => throw new System.NotImplementedException();
+
+        public ulong AggregatedTotalRuntimeWithBindingsRasp => throw new System.NotImplementedException();
+
+        public uint RaspRuleEvaluations => throw new System.NotImplementedException();
+
+        public Dictionary<string, object> ExtractSchemaDerivatives => throw new System.NotImplementedException();
+
+        public Dictionary<string, object> FingerprintDerivatives => throw new System.NotImplementedException();
+
+        public bool ShouldReportSecurityResult => throw new System.NotImplementedException();
     }
 }

tracer/test/Datadog.Trace.Security.Unit.Tests/RASP/RaspTelemetryHelperTests.cs

@@ -17,7 +17,7 @@ public class RaspTelemetryHelperTests
         [Fact]
         public void GivenARaspTelemetryHelper_WhenGenerateRaspSpanMetricTagsIsCalledWithTimeout_ThenTagIsSet()
         {
-            var raspTelemetryHelper = new RaspTelemetryHelper();
+            var raspTelemetryHelper = new RaspMetricsHelper();
             var tags = new TagsList();
             raspTelemetryHelper.AddRaspSpanMetrics(1000, 2000, true);
             raspTelemetryHelper.GenerateRaspSpanMetricTags(tags);