[Tracer] Support remote config for DD_TRACE_SAMPLING_RULES and Adaptive Sampling (#5453)

* clean up

* add mapping to DynamicConfigConfigurationSource

* add CustomSamplingRules to ImmutableDynamicSettings

* subscribe to new RCM capability

* add SamplingRuleProvenance

* add provenance to sampling rules

* return SamplingMechanism and Priority based on Provenance

* override ToString() to use in logs instead of rule name

* fix compiler error in tests

* rename provenance constant fields

* rename mechanism constant field

* update DynamicConfigurationTests

* make provenance json field not required and fix fallback value

* add unit test

* rename ImmutableDynamicSettings.CustomSamplingRules to SamplingRules

* tweak xml doc comment

* rename DynamicConfigurationTests+Config.CustomSamplingRules to SamplingRules

* update test

* fix typo in comment

* rename method

* add BuildFromRemoteConfigurationString() (not implemented yet) and call it

* rename GlobalSamplingRule to GlobalSamplingRateRule

* disabled warning for unused auto-property setter

* rename to CustomRuleConfig to LocalSamplingRuleConfig

* add RemoteSamplingRuleConfig

* implement BuildFromRemoteConfigurationString()

* rename types, add code comments

* add test for ConvertToLocalTags()

* add test for remote config json

* refactoring rules into derived classes

* add argument names

* add missing brace

* rename field

* fix build errors

* minor refactoring

* add remote rules before local rules

* add the AgentSamplingRule from outside TracerSampler

* return arrays instead of IEnumerable<CustomSamplingRule>

* expand code comments

* rename variables

* remove ISamplingRule.Priority

* fix compiler errors in unit tests

* rename json models

* fix unit tests by registering AgentSamplingRule

* push more code down from CustomSamplingRule to derived types

* remove outdated comment

* clean up code

* expand test

* clean up code

* rename test class

* add remote sampling rules to configuration log entry

* test for remote sampling rules

* don't use Should().HaveCount()

* add TraceSampler.RegisterAgentSamplingRule()

* fix xml-doc comments

* ignore additional AgentSamplingRule registrations

* add xml-doc comments

* add TraceSampler.RegisterRules()

Author: lucaspimentel

tracer/src/Datadog.Trace/Configuration/ConfigurationSources/DynamicConfigConfigurationSource.cs

@@ -24,7 +24,7 @@ internal class DynamicConfigConfigurationSource : JsonConfigurationSource
             // { ConfigurationKeys.ServiceNameMappings, "tracing_service_mapping" },
             { ConfigurationKeys.LogsInjectionEnabled, "log_injection_enabled" },
             { ConfigurationKeys.GlobalSamplingRate, "tracing_sampling_rate" },
-            // { ConfigurationKeys.CustomSamplingRules, "tracing_sampling_rules" },
+            { ConfigurationKeys.CustomSamplingRules, "tracing_sampling_rules" },
             // { ConfigurationKeys.SpanSamplingRules, "span_sampling_rules" },
             // { ConfigurationKeys.DataStreamsMonitoring.Enabled, "data_streams_enabled" },
             { ConfigurationKeys.GlobalTags, "tracing_tags" }

tracer/src/Datadog.Trace/Configuration/DynamicConfigurationManager.cs

@@ -46,6 +46,7 @@ public void Start()
                 _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingLogsInjection, true);  // 13
                 _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingSampleRate, true);     // 12
                 _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingTracingEnabled, true); // 19
+                _subscriptionManager.SetCapability(RcmCapabilitiesIndices.ApmTracingSampleRules, true);    // 29
             }
         }
 
@@ -76,7 +77,7 @@ private static void OnConfigurationChanged(ConfigurationBuilder settings)
                 TraceEnabled = settings.WithKeys(ConfigurationKeys.TraceEnabled).AsBool(),
                 // RuntimeMetricsEnabled = settings.WithKeys(ConfigurationKeys.RuntimeMetricsEnabled).AsBool(),
                 // DataStreamsMonitoringEnabled = settings.WithKeys(ConfigurationKeys.DataStreamsMonitoring.Enabled).AsBool(),
-                // CustomSamplingRules = settings.WithKeys(ConfigurationKeys.CustomSamplingRules).AsString(),
+                SamplingRules = settings.WithKeys(ConfigurationKeys.CustomSamplingRules).AsString(),
                 GlobalSamplingRate = settings.WithKeys(ConfigurationKeys.GlobalSamplingRate).AsDouble(),
                 // SpanSamplingRules = settings.WithKeys(ConfigurationKeys.SpanSamplingRules).AsString(),
                 LogsInjectionEnabled = settings.WithKeys(ConfigurationKeys.LogsInjectionEnabled).AsBool(),

tracer/src/Datadog.Trace/Configuration/ImmutableDynamicSettings.cs

@@ -20,7 +20,7 @@ internal class ImmutableDynamicSettings : IEquatable<ImmutableDynamicSettings>
 
         public double? GlobalSamplingRate { get; init; }
 
-        public string? SpanSamplingRules { get; init; }
+        public string? SamplingRules { get; init; }
 
         public bool? LogsInjectionEnabled { get; init; }
 
@@ -47,7 +47,7 @@ public bool Equals(ImmutableDynamicSettings? other)
              && RuntimeMetricsEnabled == other.RuntimeMetricsEnabled
              && DataStreamsMonitoringEnabled == other.DataStreamsMonitoringEnabled
              && Nullable.Equals(GlobalSamplingRate, other.GlobalSamplingRate)
-             && SpanSamplingRules == other.SpanSamplingRules
+             && SamplingRules == other.SamplingRules
              && LogsInjectionEnabled == other.LogsInjectionEnabled
              && AreEqual(HeaderTags, other.HeaderTags)
              && AreEqual(ServiceNameMappings, other.ServiceNameMappings)
@@ -81,15 +81,15 @@ public override int GetHashCode()
                 RuntimeMetricsEnabled,
                 DataStreamsMonitoringEnabled,
                 GlobalSamplingRate,
-                SpanSamplingRules,
+                SamplingRules,
                 LogsInjectionEnabled);
         }
 
         private static bool AreEqual(IReadOnlyDictionary<string, string>? dictionary1, IReadOnlyDictionary<string, string>? dictionary2)
         {
             if (dictionary1 == null || dictionary2 == null)
             {
-                return dictionary1 == dictionary2;
+                return ReferenceEquals(dictionary1, dictionary2);
             }
 
             if (dictionary1.Count != dictionary2.Count)

tracer/src/Datadog.Trace/Configuration/ImmutableTracerSettings.cs

@@ -38,7 +38,6 @@ public partial record ImmutableTracerSettings
         private readonly IReadOnlyDictionary<string, string> _globalTags;
         private readonly double? _globalSamplingRate;
         private readonly bool _runtimeMetricsEnabled;
-        private readonly string? _spanSamplingRules;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="ImmutableTracerSettings"/> class
@@ -99,7 +98,7 @@ internal ImmutableTracerSettings(TracerSettings settings, bool unusedParamNotToU
             MaxTracesSubmittedPerSecondInternal = settings.MaxTracesSubmittedPerSecondInternal;
             CustomSamplingRulesInternal = settings.CustomSamplingRulesInternal;
             CustomSamplingRulesFormat = settings.CustomSamplingRulesFormat;
-            _spanSamplingRules = settings.SpanSamplingRules;
+            SpanSamplingRules = settings.SpanSamplingRules;
             _globalSamplingRate = settings.GlobalSamplingRateInternal;
             IntegrationsInternal = new ImmutableIntegrationSettingsCollection(settings.IntegrationsInternal, settings.DisabledIntegrationNamesInternal);
             _headerTags = new ReadOnlyDictionary<string, string>(settings.HeaderTagsInternal);
@@ -299,6 +298,13 @@ internal ImmutableTracerSettings(TracerSettings settings, bool unusedParamNotToU
         [GeneratePublicApi(PublicApiUsage.ImmutableTracerSettings_CustomSamplingRules_Get)]
         internal string? CustomSamplingRulesInternal { get; }
 
+        /// <summary>
+        /// Gets the trace sampling rules from remote config.
+        /// These contain custom remote rules and dynamic (aka adaptive) rules.
+        /// They will be merged with local sampling rules.
+        /// </summary>
+        internal string? RemoteSamplingRules => DynamicSettings.SamplingRules;
+
         /// <summary>
         /// Gets a value indicating the format for custom sampling rules ("regex" or "glob").
         /// </summary>
@@ -309,7 +315,7 @@ internal ImmutableTracerSettings(TracerSettings settings, bool unusedParamNotToU
         /// Gets a value indicating the span sampling rules.
         /// </summary>
         /// <seealso cref="ConfigurationKeys.SpanSamplingRules"/>
-        internal string? SpanSamplingRules => DynamicSettings.SpanSamplingRules ?? _spanSamplingRules;
+        internal string? SpanSamplingRules { get; }
 
         /// <summary>
         /// Gets a value indicating a global rate for sampling.

tracer/src/Datadog.Trace/Configuration/TracerSettings.cs

@@ -213,7 +213,7 @@ internal TracerSettings(IConfigurationSource? source, IConfigurationTelemetry te
                                                    converter: value =>
                                                    {
                                                        // We intentionally report invalid values as "valid" in the converter,
-                                                       // because we don't want to automatically fallback to the
+                                                       // because we don't want to automatically fall back to the
                                                        // default value.
                                                        if (!SamplingRulesFormat.IsValid(value, out var normalizedFormat))
                                                        {

tracer/src/Datadog.Trace/Sampling/AgentSamplingRule.cs

@@ -28,11 +28,6 @@ internal class AgentSamplingRule : ISamplingRule
                                             Datadog.Trace.Sampling.SamplingMechanism.Default :
                                             Datadog.Trace.Sampling.SamplingMechanism.AgentRate;
 
-        /// <summary>
-        /// Gets the lowest possible priority
-        /// </summary>
-        public int Priority => int.MinValue;
-
         public bool IsMatch(Span span) => true;
 
         public float GetSamplingRate(Span span)

tracer/src/Datadog.Trace/Sampling/CustomSamplingRule.cs

@@ -8,15 +8,11 @@
 using System;
 using System.Collections.Generic;
 using System.Text.RegularExpressions;
-using Datadog.Trace.Logging;
-using Datadog.Trace.Vendors.Newtonsoft.Json;
 
 namespace Datadog.Trace.Sampling
 {
-    internal class CustomSamplingRule : ISamplingRule
+    internal abstract class CustomSamplingRule : ISamplingRule
     {
-        private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor<CustomSamplingRule>();
-
         private readonly float _samplingRate;
         private readonly bool _alwaysMatch;
 
@@ -28,7 +24,7 @@ internal class CustomSamplingRule : ISamplingRule
 
         private bool _regexTimedOut;
 
-        public CustomSamplingRule(
+        protected CustomSamplingRule(
             float rate,
             string patternFormat,
             string? serviceNamePattern,
@@ -54,46 +50,9 @@ _resourceNameRegex is null &&
             }
         }
 
-        public int SamplingMechanism => Datadog.Trace.Sampling.SamplingMechanism.TraceSamplingRule;
-
-        /// <summary>
-        /// Gets the priority of the rule.
-        /// Configuration rules will default to 1 as a priority and rely on order of specification.
-        /// </summary>
-        public int Priority => 1;
-
-        public static IEnumerable<CustomSamplingRule> BuildFromConfigurationString(string configuration, string patternFormat, TimeSpan timeout)
-        {
-            try
-            {
-                if (!string.IsNullOrWhiteSpace(configuration) &&
-                    JsonConvert.DeserializeObject<List<CustomRuleConfig>>(configuration) is { Count: > 0 } rules)
-                {
-                    var samplingRules = new List<CustomSamplingRule>(rules.Count);
-
-                    foreach (var r in rules)
-                    {
-                        samplingRules.Add(
-                            new CustomSamplingRule(
-                                rate: r.SampleRate,
-                                patternFormat: patternFormat,
-                                serviceNamePattern: r.Service,
-                                operationNamePattern: r.OperationName,
-                                resourceNamePattern: r.Resource,
-                                tagPatterns: r.Tags,
-                                timeout: timeout));
-                    }
+        public abstract string Provenance { get; }
 
-                    return samplingRules;
-                }
-            }
-            catch (Exception ex)
-            {
-                Log.Error(ex, "Unable to parse the trace sampling rules.");
-            }
-
-            return [];
-        }
+        public abstract int SamplingMechanism { get; }
 
         public bool IsMatch(Span span)
         {
@@ -128,32 +87,5 @@ public float GetSamplingRate(Span span)
             span.SetMetric(Metrics.SamplingRuleDecision, _samplingRate);
             return _samplingRate;
         }
-
-        public override string ToString()
-        {
-            // later this will return different values depending on the rule's provenance:
-            // local, customer (remote), or dynamic (remote)
-            return "LocalSamplingRule";
-        }
-
-        // ReSharper disable once ClassNeverInstantiated.Local
-        private class CustomRuleConfig
-        {
-            [JsonRequired]
-            [JsonProperty(PropertyName = "sample_rate")]
-            public float SampleRate { get; set; }
-
-            [JsonProperty(PropertyName = "name")]
-            public string? OperationName { get; set; }
-
-            [JsonProperty(PropertyName = "service")]
-            public string? Service { get; set; }
-
-            [JsonProperty(PropertyName = "resource")]
-            public string? Resource { get; set; }
-
-            [JsonProperty(PropertyName = "tags")]
-            public Dictionary<string, string?>? Tags { get; set; }
-        }
     }
 }

…dog.Trace/Sampling/GlobalSamplingRule.cs …Trace/Sampling/GlobalSamplingRateRule.cstracer/src/Datadog.Trace/Sampling/GlobalSamplingRule.cs renamed to tracer/src/Datadog.Trace/Sampling/GlobalSamplingRateRule.cs tracer/src/Datadog.Trace/Sampling/GlobalSamplingRule.cs renamed to tracer/src/Datadog.Trace/Sampling/GlobalSamplingRateRule.cs

@@ -1,4 +1,4 @@
-// <copyright file="GlobalSamplingRule.cs" company="Datadog">
+// <copyright file="GlobalSamplingRateRule.cs" company="Datadog">
 // Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
@@ -7,21 +7,16 @@
 
 namespace Datadog.Trace.Sampling
 {
-    internal class GlobalSamplingRule : ISamplingRule
+    internal class GlobalSamplingRateRule : ISamplingRule
     {
         private readonly float _globalRate;
 
-        public GlobalSamplingRule(float rate)
+        public GlobalSamplingRateRule(float rate)
         {
             _globalRate = rate;
         }
 
-        /// <summary>
-        /// Gets the priority which is one beneath custom rules.
-        /// </summary>
-        public int Priority => 0;
-
-        public int SamplingMechanism => Datadog.Trace.Sampling.SamplingMechanism.TraceSamplingRule;
+        public int SamplingMechanism => Datadog.Trace.Sampling.SamplingMechanism.LocalTraceSamplingRule;
 
         public bool IsMatch(Span span) => true;
 

tracer/src/Datadog.Trace/Sampling/ISamplingRule.cs

@@ -9,13 +9,6 @@ namespace Datadog.Trace.Sampling
 {
     internal interface ISamplingRule
     {
-        /// <summary>
-        /// Gets the rule priority.
-        /// Higher number means higher priority.
-        /// Not related to sampling priority.
-        /// </summary>
-        int Priority { get; }
-
         int SamplingMechanism { get; }
 
         bool IsMatch(Span span);

tracer/src/Datadog.Trace/Sampling/LocalCustomSamplingRule.cs

@@ -0,0 +1,99 @@
+// <copyright file="LocalCustomSamplingRule.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+using System;
+using System.Collections.Generic;
+using Datadog.Trace.Logging;
+using Datadog.Trace.Vendors.Newtonsoft.Json;
+
+namespace Datadog.Trace.Sampling;
+
+internal sealed class LocalCustomSamplingRule : CustomSamplingRule
+{
+    private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor<LocalCustomSamplingRule>();
+
+    public LocalCustomSamplingRule(
+        float rate,
+        string patternFormat,
+        string? serviceNamePattern,
+        string? operationNamePattern,
+        string? resourceNamePattern,
+        ICollection<KeyValuePair<string, string?>>? tagPatterns,
+        TimeSpan timeout)
+        : base(
+            rate: rate,
+            patternFormat: patternFormat,
+            serviceNamePattern: serviceNamePattern,
+            operationNamePattern: operationNamePattern,
+            resourceNamePattern: resourceNamePattern,
+            tagPatterns: tagPatterns,
+            timeout: timeout)
+    {
+    }
+
+    public override string Provenance => SamplingRuleProvenance.Local;
+
+    public override int SamplingMechanism => Sampling.SamplingMechanism.LocalTraceSamplingRule;
+
+    public static LocalCustomSamplingRule[] BuildFromConfigurationString(
+        string configuration,
+        string patternFormat,
+        TimeSpan timeout)
+    {
+        try
+        {
+            if (!string.IsNullOrWhiteSpace(configuration) &&
+                JsonConvert.DeserializeObject<List<RuleConfigJsonModel>>(configuration) is { Count: > 0 } ruleModels)
+            {
+                var samplingRules = new LocalCustomSamplingRule[ruleModels.Count];
+
+                for (var i = 0; i < ruleModels.Count; i++)
+                {
+                    var r = ruleModels[i];
+
+                    var samplingRule = new LocalCustomSamplingRule(
+                        rate: r.SampleRate,
+                        patternFormat: patternFormat, // from DD_TRACE_SAMPLING_RULES_FORMAT
+                        serviceNamePattern: r.Service,
+                        operationNamePattern: r.OperationName,
+                        resourceNamePattern: r.Resource,
+                        tagPatterns: r.Tags,
+                        timeout: timeout);
+
+                    samplingRules[i] = samplingRule;
+                }
+
+                return samplingRules;
+            }
+        }
+        catch (Exception ex)
+        {
+            Log.Error(ex, "Unable to parse the trace sampling rules.");
+        }
+
+        return [];
+    }
+
+    internal class RuleConfigJsonModel
+    {
+        [JsonRequired]
+        [JsonProperty(PropertyName = "sample_rate")]
+        public float SampleRate { get; set; }
+
+        [JsonProperty(PropertyName = "name")]
+        public string? OperationName { get; set; }
+
+        [JsonProperty(PropertyName = "service")]
+        public string? Service { get; set; }
+
+        [JsonProperty(PropertyName = "resource")]
+        public string? Resource { get; set; }
+
+        [JsonProperty(PropertyName = "tags")]
+        public Dictionary<string, string?>? Tags { get; set; }
+    }
+}