Refactor the profiling configuration reading (#7303)

## Summary of changes

Refactors how we determine the profiler's current configuration state

## Reason for change

Today, the profiler determines if it's enabled or disabled independently
of the tracer, in the native code. We then try to mirror that state in
the managed code, for several reasons

- To know whether we need to send app-domain information to the profiler
(e.g. service name, env)
- To know whether we need to send thread-context-change events to the
profiler (for CodeHotspots/EndpointProfiling)
- For telemetry reasons
- For recording the state in DSM payloads

As part of the hands-off config work, we will calculate whether to
enable the profiler based on the _full_ managed config, instead of just
using environment variables. However, when managed activation is
disabled (for back-compat investigation purposes), we _do_ need to just
use env vars, to "mirror" the state that will be calculated by the
native continuous profiler, essentially working in the existing "legacy"
state.

The refactoring in this PR is intended to lay the groundwork for the
future hands-off config work in the profiler, as well as to solve some
existing issues with inconsistent/missing telemetry

## Implementation details

- Add a `ProfilerState` `enum` to track the profiler as a 3-state
enablement which represents its real state, instead of trying to force
it as a boolean
- Use `EnvironmentConfigurationSource` to record variables read from env
in telemetry correctly.
- Add support for the `DD_PROFILER_MANAGED_ACTIVATION_ENABLED` which
will be used shortly

## Test coverage

Added unit tests for the new `ProfilerSettings` which encapsulates most
of the logic

## Other details

This PR technically requires support for
`DD_PROFILER_MANAGED_ACTIVATION_ENABLED` has been merged in order to
report the correct values. This isn't a big deal, as it will likely be
merged soon, but if it isn't, we should tweak the logic in
`ProfilerSettings` so that it's correct. Alternatively, I can split this
into two PRs - one that is correct _today_, and one that is correct only
with the profiler PR

---------

Co-authored-by: chrisnas <[email protected]>

Author: andrewlock

tracer/src/Datadog.Trace/Configuration/TracerSettings.cs

@@ -20,6 +20,7 @@
 using Datadog.Trace.ClrProfiler.ServerlessInstrumentation;
 using Datadog.Trace.Configuration.ConfigurationSources.Telemetry;
 using Datadog.Trace.Configuration.Telemetry;
+using Datadog.Trace.ContinuousProfiler;
 using Datadog.Trace.Logging;
 using Datadog.Trace.Logging.DirectSubmission;
 using Datadog.Trace.Processors;
@@ -139,28 +140,6 @@ internal TracerSettings(IConfigurationSource? source, IConfigurationTelemetry te
                 AzureAppServiceMetadata = new ImmutableAzureAppServiceSettings(source, _telemetry);
             }
 
-            // With SSI, beyond ContinuousProfiler.ConfigurationKeys.ProfilingEnabled (true or auto vs false),
-            // the profiler could be enabled via ContinuousProfiler.ConfigurationKeys.SsiDeployed:
-            //  - if it contains "profiler", the profiler is enabled after 30 seconds + at least 1 span
-            //  - if not, the profiler needed to be loaded by the CLR but no profiling will be done, only telemetry metrics will be sent
-            // So, for the Tracer, the profiler should be seen as enabled if ContinuousProfiler.ConfigurationKeys.SsiDeployed has a value
-            // (even without "profiler") so that spans will be sent to the profiler.
-            ProfilingEnabledInternal = config
-                         .WithKeys(ContinuousProfiler.ConfigurationKeys.ProfilingEnabled)
-                         .GetAs(
-                            converter: x => x switch
-                            {
-                                "auto" => true,
-                                _ when x.ToBoolean() is { } boolean => boolean,
-                                _ => ParsingResult<bool>.Failure(),
-                            },
-                            getDefaultValue: () =>
-                            {
-                                var profilingSsiDeployed = config.WithKeys(ContinuousProfiler.ConfigurationKeys.SsiDeployed).AsString();
-                                return (profilingSsiDeployed != null);
-                            },
-                            validator: null);
-
             var otelTags = config
                           .WithKeys(ConfigurationKeys.OpenTelemetry.ResourceAttributes)
                           .AsDictionaryResult(separator: '=');
@@ -890,13 +869,6 @@ static void RecordDisabledIntegrationsTelemetry(IntegrationSettingsCollection in
         /// <seealso cref="ConfigurationKeys.ApmTracingEnabled"/>
         internal bool ApmTracingEnabled => DynamicSettings.ApmTracingEnabled ?? _apmTracingEnabled;
 
-        /// <summary>
-        /// Gets a value indicating whether profiling is enabled.
-        /// Default is <c>false</c>.
-        /// </summary>
-        /// <seealso cref="ContinuousProfiler.ConfigurationKeys.ProfilingEnabled"/>
-        internal bool ProfilingEnabledInternal { get; }
-
         /// <summary>
         /// Gets the names of disabled integrations.
         /// </summary>

tracer/src/Datadog.Trace/ContinuousProfiler/ConfigurationKeys.cs

@@ -11,5 +11,6 @@ internal static class ConfigurationKeys
         public const string CodeHotspotsEnabled = "DD_PROFILING_CODEHOTSPOTS_ENABLED";
         public const string EndpointProfilingEnabled = "DD_PROFILING_ENDPOINT_COLLECTION_ENABLED";
         public const string SsiDeployed = "DD_INJECTION_ENABLED";
+        public const string ProfilerManagedActivationEnabled = "DD_PROFILER_MANAGED_ACTIVATION_ENABLED";
     }
 }

tracer/src/Datadog.Trace/ContinuousProfiler/Profiler.cs

@@ -4,24 +4,29 @@
 // </copyright>
 
 using System.Threading;
+using Datadog.Trace.Configuration;
+using Datadog.Trace.Telemetry;
 
 namespace Datadog.Trace.ContinuousProfiler
 {
     internal class Profiler
     {
         private static Profiler _instance;
 
-        internal Profiler(IContextTracker contextTracker, IProfilerStatus status)
+        internal Profiler(IContextTracker contextTracker, IProfilerStatus status, ProfilerSettings settings)
         {
             ContextTracker = contextTracker;
             Status = status;
+            Settings = settings;
         }
 
         public static Profiler Instance
         {
             get { return LazyInitializer.EnsureInitialized(ref _instance, () => Create()); }
         }
 
+        public ProfilerSettings Settings { get; }
+
         public IProfilerStatus Status { get; }
 
         public IContextTracker ContextTracker { get; }
@@ -33,9 +38,10 @@ internal static void SetInstanceForTests(Profiler value)
 
         private static Profiler Create()
         {
-            var status = new ProfilerStatus();
+            var settings = new ProfilerSettings(GlobalConfigurationSource.Instance, TelemetryFactory.Config);
+            var status = new ProfilerStatus(settings);
             var contextTracker = new ContextTracker(status);
-            return new Profiler(contextTracker, status);
+            return new Profiler(contextTracker, status, settings);
         }
     }
 }

tracer/src/Datadog.Trace/ContinuousProfiler/ProfilerSettings.cs

@@ -0,0 +1,106 @@
+// <copyright file="ProfilerSettings.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+using Datadog.Trace.Configuration;
+using Datadog.Trace.Configuration.ConfigurationSources.Telemetry;
+using Datadog.Trace.Configuration.Telemetry;
+
+namespace Datadog.Trace.ContinuousProfiler;
+
+internal class ProfilerSettings
+{
+    public ProfilerSettings(IConfigurationSource config, IConfigurationTelemetry telemetry)
+        : this(config, new EnvironmentConfigurationSource(), telemetry)
+    {
+    }
+
+    // Internal for testing only
+    internal ProfilerSettings(IConfigurationSource config, IConfigurationSource envConfig, IConfigurationTelemetry telemetry)
+    {
+        if (!IsProfilingSupported)
+        {
+            ProfilerState = ProfilerState.Disabled;
+            telemetry.Record(ConfigurationKeys.ProfilingEnabled, false, ConfigurationOrigins.Calculated);
+            return;
+        }
+
+        // If managed activation is enabled, we need to _just_ read from the environment variables,
+        // as that's all that applies
+        var envConfigBuilder = new ConfigurationBuilder(envConfig, telemetry);
+        var managedActivationEnabled = envConfigBuilder
+                                      .WithKeys(ConfigurationKeys.ProfilerManagedActivationEnabled)
+                                      .AsBool(true);
+
+        // If we're using managed activation, we use the "full" config source set.
+        // Otherwise we only read from the environment variables, to "match" the behavior of the profiler
+        var profilingConfig = managedActivationEnabled
+                                  ? new ConfigurationBuilder(config, telemetry)
+                                  : envConfigBuilder;
+
+        // With SSI, beyond ContinuousProfiler.ConfigurationKeys.ProfilingEnabled (true or auto vs false),
+        // the profiler could be enabled via ContinuousProfiler.ConfigurationKeys.SsiDeployed. If it is non-empty, then the
+        // profiler is "active", though won't begin profiling until 30 seconds have passed + at least 1 span has been generated.
+        var profilingEnabled = profilingConfig
+                              .WithKeys(ConfigurationKeys.ProfilingEnabled)
+                               // We stick with strings here instead of using the `GetAs` method,
+                               // so that telemetry continues to store true/false/auto, instead of the enum values.
+                              .AsString(
+                                   converter: x => x switch
+                                   {
+                                       "auto" => "auto",
+                                       _ when x.ToBoolean() is { } boolean => boolean ? "true" : "false",
+                                       _ => ParsingResult<string>.Failure(),
+                                   },
+                                   getDefaultValue: () =>
+                                   {
+                                       // If there's no explicit `DD_PROFILING_ENABLED` key,
+                                       // we set the state based on the SSI value, only checking env vars (not the full stack)
+                                       var isSsiDeployment = envConfigBuilder
+                                                            .WithKeys(ConfigurationKeys.SsiDeployed)
+                                                            .AsString();
+
+                                       return isSsiDeployment switch
+                                       {
+                                           { Length: > 0 } => "auto",
+                                           _ => "false",
+                                       };
+                                   },
+                                   validator: null);
+
+        ProfilerState = profilingEnabled switch
+        {
+            "auto" => ProfilerState.Auto,
+            "true" => ProfilerState.Enabled,
+            _ => ProfilerState.Disabled,
+        };
+    }
+
+    // Internal for testing only
+    internal ProfilerSettings(ProfilerState state)
+    {
+        ProfilerState = state;
+    }
+
+    /// <summary>
+    /// Gets a value indicating whether the profiler is supported on this platform at all.
+    /// If it's not supported, we should not try to P/Invoke to it or do any context tracking.
+    /// </summary>
+    public static bool IsProfilingSupported
+    {
+        get
+        {
+            var fd = FrameworkDescription.Instance;
+            return
+                (fd.OSPlatform == OSPlatformName.Windows && fd.ProcessArchitecture is ProcessArchitecture.X64 or ProcessArchitecture.X86) ||
+                (fd.OSPlatform == OSPlatformName.Linux && fd.ProcessArchitecture is ProcessArchitecture.X64);
+        }
+    }
+
+    public ProfilerState ProfilerState { get; }
+
+    public bool IsProfilerEnabled => ProfilerState != ProfilerState.Disabled;
+}

tracer/src/Datadog.Trace/ContinuousProfiler/ProfilerState.cs

@@ -0,0 +1,26 @@
+// <copyright file="ProfilerState.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+namespace Datadog.Trace.ContinuousProfiler;
+
+internal enum ProfilerState
+{
+    /// <summary>
+    /// The profiler is explicitly disabled via configuration
+    /// </summary>
+    Disabled = 0,
+
+    /// <summary>
+    /// The profiler is explicitly enabled via configuration
+    /// </summary>
+    Enabled = 1,
+
+    /// <summary>
+    /// The profiler is in "auto" mode; i.e. will start after a delay and if traces are created
+    /// </summary>
+    Auto = 2,
+}

tracer/src/Datadog.Trace/ContinuousProfiler/ProfilerStatus.cs

@@ -5,48 +5,30 @@
 
 using System;
 using System.Runtime.InteropServices;
-using Datadog.Trace.ExtensionMethods;
 using Datadog.Trace.Logging;
-using Datadog.Trace.Util;
 
 namespace Datadog.Trace.ContinuousProfiler
 {
     internal class ProfilerStatus : IProfilerStatus
     {
         private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(ProfilerStatus));
 
-        private readonly bool _isProfilingEnabled;
+        private readonly ProfilerState _profilerState;
         private readonly object _lockObj;
         private bool _isInitialized;
         private IntPtr _engineStatusPtr;
 
-        public ProfilerStatus()
+        public ProfilerStatus(ProfilerSettings settings)
         {
-            var fd = FrameworkDescription.Instance;
-            var isSupported =
-                (fd.OSPlatform == OSPlatformName.Windows && (fd.ProcessArchitecture == ProcessArchitecture.X64 || fd.ProcessArchitecture == ProcessArchitecture.X86)) ||
-                (fd.OSPlatform == OSPlatformName.Linux && fd.ProcessArchitecture == ProcessArchitecture.X64);
-
-            _isProfilingEnabled = false;
-
-            if (isSupported)
+            _profilerState = settings.ProfilerState;
+            var state = _profilerState switch
             {
-                var manualDeployement = EnvironmentHelpers.GetEnvironmentVariable(ConfigurationKeys.ProfilingEnabled);
-                if (manualDeployement != null)
-                {
-                    // it is possible that SSI installation script is setting the environment variable to "auto" to enable the profiler
-                    // instead of "true" to avoid starting the profiler immediately after the installation
-                    _isProfilingEnabled = manualDeployement.ToBoolean() ?? (manualDeployement == "auto");
-                }
-                else
-                {
-                    // the profiler is declared "enabled" just if the SSI environment variable exists to be sure that telemetry metrics
-                    // will contain the right status (i.e. we need the tracer to send the spans even if the profiler is not started yet)
-                    _isProfilingEnabled = (EnvironmentHelpers.GetEnvironmentVariable(ConfigurationKeys.SsiDeployed) != null);
-                }
-            }
+                ProfilerState.Enabled => "enabled",
+                ProfilerState.Auto => "auto",
+                _ => "disabled"
+            };
 
-            Log.Information("Continuous Profiler is {IsEnabled}.", _isProfilingEnabled ? "enabled" : "disabled");
+            Log.Information("Continuous Profiler mode = {ProfilerState}", state);
             _lockObj = new();
             _isInitialized = false;
         }
@@ -55,7 +37,7 @@ public bool IsProfilerReady
         {
             get
             {
-                if (!_isProfilingEnabled)
+                if (_profilerState == ProfilerState.Disabled)
                 {
                     return false;
                 }

tracer/src/Datadog.Trace/DataStreamsMonitoring/Aggregation/DataStreamsMessagePackFormatter.cs

@@ -9,6 +9,7 @@
 using System.Collections.Generic;
 using System.IO;
 using Datadog.Trace.Configuration;
+using Datadog.Trace.ContinuousProfiler;
 using Datadog.Trace.Vendors.Datadog.Sketches;
 using Datadog.Trace.Vendors.MessagePack;
 
@@ -51,7 +52,7 @@ internal class DataStreamsMessagePackFormatter
         private readonly byte[] _productMaskBytes = StringEncoding.UTF8.GetBytes("ProductMask");
         private readonly byte[] _isInDefaultStateBytes = StringEncoding.UTF8.GetBytes("IsInDefaultState");
 
-        public DataStreamsMessagePackFormatter(TracerSettings tracerSettings, string defaultServiceName)
+        public DataStreamsMessagePackFormatter(TracerSettings tracerSettings, ProfilerSettings profilerSettings, string defaultServiceName)
         {
             var env = tracerSettings.Environment;
             // .NET tracer doesn't yet support primary tag
@@ -60,7 +61,7 @@ public DataStreamsMessagePackFormatter(TracerSettings tracerSettings, string def
                                          ? []
                                          : StringEncoding.UTF8.GetBytes(env);
             _serviceValueBytes = StringEncoding.UTF8.GetBytes(defaultServiceName);
-            _productMask = GetProductsMask(tracerSettings);
+            _productMask = GetProductsMask(tracerSettings, profilerSettings);
             _isInDefaultState = tracerSettings.IsDataStreamsMonitoringInDefaultState;
         }
 
@@ -75,15 +76,15 @@ private enum Products : long
             Profiling = 1 << 3, // 00001000
         }
 
-        private static long GetProductsMask(TracerSettings tracerSettings)
+        private static long GetProductsMask(TracerSettings tracerSettings, ProfilerSettings profilerSettings)
         {
             var productsMask = (long)Products.Apm;
             if (tracerSettings.IsDataStreamsMonitoringEnabled)
             {
                 productsMask |= (long)Products.Dsm;
             }
 
-            if (tracerSettings.ProfilingEnabledInternal)
+            if (profilerSettings.IsProfilerEnabled)
             {
                 productsMask |= (long)Products.Profiling;
             }

tracer/src/Datadog.Trace/DataStreamsMonitoring/DataStreamsManager.cs

@@ -10,6 +10,7 @@
 using System.Threading.Tasks;
 using Datadog.Trace.Agent.DiscoveryService;
 using Datadog.Trace.Configuration;
+using Datadog.Trace.ContinuousProfiler;
 using Datadog.Trace.DataStreamsMonitoring.Aggregation;
 using Datadog.Trace.DataStreamsMonitoring.Hashes;
 using Datadog.Trace.Headers;
@@ -50,11 +51,12 @@ public DataStreamsManager(
 
     public static DataStreamsManager Create(
         TracerSettings settings,
+        ProfilerSettings profilerSettings,
         IDiscoveryService discoveryService,
         string defaultServiceName)
     {
         var writer = settings.IsDataStreamsMonitoringEnabled
-                         ? DataStreamsWriter.Create(settings, discoveryService, defaultServiceName)
+                         ? DataStreamsWriter.Create(settings, profilerSettings, discoveryService, defaultServiceName)
                          : null;
 
         return new DataStreamsManager(settings.Environment, defaultServiceName, writer, settings.IsDataStreamsMonitoringInDefaultState);

tracer/src/Datadog.Trace/DataStreamsMonitoring/DataStreamsWriter.cs

@@ -11,6 +11,7 @@
 using System.Threading.Tasks;
 using Datadog.Trace.Agent.DiscoveryService;
 using Datadog.Trace.Configuration;
+using Datadog.Trace.ContinuousProfiler;
 using Datadog.Trace.DataStreamsMonitoring.Aggregation;
 using Datadog.Trace.DataStreamsMonitoring.Transport;
 using Datadog.Trace.Logging;
@@ -70,12 +71,13 @@ public DataStreamsWriter(
 
     public static DataStreamsWriter Create(
         TracerSettings settings,
+        ProfilerSettings profilerSettings,
         IDiscoveryService discoveryService,
         string defaultServiceName)
         => new(
             settings,
             new DataStreamsAggregator(
-                new DataStreamsMessagePackFormatter(settings, defaultServiceName),
+                new DataStreamsMessagePackFormatter(settings, profilerSettings, defaultServiceName),
                 bucketDurationMs: DataStreamsConstants.DefaultBucketDurationMs),
             new DataStreamsApi(DataStreamsTransportStrategy.GetAgentIntakeFactory(settings.Exporter)),
             bucketDurationMs: DataStreamsConstants.DefaultBucketDurationMs,