Send secret masking telemetry if opted in to it (#5189)

If using the new OSS secret masker and an additional opt-in knob is turned on, then send telemetry about secrets that were masked.

An overall 'SecretMasker' event is sent with statistics such as total detections and elapsed time. Additional 'SecretMaskerDetections' events are sent mapping C3ID to pattern monikers. These are batched to keep event sizes capped and the total number of events are also capped.

This also includes an update to the Microsoft.Security.Utilities.Core dependency from v1.18 to v1.19. Release notes: https://github.com/microsoft/security-utilities/blob/release/v1.19.0/docs/ReleaseHistory.md

Author: nguerrera

src/Agent.Sdk/Agent.Sdk.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Security.Utilities.Core" Version="1.18.0" />
+    <PackageReference Include="Microsoft.Security.Utilities.Core" Version="1.19.0" />
     <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageReference Include="System.IO.FileSystem.AccessControl" Version="6.0.0-preview.5.21301.5" />
     <PackageReference Include="System.Management" Version="4.7.0" />

src/Agent.Sdk/Knob/AgentKnobs.cs

@@ -681,6 +681,13 @@ public class AgentKnobs
             new EnvironmentKnobSource("AZP_ENABLE_NEW_MASKER_AND_REGEXES"),
             new BuiltInDefaultKnobSource("false"));
 
+        public static readonly Knob SendSecretMaskerTelemetry = new Knob(
+            nameof(SendSecretMaskerTelemetry),
+            "If true, the agent will send telemetry about secret masking",
+            new RuntimeKnobSource("AZP_SEND_SECRET_MASKER_TELEMETRY"),
+            new EnvironmentKnobSource("AZP_SEND_SECRET_MASKER_TELEMETRY"),
+            new BuiltInDefaultKnobSource("false"));
+
         public static readonly Knob AddDockerInitOption = new Knob(
             nameof(AddDockerInitOption),
             "If true, the agent will create docker container with the --init option.",

src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs

@@ -2,12 +2,19 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Collections.Generic;
 
 namespace Agent.Sdk.SecretMasking
 {
     /// <summary>
-    /// Extended ISecretMasker interface that adds support for logging the origin of
-    /// regexes, encoders and literal secret values.
+    /// An action that publishes the given data corresonding to the given
+    /// feature to a telemetry channel.
+    /// </summary>
+    public delegate void PublishSecretMaskerTelemetryAction(string feature, Dictionary<string, string> data);
+
+    /// <summary>
+    /// Extended ISecretMasker interface that adds support for telemetry and
+    /// logging the origin of regexes, encoders and literal secret values.
     /// </summary>
     public interface ILoggedSecretMasker : IDisposable
     {
@@ -19,5 +26,34 @@ public interface ILoggedSecretMasker : IDisposable
         string MaskSecrets(string input);
         void RemoveShortSecretsFromDictionary();
         void SetTrace(ITraceWriter trace);
+
+        /// <summary>
+        /// Begin collecting data for secret masking telemetry.
+        /// </summary>
+        /// <remarks>
+        /// This is a no-op if <see cref="LegacySecretMasker"/> is being used,
+        /// only <see cref="OssSecretMasker"/> supports telemetry. Also, the
+        /// agent will only call this if a feature flag that opts in to secret
+        /// masking telemetry is enabled..
+        /// </remarks>
+        /// <param name="maxUniqueCorrelatingIds">
+        /// The maximum number of unique correlating IDs to collect.
+        /// </param>
+        void StartTelemetry(int maxUniqueCorrelatingIds);
+
+        /// <summary>
+        /// Stop collecting data for secret masking telemetry and publish the
+        /// telemetry events.
+        /// </summary>
+        /// <remarks>
+        /// This is a no-op if <see cref="LegacySecretMasker"/> is being used,
+        /// only <see cref="OssSecretMasker"/> supports telemetry.
+        /// <param name="maxCorrelatingIdsPerEvent">
+        /// The maximum number of correlating IDs to report in a single
+        /// telemetry event.
+        /// <param name="publishAction">
+        /// Callback to publish the telemetry data.
+        /// </param>
+        void StopAndPublishTelemetry(int maxCorrelatingIdsPerEvent, PublishSecretMaskerTelemetryAction publishAction);
     }
 }

src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs

@@ -1,4 +1,3 @@
-
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
@@ -143,6 +142,16 @@ public void Dispose()
             GC.SuppressFinalize(this);
         }
 
+        public void StartTelemetry(int maxDetections)
+        {
+            (_secretMasker as OssSecretMasker)?.StartTelemetry(maxDetections);
+        }
+
+        public void StopAndPublishTelemetry(int maxDetectionsPerEvent, PublishSecretMaskerTelemetryAction publishAction)
+        {
+            (_secretMasker as OssSecretMasker)?.StopAndPublishTelemetry(publishAction, maxDetectionsPerEvent);
+        }
+
         protected virtual void Dispose(bool disposing)
         {
             if (disposing)

src/Agent.Sdk/SecretMasking/OssSecretMasker.cs

@@ -1,27 +1,28 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Text.RegularExpressions;
+using System.Threading;
+
 using Microsoft.Security.Utilities;
 
 namespace Agent.Sdk.SecretMasking;
 
 public sealed class OssSecretMasker : IRawSecretMasker
 {
     private SecretMasker _secretMasker;
+    private Telemetry _telemetry;
 
-    public OssSecretMasker() : this(Array.Empty<RegexPattern>())
-    {
-    }
-
-    public OssSecretMasker(IEnumerable<RegexPattern> patterns)
+    public OssSecretMasker(IEnumerable<RegexPattern> patterns = null)
     {
-        _secretMasker = new SecretMasker(patterns, generateCorrelatingIds: true);
-        _secretMasker.DefaultRegexRedactionToken = "***";
+        _secretMasker = new SecretMasker(patterns,
+                                         generateCorrelatingIds: true,
+                                         defaultRegexRedactionToken: "***");
     }
 
-
     /// <summary>
     /// This property allows to set the minimum length of a secret for masking
     /// </summary>
@@ -31,9 +32,6 @@ public int MinSecretLength
         set => _secretMasker.MinimumSecretLength = value;
     }
 
-    /// <summary>
-    /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time.
-    /// </summary>
     public void AddRegex(string pattern)
     {
         // NOTE: This code path is used for regexes sent to the agent via
@@ -52,95 +50,189 @@ public void AddRegex(string pattern)
         _secretMasker.AddRegex(regexPattern);
     }
 
-    /// <summary>
-    /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time.
-    /// </summary>
     public void AddValue(string test)
     {
         _secretMasker.AddValue(test);
     }
 
-    /// <summary>
-    /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time.
-    /// </summary>
     public void AddValueEncoder(Func<string, string> encoder)
     {
-       _secretMasker.AddLiteralEncoder(x => encoder(x));
+        _secretMasker.AddLiteralEncoder(x => encoder(x));
     }
 
     public void Dispose()
     {
         _secretMasker?.Dispose();
         _secretMasker = null;
+        _telemetry = null;
     }
 
     public string MaskSecrets(string input)
     {
-        return _secretMasker.MaskSecrets(input);
+        _secretMasker.SyncObject.EnterReadLock();
+        try
+        {
+            _telemetry?.ProcessInput(input);
+            return _secretMasker.MaskSecrets(input, _telemetry?.ProcessDetection);
+        }
+        finally
+        {
+            _secretMasker.SyncObject.ExitReadLock();
+        }
     }
 
-    /// <summary>
-    /// Removes secrets from the dictionary shorter than the MinSecretLength property.
-    /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time.
-    /// </summary>
-    public void RemoveShortSecretsFromDictionary()
+    public void StartTelemetry(int maxUniqueCorrelatingIds)
     {
-        var filteredValueSecrets = new HashSet<SecretLiteral>();
-        var filteredRegexSecrets = new HashSet<RegexPattern>();
-
+        _secretMasker.SyncObject.EnterWriteLock();
         try
         {
-            _secretMasker.SyncObject.EnterReadLock();
+            _telemetry ??= new Telemetry(maxUniqueCorrelatingIds);
+        }
+        finally
+        {
+            _secretMasker.SyncObject.ExitWriteLock();
+        }
+    }
 
-            foreach (var secret in _secretMasker.EncodedSecretLiterals)
-            {
-                if (secret.Value.Length < MinSecretLength)
-                {
-                    filteredValueSecrets.Add(secret);
-                }
-            }
+    public void StopAndPublishTelemetry(PublishSecretMaskerTelemetryAction publishAction, int maxCorrelatingIdsPerEvent)
+    {
+        Telemetry telemetry;
 
-            foreach (var secret in _secretMasker.RegexPatterns)
-            {
-                if (secret.Pattern.Length < MinSecretLength)
-                {
-                    filteredRegexSecrets.Add(secret);
-                }
-            }
+        _secretMasker.SyncObject.EnterWriteLock();
+        try
+        {
+            telemetry = _telemetry;
+            _telemetry = null;
         }
         finally
         {
-            if (_secretMasker.SyncObject.IsReadLockHeld)
-            {
-                _secretMasker.SyncObject.ExitReadLock();
-            }
+            _secretMasker.SyncObject.ExitWriteLock();
         }
 
-        try
+        telemetry?.Publish(publishAction, _secretMasker.ElapsedMaskingTime, maxCorrelatingIdsPerEvent);
+    }
+
+    private sealed class Telemetry
+    {
+        // NOTE: Telemetry does not fit into the reader-writer lock model of the
+        // SecretMasker API because we *write* telemetry during *read*
+        // operations. We therefore use separate interlocked operations and a
+        // concurrent dictionary when writing to telemetry.
+
+        // Key=CrossCompanyCorrelatingId (C3ID), Value=Rule Moniker C3ID is a
+        // non-reversible seeded hash and only available when detection is made
+        // by a high-confidence rule that matches secrets with high entropy.
+        private readonly ConcurrentDictionary<string, string> _correlationData;
+        private readonly int _maxUniqueCorrelatingIds;
+        private long _charsScanned;
+        private long _stringsScanned;
+        private long _totalDetections;
+
+        public Telemetry(int maxDetections)
         {
-            _secretMasker.SyncObject.EnterWriteLock();
+            _correlationData = new ConcurrentDictionary<string, string>();
+            _maxUniqueCorrelatingIds = maxDetections;
+            ProcessDetection = ProcessDetectionImplementation;
+        }
 
-            foreach (var secret in filteredValueSecrets)
-            {
-                _secretMasker.EncodedSecretLiterals.Remove(secret);
-            }
+        public void ProcessInput(string input)
+        {
+            Interlocked.Add(ref _charsScanned, input.Length);
+            Interlocked.Increment(ref _stringsScanned);
+        }
 
-            foreach (var secret in filteredRegexSecrets)
-            {
-                _secretMasker.RegexPatterns.Remove(secret);
-            }
+        public Action<Detection> ProcessDetection { get; }
 
-            foreach (var secret in filteredValueSecrets)
+        private void ProcessDetectionImplementation(Detection detection)
+        {
+            Interlocked.Increment(ref _totalDetections);
+
+            // NOTE: We cannot prevent the concurrent dictionary from exceeding
+            // the maximum detection count when multiple threads add detections
+            // in parallel. The condition here is therefore a best effort to
+            // constrain the memory consumed by excess detections that will not
+            // be published. Furthermore, it is deliberate that we use <=
+            // instead of < here as it allows us to detect the case where the
+            // maximum number of events have been exceeded without adding any
+            // additional state.
+            if (_correlationData.Count <= _maxUniqueCorrelatingIds &&
+                detection.CrossCompanyCorrelatingId != null)
             {
-                _secretMasker.ExplicitlyAddedSecretLiterals.Remove(secret);
+                _correlationData.TryAdd(detection.CrossCompanyCorrelatingId, detection.Moniker);
             }
         }
-        finally
+
+        public void Publish(PublishSecretMaskerTelemetryAction publishAction, TimeSpan elapsedMaskingTime, int maxCorrelatingIdsPerEvent)
         {
-            if (_secretMasker.SyncObject.IsWriteLockHeld)
+            Dictionary<string, string> correlationData = null;
+            int uniqueCorrelatingIds = 0;
+            bool correlationDataIsIncomplete = false;
+
+            // Publish 'SecretMaskerCorrelation' events mapping unique C3IDs to
+            // rule moniker. No more than 'maxCorrelatingIdsPerEvent' are
+            // published in a single event.
+            foreach (var pair in _correlationData)
             {
-                _secretMasker.SyncObject.ExitWriteLock();
+                if (uniqueCorrelatingIds >= _maxUniqueCorrelatingIds)
+                {
+                    correlationDataIsIncomplete = true;
+                    break;
+                }
+
+                correlationData ??= new Dictionary<string, string>(maxCorrelatingIdsPerEvent);
+                correlationData.Add(pair.Key, pair.Value);
+                uniqueCorrelatingIds++;
+
+                if (correlationData.Count >= maxCorrelatingIdsPerEvent)
+                {
+                    publishAction("SecretMaskerCorrelation", correlationData);
+                    correlationData = null;
+                }
             }
+
+            if (correlationData != null)
+            {
+                publishAction("SecretMaskerCorrelation", correlationData);
+                correlationData = null;
+            }
+
+            // Send overall information in a 'SecretMasker' event.
+            var overallData = new Dictionary<string, string> {
+                // The version of Microsoft.Security.Utilities.Core used.
+                { "Version", SecretMasker.Version.ToString() },
+
+                // The total number number of characters scanned by the secret masker.
+                { "CharsScanned", _charsScanned.ToString(CultureInfo.InvariantCulture) },
+                
+                // The total number of strings scanned by the secret masker.
+                { "StringsScanned", _stringsScanned.ToString(CultureInfo.InvariantCulture) },
+
+                // The total number of detections made by the secret masker.
+                // This includes duplicate detections and detections without
+                // correlating IDs such as those made by literal values.
+                { "TotalDetections", _totalDetections.ToString(CultureInfo.InvariantCulture) },
+
+                // The total amount of time spent masking secrets.
+                { "ElapsedMaskingTimeInMilliseconds", elapsedMaskingTime.TotalMilliseconds.ToString(CultureInfo.InvariantCulture) },
+
+                // Whether the 'maxUniqueCorrelatingIds' limit was exceeded and
+                // therefore the 'SecretMaskerDetectionCorrelation' events does
+                // not contain all unique correlating IDs detected.
+                { "CorrelationDataIsIncomplete", correlationDataIsIncomplete.ToString(CultureInfo.InvariantCulture) },
+
+                // The total number of unique correlating IDs reported in
+                // 'SecretMaskerCorrelation' events.
+                //
+                // NOTE: This may be less than the total number of unique
+                // correlating IDs if the maximum was exceeded. See above.
+                { "UniqueCorrelatingIds", uniqueCorrelatingIds.ToString(CultureInfo.InvariantCulture) },
+            };
+
+            publishAction("SecretMasker", overallData);
         }
     }
+
+    // This is a no-op for the OSS SecretMasker because it respects
+    // MinimumSecretLength immediately without requiring an extra API call.
+    void IRawSecretMasker.RemoveShortSecretsFromDictionary() { }
 }