Integrate secrets detection library with telemetry (Azure#26399)

vidai-msft · web-flow · commit 5ece24824a5d · 2024-10-23T10:10:16.000+08:00
diff --git a/src/Accounts/Accounts/CommonModule/AzModule.cs b/src/Accounts/Accounts/CommonModule/AzModule.cs
@@ -385,10 +385,10 @@ public Dictionary<string, string> GetTelemetryInfo(string telemetryId)
             Dictionary<string, string> telemetryInfo = null;
             if (_telemetry.TryGetValue(telemetryId, out var qos))
             {
-                if (qos?.SanitizerInfo?.DetectedProperties?.Count > 0)
+                if (qos?.SanitizerInfo?.DetectedProperties.IsEmpty == false)
                 {
                     var showSecretsWarning = qos.SanitizerInfo.ShowSecretsWarning && qos.SanitizerInfo.SecretsDetected;
-                    var sanitizedProperties = string.Join(", ", qos.SanitizerInfo.DetectedProperties);
+                    var sanitizedProperties = string.Join(", ", qos.SanitizerInfo.DetectedProperties.PropertyNames);
                     var invocationName = qos.InvocationName;
                     telemetryInfo = new Dictionary<string, string>
                     {
diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerCollectionProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerCollectionProvider.cs
@@ -39,13 +39,16 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti
                         var collItemType = collItem.GetType();
                         if (collItemType == typeof(string))
                         {
-                            if (Service.TrySanitizeData(collItem as string, out string sanitizedData))
+                            if (Service.TrySanitizeData(collItem as string, out var detections, out _))
                             {
                                 telemetry.SecretsDetected = true;
                                 var propertyPath = ResolvePropertyPath(property);
                                 if (!string.IsNullOrEmpty(propertyPath))
                                 {
-                                    telemetry.DetectedProperties.Add(ResolvePropertyPath(property));
+                                    foreach (var detection in detections)
+                                    {
+                                        telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);
+                                    }
                                 }
                             }
                         }
diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerDictionaryProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerDictionaryProvider.cs
@@ -40,14 +40,17 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti
                         var dicItemValueType = dictItemValue.GetType();
                         if (dicItemValueType == typeof(string))
                         {
-                            if (Service.TrySanitizeData(dictItemValue as string, out string sanitizedData))
+                            if (Service.TrySanitizeData(dictItemValue as string, out var detections, out _))
                             {
                                 // Sanitize dictionary item value
                                 telemetry.SecretsDetected = true;
                                 var propertyPath = ResolvePropertyPath(property);
                                 if (!string.IsNullOrEmpty(propertyPath))
                                 {
-                                    telemetry.DetectedProperties.Add(propertyPath);
+                                    foreach (var detection in detections)
+                                    {
+                                        telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);
+                                    }
                                 }
                             }
                         }
diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonArrayProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonArrayProvider.cs
@@ -37,13 +37,16 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti
                         switch (jItem.Type)
                         {
                             case JTokenType.String:
-                                if (Service.TrySanitizeData(jItem.Value<string>(), out string sanitizedData))
+                                if (Service.TrySanitizeData(jItem.Value<string>(), out var detections, out _))
                                 {
                                     telemetry.SecretsDetected = true;
                                     var propertyPath = ResolvePropertyPath(property);
                                     if (!string.IsNullOrEmpty(propertyPath))
                                     {
-                                        telemetry.DetectedProperties.Add(propertyPath);
+                                        foreach (var detection in detections)
+                                        {
+                                            telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);
+                                        }
                                     }
                                 }
                                 break;
diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonObjectProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerJsonObjectProvider.cs
@@ -37,13 +37,16 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti
                         switch (propValue.Type)
                         {
                             case JTokenType.String:
-                                if (Service.TrySanitizeData(propValue.Value<string>(), out string sanitizedData))
+                                if (Service.TrySanitizeData(propValue.Value<string>(), out var detections, out _))
                                 {
                                     telemetry.SecretsDetected = true;
                                     var propertyPath = ResolvePropertyPath(property);
                                     if (!string.IsNullOrEmpty(propertyPath))
                                     {
-                                        telemetry.DetectedProperties.Add(propertyPath);
+                                        foreach (var detection in detections)
+                                        {
+                                            telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);
+                                        }
                                     }
                                 }
                                 break;
diff --git a/src/Accounts/Authentication/Sanitizer/Providers/SanitizerStringProvider.cs b/src/Accounts/Authentication/Sanitizer/Providers/SanitizerStringProvider.cs
@@ -12,9 +12,9 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
-using System.Collections.Generic;
 using Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Services;
 using Microsoft.WindowsAzure.Commands.Common.Sanitizer;
+using System.Collections.Generic;
 
 namespace Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Providers
 {
@@ -29,13 +29,16 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti
             var propertyValue = property?.GetValue(sanitizingObject) ?? sanitizingObject;
             if (propertyValue is string data)
             {
-                if (Service.TrySanitizeData(data, out string sanitizedData))
+                if (Service.TrySanitizeData(data, out var detections, out _))
                 {
                     telemetry.SecretsDetected = true;
                     var propertyPath = ResolvePropertyPath(property);
                     if (!string.IsNullOrEmpty(propertyPath))
                     {
-                        telemetry.DetectedProperties.Add(propertyPath);
+                        foreach (var detection in detections)
+                        {
+                            telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);
+                        }
                     }
                 }
             }
diff --git a/src/Accounts/Authentication/Sanitizer/Services/DefaultSanitizerService.cs b/src/Accounts/Authentication/Sanitizer/Services/DefaultSanitizerService.cs
@@ -45,19 +45,20 @@ internal class DefaultSanitizerService : ISanitizerService
             { "Microsoft.Azure.Storage.File.CloudFileDirectory", new[] { "Parent" } },
         };
 
-        private readonly SecretMasker _secretMasker = new SecretMasker(WellKnownRegexPatterns.HighConfidenceMicrosoftSecurityModels, generateCorrelatingIds: true);
+        private readonly SecretMasker _secretMasker = new SecretMasker(WellKnownRegexPatterns.HighConfidenceMicrosoftSecurityModels);
 
-        public bool TrySanitizeData(string data, out string sanitizedData)
+        public bool TrySanitizeData(string data, out IEnumerable<Detection> detections, out string sanitizedData)
         {
             sanitizedData = string.Empty;
 
-            if (!string.IsNullOrWhiteSpace(data))
+            if (string.IsNullOrWhiteSpace(data))
             {
-                var detections = _secretMasker.DetectSecrets(data);
-                return detections.Any();
+                detections = Enumerable.Empty<Detection>();
+                return false;
             }
 
-            return false;
+            detections = _secretMasker.DetectSecrets(data);
+            return detections.Any();
         }
     }
 }
diff --git a/src/Accounts/Authentication/Sanitizer/Services/ISanitizerService.cs b/src/Accounts/Authentication/Sanitizer/Services/ISanitizerService.cs
@@ -12,6 +12,7 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using Microsoft.Security.Utilities;
 using System.Collections.Generic;
 
 namespace Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Services
@@ -20,6 +21,6 @@ public interface ISanitizerService
     {
         IReadOnlyDictionary<string, IEnumerable<string>> IgnoredProperties { get; }
 
-        bool TrySanitizeData(string data, out string sanitizedData);
+        bool TrySanitizeData(string data, out IEnumerable<Detection> detections, out string sanitizedData);
     }
 }
diff --git a/tools/Common.Netcore.Dependencies.targets b/tools/Common.Netcore.Dependencies.targets
@@ -3,22 +3,22 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.24"/>
     <PackageReference Include="Microsoft.Rest.ClientRuntime.Azure" Version="3.3.19"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Aks" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Authentication.Abstractions" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Authorization" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Common" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Compute" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Graph.Rbac" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.KeyVault" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Monitor" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Network" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.PolicyInsights" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.ResourceManager" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Storage" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Storage.Management" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Strategies" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Websites" Version="1.3.101-preview"/>
-    <PackageReference Include="Microsoft.Azure.PowerShell.Common.Share" Version="1.3.101-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Aks" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Authentication.Abstractions" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Authorization" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Common" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Compute" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Graph.Rbac" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.KeyVault" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Monitor" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Network" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.PolicyInsights" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.ResourceManager" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Storage" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Storage.Management" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Strategies" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Clients.Websites" Version="1.3.102-preview"/>
+    <PackageReference Include="Microsoft.Azure.PowerShell.Common.Share" Version="1.3.102-preview"/>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
   </ItemGroup>
   <ItemGroup>
@@ -37,7 +37,7 @@
     <PackageReference Include="PowerShellStandard.Library" Version="5.1.0" PrivateAssets="All" />
   </ItemGroup>
   <PropertyGroup>
-    <StorageToolsPath>$(NugetPackageRoot)\microsoft.azure.powershell.storage\1.3.101-preview\tools\</StorageToolsPath>
+    <StorageToolsPath>$(NugetPackageRoot)\microsoft.azure.powershell.storage\1.3.102-preview\tools\</StorageToolsPath>
   </PropertyGroup>
   <ItemGroup Condition="'$(OmitJsonPackage)' != 'true'">
     <PackageReference Include="Newtonsoft.Json" Version="13.0.2"/>

Original file line number	Diff line number	Diff line change
`@@ -385,10 +385,10 @@ public Dictionary<string, string> GetTelemetryInfo(string telemetryId)`
`385`	`385`	`Dictionary<string, string> telemetryInfo = null;`
`386`	`386`	`if (_telemetry.TryGetValue(telemetryId, out var qos))`
`387`	`387`	`{`
`388`		`- if (qos?.SanitizerInfo?.DetectedProperties?.Count > 0)`
	`388`	`+ if (qos?.SanitizerInfo?.DetectedProperties.IsEmpty == false)`
`389`	`389`	`{`
`390`	`390`	`var showSecretsWarning = qos.SanitizerInfo.ShowSecretsWarning && qos.SanitizerInfo.SecretsDetected;`
`391`		`- var sanitizedProperties = string.Join(", ", qos.SanitizerInfo.DetectedProperties);`
	`391`	`+ var sanitizedProperties = string.Join(", ", qos.SanitizerInfo.DetectedProperties.PropertyNames);`
`392`	`392`	`var invocationName = qos.InvocationName;`
`393`	`393`	`telemetryInfo = new Dictionary<string, string>`
`394`	`394`	`{`
Original file line number	Diff line number	Diff line change
`@@ -39,13 +39,16 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti`
`39`	`39`	`var collItemType = collItem.GetType();`
`40`	`40`	`if (collItemType == typeof(string))`
`41`	`41`	`{`
`42`		`- if (Service.TrySanitizeData(collItem as string, out string sanitizedData))`
	`42`	`+ if (Service.TrySanitizeData(collItem as string, out var detections, out _))`
`43`	`43`	`{`
`44`	`44`	`telemetry.SecretsDetected = true;`
`45`	`45`	`var propertyPath = ResolvePropertyPath(property);`
`46`	`46`	`if (!string.IsNullOrEmpty(propertyPath))`
`47`	`47`	`{`
`48`		`- telemetry.DetectedProperties.Add(ResolvePropertyPath(property));`
	`48`	`+ foreach (var detection in detections)`
	`49`	`+ {`
	`50`	`+ telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);`
	`51`	`+ }`
`49`	`52`	`}`
`50`	`53`	`}`
`51`	`54`	`}`
Original file line number	Diff line number	Diff line change
`@@ -40,14 +40,17 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti`
`40`	`40`	`var dicItemValueType = dictItemValue.GetType();`
`41`	`41`	`if (dicItemValueType == typeof(string))`
`42`	`42`	`{`
`43`		`- if (Service.TrySanitizeData(dictItemValue as string, out string sanitizedData))`
	`43`	`+ if (Service.TrySanitizeData(dictItemValue as string, out var detections, out _))`
`44`	`44`	`{`
`45`	`45`	`// Sanitize dictionary item value`
`46`	`46`	`telemetry.SecretsDetected = true;`
`47`	`47`	`var propertyPath = ResolvePropertyPath(property);`
`48`	`48`	`if (!string.IsNullOrEmpty(propertyPath))`
`49`	`49`	`{`
`50`		`- telemetry.DetectedProperties.Add(propertyPath);`
	`50`	`+ foreach (var detection in detections)`
	`51`	`+ {`
	`52`	`+ telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);`
	`53`	`+ }`
`51`	`54`	`}`
`52`	`55`	`}`
`53`	`56`	`}`
Original file line number	Diff line number	Diff line change
`@@ -37,13 +37,16 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti`
`37`	`37`	`switch (jItem.Type)`
`38`	`38`	`{`
`39`	`39`	`case JTokenType.String:`
`40`		`- if (Service.TrySanitizeData(jItem.Value<string>(), out string sanitizedData))`
	`40`	`+ if (Service.TrySanitizeData(jItem.Value<string>(), out var detections, out _))`
`41`	`41`	`{`
`42`	`42`	`telemetry.SecretsDetected = true;`
`43`	`43`	`var propertyPath = ResolvePropertyPath(property);`
`44`	`44`	`if (!string.IsNullOrEmpty(propertyPath))`
`45`	`45`	`{`
`46`		`- telemetry.DetectedProperties.Add(propertyPath);`
	`46`	`+ foreach (var detection in detections)`
	`47`	`+ {`
	`48`	`+ telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);`
	`49`	`+ }`
`47`	`50`	`}`
`48`	`51`	`}`
`49`	`52`	`break;`
Original file line number	Diff line number	Diff line change
`@@ -12,9 +12,9 @@`
`12`	`12`	`// limitations under the License.`
`13`	`13`	`// ----------------------------------------------------------------------------------`
`14`	`14`
`15`		`-using System.Collections.Generic;`
`16`	`15`	`using Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Services;`
`17`	`16`	`using Microsoft.WindowsAzure.Commands.Common.Sanitizer;`
	`17`	`+using System.Collections.Generic;`
`18`	`18`
`19`	`19`	`namespace Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Providers`
`20`	`20`	`{`
`@@ -29,13 +29,16 @@ public override void SanitizeValue(object sanitizingObject, Stack<object> saniti`
`29`	`29`	`var propertyValue = property?.GetValue(sanitizingObject) ?? sanitizingObject;`
`30`	`30`	`if (propertyValue is string data)`
`31`	`31`	`{`
`32`		`- if (Service.TrySanitizeData(data, out string sanitizedData))`
	`32`	`+ if (Service.TrySanitizeData(data, out var detections, out _))`
`33`	`33`	`{`
`34`	`34`	`telemetry.SecretsDetected = true;`
`35`	`35`	`var propertyPath = ResolvePropertyPath(property);`
`36`	`36`	`if (!string.IsNullOrEmpty(propertyPath))`
`37`	`37`	`{`
`38`		`- telemetry.DetectedProperties.Add(propertyPath);`
	`38`	`+ foreach (var detection in detections)`
	`39`	`+ {`
	`40`	`+ telemetry.DetectedProperties.AddPropertyInfo(propertyPath, detection.Moniker);`
	`41`	`+ }`
`39`	`42`	`}`
`40`	`43`	`}`
`41`	`44`	`}`
Original file line number	Diff line number	Diff line change
`@@ -45,19 +45,20 @@ internal class DefaultSanitizerService : ISanitizerService`
`45`	`45`	`{ "Microsoft.Azure.Storage.File.CloudFileDirectory", new[] { "Parent" } },`
`46`	`46`	`};`
`47`	`47`
`48`		`- private readonly SecretMasker _secretMasker = new SecretMasker(WellKnownRegexPatterns.HighConfidenceMicrosoftSecurityModels, generateCorrelatingIds: true);`
	`48`	`+ private readonly SecretMasker _secretMasker = new SecretMasker(WellKnownRegexPatterns.HighConfidenceMicrosoftSecurityModels);`
`49`	`49`
`50`		`- public bool TrySanitizeData(string data, out string sanitizedData)`
	`50`	`+ public bool TrySanitizeData(string data, out IEnumerable<Detection> detections, out string sanitizedData)`
`51`	`51`	`{`
`52`	`52`	`sanitizedData = string.Empty;`
`53`	`53`
`54`		`- if (!string.IsNullOrWhiteSpace(data))`
	`54`	`+ if (string.IsNullOrWhiteSpace(data))`
`55`	`55`	`{`
`56`		`- var detections = _secretMasker.DetectSecrets(data);`
`57`		`- return detections.Any();`
	`56`	`+ detections = Enumerable.Empty<Detection>();`
	`57`	`+ return false;`
`58`	`58`	`}`
`59`	`59`
`60`		`- return false;`
	`60`	`+ detections = _secretMasker.DetectSecrets(data);`
	`61`	`+ return detections.Any();`
`61`	`62`	`}`
`62`	`63`	`}`
`63`	`64`	`}`
Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,7 @@`
`12`	`12`	`// limitations under the License.`
`13`	`13`	`// ----------------------------------------------------------------------------------`
`14`	`14`
	`15`	`+using Microsoft.Security.Utilities;`
`15`	`16`	`using System.Collections.Generic;`
`16`	`17`
`17`	`18`	`namespace Microsoft.Azure.Commands.Common.Authentication.Sanitizer.Services`
`@@ -20,6 +21,6 @@ public interface ISanitizerService`
`20`	`21`	`{`
`21`	`22`	`IReadOnlyDictionary<string, IEnumerable<string>> IgnoredProperties { get; }`
`22`	`23`
`23`		`- bool TrySanitizeData(string data, out string sanitizedData);`
	`24`	`+ bool TrySanitizeData(string data, out IEnumerable<Detection> detections, out string sanitizedData);`
`24`	`25`	`}`
`25`	`26`	`}`