Implement new interface IAzurePSSanitizer and ISanitizerService for secrets detection feature (#24249)

* Register IAzurePSSanitizer and ISanitizerService components and implement the functions defined in the interfaces.

* Upload new localfeed and test coverage pipeline to test all scenario tests

* Update live test to include sanitizer

* Register IAzurePSSanitizer and ISanitizerService components and implement the functions defined in the interfaces.

* Enable secrets warning on CI

* Add local build of common lib

* Move secrets detection related objects from common repo to Authentication

* Add change log

* Update config docs and fix issues

* Update src/Accounts/Accounts/ChangeLog.md

Co-authored-by: Yeming Liu <[email protected]>

---------

Co-authored-by: Yeming Liu <[email protected]>

Author: vidai-msft

.azure-pipelines/powershell-core.yml

@@ -20,6 +20,7 @@ variables:
   EnableTestCoverage: true
   TestCoverageLocation: $(Build.SourcesDirectory)/artifacts
   PowerShellPlatform: PowerShell Core
+  AZURE_CLIENTS_SHOW_SECRETS_WARNING: true
 
 trigger: none
 

.azure-pipelines/windows-powershell.yml

@@ -12,6 +12,7 @@ variables:
   EnableTestCoverage: true
   TestCoverageLocation: $(Build.SourcesDirectory)/artifacts
   PowerShellPlatform: Windows PowerShell
+  AZURE_CLIENTS_SHOW_SECRETS_WARNING: true
 
 trigger: none
 

src/Accounts/Accounts/Account/ConnectAzureRmAccount.cs

@@ -46,6 +46,8 @@
 using Microsoft.Azure.PowerShell.Common.Share.Survey;
 using Microsoft.Azure.Commands.Profile.Utilities;
 using System.Management.Automation.Runspaces;
+using Microsoft.WindowsAzure.Commands.Common.Sanitizer;
+using Microsoft.Azure.Commands.Common.Authentication.Sanitizer;
 
 namespace Microsoft.Azure.Commands.Profile
 {
@@ -787,6 +789,7 @@ public void OnImport()
                 AzureSession.Instance.RegisterComponent(nameof(AzureCredentialFactory), () => new AzureCredentialFactory());
                 AzureSession.Instance.RegisterComponent(nameof(MsalAccessTokenAcquirerFactory), () => new MsalAccessTokenAcquirerFactory());
                 AzureSession.Instance.RegisterComponent<ISshCredentialFactory>(nameof(ISshCredentialFactory), () => new SshCredentialFactory());
+                AzureSession.Instance.RegisterComponent<IOutputSanitizer>(nameof(IOutputSanitizer), () => new OutputSanitizer());
 #if DEBUG || TESTCOVERAGE
                 AzureSession.Instance.RegisterComponent<ITestCoverage>(nameof(ITestCoverage), () => new TestCoverage());
 #endif

src/Accounts/Accounts/ChangeLog.md

@@ -19,6 +19,7 @@
 -->
 
 ## Upcoming Release
+* Added a preview feature to detect secrets and sensitive information from the output of Azure PowerShell cmdlets to prevent leakage. Enable it by `Set-AzConfig -DisplaySecretsWarning $true`. Learn more at https://go.microsoft.com/fwlink/?linkid=2258844
 * Fixed `CacheDirectory` and `CacheFile` out-of-sync issue in AzureRmContextSettings.json and the customers are not allowed to change these 2 properties.
 * Redirected device code login messages from warning stream to information stream if use device authentication in `Connect-AzAccount`.
 

src/Accounts/Accounts/help/Clear-AzConfig.md

@@ -21,10 +21,10 @@ Clear-AzConfig [-Force] [-PassThru] [-AppliesTo <String>] [-Scope <ConfigScope>]
 ### ClearByKey
 ```
 Clear-AzConfig [-PassThru] [-AppliesTo <String>] [-Scope <ConfigScope>]
- [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [-CheckForUpgrade]
- [-DefaultSubscriptionForLogin] [-DisableErrorRecordsPersistence] [-DisplayBreakingChangeWarning]
- [-DisplayRegionIdentified] [-DisplaySurveyMessage] [-EnableDataCollection] [-EnableLoginByWam]
- [<CommonParameters>]
+ [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm]
+ [-CheckForUpgrade] [-DefaultSubscriptionForLogin] [-DisableErrorRecordsPersistence]
+ [-DisplayBreakingChangeWarning] [-DisplayRegionIdentified] [-DisplaySecretsWarning]
+ [-DisplaySurveyMessage] [-EnableDataCollection] [-EnableLoginByWam] [<CommonParameters>]
 ```
 
 ## DESCRIPTION
@@ -161,6 +161,21 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -DisplaySecretsWarning
+When enabled, a warning message will be displayed when the cmdlet output contains secrets. Learn more at https://go.microsoft.com/fwlink/?linkid=2258844
+
+```yaml
+Type: System.Management.Automation.SwitchParameter
+Parameter Sets: ClearByKey
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -DisplaySurveyMessage
 When enabled, you are prompted infrequently to participate in user experience surveys for Azure PowerShell.
 

src/Accounts/Accounts/help/Get-AzConfig.md

@@ -15,8 +15,8 @@ Gets the configs of Azure PowerShell.
 ```
 Get-AzConfig [-AppliesTo <String>] [-Scope <ConfigScope>] [-DefaultProfile <IAzureContextContainer>]
  [-CheckForUpgrade] [-DefaultSubscriptionForLogin] [-DisableErrorRecordsPersistence]
- [-DisplayBreakingChangeWarning] [-DisplayRegionIdentified] [-DisplaySurveyMessage] [-EnableDataCollection]
- [-EnableLoginByWam] [<CommonParameters>]
+ [-DisplayBreakingChangeWarning] [-DisplayRegionIdentified] [-DisplaySecretsWarning]
+ [-DisplaySurveyMessage] [-EnableDataCollection] [-EnableLoginByWam] [<CommonParameters>]
 ```
 
 ## DESCRIPTION
@@ -172,6 +172,21 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -DisplaySecretsWarning
+When enabled, a warning message will be displayed when the cmdlet output contains secrets. Learn more at https://go.microsoft.com/fwlink/?linkid=2258844
+
+```yaml
+Type: System.Management.Automation.SwitchParameter
+Parameter Sets: (All)
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -DisplaySurveyMessage
 When enabled, you are prompted infrequently to participate in user experience surveys for Azure PowerShell.
 

src/Accounts/Accounts/help/Update-AzConfig.md

@@ -16,7 +16,8 @@ Updates the configs of Azure PowerShell.
 Update-AzConfig [-AppliesTo <String>] [-Scope <ConfigScope>] [-DefaultProfile <IAzureContextContainer>]
  [-WhatIf] [-Confirm] [-CheckForUpgrade <Boolean>] [-DefaultSubscriptionForLogin <String>]
  [-DisableErrorRecordsPersistence <Boolean>] [-DisplayBreakingChangeWarning <Boolean>]
- [-DisplayRegionIdentified <Boolean>] [-DisplaySurveyMessage <Boolean>] [-EnableDataCollection <Boolean>]
+ [-DisplayRegionIdentified <Boolean>] [-DisplaySecretsWarning <Boolean>]
+ [-DisplaySurveyMessage <Boolean>] [-EnableDataCollection <Boolean>]
  [-EnableLoginByWam <Boolean>] [<CommonParameters>]
 ```
 
@@ -69,6 +70,19 @@ EnableDataCollection True  Az         CurrentUser When enabled, Azure PowerShell
 Sets the "EnableDataCollection" config as "$true". This enables sending the telemetry data.
 Setting this config is equivalent to `Enable-AzDataCollection` and `Disable-AzDataCollection`.
 
+### Example 4
+```powershell
+Update-AzConfig -DisplaySecretsWarning $true
+```
+
+```output
+Key                   Value Applies To Scope       Help Message
+---                   ----- ---------- -----       ------------
+DisplaySecretsWarning True  Az         CurrentUser When enabled, a warning message for secrets redaction will be displ…
+```
+
+Sets the "DisplaySecretsWarning" config as "$true". This enables the secrets detection during the cmdlet execution and displays a warning message if any secrets are found in the output.
+
 ## PARAMETERS
 
 ### -AppliesTo
@@ -184,6 +198,21 @@ Accept pipeline input: True (ByPropertyName)
 Accept wildcard characters: False
 ```
 
+### -DisplaySecretsWarning
+When enabled, a warning message will be displayed when the cmdlet output contains secrets. Learn more at https://go.microsoft.com/fwlink/?linkid=2258844
+
+```yaml
+Type: System.Boolean
+Parameter Sets: (All)
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: True (ByPropertyName)
+Accept wildcard characters: False
+```
+
 ### -DisplaySurveyMessage
 When enabled, you are prompted infrequently to participate in user experience surveys for Azure PowerShell.
 
@@ -286,10 +315,10 @@ This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable
 
 ## INPUTS
 
-### System.String
-
 ### System.Boolean
 
+### System.String
+
 ## OUTPUTS
 
 ### Microsoft.Azure.Commands.Profile.Models.PSConfig

src/Accounts/Authentication/Config/ConfigInitializer.cs

@@ -226,6 +226,12 @@ private void RegisterConfigs(IConfigManager configManager)
             configManager.RegisterConfig(new EnableLoginByWamConfig());
             configManager.RegisterConfig(new EnableInterceptSurveyConfig());
             configManager.RegisterConfig(new DisplayBreakingChangeWarningsConfig());
+            configManager.RegisterConfig(new SimpleTypedConfig<bool>(
+                ConfigKeys.DisplaySecretsWarning,
+                Resources.HelpMessageOfDisplaySecretsWarning,
+                false,
+                "AZURE_CLIENTS_SHOW_SECRETS_WARNING",
+                new[] { AppliesTo.Az }));
         }
     }
 }

src/Accounts/Authentication/Properties/Resources.Designer.cs

@@ -402,4 +402,7 @@
   <data name="FailedToMigrateMsalCacheWithLegayName" xml:space="preserve">
     <value>INITIALIZATION: Failed to migrate MSAL token cache of the legacy name with error : {0}</value>
   </data>
+  <data name="HelpMessageOfDisplaySecretsWarning" xml:space="preserve">
+    <value>When enabled, a warning message will be displayed when the cmdlet output contains secrets. Learn more at https://go.microsoft.com/fwlink/?linkid=2258844</value>
+  </data>
 </root>