[v3.x] Filter the secret for preventing logging - cherry pick PR (#8953)

* Filter the secret for preventing logging

* Remove Sanitizer testing as the same as before

Author: TsuyoshiUshio

src/WebJobs.Script.WebHost/Management/LinuxSpecialization/BashCommandHandler.cs

@@ -3,8 +3,7 @@
 
 using System;
 using System.Diagnostics;
-using System.IO;
-using System.IO.Compression;
+using Microsoft.Azure.WebJobs.Logging;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Extensions.Logging;
 
@@ -41,7 +40,7 @@ public BashCommandHandler(IMetricsLogger metricsLogger, ILogger<BashCommandHandl
                             CreateNoWindow = true
                         }
                     };
-                    _logger.LogInformation($"Running: {process.StartInfo.FileName} {process.StartInfo.Arguments}");
+                    _logger.LogInformation($"Running: bash.exe (arguments omitted)");
                     process.Start();
                     var output = process.StandardOutput.ReadToEnd().Trim();
                     var error = process.StandardError.ReadToEnd().Trim();

src/WebJobs.Script.WebHost/Management/LinuxSpecialization/PackageDownloadHandler.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.IO.Abstractions;
 using System.Net;
@@ -10,6 +11,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using DryIoc;
+using Microsoft.Azure.WebJobs.Logging;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.WebHost.Models;
 using Microsoft.Extensions.Logging;
@@ -124,15 +126,18 @@ private async Task<string> Download(RunFromPackageContext pkgContext, Uri zipUri
 
         private void AriaDownload(string directory, string fileName, Uri zipUri, bool isWarmupRequest, string downloadMetricName)
         {
+            var command = $"{Aria2CExecutable} --allow-overwrite -x12 -d {directory} -o {fileName} '{zipUri}'";
             (string stdout, string stderr, int exitCode) = _bashCommandHandler.RunBashCommand(
-                $"{Aria2CExecutable} --allow-overwrite -x12 -d {directory} -o {fileName} '{zipUri}'",
+                command,
                 downloadMetricName);
             if (exitCode != 0)
             {
                 var msg = $"Error downloading package. stdout: {stdout}, stderr: {stderr}, exitCode: {exitCode}";
                 _logger.LogError(msg);
                 throw new InvalidOperationException(msg);
             }
+            _logger.LogInformation($"Executed: {Sanitizer.Sanitize(command)}");
+
             var fileInfo = FileUtility.FileInfoFromFileName(Path.Combine(directory, fileName));
             _logger.LogInformation("'{fileInfo.Length}' bytes downloaded. IsWarmupRequest = '{isWarmupRequest}'",
                 fileInfo.Length, isWarmupRequest);

src/WebJobs.Script.WebHost/Management/LinuxSpecialization/RunFromPackageHandler.cs

@@ -2,12 +2,15 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Logging;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.WebHost.Models;
 using Microsoft.Extensions.Logging;
+using NuGet.Protocol.Plugins;
 
 namespace Microsoft.Azure.WebJobs.Script.WebHost.Management.LinuxSpecialization
 {
@@ -149,6 +152,7 @@ private CodePackageType GetPackageType(string filePath, RunFromPackageContext pk
 
             // Check file magic-number using `file` command.
             (var output, _, _) = _bashCommandHandler.RunBashCommand($"{BashCommandHandler.FileCommand} -b {filePath}", MetricEventNames.LinuxContainerSpecializationFileCommand);
+            _logger.LogInformation(Sanitizer.Sanitize($"Executed: {BashCommandHandler.FileCommand} -b {filePath} {MetricEventNames.LinuxContainerSpecializationFileCommand}"));
             if (output.StartsWith(SquashfsPrefix, StringComparison.OrdinalIgnoreCase))
             {
                 return CodePackageType.Squashfs;
@@ -177,9 +181,9 @@ private async Task CreateBindMount(string sourcePath, string targetPath)
         private void UnsquashImage(string filePath, string scriptPath)
         {
             _logger.LogDebug($"Unsquashing remote zip to {scriptPath}");
-
-            _bashCommandHandler.RunBashCommand($"{UnsquashFSExecutable} -f -d '{scriptPath}' '{filePath}'",
-                MetricEventNames.LinuxContainerSpecializationUnsquash);
+            var command = $"{UnsquashFSExecutable} -f -d '{scriptPath}' '{filePath}'";
+            _bashCommandHandler.RunBashCommand(command, MetricEventNames.LinuxContainerSpecializationUnsquash);
+            _logger.LogInformation(Sanitizer.Sanitize($"Executed: {command}"));
         }
 
         public async Task<bool> MountAzureFileShare(HostAssignmentContext assignmentContext)

src/WebJobs.Script/Sanitizer.cs

@@ -16,7 +16,7 @@ internal static class Sanitizer
 
         // List of keywords that should not be replaced with [Hidden Credential]
         private static readonly string[] AllowedTokens = new string[] { "PublicKeyToken=" };
-        private static readonly string[] CredentialTokens = new string[] { "Token=", "DefaultEndpointsProtocol=http", "AccountKey=", "Data Source=", "Server=", "Password=", "pwd=", "&sig=", "SharedAccessKey=" };
+        internal static readonly string[] CredentialTokens = new string[] { "Token=", "DefaultEndpointsProtocol=http", "AccountKey=", "Data Source=", "Server=", "Password=", "pwd=", "&sig=", "&sig=", "SharedAccessKey=" };
 
         /// <summary>
         /// Removes well-known credential strings from strings.

test/WebJobs.Script.Tests.Integration/Management/InstanceManagerTests.cs

@@ -194,6 +194,7 @@ public async void StartAssignment_Succeeds_With_NonEmpty_ScmRunFromPackage_Blob(
 
             IConfiguration configuration = TestHelpers.GetTestConfiguration();
             string connectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
+
             Uri sasUri = await TestHelpers.CreateBlobSas(connectionString, zipFilePath, "scm-run-from-pkg-test", "NonEmpty.zip");
 
             _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
@@ -226,7 +227,7 @@ public async void StartAssignment_Succeeds_With_NonEmpty_ScmRunFromPackage_Blob(
 
             var logs = _loggerProvider.GetAllLogMessages().Select(p => p.FormattedMessage).ToArray();
 
-            if (logs.Length == 9)
+            if (logs.Length == 10)
             {
                 Assert.Collection(logs,
                     p => Assert.StartsWith("Starting Assignment", p),
@@ -237,6 +238,7 @@ public async void StartAssignment_Succeeds_With_NonEmpty_ScmRunFromPackage_Blob(
                     p => Assert.StartsWith("Output:", p),
                     p => Assert.True(true), // this line varies depending on whether WSL is on the machine; just ignore it
                     p => Assert.StartsWith("exitCode:", p),
+                    p => Assert.StartsWith("Executed: ", p),
                     p => Assert.StartsWith("Triggering specialization", p));
             }
             else
@@ -248,6 +250,7 @@ public async void StartAssignment_Succeeds_With_NonEmpty_ScmRunFromPackage_Blob(
                     p => Assert.StartsWith("Unsquashing remote zip", p),
                     p => Assert.StartsWith("Running: ", p),
                     p => Assert.StartsWith("Error running bash", p),
+                    p => Assert.StartsWith("Executed: ", p),
                     p => Assert.StartsWith("Triggering specialization", p));
             }
         }