Skip to content

Commit d248676

Browse files
Francisco-GaminoFrancisco-Gamino
authored
Add diagnostic events for scenarios that prevent a function app from starting (#9597)
* Fix typo in HostIdCollisionErrorCode * Add diagnostic event for when the function app has 10 non-decryptable secrets backups * Add a test case for a diagnostic event when the function app has 10 non-decryptable secrets backups * Log a diagnostic event if the read operation failed because the blob access tier is set to archived * Add BlobRepository_TierSetToArchive_ReadAsync_Logs_DiagnosticEvent * Add a diagnostic event for parsing errors in the host configuration file * Add InvalidHostJsonLogsDiagnosticEvent test case * Update Moq version to 4.20.69 * Add helper method to validate that the expected diagnostic event is present
1 parent d6a79bb commit d248676

File tree

11 files changed

+250
-5
lines changed

11 files changed

+250
-5
lines changed

‎src/WebJobs.Script.WebHost/Properties/Resources.Designer.cs‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/WebJobs.Script.WebHost/Properties/Resources.resx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,9 @@
139139
"uri": "{uri}"
140140
}}</value>
141141
</data>
142+
<data name="FailedToReadBlobSecretRepositoryTierSetToArchive" xml:space="preserve">
143+
<value>Failed to read from Blob Storage Secret Repository because its access tier is set to archive.</value>
144+
</data>
142145
<data name="FunctionSecretsSchemaV0" xml:space="preserve">
143146
<value>{
144147
"type": "object",

‎src/WebJobs.Script.WebHost/Security/KeyManagement/BlobStorageSecretsRepository.cs‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
using Azure.Storage.Blobs;
1010
using Azure.Storage.Blobs.Models;
1111
using Microsoft.Azure.WebJobs.Host.Storage;
12+
using Microsoft.Azure.WebJobs.Script.Diagnostics;
1213
using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics.Extensions;
14+
using Microsoft.Azure.WebJobs.Script.WebHost.Properties;
1315
using Microsoft.Extensions.Logging;
1416

1517
namespace Microsoft.Azure.WebJobs.Script.WebHost
@@ -19,6 +21,7 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost
1921
/// </summary>
2022
public class BlobStorageSecretsRepository : BaseSecretsRepository
2123
{
24+
private readonly string _blobArchivedName = "BlobArchived";
2225
private readonly string _secretsBlobPath;
2326
private readonly string _hostSecretsBlobPath;
2427
private readonly string _secretsContainerName = "azure-webjobs-secrets";
@@ -82,6 +85,7 @@ public override async Task<ScriptSecrets> ReadAsync(ScriptSecretsType type, stri
8285
{
8386
string secretsContent = null;
8487
string blobPath = GetSecretsBlobPath(type, functionName);
88+
const string Operation = "read";
8589
try
8690
{
8791
BlobClient secretBlobClient = Container.GetBlobClient(blobPath);
@@ -94,9 +98,24 @@ public override async Task<ScriptSecrets> ReadAsync(ScriptSecretsType type, stri
9498
}
9599
}
96100
}
101+
catch (RequestFailedException rfex) when (rfex.Status == 409)
102+
{
103+
// If the read operation failed because the blob access tier is set to archived, log a diagnostic event.
104+
if (rfex.ErrorCode.Equals(_blobArchivedName, StringComparison.OrdinalIgnoreCase))
105+
{
106+
Logger?.LogDiagnosticEventError(
107+
DiagnosticEventConstants.FailedToReadBlobStorageRepositoryErrorCode,
108+
Resources.FailedToReadBlobSecretRepositoryTierSetToArchive,
109+
DiagnosticEventConstants.FailedToReadBlobStorageRepositoryHelpLink,
110+
rfex);
111+
}
112+
113+
LogErrorMessage(Operation, rfex);
114+
throw;
115+
}
97116
catch (Exception ex)
98117
{
99-
LogErrorMessage("read", ex);
118+
LogErrorMessage(Operation, ex);
100119
throw;
101120
}
102121

‎src/WebJobs.Script.WebHost/Security/KeyManagement/SecretManager.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -590,7 +590,11 @@ private async Task PersistSecretsAsync<T>(T secrets, string keyScope = null, boo
590590
{
591591
string message = string.Format(Resources.ErrorTooManySecretBackups, ScriptConstants.MaximumSecretBackupCount, string.IsNullOrEmpty(keyScope) ? "host" : keyScope, await AnalyzeSnapshots(secretBackups));
592592
_logger?.LogDebug(message);
593-
throw new InvalidOperationException(message);
593+
594+
var exception = new InvalidOperationException(message);
595+
_logger?.LogDiagnosticEventError(DiagnosticEventConstants.MaximumSecretBackupCountErrorCode, message, DiagnosticEventConstants.MaximumSecretBackupCountHelpLink, exception);
596+
597+
throw exception;
594598
}
595599
await _repository.WriteSnapshotAsync(secretsType, keyScope, secrets);
596600
}

‎src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,16 @@ internal JObject LoadHostConfig(string configFilePath)
205205
}
206206
catch (JsonException ex)
207207
{
208-
throw new FormatException($"Unable to parse host configuration file '{configFilePath}'.", ex);
208+
var message = $"Unable to parse host configuration file '{configFilePath}'.";
209+
var formatException = new FormatException(message, ex);
210+
211+
_logger.LogDiagnosticEventError(
212+
DiagnosticEventConstants.UnableToParseHostConfigurationFileErrorCode,
213+
message,
214+
DiagnosticEventConstants.UnableToParseHostConfigurationFileHelpLink,
215+
formatException);
216+
217+
throw formatException;
209218
}
210219
catch (Exception ex) when (ex is FileNotFoundException || ex is DirectoryNotFoundException)
211220
{

‎src/WebJobs.Script/Diagnostics/DiagnosticEventConstants.cs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,22 @@ namespace Microsoft.Azure.WebJobs.Script
55
{
66
internal static class DiagnosticEventConstants
77
{
8-
public const string HostIdCollisionErrorCode = "AZFD004";
8+
public const string HostIdCollisionErrorCode = "AZFD0004";
99
public const string HostIdCollisionHelpLink = "https://go.microsoft.com/fwlink/?linkid=2224100";
1010

1111
public const string ExternalStartupErrorCode = "AZFD0005";
1212
public const string ExternalStartupErrorHelpLink = "https://go.microsoft.com/fwlink/?linkid=2224847";
1313

1414
public const string SasTokenExpiringErrorCode = "AZFD0006";
1515
public const string SasTokenExpiringErrorHelpLink = "https://go.microsoft.com/fwlink/?linkid=2244092";
16+
17+
public const string MaximumSecretBackupCountErrorCode = "AZFD0007";
18+
public const string MaximumSecretBackupCountHelpLink = "https://go.microsoft.com/fwlink/?linkid=2241600";
19+
20+
public const string FailedToReadBlobStorageRepositoryErrorCode = "AZFD0008";
21+
public const string FailedToReadBlobStorageRepositoryHelpLink = "https://go.microsoft.com/fwlink/?linkid=2241601";
22+
23+
public const string UnableToParseHostConfigurationFileErrorCode = "AZFD0009";
24+
public const string UnableToParseHostConfigurationFileHelpLink = "https://go.microsoft.com/fwlink/?linkid=2248917";
1625
}
1726
}

‎test/WebJobs.Script.Tests.Integration/Diagnostics/DiagnosticEventTestUtils.cs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using Microsoft.Extensions.Logging;
5+
using System.Collections.Generic;
6+
using System;
7+
using Xunit;
8+
using Microsoft.WebJobs.Script.Tests;
9+
using Microsoft.Azure.AppService.Proxy.Common.Extensions;
10+
11+
namespace Microsoft.Azure.WebJobs.Script.Tests.Integration.Diagnostics
12+
{
13+
public class DiagnosticEventTestUtils
14+
{
15+
public static void ValidateThatTheExpectedDiagnosticEventIsPresent(
16+
TestLoggerProvider loggerProvider,
17+
string expectedMessage,
18+
LogLevel logLevel,
19+
string helpLink,
20+
string errorCode
21+
)
22+
{
23+
LogMessage actualEvent = null;
24+
25+
// Find the expected diagnostic event
26+
foreach (var message in loggerProvider.GetAllLogMessages())
27+
{
28+
if (message.FormattedMessage.IndexOf(expectedMessage, StringComparison.OrdinalIgnoreCase) > -1 &&
29+
message.Level == logLevel &&
30+
message.State is Dictionary<string, object> dictionary &&
31+
dictionary.ContainsKey("MS_HelpLink") && dictionary.ContainsKey("MS_ErrorCode") &&
32+
dictionary.GetValueOrDefault("MS_HelpLink").ToString().Equals(helpLink, StringComparison.OrdinalIgnoreCase) &&
33+
dictionary.GetValueOrDefault("MS_ErrorCode").ToString().Equals(errorCode, StringComparison.OrdinalIgnoreCase))
34+
{
35+
actualEvent = message;
36+
break;
37+
}
38+
}
39+
40+
// Make sure that the expected event was found
41+
Assert.NotNull(actualEvent);
42+
}
43+
}
44+
}

‎test/WebJobs.Script.Tests.Integration/Host/BlobStorageSecretsRepositoryTests.cs‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System.IO;
5+
using System.Threading.Tasks;
6+
using Azure;
7+
using Microsoft.Azure.WebJobs.Host.Storage;
8+
using Microsoft.Azure.WebJobs.Script.WebHost;
9+
using Microsoft.Extensions.Logging;
10+
using Microsoft.WebJobs.Script.Tests;
11+
using Xunit;
12+
using Moq;
13+
using Azure.Storage.Blobs;
14+
using Microsoft.Azure.WebJobs.Script.WebHost.Properties;
15+
using LogLevel = Microsoft.Extensions.Logging.LogLevel;
16+
using System.Threading;
17+
using Microsoft.Azure.WebJobs.Script.Tests.Integration.Diagnostics;
18+
19+
namespace Microsoft.Azure.WebJobs.Script.Tests.Integration.Host
20+
{
21+
public class BlobStorageSecretsRepositoryTests
22+
{
23+
[Fact]
24+
public async Task BlobRepository_TierSetToArchive_ReadAsync_Logs_DiagnosticEvent()
25+
{
26+
var mockBlobStorageProvider = new Mock<IAzureBlobStorageProvider>();
27+
var mockBlobContainerClient = new Mock<BlobContainerClient>();
28+
var mockBlobClient = new Mock<BlobClient>();
29+
var mockBlobServiceClient = new Mock<BlobServiceClient>();
30+
31+
var exception = new RequestFailedException(409, "Conflict", "BlobArchived", null);
32+
Response<bool> response = Response.FromValue(true, default);
33+
Task<Response<bool>> taskResponse = Task.FromResult(response);
34+
35+
mockBlobStorageProvider
36+
.Setup(provider => provider.TryCreateBlobServiceClientFromConnection(It.IsAny<string>(), out It.Ref<BlobServiceClient>.IsAny))
37+
.Returns((string connection, out BlobServiceClient client) =>
38+
{
39+
client = mockBlobServiceClient.Object;
40+
return true;
41+
});
42+
43+
mockBlobContainerClient
44+
.Setup(client => client.GetBlobClient(It.IsAny<string>()))
45+
.Returns(mockBlobClient.Object);
46+
47+
mockBlobContainerClient
48+
.Setup(client => client.Exists(default))
49+
.Returns(response);
50+
51+
mockBlobServiceClient
52+
.Setup(client => client.GetBlobContainerClient(It.IsAny<string>()))
53+
.Returns(mockBlobContainerClient.Object);
54+
55+
mockBlobClient
56+
.Setup(client => client.DownloadAsync())
57+
.Throws(exception);
58+
59+
mockBlobClient
60+
.Setup(client => client.ExistsAsync(It.IsAny<CancellationToken>()))
61+
.Returns(taskResponse);
62+
63+
// Create logger
64+
var loggerProvider = new TestLoggerProvider();
65+
var loggerFactory = new LoggerFactory();
66+
loggerFactory.AddProvider(loggerProvider);
67+
var logger = loggerFactory.CreateLogger<BlobStorageSecretsRepository>();
68+
69+
// BlobStorageSecretsRepository settings:
70+
var environment = new TestEnvironment();
71+
var secretSentinelDirectoryPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
72+
var appName = "Test_test";
73+
var testFunctionName = "Function1";
74+
75+
var secretsRepository = new BlobStorageSecretsRepository(
76+
secretSentinelDirectoryPath,
77+
ConnectionStringNames.Storage,
78+
appName,
79+
logger,
80+
environment,
81+
mockBlobStorageProvider.Object);
82+
83+
await Assert.ThrowsAsync<RequestFailedException>(async () =>
84+
{
85+
await secretsRepository.ReadAsync(ScriptSecretsType.Host, testFunctionName);
86+
});
87+
88+
DiagnosticEventTestUtils.ValidateThatTheExpectedDiagnosticEventIsPresent(
89+
loggerProvider,
90+
Resources.FailedToReadBlobSecretRepositoryTierSetToArchive,
91+
LogLevel.Error,
92+
DiagnosticEventConstants.FailedToReadBlobStorageRepositoryHelpLink,
93+
DiagnosticEventConstants.FailedToReadBlobStorageRepositoryErrorCode
94+
);
95+
}
96+
}
97+
}

‎test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
<PackageReference Include="Microsoft.Azure.ServiceBus" Version="4.2.1" />
5050
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
5151
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.4" />
52-
<PackageReference Include="Moq" Version="4.9.0" />
52+
<PackageReference Include="Moq" Version="4.20.69" />
5353
<PackageReference Include="xunit" Version="2.4.1" />
5454
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
5555
<PrivateAssets>all</PrivateAssets>

‎test/WebJobs.Script.Tests/Configuration/HostJsonFileConfigurationSourceTests.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,42 @@ public void Initialize_Sanitizes_HostJsonLog()
223223
Assert.Equal($"Host configuration file read:{Environment.NewLine}{hostJson}", logMessage);
224224
}
225225

226+
[Fact]
227+
public void InvalidHostJsonLogsDiagnosticEvent()
228+
{
229+
Assert.False(File.Exists(_hostJsonFile));
230+
231+
var hostJsonContent = " { fooBar";
232+
TestMetricsLogger testMetricsLogger = new TestMetricsLogger();
233+
234+
File.WriteAllText(_hostJsonFile, hostJsonContent);
235+
Assert.True(File.Exists(_hostJsonFile));
236+
237+
var ex = Assert.Throws<FormatException>(() => BuildHostJsonConfiguration(testMetricsLogger));
238+
239+
var expectedTraceMessage = $"Unable to parse host configuration file '{_hostJsonFile}'.";
240+
241+
LogMessage actualEvent = null;
242+
243+
// Find the expected diagnostic event
244+
foreach (var message in _loggerProvider.GetAllLogMessages())
245+
{
246+
if (message.FormattedMessage.IndexOf(expectedTraceMessage, StringComparison.OrdinalIgnoreCase) > -1 &&
247+
message.Level == LogLevel.Error &&
248+
message.State is Dictionary<string, object> dictionary &&
249+
dictionary.ContainsKey("MS_HelpLink") && dictionary.ContainsKey("MS_ErrorCode") &&
250+
dictionary.GetValueOrDefault("MS_HelpLink").ToString().Equals(DiagnosticEventConstants.UnableToParseHostConfigurationFileHelpLink.ToString(), StringComparison.OrdinalIgnoreCase) &&
251+
dictionary.GetValueOrDefault("MS_ErrorCode").ToString().Equals(DiagnosticEventConstants.UnableToParseHostConfigurationFileErrorCode.ToString(), StringComparison.OrdinalIgnoreCase))
252+
{
253+
actualEvent = message;
254+
break;
255+
}
256+
}
257+
258+
// Make sure that the expected event was found
259+
Assert.NotNull(actualEvent);
260+
}
261+
226262
private IConfiguration BuildHostJsonConfiguration(TestMetricsLogger testMetricsLogger, IEnvironment environment = null)
227263
{
228264
environment = environment ?? new TestEnvironment();

0 commit comments

Comments
 (0)