Skip to content

Commit 575890a

Browse files
RohitRanjanMSRohitRanjanMS
authored
Updating startup failure logging. (#11055)
* Updating startup failure logging.
1 parent 62f8d8a commit 575890a

File tree

4 files changed

+184
-54
lines changed

4 files changed

+184
-54
lines changed

‎release_notes.md

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1-
### Release notes
2-
3-
<!-- Please add your release notes in the following format:
4-
- My change description (#PR)
5-
-->
6-
- Memory allocation optimizations in `ScriptStartupTypeLocator.GetExtensionsStartupTypesAsync` (#11012)
7-
- Fix invocation timeout when incoming request contains "x-ms-invocation-id" header (#10980)
8-
- Warn if .azurefunctions folder does not exist (#10967)
9-
- Memory allocation & CPU optimizations in `GrpcMessageExtensionUtilities.ConvertFromHttpMessageToExpando` (#11054)
10-
- Memory allocation optimizations in `ReadLanguageWorkerFile` by reading files in buffered chunks, preventing LOH allocations (#11069)
1+
### Release notes
2+
3+
<!-- Please add your release notes in the following format:
4+
- My change description (#PR)
5+
-->
6+
- Memory allocation optimizations in `ScriptStartupTypeLocator.GetExtensionsStartupTypesAsync` (#11012)
7+
- Fix invocation timeout when incoming request contains "x-ms-invocation-id" header (#10980)
8+
- Warn if .azurefunctions folder does not exist (#10967)
9+
- Memory allocation & CPU optimizations in `GrpcMessageExtensionUtilities.ConvertFromHttpMessageToExpando` (#11054)
10+
- Memory allocation optimizations in `ReadLanguageWorkerFile` by reading files in buffered chunks, preventing LOH allocations (#11069)
11+
- Enhancing the capability to send startup failure logs to AppInsights/Otel. (#11055)

‎src/WebJobs.Script.WebHost/Diagnostics/CompositeLogger.cs

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Linq;
6+
using Microsoft.Extensions.Logging;
7+
8+
namespace Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics
9+
{
10+
internal sealed class CompositeLogger : ILogger
11+
{
12+
private readonly ILogger[] _loggers;
13+
14+
public CompositeLogger(params ILogger[] loggers)
15+
{
16+
_loggers = loggers is { Length: > 0 } ? loggers : throw new ArgumentNullException(nameof(loggers));
17+
}
18+
19+
public IDisposable BeginScope<TState>(TState state)
20+
{
21+
// Create a composite scope that wraps scopes from all loggers
22+
var scopes = new IDisposable[_loggers.Length];
23+
for (int i = 0; i < _loggers.Length; i++)
24+
{
25+
scopes[i] = _loggers[i].BeginScope(state);
26+
}
27+
28+
return new CompositeScope(scopes);
29+
}
30+
31+
public bool IsEnabled(LogLevel logLevel)
32+
{
33+
return _loggers.Any(l => l.IsEnabled(logLevel));
34+
}
35+
36+
public void Log<TState>(
37+
LogLevel logLevel, EventId eventId, TState state,
38+
Exception exception, Func<TState, Exception, string> formatter)
39+
{
40+
foreach (var logger in _loggers)
41+
{
42+
if (logger.IsEnabled(logLevel))
43+
{
44+
logger.Log(logLevel, eventId, state, exception, formatter);
45+
}
46+
}
47+
}
48+
49+
private sealed class CompositeScope : IDisposable
50+
{
51+
private readonly IDisposable[] _scopes;
52+
53+
public CompositeScope(IDisposable[] scopes)
54+
=> _scopes = scopes;
55+
56+
public void Dispose()
57+
{
58+
foreach (ref readonly var scope in _scopes.AsSpan())
59+
{
60+
scope?.Dispose();
61+
}
62+
}
63+
}
64+
}
65+
}

‎src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs

Lines changed: 105 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,11 @@
66
using System.Collections.ObjectModel;
77
using System.IO;
88
using System.Linq;
9+
using System.Reflection;
910
using System.Threading;
1011
using System.Threading.Tasks;
12+
using Azure.Identity;
13+
using Azure.Monitor.OpenTelemetry.Exporter;
1114
using Microsoft.ApplicationInsights.AspNetCore;
1215
using Microsoft.ApplicationInsights.AspNetCore.Extensions;
1316
using Microsoft.ApplicationInsights.DependencyCollector;
@@ -36,6 +39,7 @@
3639
using OpenTelemetry.Metrics;
3740
using OpenTelemetry.Trace;
3841
using static Microsoft.Azure.WebJobs.Script.EnvironmentSettingNames;
42+
using AppInsightsCredentialOptions = Microsoft.Azure.WebJobs.Logging.ApplicationInsights.TokenCredentialOptions;
3943
using IApplicationLifetime = Microsoft.AspNetCore.Hosting.IApplicationLifetime;
4044

4145
namespace Microsoft.Azure.WebJobs.Script.WebHost
@@ -224,7 +228,7 @@ public async Task StartAsync(CancellationToken cancellationToken)
224228
}
225229
}
226230

227-
private void CheckFileSystem()
231+
private async Task CheckFileSystemAsync()
228232
{
229233
if (_environment.ZipDeploymentAppSettingsExist())
230234
{
@@ -247,7 +251,10 @@ private void CheckFileSystem()
247251
}
248252
finally
249253
{
250-
_logger.LogError(errorPrefix + errorSuffix);
254+
var errorMessage = $"{errorPrefix}{errorSuffix}";
255+
256+
_logger.LogError(errorMessage);
257+
await LogErrorWithTransientOtelLoggerAsync(errorMessage);
251258
}
252259
_applicationLifetime.StopApplication();
253260
}
@@ -319,7 +326,7 @@ private async Task StartHostAsync(CancellationToken cancellationToken, int attem
319326
/// </summary>
320327
private async Task UnsynchronizedStartHostAsync(ScriptHostStartupOperation activeOperation, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal)
321328
{
322-
CheckFileSystem();
329+
await CheckFileSystemAsync();
323330
if (ShutdownRequested)
324331
{
325332
return;
@@ -736,7 +743,20 @@ private ILogger GetHostLogger(IHost host)
736743

737744
// Attempt to get the host logger with JobHost configuration applied
738745
// using the default logger as a fallback
739-
return hostLoggerFactory?.CreateLogger(LogCategories.Startup) ?? _logger;
746+
if (hostLoggerFactory is not null)
747+
{
748+
return hostLoggerFactory?.CreateLogger(LogCategories.Startup);
749+
}
750+
751+
// An error occurred before the host was built; a minimal logger factory is being created to send telemetry to AppInsights/Otel.
752+
var otelLoggerFactory = BuildOtelLoggerFactory();
753+
754+
// If the Otel logger factory is null, use the fallback logger instead. These logs will not be accessible in AppInsights/Otel.
755+
if (otelLoggerFactory is null)
756+
{
757+
return _logger;
758+
}
759+
return new CompositeLogger(_logger, otelLoggerFactory.CreateLogger(LogCategories.Startup));
740760
}
741761

742762
private void LogInitialization(IHost host, bool isOffline, int attemptCount, int startCount, Guid operationId)
@@ -1047,6 +1067,87 @@ private void EndStartupOperation(ScriptHostStartupOperation operation)
10471067
_logger.StartupOperationCompleted(operation.Id);
10481068
}
10491069

1070+
private async Task LogErrorWithTransientOtelLoggerAsync(string log)
1071+
{
1072+
var loggerFactory = BuildOtelLoggerFactory();
1073+
1074+
if (loggerFactory is not null)
1075+
{
1076+
var logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryHostGeneral);
1077+
logger.LogError(log);
1078+
1079+
// Delay increases the chance that the log is sent to AppInsights/Otel before the logger factory is disposed
1080+
await Task.Delay(2000);
1081+
// Do a force flush as the host is shutting down and we want to ensure the logs are sent before disposing the logger factory
1082+
ForceFlush(loggerFactory);
1083+
// Give some time for the logger to flush
1084+
await Task.Delay(4000);
1085+
}
1086+
}
1087+
1088+
private ILoggerFactory BuildOtelLoggerFactory()
1089+
{
1090+
var appInsightsConnStr = GetConfigurationValue(AppInsightsConnectionString, _config);
1091+
var otlpEndpoint = GetConfigurationValue(OtlpEndpoint, _config);
1092+
if (appInsightsConnStr is not { Length: > 0 } && otlpEndpoint is not { Length: > 0 })
1093+
{
1094+
return null; // Nothing configured
1095+
}
1096+
1097+
// Create a minimal logger factory with OpenTelemetry and Azure Monitor exporter
1098+
return LoggerFactory.Create(builder =>
1099+
{
1100+
builder.AddOpenTelemetry(logging =>
1101+
{
1102+
logging.IncludeScopes = true;
1103+
logging.IncludeFormattedMessage = true;
1104+
1105+
if (appInsightsConnStr is { Length: > 0 })
1106+
{
1107+
logging.AddAzureMonitorLogExporter(options =>
1108+
{
1109+
options.ConnectionString = appInsightsConnStr;
1110+
1111+
var appInsightsAuthStr = GetConfigurationValue(AppInsightsAuthenticationString, _config);
1112+
if (appInsightsAuthStr is { Length: > 0 })
1113+
{
1114+
var credOptions = AppInsightsCredentialOptions.ParseAuthenticationString(appInsightsAuthStr);
1115+
options.Credential = new ManagedIdentityCredential(credOptions.ClientId);
1116+
}
1117+
});
1118+
}
1119+
1120+
if (otlpEndpoint is { Length: > 0 })
1121+
{
1122+
logging.AddOtlpExporter();
1123+
}
1124+
});
1125+
});
1126+
}
1127+
1128+
private void ForceFlush(ILoggerFactory loggerFactory)
1129+
{
1130+
var serviceProvider = (IServiceProvider)loggerFactory.GetType()
1131+
.GetField("_serviceProvider", BindingFlags.NonPublic | BindingFlags.Instance)
1132+
.GetValue(loggerFactory);
1133+
1134+
// Get all logger providers from the service provider
1135+
var providers = serviceProvider?.GetServices<ILoggerProvider>() ?? Enumerable.Empty<ILoggerProvider>();
1136+
1137+
foreach (var provider in providers)
1138+
{
1139+
if (provider is OpenTelemetryLoggerProvider otelProvider)
1140+
{
1141+
otelProvider.Dispose();
1142+
}
1143+
}
1144+
}
1145+
1146+
private static string GetConfigurationValue(string key, IConfiguration configuration = null)
1147+
{
1148+
return configuration?[key] ?? Environment.GetEnvironmentVariable(key);
1149+
}
1150+
10501151
protected virtual void Dispose(bool disposing)
10511152
{
10521153
if (!_disposed)

‎src/WebJobs.Script/ScriptHostBuilderExtensions.cs

Lines changed: 3 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,7 @@
66
using System.ComponentModel;
77
using System.Diagnostics.Tracing;
88
using System.Runtime.InteropServices;
9-
using System.Threading;
10-
using Azure.Identity;
11-
using Microsoft.ApplicationInsights;
12-
using Microsoft.ApplicationInsights.Channel;
13-
using Microsoft.ApplicationInsights.DataContracts;
149
using Microsoft.ApplicationInsights.Extensibility;
15-
using Microsoft.ApplicationInsights.Extensibility.Implementation;
1610
using Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel;
1711
using Microsoft.Azure.WebJobs.Host;
1812
using Microsoft.Azure.WebJobs.Host.Executors;
@@ -251,9 +245,7 @@ public static IHostBuilder AddScriptHostCore(this IHostBuilder builder, ScriptAp
251245
}
252246
catch (Exception ex)
253247
{
254-
string appInsightsConnStr = GetConfigurationValue(EnvironmentSettingNames.AppInsightsConnectionString, context.Configuration);
255-
string appInsightsAuthStr = GetConfigurationValue(EnvironmentSettingNames.AppInsightsAuthenticationString, context.Configuration);
256-
RecordAndThrowExternalStartupException("Error configuring services in an external startup class.", ex, loggerFactory, appInsightsConnStr, appInsightsAuthStr);
248+
RecordAndThrowExternalStartupException("Error configuring services in an external startup class.", ex, loggerFactory);
257249
}
258250
}
259251

@@ -279,10 +271,7 @@ public static IHostBuilder AddScriptHostCore(this IHostBuilder builder, ScriptAp
279271
}
280272
catch (Exception ex)
281273
{
282-
// Go directly to the environment; We have no valid configuration from the customer at this point.
283-
string appInsightsConnStr = GetConfigurationValue(EnvironmentSettingNames.AppInsightsConnectionString);
284-
string appInsightsAuthStr = GetConfigurationValue(EnvironmentSettingNames.AppInsightsAuthenticationString);
285-
RecordAndThrowExternalStartupException("Error building configuration in an external startup class.", ex, loggerFactory, appInsightsConnStr, appInsightsAuthStr);
274+
RecordAndThrowExternalStartupException("Error building configuration in an external startup class.", ex, loggerFactory);
286275
}
287276
});
288277
}
@@ -578,38 +567,12 @@ private static T GetAndRemove<T>(this IDictionary<object, object> dictionary, st
578567
}
579568
}
580569

581-
private static void RecordAndThrowExternalStartupException(string message, Exception ex, ILoggerFactory loggerFactory, string appInsightsConnStr, string appInsightsAuthString)
570+
private static void RecordAndThrowExternalStartupException(string message, Exception ex, ILoggerFactory loggerFactory)
582571
{
583572
var startupEx = new ExternalStartupException(message, ex);
584573

585574
var logger = loggerFactory.CreateLogger(LogCategories.Startup);
586575
logger.LogDiagnosticEventError(DiagnosticEventConstants.ExternalStartupErrorCode, message, DiagnosticEventConstants.ExternalStartupErrorHelpLink, startupEx);
587-
588-
// Send the error to App Insights if possible. This is happening during ScriptHost construction so we
589-
// have no existing TelemetryClient to use. Create a one-off client and flush it ASAP.
590-
if (!string.IsNullOrEmpty(appInsightsConnStr))
591-
{
592-
using TelemetryConfiguration telemetryConfiguration = new()
593-
{
594-
ConnectionString = appInsightsConnStr,
595-
TelemetryChannel = new InMemoryChannel()
596-
};
597-
598-
if (!string.IsNullOrEmpty(appInsightsAuthString))
599-
{
600-
TokenCredentialOptions credentialOptions = TokenCredentialOptions.ParseAuthenticationString(appInsightsAuthString);
601-
// Default is connection string based ingestion
602-
telemetryConfiguration.SetAzureTokenCredential(new ManagedIdentityCredential(credentialOptions.ClientId));
603-
}
604-
605-
TelemetryClient telemetryClient = new(telemetryConfiguration);
606-
telemetryClient.Context.GetInternalContext().SdkVersion = new ApplicationInsightsSdkVersionProvider().GetSdkVersion();
607-
telemetryClient.TrackTrace(startupEx.ToString(), SeverityLevel.Error);
608-
telemetryClient.TrackException(startupEx);
609-
telemetryClient.Flush();
610-
// Flush is async, sleep for 1 sec to give it time to flush
611-
Thread.Sleep(1000);
612-
}
613576
throw startupEx;
614577
}
615578

0 commit comments

Comments
 (0)