Skip to content

Commit c6f6ecb

Browse files
authored
Add changes for log analytics / azure monitoring for Linux Dedicated (#5212)
1 parent 8e37c6f commit c6f6ecb

18 files changed

+82
-51
lines changed

src/WebJobs.Script.WebHost/Diagnostics/LinuxAppServiceEventGenerator.cs

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,15 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics
99
internal class LinuxAppServiceEventGenerator : LinuxEventGenerator
1010
{
1111
private readonly LinuxAppServiceFileLoggerFactory _loggerFactory;
12+
private readonly HostNameProvider _hostNameProvider;
1213

13-
public LinuxAppServiceEventGenerator(LinuxAppServiceFileLoggerFactory loggerFactory)
14+
public LinuxAppServiceEventGenerator(LinuxAppServiceFileLoggerFactory loggerFactory, HostNameProvider hostNameProvider)
1415
{
1516
_loggerFactory = loggerFactory;
17+
_hostNameProvider = hostNameProvider ?? throw new ArgumentNullException(nameof(hostNameProvider));
1618
}
1719

18-
public static string TraceEventRegex { get; } = $"(?<Level>[0-6]),(?<SubscriptionId>[^,]*),(?<AppName>[^,]*),(?<FunctionName>[^,]*),(?<EventName>[^,]*),(?<Source>[^,]*),\"(?<Details>.*)\",\"(?<Summary>.*)\",(?<HostVersion>[^,]*),(?<EventTimestamp>[^,]+),(?<ExceptionType>[^,]*),\"(?<ExceptionMessage>.*)\",(?<FunctionInvocationId>[^,]*),(?<HostInstanceId>[^,]*),(?<ActivityId>[^,\"]*)";
20+
public static string TraceEventRegex { get; } = $"(?<Level>[0-6]),(?<SubscriptionId>[^,]*),(?<HostName>[^,]*),(?<AppName>[^,]*),(?<FunctionName>[^,]*),(?<EventName>[^,]*),(?<Source>[^,]*),\"(?<Details>.*)\",\"(?<Summary>.*)\",(?<HostVersion>[^,]*),(?<EventTimestamp>[^,]+),(?<ExceptionType>[^,]*),\"(?<ExceptionMessage>.*)\",(?<FunctionInvocationId>[^,]*),(?<HostInstanceId>[^,]*),(?<ActivityId>[^,\"]*)";
1921

2022
public static string MetricEventRegex { get; } = $"(?<SubscriptionId>[^,]*),(?<AppName>[^,]*),(?<FunctionName>[^,]*),(?<EventName>[^,]*),(?<Average>\\d*),(?<Min>\\d*),(?<Max>\\d*),(?<Count>\\d*),(?<HostVersion>[^,]*),(?<EventTimestamp>[^,]+),(?<Details>[^,\"]*)";
2123

@@ -27,10 +29,11 @@ public override void LogFunctionTraceEvent(LogLevel level, string subscriptionId
2729
{
2830
var eventTimestamp = DateTime.UtcNow.ToString(EventTimestampFormat);
2931
var hostVersion = ScriptHost.Version;
32+
var hostName = _hostNameProvider.Value;
3033
FunctionsSystemLogsEventSource.Instance.SetActivityId(activityId);
3134

3235
var logger = _loggerFactory.GetOrCreate(FunctionsLogsCategory);
33-
WriteEvent(logger, $"{(int)ToEventLevel(level)},{subscriptionId},{appName},{functionName},{eventName},{source},{NormalizeString(details)},{NormalizeString(summary)},{hostVersion},{eventTimestamp},{exceptionType},{NormalizeString(exceptionMessage)},{functionInvocationId},{hostInstanceId},{activityId}");
36+
WriteEvent(logger, $"{(int)ToEventLevel(level)},{subscriptionId},{hostName},{appName},{functionName},{eventName},{source},{NormalizeString(details)},{NormalizeString(summary)},{hostVersion},{eventTimestamp},{exceptionType},{NormalizeString(exceptionMessage)},{functionInvocationId},{hostInstanceId},{activityId}");
3437
}
3538

3639
public override void LogFunctionMetricEvent(string subscriptionId, string appName, string functionName, string eventName, long average,

src/WebJobs.Script.WebHost/Diagnostics/LinuxEventGenerator.cs

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,23 @@ public abstract class LinuxEventGenerator : IEventGenerator
1212
public static readonly string EventTimestampFormat = "MM/dd/yyyy hh:mm:ss.fff tt";
1313

1414
// These names should match the source file names for fluentd
15-
public static readonly string FunctionsLogsCategory = "functionslogs";
15+
public static readonly string FunctionsLogsCategory = "functionslogsv2";
1616
public static readonly string FunctionsMetricsCategory = "functionsmetrics";
1717
public static readonly string FunctionsDetailsCategory = "functionsdetails";
1818
public static readonly string FunctionsExecutionEventsCategory = "functionexecutionevents";
1919

2020
internal static string NormalizeString(string value)
2121
{
22-
// need to remove newlines for csv output
22+
// Need to remove newlines for csv output
2323
value = value.Replace(Environment.NewLine, " ");
2424

25+
// Need to replace double quotes with single quotes as
26+
// our regex query looks at double quotes as delimeter for
27+
// individual column
28+
// TODO: Once the regex takes into account for quotes, we can
29+
// safely remove this
30+
value = value.Replace("\"", "'");
31+
2532
// Wrap string literals in enclosing quotes
2633
// For string columns that may contain quotes and/or
2734
// our delimiter ',', before writing the value we

src/WebJobs.Script.WebHost/HostNameProvider.cs

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,11 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost
2121
public class HostNameProvider
2222
{
2323
private readonly IEnvironment _environment;
24-
private readonly ILogger _logger;
2524
private string _hostName;
2625

27-
public HostNameProvider(IEnvironment environment, ILogger<HostNameProvider> logger)
26+
public HostNameProvider(IEnvironment environment)
2827
{
2928
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
30-
_logger = logger;
3129
}
3230

3331
public virtual string Value
@@ -52,17 +50,17 @@ public virtual string Value
5250
}
5351
}
5452

55-
public virtual void Synchronize(HttpRequest request)
53+
public virtual void Synchronize(HttpRequest request, ILogger logger)
5654
{
5755
string hostNameHeaderValue = request.Headers[ScriptConstants.AntaresDefaultHostNameHeader];
5856
if (!string.IsNullOrEmpty(hostNameHeaderValue) &&
5957
string.Compare(Value, hostNameHeaderValue) != 0)
6058
{
61-
if (string.Compare(Value, hostNameHeaderValue) != 0)
62-
{
63-
_logger.LogInformation("HostName updated from '{0}' to '{1}'", Value, hostNameHeaderValue);
64-
_hostName = hostNameHeaderValue;
65-
}
59+
if (string.Compare(Value, hostNameHeaderValue) != 0)
60+
{
61+
logger.LogInformation("HostName updated from '{0}' to '{1}'", Value, hostNameHeaderValue);
62+
_hostName = hostNameHeaderValue;
63+
}
6664
}
6765
}
6866

src/WebJobs.Script.WebHost/Middleware/HostnameFixupMiddleware.cs

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,26 @@
33

44
using System.Threading.Tasks;
55
using Microsoft.AspNetCore.Http;
6+
using Microsoft.Extensions.Logging;
67

78
namespace Microsoft.Azure.WebJobs.Script.WebHost.Middleware
89
{
910
public class HostnameFixupMiddleware
1011
{
1112
private readonly RequestDelegate _next;
1213
private readonly HostNameProvider _hostNameProvider;
14+
private readonly ILogger _logger;
1315

14-
public HostnameFixupMiddleware(RequestDelegate next, HostNameProvider hostNameProvider)
16+
public HostnameFixupMiddleware(RequestDelegate next, HostNameProvider hostNameProvider, ILogger<HostnameFixupMiddleware> logger)
1517
{
1618
_next = next;
1719
_hostNameProvider = hostNameProvider;
20+
_logger = logger;
1821
}
1922

2023
public async Task Invoke(HttpContext context)
2124
{
22-
_hostNameProvider.Synchronize(context.Request);
23-
25+
_hostNameProvider.Synchronize(context.Request, _logger);
2426
await _next.Invoke(context);
2527
}
2628
}

src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,8 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi
9191
}
9292
else if (SystemEnvironment.Instance.IsLinuxAppService())
9393
{
94-
return new LinuxAppServiceEventGenerator(new LinuxAppServiceFileLoggerFactory());
94+
var hostNameProvider = p.GetService<HostNameProvider>();
95+
return new LinuxAppServiceEventGenerator(new LinuxAppServiceFileLoggerFactory(), hostNameProvider);
9596
}
9697
else
9798
{

test/WebJobs.Script.Tests.Integration/Management/FunctionsSyncManagerTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ public FunctionsSyncManagerTests()
115115
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteHostName)).Returns(testHostName);
116116
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.SkipSslValidation)).Returns((string)null);
117117
_mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsSecretStorageType)).Returns("blob");
118-
_hostNameProvider = new HostNameProvider(_mockEnvironment.Object, loggerFactory.CreateLogger<HostNameProvider>());
118+
_hostNameProvider = new HostNameProvider(_mockEnvironment.Object);
119119

120120
var functionMetadataProvider = new FunctionMetadataProvider(optionsMonitor, new OptionsWrapper<LanguageWorkerOptions>(CreateLanguageWorkerConfigSettings()), NullLogger<FunctionMetadataProvider>.Instance, new TestMetricsLogger());
121121
_functionsSyncManager = new FunctionsSyncManager(configuration, hostIdProviderMock.Object, optionsMonitor, loggerFactory.CreateLogger<FunctionsSyncManager>(), httpClient, secretManagerProviderMock.Object, _mockWebHostEnvironment.Object, _mockEnvironment.Object, _hostNameProvider, functionMetadataProvider);

test/WebJobs.Script.Tests.Integration/Security/SecretManagerProviderTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public SecretManagerProviderTests()
4242
var loggerProvider = new TestLoggerProvider();
4343
var loggerFactory = new LoggerFactory();
4444
loggerFactory.AddProvider(loggerProvider);
45-
var hostNameProvider = new HostNameProvider(environment, loggerFactory.CreateLogger<HostNameProvider>());
45+
var hostNameProvider = new HostNameProvider(environment);
4646

4747
_provider = new DefaultSecretManagerProvider(optionsMonitor, mockIdProvider.Object, config,
4848
new TestEnvironment(), NullLoggerFactory.Instance, new TestMetricsLogger(), hostNameProvider);

test/WebJobs.Script.Tests/DefaultScriptWebHookProviderTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public DefaultScriptWebHookProviderTests()
3535
loggerFactory.AddProvider(loggerProvider);
3636
var testEnvironment = new TestEnvironment();
3737
testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteHostName, TestHostName);
38-
var hostNameProvider = new HostNameProvider(testEnvironment, loggerFactory.CreateLogger<HostNameProvider>());
38+
var hostNameProvider = new HostNameProvider(testEnvironment);
3939
_webHookProvider = new DefaultScriptWebHookProvider(mockSecretManagerProvider.Object, hostNameProvider);
4040
}
4141

test/WebJobs.Script.Tests/Diagnostics/LinuxAppServiceEventGeneratorTests.cs

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Collections.Generic;
66
using System.Linq;
77
using System.Text.RegularExpressions;
8+
using Microsoft.Azure.WebJobs.Script.WebHost;
89
using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics;
910
using Microsoft.Extensions.Logging;
1011
using Moq;
@@ -14,6 +15,8 @@ namespace Microsoft.Azure.WebJobs.Script.Tests.Diagnostics
1415
{
1516
public class LinuxAppServiceEventGeneratorTests
1617
{
18+
private const string _hostNameDefault = "SimpleApp";
19+
1720
private readonly LinuxAppServiceEventGenerator _generator;
1821
private readonly Dictionary<string, MockLinuxAppServiceFileLogger> _loggers;
1922

@@ -32,7 +35,12 @@ public LinuxAppServiceEventGeneratorTests()
3235
var loggerFactoryMock = new Mock<LinuxAppServiceFileLoggerFactory>(MockBehavior.Strict);
3336
loggerFactoryMock.Setup(f => f.GetOrCreate(It.IsAny<string>())).Returns<string>(s => _loggers[s]);
3437

35-
_generator = new LinuxAppServiceEventGenerator(loggerFactoryMock.Object);
38+
var environmentMock = new Mock<IEnvironment>();
39+
environmentMock.Setup(f => f.GetEnvironmentVariable(It.Is<string>(v => v == "WEBSITE_HOSTNAME")))
40+
.Returns<string>(s => _hostNameDefault);
41+
42+
var hostNameProvider = new HostNameProvider(environmentMock.Object);
43+
_generator = new LinuxAppServiceEventGenerator(loggerFactoryMock.Object, hostNameProvider);
3644
}
3745

3846
[Theory]
@@ -47,23 +55,24 @@ public void ParseLogEvents(LogLevel level, string subscriptionId, string appName
4755
var match = regex.Match(evt);
4856

4957
Assert.True(match.Success);
50-
Assert.Equal(16, match.Groups.Count);
58+
Assert.Equal(17, match.Groups.Count);
5159

5260
DateTime dt;
5361
var groupMatches = match.Groups.Select(p => p.Value).Skip(1).ToArray();
5462
Assert.Collection(groupMatches,
5563
p => Assert.Equal((int)LinuxEventGenerator.ToEventLevel(level), int.Parse(p)),
5664
p => Assert.Equal(subscriptionId, p),
65+
p => Assert.Equal(_hostNameDefault, p),
5766
p => Assert.Equal(appName, p),
5867
p => Assert.Equal(functionName, p),
5968
p => Assert.Equal(eventName, p),
6069
p => Assert.Equal(source, p),
61-
p => Assert.Equal(details, p),
62-
p => Assert.Equal(summary, p),
70+
p => Assert.Equal(details, LinuxContainerEventGeneratorTests.UnNormalize(p)),
71+
p => Assert.Equal(summary, LinuxContainerEventGeneratorTests.UnNormalize(p)),
6372
p => Assert.Equal(ScriptHost.Version, p),
6473
p => Assert.True(DateTime.TryParse(p, out dt)),
6574
p => Assert.Equal(exceptionType, p),
66-
p => Assert.Equal(exceptionMessage, p),
75+
p => Assert.Equal(exceptionMessage, LinuxContainerEventGeneratorTests.UnNormalize(p)),
6776
p => Assert.Equal(functionInvocationId, p),
6877
p => Assert.Equal(hostInstanceId, p),
6978
p => Assert.Equal(activityId, p));
@@ -117,8 +126,8 @@ public void ParseDetailsEvents(string siteName, string functionName, string inpu
117126
Assert.Collection(groupMatches,
118127
p => Assert.Equal(siteName, p),
119128
p => Assert.Equal(functionName, p),
120-
p => Assert.Equal(inputBindings, p),
121-
p => Assert.Equal(outputBindings, p),
129+
p => Assert.Equal(inputBindings, LinuxContainerEventGeneratorTests.UnNormalize(p)),
130+
p => Assert.Equal(outputBindings, LinuxContainerEventGeneratorTests.UnNormalize(p)),
122131
p => Assert.Equal(scriptType, p),
123132
p => Assert.Equal(isDisabled ? "1" : "0", p));
124133
}

test/WebJobs.Script.Tests/Diagnostics/LinuxContainerEventGeneratorTests.cs

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,12 @@ public void ParseLogEvents(LogLevel level, string subscriptionId, string appName
7272
p => Assert.Equal(functionName, p),
7373
p => Assert.Equal(eventName, p),
7474
p => Assert.Equal(source, p),
75-
p => Assert.Equal(details, JsonUnescape(p)),
76-
p => Assert.Equal(summary, JsonUnescape(p)),
75+
p => Assert.Equal(details, UnNormalize(JsonUnescape(p))),
76+
p => Assert.Equal(summary, UnNormalize(JsonUnescape(p))),
7777
p => Assert.Equal(ScriptHost.Version, p),
7878
p => Assert.True(DateTime.TryParse(p, out dt)),
7979
p => Assert.Equal(exceptionType, p),
80-
p => Assert.Equal(exceptionMessage, JsonUnescape(p)),
80+
p => Assert.Equal(exceptionMessage, UnNormalize(JsonUnescape(p))),
8181
p => Assert.Equal(functionInvocationId, p),
8282
p => Assert.Equal(hostInstanceId, p),
8383
p => Assert.Equal(activityId, p),
@@ -93,6 +93,14 @@ private static string JsonUnescape(string value)
9393
return value.Replace("\\", string.Empty);
9494
}
9595

96+
public static string UnNormalize(string normalized)
97+
{
98+
// We replace all double quotes to single before the writing the logs
99+
// to avoid our logging agents parsing break
100+
// TODO: we can remove this once platform is able to handle quotes in logs
101+
return normalized.Replace("'", "\"");
102+
}
103+
96104
private static string JsonSerializeEvent(string evt)
97105
{
98106
// the logging pipeline currently wraps our raw log data with JSON
@@ -166,8 +174,8 @@ public void ParseDetailsEvents(string siteName, string functionName, string inpu
166174
Assert.Collection(groupMatches,
167175
p => Assert.Equal(siteName, p),
168176
p => Assert.Equal(functionName, p),
169-
p => Assert.Equal(inputBindings, JsonUnescape(p)),
170-
p => Assert.Equal(outputBindings, JsonUnescape(p)),
177+
p => Assert.Equal(inputBindings, UnNormalize(JsonUnescape(p))),
178+
p => Assert.Equal(outputBindings, UnNormalize(JsonUnescape(p))),
171179
p => Assert.Equal(scriptType, p),
172180
p => Assert.Equal(isDisabled ? "1" : "0", p));
173181
}

0 commit comments

Comments
 (0)