Skip to content

Commit 0ee99cb

Browse files
mathewcmathewc
authored
Linux Consumption metrics publisher for Legion (#10932)
1 parent cb16901 commit 0ee99cb

21 files changed

+980
-261
lines changed

‎src/WebJobs.Script.WebHost/Configuration/LinuxConsumptionLegionMetricsPublisherOptions.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
namespace Microsoft.Azure.WebJobs.Script.WebHost.Configuration
5+
{
6+
public sealed class LinuxConsumptionLegionMetricsPublisherOptions
7+
{
8+
internal const int DefaultMetricsPublishIntervalMS = 30 * 1000;
9+
10+
public int MetricsPublishIntervalMS { get; set; } = DefaultMetricsPublishIntervalMS;
11+
12+
public string ContainerName { get; set; }
13+
14+
public string MetricsFilePath { get; set; }
15+
}
16+
}

‎src/WebJobs.Script.WebHost/Configuration/LinuxConsumptionLegionMetricsPublisherOptionsSetup.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using Microsoft.Extensions.Options;
5+
6+
namespace Microsoft.Azure.WebJobs.Script.WebHost.Configuration
7+
{
8+
public class LinuxConsumptionLegionMetricsPublisherOptionsSetup : IConfigureOptions<LinuxConsumptionLegionMetricsPublisherOptions>
9+
{
10+
private readonly IEnvironment _environment;
11+
12+
public LinuxConsumptionLegionMetricsPublisherOptionsSetup(IEnvironment environment)
13+
{
14+
_environment = environment;
15+
}
16+
17+
public void Configure(LinuxConsumptionLegionMetricsPublisherOptions options)
18+
{
19+
options.ContainerName = _environment.GetEnvironmentVariable(EnvironmentSettingNames.ContainerName);
20+
options.MetricsFilePath = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsMetricsPublishPath);
21+
}
22+
}
23+
}

‎src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs‎

Lines changed: 10 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ public class FlexConsumptionMetricsPublisher : IMetricsPublisher, IDisposable
2626
private readonly IHostMetricsProvider _metricsProvider;
2727
private readonly object _lock = new object();
2828
private readonly IFileSystem _fileSystem;
29+
private readonly LegionMetricsFileManager _metricsFileManager;
2930

3031
private Timer _metricsPublisherTimer;
3132
private bool _started = false;
@@ -44,6 +45,7 @@ public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor
4445
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
4546
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
4647
_fileSystem = fileSystem ?? new FileSystem();
48+
_metricsFileManager = new LegionMetricsFileManager(_options.MetricsFilePath, _fileSystem, _logger, _options.MaxFileCount);
4749
_metricsProvider = metricsProvider ?? throw new ArgumentNullException(nameof(metricsProvider));
4850

4951
if (_standbyOptions.CurrentValue.InStandbyMode)
@@ -66,13 +68,13 @@ public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor
6668

6769
internal bool IsAlwaysReady { get; set; }
6870

69-
internal string MetricsFilePath { get; set; }
71+
internal LegionMetricsFileManager MetricsFileManager => _metricsFileManager;
7072

7173
public void Start()
7274
{
7375
Initialize();
7476

75-
_logger.LogInformation($"Starting metrics publisher (AlwaysReady={IsAlwaysReady}, MetricsPath='{MetricsFilePath}').");
77+
_logger.LogInformation($"Starting metrics publisher (AlwaysReady={IsAlwaysReady}, MetricsPath='{_metricsFileManager.MetricsFilePath}').");
7678

7779
_metricsPublisherTimer = new Timer(OnFunctionMetricsPublishTimer, null, _initialPublishDelay, _metricPublishInterval);
7880
_started = true;
@@ -86,7 +88,6 @@ internal void Initialize()
8688
_metricPublishInterval = TimeSpan.FromMilliseconds(_options.MetricsPublishIntervalMS);
8789
_initialPublishDelay = TimeSpan.FromMilliseconds(_options.InitialPublishDelayMS);
8890
_intervalStopwatch = ValueStopwatch.StartNew();
89-
MetricsFilePath = _options.MetricsFilePath;
9091

9192
IsAlwaysReady = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsAlwaysReadyInstance) == "1";
9293
}
@@ -136,7 +137,12 @@ internal async Task OnPublishMetrics(DateTime now)
136137
FunctionExecutionTimeMS = FunctionExecutionCount = 0;
137138
}
138139

139-
await PublishMetricsAsync(metrics);
140+
await _metricsFileManager.PublishMetricsAsync(metrics);
141+
}
142+
catch (Exception ex) when (!ex.IsFatal())
143+
{
144+
// ensure no background exceptions escape
145+
_logger.LogError(ex, $"Error publishing metrics.");
140146
}
141147
finally
142148
{
@@ -149,84 +155,6 @@ private async void OnFunctionMetricsPublishTimer(object state)
149155
await OnPublishMetrics(DateTime.UtcNow);
150156
}
151157

152-
private async Task PublishMetricsAsync(Metrics metrics)
153-
{
154-
string fileName = string.Empty;
155-
156-
try
157-
{
158-
bool metricsPublishEnabled = !string.IsNullOrEmpty(MetricsFilePath);
159-
if (metricsPublishEnabled && !PrepareDirectoryForFile())
160-
{
161-
return;
162-
}
163-
164-
string metricsContent = JsonConvert.SerializeObject(metrics);
165-
_logger.PublishingMetrics(metricsContent);
166-
167-
if (metricsPublishEnabled)
168-
{
169-
fileName = $"{Guid.NewGuid().ToString().ToLower()}.json";
170-
string filePath = Path.Combine(MetricsFilePath, fileName);
171-
172-
using (var streamWriter = _fileSystem.File.CreateText(filePath))
173-
{
174-
await streamWriter.WriteAsync(metricsContent);
175-
}
176-
}
177-
}
178-
catch (Exception ex) when (!ex.IsFatal())
179-
{
180-
// TODO: consider using a retry strategy here
181-
_logger.LogError(ex, $"Error writing metrics file '{fileName}'.");
182-
}
183-
}
184-
185-
private bool PrepareDirectoryForFile()
186-
{
187-
if (string.IsNullOrEmpty(MetricsFilePath))
188-
{
189-
return false;
190-
}
191-
192-
// ensure the directory exists
193-
_fileSystem.Directory.CreateDirectory(MetricsFilePath);
194-
195-
var metricsDirectoryInfo = _fileSystem.DirectoryInfo.FromDirectoryName(MetricsFilePath);
196-
var files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();
197-
198-
// ensure we're under the max file count
199-
if (files.Count < _options.MaxFileCount)
200-
{
201-
return true;
202-
}
203-
204-
// we're at or over limit
205-
// delete enough files that we have space to write a new one
206-
int numToDelete = files.Count - _options.MaxFileCount + 1;
207-
var filesToDelete = files.Take(numToDelete).ToArray();
208-
209-
_logger.LogDebug($"Deleting {filesToDelete.Length} metrics file(s).");
210-
211-
foreach (var file in filesToDelete)
212-
{
213-
try
214-
{
215-
file.Delete();
216-
}
217-
catch (Exception ex) when (!ex.IsFatal())
218-
{
219-
// best effort
220-
_logger.LogError(ex, $"Error deleting metrics file '{file.FullName}'.");
221-
}
222-
}
223-
224-
files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();
225-
226-
// return true if we have space for a new file
227-
return files.Count < _options.MaxFileCount;
228-
}
229-
230158
private void OnStandbyOptionsChange()
231159
{
232160
if (!_standbyOptions.CurrentValue.InStandbyMode)

‎src/WebJobs.Script.WebHost/Metrics/LegionMetricsFileManager.cs‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.IO;
6+
using System.IO.Abstractions;
7+
using System.Linq;
8+
using System.Threading.Tasks;
9+
using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions;
10+
using Microsoft.Extensions.Logging;
11+
using Newtonsoft.Json;
12+
13+
namespace Microsoft.Azure.WebJobs.Script.WebHost.Metrics
14+
{
15+
internal sealed class LegionMetricsFileManager
16+
{
17+
private readonly IFileSystem _fileSystem;
18+
private readonly int _maxFileCount;
19+
private readonly ILogger _logger;
20+
private readonly JsonSerializerSettings _serializerSettings;
21+
22+
public LegionMetricsFileManager(string metricsFilePath, IFileSystem fileSystem, ILogger logger, int maxFileCount)
23+
{
24+
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
25+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
26+
MetricsFilePath = metricsFilePath;
27+
_maxFileCount = maxFileCount;
28+
29+
_serializerSettings = new JsonSerializerSettings
30+
{
31+
NullValueHandling = NullValueHandling.Ignore
32+
};
33+
}
34+
35+
// Internal access for testing only
36+
internal string MetricsFilePath { get; set; }
37+
38+
private bool PrepareDirectoryForFile()
39+
{
40+
if (string.IsNullOrEmpty(MetricsFilePath))
41+
{
42+
return false;
43+
}
44+
45+
// ensure the directory exists
46+
var metricsDirectoryInfo = _fileSystem.Directory.CreateDirectory(MetricsFilePath);
47+
48+
// ensure we're under the max file count
49+
var files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();
50+
if (files.Count < _maxFileCount)
51+
{
52+
return true;
53+
}
54+
55+
// we're at or over limit
56+
// delete enough files that we have space to write a new one
57+
int numToDelete = files.Count - _maxFileCount + 1;
58+
var filesToDelete = files.Take(numToDelete).ToArray();
59+
60+
_logger.LogDebug($"Deleting {filesToDelete.Length} metrics file(s).");
61+
62+
Parallel.ForEach(filesToDelete, file =>
63+
{
64+
try
65+
{
66+
file.Delete();
67+
}
68+
catch (Exception ex) when (!ex.IsFatal())
69+
{
70+
// best effort
71+
_logger.LogError(ex, $"Error deleting metrics file '{file.FullName}'.");
72+
}
73+
});
74+
75+
files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();
76+
77+
// return true if we have space for a new file
78+
return files.Count < _maxFileCount;
79+
}
80+
81+
public async Task PublishMetricsAsync(object metrics)
82+
{
83+
string fileName = string.Empty;
84+
85+
try
86+
{
87+
bool metricsPublishEnabled = !string.IsNullOrEmpty(MetricsFilePath);
88+
if (metricsPublishEnabled && !PrepareDirectoryForFile())
89+
{
90+
return;
91+
}
92+
93+
string metricsContent = JsonConvert.SerializeObject(metrics, _serializerSettings);
94+
_logger.PublishingMetrics(metricsContent);
95+
96+
if (metricsPublishEnabled)
97+
{
98+
fileName = $"{Guid.NewGuid().ToString().ToLowerInvariant()}.json";
99+
string filePath = Path.Combine(MetricsFilePath, fileName);
100+
101+
using var streamWriter = _fileSystem.File.CreateText(filePath);
102+
await streamWriter.WriteAsync(metricsContent);
103+
}
104+
}
105+
catch (Exception ex) when (!ex.IsFatal())
106+
{
107+
// TODO: consider using a retry strategy here
108+
_logger.LogError(ex, $"Error writing metrics file '{fileName}'.");
109+
}
110+
}
111+
}
112+
}

0 commit comments

Comments
 (0)