Skip to content

Commit da70f86

Browse files
jviaujviau
committed
Merge remote-tracking branch 'upstream/dev' into jviau/abort-channel
2 parents 3cd320c + 7e3e7b7 commit da70f86

24 files changed

+314
-181
lines changed

‎eng/build/Workers.Powershell.props‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@
33
<ItemGroup>
44
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.0" Version="4.0.3148" />
55
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.2" Version="4.0.4025" />
6-
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.4" Version="4.0.4134" />
6+
<PackageReference Include="Microsoft.Azure.Functions.PowerShellWorker.PS7.4" Version="4.0.4206" />
77
</ItemGroup>
88

99
<Target Name="RemovePowershellWorkerRuntimes" BeforeTargets="AssignTargetPaths" Condition="'$(RuntimeIdentifier)' != ''">
1010
<ItemGroup>
1111
<_KeepPowerShellRuntime Include="win;win-x86;win10-x86;win-x64;win10-x64" Condition="$(RuntimeIdentifier.StartsWith(win))" />
12-
<_KeepPowerShellRuntime Include="linux;linux-x64" Condition="$(RuntimeIdentifier.StartsWith(linux))" />
12+
<_KeepPowerShellRuntime Include="linux;linux-x64;unix" Condition="$(RuntimeIdentifier.StartsWith(linux))" />
1313
</ItemGroup>
1414

1515
<PropertyGroup>

‎release_notes.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,9 @@
33
<!-- Please add your release notes in the following format:
44
- My change description (#PR)
55
-->
6-
- Improved memory metrics reporting using CGroup data for Linux consumption (#10968)
6+
- Improved memory metrics reporting using CGroup data for Linux consumption (#10968)
7+
- Memory allocation optimizations in `RpcWorkerConfigFactory.AddProviders` (#10959)
8+
- Fixing GrpcWorkerChannel concurrency bug (#10998)
9+
- Avoid circular dependency when resolving LinuxContainerLegionMetricsPublisher. (#10991)
10+
- Add 'unix' to the list of runtimes kept when importing PowerShell worker for Linux builds
11+
- Update PowerShell 7.4 worker to 4.0.4206

‎src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ internal partial class GrpcWorkerChannel : IRpcWorkerChannel, IDisposable
6767
private bool _disposing;
6868
private WorkerInitResponse _initMessage;
6969
private RpcWorkerChannelState _state;
70-
private IDictionary<string, Exception> _functionLoadErrors = new Dictionary<string, Exception>();
71-
private IDictionary<string, Exception> _metadataRequestErrors = new Dictionary<string, Exception>();
70+
private IDictionary<string, Exception> _functionLoadErrors = new ConcurrentDictionary<string, Exception>();
71+
private IDictionary<string, Exception> _metadataRequestErrors = new ConcurrentDictionary<string, Exception>();
7272
private ConcurrentDictionary<string, ExecutingInvocation> _executingInvocations = new();
7373
private IDictionary<string, BufferBlock<ScriptInvocationContext>> _functionInputBuffers = new ConcurrentDictionary<string, BufferBlock<ScriptInvocationContext>>();
7474
private ConcurrentDictionary<string, TaskCompletionSource<bool>> _workerStatusRequests = new ConcurrentDictionary<string, TaskCompletionSource<bool>>();

‎src/WebJobs.Script.WebHost/Metrics/LinuxContainerLegionMetricsPublisher.cs‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
using Microsoft.Azure.WebJobs.Script.Config;
1111
using Microsoft.Azure.WebJobs.Script.Diagnostics;
1212
using Microsoft.Azure.WebJobs.Script.WebHost.Configuration;
13+
using Microsoft.Extensions.DependencyInjection;
1314
using Microsoft.Extensions.Logging;
1415
using Microsoft.Extensions.Options;
1516

@@ -26,7 +27,7 @@ public sealed class LinuxContainerLegionMetricsPublisher : IMetricsPublisher, ID
2627
private readonly IDisposable _hostingConfigOptionsOnChangeListener;
2728
private readonly IEnvironment _environment;
2829
private readonly ILogger _logger;
29-
private readonly IScriptHostManager _scriptHostManager;
30+
private readonly IServiceProvider _serviceProvider;
3031
private readonly string _containerName;
3132
private readonly IOptionsMonitor<FunctionsHostingConfigOptions> _hostingConfigOptions;
3233

@@ -40,15 +41,15 @@ public sealed class LinuxContainerLegionMetricsPublisher : IMetricsPublisher, ID
4041

4142
public LinuxContainerLegionMetricsPublisher(IEnvironment environment, IOptionsMonitor<StandbyOptions> standbyOptions,
4243
IOptions<LinuxConsumptionLegionMetricsPublisherOptions> options, ILogger<LinuxContainerLegionMetricsPublisher> logger,
43-
IFileSystem fileSystem, ILinuxConsumptionMetricsTracker metricsTracker, IScriptHostManager scriptHostManager,
44+
IFileSystem fileSystem, ILinuxConsumptionMetricsTracker metricsTracker, IServiceProvider serviceProvider,
4445
IOptionsMonitor<FunctionsHostingConfigOptions> functionsHostingConfigOptions,
4546
int? metricsPublishIntervalMS = null)
4647
{
4748
_standbyOptions = standbyOptions ?? throw new ArgumentNullException(nameof(standbyOptions));
4849
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
4950
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
5051
_metricsTracker = metricsTracker ?? throw new ArgumentNullException(nameof(metricsTracker));
51-
_scriptHostManager = scriptHostManager ?? throw new ArgumentNullException(nameof(scriptHostManager));
52+
_serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
5253
_hostingConfigOptions = functionsHostingConfigOptions ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions));
5354
_containerName = options.Value.ContainerName;
5455

@@ -75,21 +76,7 @@ public LinuxContainerLegionMetricsPublisher(IEnvironment environment, IOptionsMo
7576
_hostingConfigOptionsOnChangeListener = _hostingConfigOptions.OnChange(OnHostingConfigOptionsChanged);
7677
}
7778

78-
public IMetricsLogger MetricsLogger
79-
{
80-
get
81-
{
82-
if (_metricsLogger == null)
83-
{
84-
if (!Utility.TryGetHostService<IMetricsLogger>(_scriptHostManager, out _metricsLogger))
85-
{
86-
throw new InvalidOperationException($"Unable to resolve {nameof(IMetricsLogger)} service.");
87-
}
88-
}
89-
90-
return _metricsLogger;
91-
}
92-
}
79+
private IMetricsLogger MetricsLogger => _metricsLogger ??= _serviceProvider.GetRequiredService<IMetricsLogger>();
9380

9481
private void OnHostingConfigOptionsChanged(FunctionsHostingConfigOptions newOptions)
9582
{

‎src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the MIT License. See License.txt in the project root for license information.
33

4+
using System;
45
using System.IO.Abstractions;
56
using System.Net.Http;
67
using System.Runtime.InteropServices;
@@ -304,9 +305,9 @@ private static void AddLinuxContainerServices(this IServiceCollection services)
304305
var logger = s.GetService<ILogger<LinuxContainerLegionMetricsPublisher>>();
305306
var metricsTracker = s.GetService<ILinuxConsumptionMetricsTracker>();
306307
var standbyOptions = s.GetService<IOptionsMonitor<StandbyOptions>>();
307-
var scriptHostManager = s.GetService<IScriptHostManager>();
308+
var serviceProvider = s.GetService<IServiceProvider>();
308309
var hostingConfigOptions = s.GetService<IOptionsMonitor<FunctionsHostingConfigOptions>>();
309-
return new LinuxContainerLegionMetricsPublisher(environment, standbyOptions, options, logger, new FileSystem(), metricsTracker, scriptHostManager, hostingConfigOptions);
310+
return new LinuxContainerLegionMetricsPublisher(environment, standbyOptions, options, logger, new FileSystem(), metricsTracker, serviceProvider, hostingConfigOptions);
310311
}
311312
else if (environment.IsFlexConsumptionSku())
312313
{

‎src/WebJobs.Script/Workers/Profiles/Conditions/EnvironmentCondition.cs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,15 @@ internal EnvironmentCondition(ILogger logger, IEnvironment environment, WorkerPr
3030
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
3131
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
3232

33-
descriptor.Properties.TryGetValue(WorkerConstants.WorkerDescriptionProfileConditionName, out _name);
34-
descriptor.Properties.TryGetValue(WorkerConstants.WorkerDescriptionProfileConditionExpression, out _expression);
33+
if (descriptor.Properties.TryGetValue(WorkerConstants.WorkerDescriptionProfileConditionName, out var conditionNameElement))
34+
{
35+
_name = conditionNameElement.GetString();
36+
}
37+
38+
if (descriptor.Properties.TryGetValue(WorkerConstants.WorkerDescriptionProfileConditionExpression, out var conditionExpressionElement))
39+
{
40+
_expression = conditionExpressionElement.GetString();
41+
}
3542

3643
Validate();
3744
}

‎src/WebJobs.Script/Workers/Profiles/Conditions/HostPropertyCondition.cs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,15 @@ public HostPropertyCondition(ILogger logger, ISystemRuntimeInformation systemRun
3232
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
3333
_systemRuntimeInformation = systemRuntimeInformation ?? throw new ArgumentNullException(nameof(systemRuntimeInformation));
3434

35-
descriptor.Properties.TryGetValue(WorkerConstants.WorkerDescriptionProfileConditionName, out _name);
36-
descriptor.Properties.TryGetValue(WorkerConstants.WorkerDescriptionProfileConditionExpression, out _expression);
35+
if (descriptor.Properties.TryGetValue(WorkerConstants.WorkerDescriptionProfileConditionName, out var conditionNameElement))
36+
{
37+
_name = conditionNameElement.GetString();
38+
}
39+
40+
if (descriptor.Properties.TryGetValue(WorkerConstants.WorkerDescriptionProfileConditionExpression, out var conditionExpressionElement))
41+
{
42+
_expression = conditionExpressionElement.GetString();
43+
}
3744

3845
Validate();
3946
}

‎src/WebJobs.Script/Workers/Profiles/WorkerProfileConditionDescriptor.cs‎

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,35 +2,18 @@
22
// Licensed under the MIT License. See License.txt in the project root for license information.
33

44
using System.Collections.Generic;
5-
using System.Linq;
6-
using Newtonsoft.Json;
7-
using Newtonsoft.Json.Linq;
5+
using System.Text.Json;
6+
using System.Text.Json.Serialization;
87

98
namespace Microsoft.Azure.WebJobs.Script.Workers.Profiles
109
{
1110
public sealed class WorkerProfileConditionDescriptor
1211
{
13-
[JsonExtensionData]
14-
#pragma warning disable CS0649 // The value is assigned by the serializer
15-
private IDictionary<string, JToken> _extensionData;
16-
#pragma warning restore CS0649
17-
18-
private IDictionary<string, string> _properties;
19-
20-
[JsonProperty(Required = Required.Always, PropertyName = WorkerConstants.WorkerDescriptionProfileConditionType)]
12+
[JsonRequired]
13+
[JsonPropertyName(WorkerConstants.WorkerDescriptionProfileConditionType)]
2114
public string Type { get; set; }
2215

23-
public IDictionary<string, string> Properties
24-
{
25-
get
26-
{
27-
if (_properties == null)
28-
{
29-
_properties = _extensionData?.ToDictionary(kv => kv.Key, kv => kv.Value.ToString()) ?? new Dictionary<string, string>();
30-
}
31-
32-
return _properties;
33-
}
34-
}
16+
[JsonExtensionData]
17+
public Dictionary<string, JsonElement> Properties { get; set; } = new Dictionary<string, JsonElement>();
3518
}
3619
}

‎src/WebJobs.Script/Workers/Rpc/Configuration/RpcWorkerConfigFactory.cs‎

Lines changed: 48 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@
55
using System.Collections.Generic;
66
using System.IO;
77
using System.Linq;
8+
using System.Text.Json;
89
using System.Text.RegularExpressions;
910
using Microsoft.Azure.WebJobs.Script.Diagnostics;
1011
using Microsoft.Azure.WebJobs.Script.Workers.Profiles;
1112
using Microsoft.Extensions.Configuration;
1213
using Microsoft.Extensions.Logging;
13-
using Newtonsoft.Json.Linq;
1414

1515
namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc
1616
{
@@ -24,6 +24,10 @@ internal class RpcWorkerConfigFactory
2424
private readonly IMetricsLogger _metricsLogger;
2525
private readonly string _workerRuntime;
2626
private readonly IEnvironment _environment;
27+
private readonly JsonSerializerOptions _jsonSerializerOptions = new()
28+
{
29+
PropertyNameCaseInsensitive = true
30+
};
2731

2832
private Dictionary<string, RpcWorkerConfig> _workerDescriptionDictionary = new Dictionary<string, RpcWorkerConfig>();
2933

@@ -116,6 +120,18 @@ internal void AddProvider(string workerDir)
116120
{
117121
try
118122
{
123+
// After specialization, load worker config only for the specified runtime unless it's a multi-language app.
124+
if (!string.IsNullOrWhiteSpace(_workerRuntime) && !_environment.IsPlaceholderModeEnabled() && !_environment.IsMultiLanguageRuntimeEnvironment())
125+
{
126+
string workerRuntime = Path.GetFileName(workerDir);
127+
// Only skip worker directories that don't match the current runtime.
128+
// Do not skip non-worker directories like the function app payload directory
129+
if (!workerRuntime.Equals(_workerRuntime, StringComparison.OrdinalIgnoreCase) && workerDir.StartsWith(WorkersDirPath))
130+
{
131+
return;
132+
}
133+
}
134+
119135
string workerConfigPath = Path.Combine(workerDir, RpcWorkerConstants.WorkerConfigFileName);
120136

121137
if (!File.Exists(workerConfigPath))
@@ -126,15 +142,13 @@ internal void AddProvider(string workerDir)
126142

127143
_logger.LogDebug("Found worker config: {workerConfigPath}", workerConfigPath);
128144

129-
// Parse worker config file
130-
string json = File.ReadAllText(workerConfigPath);
131-
JObject workerConfig = JObject.Parse(json);
132-
RpcWorkerDescription workerDescription = workerConfig.Property(WorkerConstants.WorkerDescription).Value.ToObject<RpcWorkerDescription>();
145+
var workerConfig = GetWorkerConfigJsonElement(workerConfigPath);
146+
var workerDescriptionElement = workerConfig.GetProperty(WorkerConstants.WorkerDescription);
147+
var workerDescription = workerDescriptionElement.Deserialize<RpcWorkerDescription>(_jsonSerializerOptions);
133148
workerDescription.WorkerDirectory = workerDir;
134149

135-
//Read the profiles from worker description and load the profile for which the conditions match
136-
JToken profiles = workerConfig.GetValue(WorkerConstants.WorkerDescriptionProfiles);
137-
if (profiles != null)
150+
// Read the profiles from worker description and load the profile for which the conditions match
151+
if (workerConfig.TryGetProperty(WorkerConstants.WorkerDescriptionProfiles, out var profiles))
138152
{
139153
List<WorkerDescriptionProfile> workerDescriptionProfiles = ReadWorkerDescriptionProfiles(profiles);
140154
if (workerDescriptionProfiles.Count > 0)
@@ -146,7 +160,7 @@ internal void AddProvider(string workerDir)
146160

147161
// Check if any app settings are provided for that language
148162
var languageSection = _config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{workerDescription.Language}");
149-
workerDescription.Arguments = workerDescription.Arguments ?? new List<string>();
163+
workerDescription.Arguments ??= new List<string>();
150164
GetWorkerDescriptionFromAppSettings(workerDescription, languageSection);
151165
AddArgumentsFromAppSettings(workerDescription, languageSection);
152166

@@ -197,9 +211,24 @@ internal void AddProvider(string workerDir)
197211
}
198212
}
199213

200-
private List<WorkerDescriptionProfile> ReadWorkerDescriptionProfiles(JToken profilesJToken)
214+
private static JsonElement GetWorkerConfigJsonElement(string workerConfigPath)
201215
{
202-
var profiles = profilesJToken.ToObject<IList<WorkerProfileDescriptor>>();
216+
ReadOnlySpan<byte> jsonSpan = File.ReadAllBytes(workerConfigPath);
217+
218+
if (jsonSpan.StartsWith<byte>([0xEF, 0xBB, 0xBF]))
219+
{
220+
jsonSpan = jsonSpan[3..]; // Skip UTF-8 Byte Order Mark (BOM) if present at the beginning of the file.
221+
}
222+
223+
var reader = new Utf8JsonReader(jsonSpan, isFinalBlock: true, state: default);
224+
using var doc = JsonDocument.ParseValue(ref reader);
225+
226+
return doc.RootElement.Clone();
227+
}
228+
229+
private List<WorkerDescriptionProfile> ReadWorkerDescriptionProfiles(JsonElement profilesElement)
230+
{
231+
var profiles = profilesElement.Deserialize<IList<WorkerProfileDescriptor>>(_jsonSerializerOptions);
203232

204233
if (profiles == null || profiles.Count <= 0)
205234
{
@@ -237,11 +266,16 @@ private List<WorkerDescriptionProfile> ReadWorkerDescriptionProfiles(JToken prof
237266
return descriptionProfiles;
238267
}
239268

240-
internal WorkerProcessCountOptions GetWorkerProcessCount(JObject workerConfig)
269+
internal WorkerProcessCountOptions GetWorkerProcessCount(JsonElement workerConfig)
241270
{
242-
WorkerProcessCountOptions workerProcessCount = workerConfig.Property(WorkerConstants.ProcessCount)?.Value.ToObject<WorkerProcessCountOptions>();
271+
WorkerProcessCountOptions workerProcessCount = null;
272+
273+
if (workerConfig.TryGetProperty(WorkerConstants.ProcessCount, out var processCountElement))
274+
{
275+
workerProcessCount = processCountElement.Deserialize<WorkerProcessCountOptions>();
276+
}
243277

244-
workerProcessCount = workerProcessCount ?? new WorkerProcessCountOptions();
278+
workerProcessCount ??= new WorkerProcessCountOptions();
245279

246280
if (workerProcessCount.SetProcessCountToNumberOfCpuCores)
247281
{

‎test/WebJobs.Script.Tests.Integration/Host/StandbyManager/StandbyManagerE2ETestBase.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
using System.Web.Http;
1313
using Microsoft.AspNetCore.Hosting;
1414
using Microsoft.AspNetCore.TestHost;
15+
using Microsoft.Azure.Functions.Platform.Metrics.LinuxConsumption;
1516
using Microsoft.Azure.WebJobs.Extensions.Http;
1617
using Microsoft.Azure.WebJobs.Host.Executors;
1718
using Microsoft.Azure.WebJobs.Script.Diagnostics;
@@ -73,6 +74,7 @@ protected async Task<IWebHostBuilder> CreateWebHostBuilderAsync(string testDirNa
7374
environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteName, websiteSiteName);
7475
}
7576

77+
var testMetricsTracker = new TestMetricsTracker();
7678
var webHostBuilder = Program.CreateWebHostBuilder()
7779
.ConfigureAppConfiguration(c =>
7880
{
@@ -102,6 +104,7 @@ protected async Task<IWebHostBuilder> CreateWebHostBuilderAsync(string testDirNa
102104

103105
c.AddSingleton<IEnvironment>(_ => environment);
104106
c.AddSingleton<IMetricsLogger>(_ => _metricsLogger);
107+
c.AddSingleton<ILinuxConsumptionMetricsTracker>(_ => testMetricsTracker);
105108
})
106109
.ConfigureScriptHostLogging(b =>
107110
{
@@ -121,7 +124,7 @@ protected async Task<IWebHostBuilder> CreateWebHostBuilderAsync(string testDirNa
121124
return webHostBuilder;
122125
}
123126

124-
protected async Task InitializeTestHostAsync(string testDirName, IEnvironment environment, string websiteSiteName = TestSiteName)
127+
protected async Task<IWebHost> InitializeTestHostAsync(string testDirName, IEnvironment environment, string websiteSiteName = TestSiteName)
125128
{
126129
var webHostBuilder = await CreateWebHostBuilderAsync(testDirName, environment, websiteSiteName);
127130
_httpServer = new TestServer(webHostBuilder);
@@ -135,6 +138,8 @@ protected async Task InitializeTestHostAsync(string testDirName, IEnvironment en
135138
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Host is in standby mode")));
136139

137140
_expectedHostId = await _httpServer.Host.Services.GetService<IHostIdProvider>().GetHostIdAsync(CancellationToken.None);
141+
142+
return _httpServer.Host;
138143
}
139144

140145

0 commit comments

Comments
 (0)