Skip to content

Commit b4466aa

Browse files
soninarensoninaren
authored
Restart workers when we need to refresh the worker metadata (#9244)
1 parent e9637fc commit b4466aa

File tree

5 files changed

+220
-3
lines changed

5 files changed

+220
-3
lines changed

‎src/WebJobs.Script/Host/WorkerFunctionMetadataProvider.cs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,15 @@ namespace Microsoft.Azure.WebJobs.Script
2020
internal class WorkerFunctionMetadataProvider : IWorkerFunctionMetadataProvider
2121
{
2222
private readonly Dictionary<string, ICollection<string>> _functionErrors = new Dictionary<string, ICollection<string>>();
23-
private readonly IOptions<ScriptJobHostOptions> _scriptOptions;
23+
private readonly IOptionsMonitor<ScriptApplicationHostOptions> _scriptOptions;
2424
private readonly ILogger _logger;
2525
private readonly IEnvironment _environment;
2626
private readonly IWebHostRpcWorkerChannelManager _channelManager;
2727
private string _workerRuntime;
2828
private ImmutableArray<FunctionMetadata> _functions;
2929

3030
public WorkerFunctionMetadataProvider(
31-
IOptions<ScriptJobHostOptions> scriptOptions,
31+
IOptionsMonitor<ScriptApplicationHostOptions> scriptOptions,
3232
ILogger<WorkerFunctionMetadataProvider> logger,
3333
IEnvironment environment,
3434
IWebHostRpcWorkerChannelManager webHostRpcWorkerChannelManager)
@@ -61,6 +61,16 @@ public async Task<FunctionMetadataResult> GetFunctionMetadataAsync(IEnumerable<R
6161
throw new InvalidOperationException(nameof(_channelManager));
6262
}
6363

64+
// Scenario: Restart worker for hot reload on a readwrite file system
65+
// We reuse the worker started in placeholderMode only when the fileSystem is readonly
66+
// otherwise we shutdown the channel in which case the channel should not have any channels anyway
67+
// forceRefresh in only true once in the script host intialization flow.
68+
// forceRefresh will be false when bundle is not used (dotnet and dotnet-isolated).
69+
if (!_environment.IsPlaceholderModeEnabled() && forceRefresh && !_scriptOptions.CurrentValue.IsFileSystemReadOnly)
70+
{
71+
_channelManager.ShutdownChannelsAsync().GetAwaiter().GetResult();
72+
}
73+
6474
var channels = _channelManager.GetChannels(_workerRuntime);
6575

6676
if (channels?.Any() != true)
@@ -95,7 +105,7 @@ public async Task<FunctionMetadataResult> GetFunctionMetadataAsync(IEnumerable<R
95105
_logger.FunctionMetadataProviderFunctionFound(_functions.IsDefault ? 0 : _functions.Count());
96106

97107
// Validate if the app has functions in legacy format and add in logs to inform about the mixed app
98-
_ = Task.Delay(TimeSpan.FromMinutes(1)).ContinueWith(t => ValidateFunctionAppFormat(_scriptOptions.Value.RootScriptPath, _logger, _environment));
108+
_ = Task.Delay(TimeSpan.FromMinutes(1)).ContinueWith(t => ValidateFunctionAppFormat(_scriptOptions.CurrentValue.ScriptPath, _logger, _environment));
99109

100110
break;
101111
}

‎test/WebJobs.Script.Tests.Integration/TestScripts/NodeWithBundles/HttpTriggerNoAuth/function.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"bindings": [
3+
{
4+
"type": "httpTrigger",
5+
"name": "request",
6+
"direction": "in",
7+
"methods": [ "get", "post" ],
8+
"authLevel": "anonymous"
9+
},
10+
{
11+
"type": "http",
12+
"name": "response",
13+
"direction": "out"
14+
}
15+
]
16+
}

‎test/WebJobs.Script.Tests.Integration/TestScripts/NodeWithBundles/HttpTriggerNoAuth/index.js‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
var util = require('util');
2+
3+
module.exports = function (context, req) {
4+
context.log('Node.js HttpTrigger function invoked.');
5+
6+
context.res = {
7+
status: 200,
8+
body: {
9+
reqBodyType: 'Node.js HttpTrigger function invoked.',
10+
reqBodyIsArray: util.isArray(req.body),
11+
reqBody: req.body,
12+
reqRawBodyType: typeof req.rawBody,
13+
reqRawBody: req.rawBody,
14+
reqHeaders: req.headers,
15+
bindingData: context.bindingData,
16+
reqOriginalUrl: req.originalUrl
17+
},
18+
headers: {
19+
'test-header': 'Test Response Header',
20+
"Content-Type": "application/json; charset=utf-8"
21+
}
22+
};
23+
24+
context.done();
25+
};

‎test/WebJobs.Script.Tests.Integration/TestScripts/NodeWithBundles/host.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"version": "2.0",
3+
"extensionBundle": {
4+
"id": "Microsoft.Azure.Functions.ExtensionBundle",
5+
"version": "[3.*, 4.0.0)"
6+
}
7+
}

‎test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SpecializationE2ETests.cs‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
using Microsoft.Extensions.Logging;
3333
using Microsoft.Extensions.Options;
3434
using Microsoft.WebJobs.Script.Tests;
35+
using TestFunctions;
3536
using Xunit;
3637
using Xunit.Abstractions;
3738
using IApplicationLifetime = Microsoft.AspNetCore.Hosting.IApplicationLifetime;
@@ -249,6 +250,164 @@ public async Task Specialization_ResetsSharedLoadContext()
249250
}
250251
}
251252

253+
[Fact]
254+
public async Task ForNonReadOnlyFileSystem_RestartWorkerForSpecializationAndHotReload()
255+
{
256+
_environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, "node");
257+
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableWorkerIndexing);
258+
259+
var builder = CreateStandbyHostBuilder("HttpTriggerNoAuth");
260+
261+
builder.ConfigureAppConfiguration(config =>
262+
{
263+
string scriptRootConfigPath = ConfigurationPath.Combine(ConfigurationSectionNames.WebHost, nameof(ScriptApplicationHostOptions.ScriptPath));
264+
config.AddInMemoryCollection(new Dictionary<string, string>
265+
{
266+
{ _scriptRootConfigPath, Path.GetFullPath(@"TestScripts\NodeWithBundles") }
267+
});
268+
});
269+
270+
using var testServer = new TestServer(builder);
271+
272+
var client = testServer.CreateClient();
273+
274+
var response = await client.GetAsync("api/warmup");
275+
response.EnsureSuccessStatusCode();
276+
277+
var webChannelManager = testServer.Services.GetService<IWebHostRpcWorkerChannelManager>();
278+
var channel = await webChannelManager.GetChannels("node").Single().Value.Task;
279+
var processId = channel.WorkerProcess.Process.Id;
280+
281+
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteContainerReady, "1");
282+
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0");
283+
284+
response = await client.GetAsync("api/HttpTriggerNoAuth");
285+
response.EnsureSuccessStatusCode();
286+
var responseContent = await response.Content.ReadAsStringAsync();
287+
288+
string content = "Node.js HttpTrigger function invoked.";
289+
responseContent.Contains(content);
290+
291+
channel = await webChannelManager.GetChannels("node").Single().Value.Task;
292+
var newProcessId = channel.WorkerProcess.Process.Id;
293+
Assert.NotEqual(processId, newProcessId);
294+
Assert.Contains(content, responseContent);
295+
296+
var indexJS = Path.GetFullPath(@"TestScripts\NodeWithBundles\HttpTriggerNoAuth\index.js");
297+
298+
string fileContent = File.ReadAllText(indexJS);
299+
string newContent = "Updated Node.js HttpTrigger function invoked.";
300+
string updatedContent = fileContent.Replace(content, newContent);
301+
File.WriteAllText(indexJS, updatedContent);
302+
303+
var manager = testServer.Host.Services.GetService<IScriptHostManager>();
304+
var hostService = manager as WebJobsScriptHostService;
305+
306+
await TestHelpers.Await(() =>
307+
{
308+
return hostService.State == ScriptHostState.Default;
309+
}, 5000);
310+
311+
await TestHelpers.Await(() =>
312+
{
313+
return hostService.State == ScriptHostState.Running;
314+
}, 5000);
315+
316+
response = await client.GetAsync("api/HttpTriggerNoAuth");
317+
response.EnsureSuccessStatusCode();
318+
responseContent = await response.Content.ReadAsStringAsync();
319+
responseContent.Contains(newContent);
320+
321+
channel = await webChannelManager.GetChannels("node").Single().Value.Task;
322+
var hotReloadProcessId = channel.WorkerProcess.Process.Id;
323+
Assert.NotEqual(hotReloadProcessId, newProcessId);
324+
Assert.Contains(newContent, responseContent);
325+
}
326+
327+
[Fact]
328+
public async Task Specialization_RestartsWorkerForNonReadOnlyFileSystem()
329+
{
330+
_environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, "node");
331+
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableWorkerIndexing);
332+
333+
var builder = CreateStandbyHostBuilder("HttpTriggerNoAuth");
334+
335+
builder.ConfigureAppConfiguration(config =>
336+
{
337+
string scriptRootConfigPath = ConfigurationPath.Combine(ConfigurationSectionNames.WebHost, nameof(ScriptApplicationHostOptions.ScriptPath));
338+
config.AddInMemoryCollection(new Dictionary<string, string>
339+
{
340+
{ _scriptRootConfigPath, Path.GetFullPath(@"TestScripts\NodeWithBundles") }
341+
});
342+
});
343+
344+
using var testServer = new TestServer(builder);
345+
346+
var client = testServer.CreateClient();
347+
348+
var response = await client.GetAsync("api/warmup");
349+
response.EnsureSuccessStatusCode();
350+
351+
var placeholderContext = FunctionAssemblyLoadContext.Shared;
352+
353+
var webChannelManager = testServer.Services.GetService<IWebHostRpcWorkerChannelManager>();
354+
var channel = await webChannelManager.GetChannels("node").Single().Value.Task;
355+
var processId = channel.WorkerProcess.Process.Id;
356+
357+
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteContainerReady, "1");
358+
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0");
359+
360+
//await _pauseBeforeHostBuild.WaitAsync(10000);
361+
response = await client.GetAsync("api/HttpTriggerNoAuth");
362+
response.EnsureSuccessStatusCode();
363+
364+
channel = await webChannelManager.GetChannels("node").Single().Value.Task;
365+
var newProcessId = channel.WorkerProcess.Process.Id;
366+
Assert.NotEqual(processId, newProcessId);
367+
}
368+
369+
370+
[Fact]
371+
public async Task Specialization_UsePlaceholderWorkerforReadOnlyFileSystem()
372+
{
373+
_environment.SetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName, "node");
374+
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableWorkerIndexing);
375+
376+
var builder = CreateStandbyHostBuilder("HttpTriggerNoAuth");
377+
string isFileSystemReadOnly = ConfigurationPath.Combine(ConfigurationSectionNames.WebHost, nameof(ScriptApplicationHostOptions.IsFileSystemReadOnly));
378+
379+
builder.ConfigureAppConfiguration(config =>
380+
{
381+
config.AddInMemoryCollection(new Dictionary<string, string>
382+
{
383+
{ _scriptRootConfigPath, Path.GetFullPath(@"TestScripts\NodeWithBundles") }
384+
});
385+
});
386+
387+
388+
using var testServer = new TestServer(builder);
389+
390+
var client = testServer.CreateClient();
391+
392+
var response = await client.GetAsync("api/warmup");
393+
response.EnsureSuccessStatusCode();
394+
395+
var webChannelManager = testServer.Services.GetService<IWebHostRpcWorkerChannelManager>();
396+
var channel = await webChannelManager.GetChannels("node").Single().Value.Task;
397+
var processId = channel.WorkerProcess.Process.Id;
398+
399+
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteRunFromPackage, "1");
400+
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteContainerReady, "1");
401+
_environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0");
402+
403+
response = await client.GetAsync("api/HttpTriggerNoAuth");
404+
response.EnsureSuccessStatusCode();
405+
406+
channel = await webChannelManager.GetChannels("node").Single().Value.Task;
407+
var newProcessId = channel.WorkerProcess.Process.Id;
408+
Assert.Equal(processId, newProcessId);
409+
}
410+
252411
[Fact]
253412
public async Task Specialization_GCMode()
254413
{

0 commit comments

Comments
 (0)