Skip to content

Commit 12fff58

Browse files
mhoegermhoeger
authored
Sequential JobHost restart if config set (#6883)
* Sequential JobHost restart if config set * Add test * fix unit test
1 parent 4358227 commit 12fff58

File tree

3 files changed

+91
-9
lines changed

3 files changed

+91
-9
lines changed

‎src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,11 @@
1212
using Microsoft.ApplicationInsights.Extensibility;
1313
using Microsoft.ApplicationInsights.Extensibility.Implementation.ApplicationId;
1414
using Microsoft.Azure.WebJobs.Logging;
15+
using Microsoft.Azure.WebJobs.Script.Configuration;
1516
using Microsoft.Azure.WebJobs.Script.Diagnostics;
1617
using Microsoft.Azure.WebJobs.Script.Scale;
1718
using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics.Extensions;
19+
using Microsoft.Extensions.Configuration;
1820
using Microsoft.Extensions.DependencyInjection;
1921
using Microsoft.Extensions.Hosting;
2022
using Microsoft.Extensions.Logging;
@@ -34,6 +36,7 @@ public class WebJobsScriptHostService : IHostedService, IScriptHostManager, ISer
3436
private readonly IMetricsLogger _metricsLogger;
3537
private readonly HostPerformanceManager _performanceManager;
3638
private readonly IOptions<HostHealthMonitorOptions> _healthMonitorOptions;
39+
private readonly IConfiguration _config;
3740
private readonly SlidingWindow<bool> _healthCheckWindow;
3841
private readonly Timer _hostHealthCheckTimer;
3942
private readonly SemaphoreSlim _hostStartSemaphore = new SemaphoreSlim(1, 1);
@@ -53,7 +56,8 @@ public class WebJobsScriptHostService : IHostedService, IScriptHostManager, ISer
5356

5457
public WebJobsScriptHostService(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, IScriptHostBuilder scriptHostBuilder, ILoggerFactory loggerFactory,
5558
IScriptWebHostEnvironment scriptWebHostEnvironment, IEnvironment environment,
56-
HostPerformanceManager hostPerformanceManager, IOptions<HostHealthMonitorOptions> healthMonitorOptions, IMetricsLogger metricsLogger, IApplicationLifetime applicationLifetime)
59+
HostPerformanceManager hostPerformanceManager, IOptions<HostHealthMonitorOptions> healthMonitorOptions,
60+
IMetricsLogger metricsLogger, IApplicationLifetime applicationLifetime, IConfiguration config)
5761
{
5862
if (loggerFactory == null)
5963
{
@@ -74,6 +78,7 @@ public WebJobsScriptHostService(IOptionsMonitor<ScriptApplicationHostOptions> ap
7478
_performanceManager = hostPerformanceManager ?? throw new ArgumentNullException(nameof(hostPerformanceManager));
7579
_healthMonitorOptions = healthMonitorOptions ?? throw new ArgumentNullException(nameof(healthMonitorOptions));
7680
_logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryHostGeneral);
81+
_config = config ?? throw new ArgumentNullException(nameof(config));
7782

7883
_hostStarted = _hostStartedSource.Task;
7984

@@ -461,8 +466,22 @@ public async Task RestartHostAsync(CancellationToken cancellationToken)
461466

462467
using (var activeOperation = ScriptHostStartupOperation.Create(cancellationToken, _logger))
463468
{
464-
Task startTask = UnsynchronizedStartHostAsync(activeOperation);
465-
Task stopTask = Orphan(previousHost, cancellationToken);
469+
Task startTask, stopTask;
470+
471+
// If we are running in development mode with core tools, do not overlap the restarts.
472+
// Overlapping restarts are problematic when language worker processes are listening
473+
// to the same debug port
474+
if (ShouldEnforceSequentialRestart())
475+
{
476+
stopTask = Orphan(previousHost, cancellationToken);
477+
await stopTask;
478+
startTask = UnsynchronizedStartHostAsync(activeOperation);
479+
}
480+
else
481+
{
482+
startTask = UnsynchronizedStartHostAsync(activeOperation);
483+
stopTask = Orphan(previousHost, cancellationToken);
484+
}
466485

467486
await startTask;
468487
}
@@ -487,6 +506,18 @@ public async Task RestartHostAsync(CancellationToken cancellationToken)
487506
}
488507
}
489508

509+
internal bool ShouldEnforceSequentialRestart()
510+
{
511+
var sequentialRestartSetting = _config.GetSection(ConfigurationSectionNames.SequentialJobHostRestart);
512+
if (sequentialRestartSetting != null)
513+
{
514+
bool.TryParse(sequentialRestartSetting.Value, out bool enforceSequentialOrder);
515+
return enforceSequentialOrder;
516+
}
517+
518+
return false;
519+
}
520+
490521
private void OnHostInitializing(object sender, EventArgs e)
491522
{
492523
// we check host health before starting to avoid starting

‎src/WebJobs.Script/Config/ConfigurationSectionNames.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,6 @@ public static class ConfigurationSectionNames
2222
public const string CustomHttpHeaders = Http + ":customHeaders";
2323
public const string EasyAuth = "easyauth";
2424
public const string Retry = "retry";
25+
public const string SequentialJobHostRestart = JobHost + ":sequentialRestart";
2526
}
2627
}

‎test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs‎

Lines changed: 56 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Licensed under the MIT License. See License.txt in the project root for license information.
33

44
using System;
5+
using System.Collections.Generic;
56
using System.Linq;
67
using System.Threading;
78
using System.Threading.Tasks;
@@ -33,6 +34,7 @@ public class WebJobsScriptHostServiceTests
3334
private ILoggerFactory _loggerFactory;
3435
private Mock<IScriptWebHostEnvironment> _mockScriptWebHostEnvironment;
3536
private Mock<IEnvironment> _mockEnvironment;
37+
private IConfiguration _mockConfig;
3638
private OptionsWrapper<HostHealthMonitorOptions> _healthMonitorOptions;
3739
private HostPerformanceManager _hostPerformanceManager;
3840
private Mock<IHost> _host;
@@ -55,6 +57,7 @@ public WebJobsScriptHostServiceTests()
5557
_healthMonitorOptions = new OptionsWrapper<HostHealthMonitorOptions>(new HostHealthMonitorOptions());
5658
var serviceProviderMock = new Mock<IServiceProvider>(MockBehavior.Strict);
5759
_hostPerformanceManager = new HostPerformanceManager(_mockEnvironment.Object, _healthMonitorOptions, serviceProviderMock.Object);
60+
_mockConfig = new Mock<IConfiguration>().Object;
5861
}
5962

6063
private Mock<IHost> CreateMockHost(SemaphoreSlim disposedSemaphore = null)
@@ -104,7 +107,9 @@ public async Task HostInitialization_OnInitializationException_MaintainsErrorInf
104107

105108
_hostService = new WebJobsScriptHostService(
106109
_monitor, hostBuilder.Object, NullLoggerFactory.Instance,
107-
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock<IApplicationLifetime>().Object);
110+
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object,
111+
_hostPerformanceManager, _healthMonitorOptions, metricsLogger,
112+
new Mock<IApplicationLifetime>().Object, _mockConfig);
108113

109114
await _hostService.StartAsync(CancellationToken.None);
110115
Assert.True(AreRequiredMetricsGenerated(metricsLogger));
@@ -129,7 +134,10 @@ public async Task HostRestart_Specialization_Succeeds()
129134

130135
_hostService = new WebJobsScriptHostService(
131136
_monitor, hostBuilder.Object, _loggerFactory,
132-
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock<IApplicationLifetime>().Object);
137+
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object,
138+
_hostPerformanceManager, _healthMonitorOptions,
139+
metricsLogger, new Mock<IApplicationLifetime>().Object,
140+
_mockConfig);
133141

134142
await _hostService.StartAsync(CancellationToken.None);
135143
Assert.True(AreRequiredMetricsGenerated(metricsLogger));
@@ -181,7 +189,10 @@ public async Task HostRestart_DuringInitializationWithError_Recovers()
181189

182190
_hostService = new WebJobsScriptHostService(
183191
_monitor, hostBuilder.Object, _loggerFactory,
184-
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock<IApplicationLifetime>().Object);
192+
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object,
193+
_hostPerformanceManager, _healthMonitorOptions,
194+
metricsLogger, new Mock<IApplicationLifetime>().Object,
195+
_mockConfig);
185196

186197
TestLoggerProvider hostALogger = hostA.Object.GetTestLoggerProvider();
187198
TestLoggerProvider hostBLogger = hostB.Object.GetTestLoggerProvider();
@@ -254,7 +265,10 @@ public async Task HostRestart_DuringInitialization_Cancels()
254265

255266
_hostService = new WebJobsScriptHostService(
256267
_monitor, hostBuilder.Object, _loggerFactory,
257-
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock<IApplicationLifetime>().Object);
268+
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object,
269+
_hostPerformanceManager, _healthMonitorOptions,
270+
metricsLogger, new Mock<IApplicationLifetime>().Object,
271+
_mockConfig);
258272

259273
TestLoggerProvider hostALogger = hostA.Object.GetTestLoggerProvider();
260274

@@ -321,7 +335,9 @@ public async Task DisposedHost_ServicesNotExposed()
321335

322336
_hostService = new WebJobsScriptHostService(
323337
_monitor, hostBuilder.Object, NullLoggerFactory.Instance,
324-
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock<IApplicationLifetime>().Object);
338+
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object,
339+
_hostPerformanceManager, _healthMonitorOptions, metricsLogger,
340+
new Mock<IApplicationLifetime>().Object, _mockConfig);
325341

326342
Task startTask = _hostService.StartAsync(CancellationToken.None);
327343

@@ -351,7 +367,9 @@ public async Task HostRestart_BeforeStart_WaitsForStartToContinue()
351367

352368
_hostService = new WebJobsScriptHostService(
353369
_monitor, hostBuilder.Object, _loggerFactory,
354-
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions, metricsLogger, new Mock<IApplicationLifetime>().Object);
370+
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object,
371+
_hostPerformanceManager, _healthMonitorOptions, metricsLogger,
372+
new Mock<IApplicationLifetime>().Object, _mockConfig);
355373

356374
// Simulate a call to specialize coming from the PlaceholderSpecializationMiddleware. This
357375
// can happen before we ever start the service, which could create invalid state.
@@ -379,6 +397,38 @@ public async Task HostRestart_BeforeStart_WaitsForStartToContinue()
379397
_host.Verify(p => p.StartAsync(It.IsAny<CancellationToken>()), Times.Once);
380398
}
381399

400+
[Theory]
401+
[InlineData("1", false)]
402+
[InlineData("0", false)]
403+
[InlineData("true", true)]
404+
[InlineData("True", true)]
405+
[InlineData("false", false)]
406+
public void ShouldEnforceSequentialRestart_WithCorrectConfig(string value, bool expectedResult)
407+
{
408+
var metricsLogger = new TestMetricsLogger();
409+
_host.Setup(h => h.StartAsync(It.IsAny<CancellationToken>()))
410+
.Returns(Task.CompletedTask);
411+
412+
var hostBuilder = new Mock<IScriptHostBuilder>();
413+
hostBuilder.Setup(b => b.BuildHost(It.IsAny<bool>(), It.IsAny<bool>()))
414+
.Returns(_host.Object);
415+
416+
IConfiguration config = new ConfigurationBuilder()
417+
.AddInMemoryCollection(new Dictionary<string, string>
418+
{
419+
{ "AzureFunctionsJobHost:SequentialRestart", value },
420+
})
421+
.Build();
422+
423+
_hostService = new WebJobsScriptHostService(
424+
_monitor, hostBuilder.Object, NullLoggerFactory.Instance,
425+
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object,
426+
_hostPerformanceManager, _healthMonitorOptions, metricsLogger,
427+
new Mock<IApplicationLifetime>().Object, config);
428+
429+
Assert.Equal(expectedResult, _hostService.ShouldEnforceSequentialRestart());
430+
}
431+
382432
public void RestartHost()
383433
{
384434
_hostService.RestartHostAsync(CancellationToken.None).Wait();

0 commit comments

Comments
 (0)