Skip to content

Commit 1817b41

Browse files
pragnagopapragnagopa
authored
Always restart host if RestartHost is called (#4035)
1 parent 940a0f8 commit 1817b41

File tree

2 files changed

+90
-32
lines changed

2 files changed

+90
-32
lines changed

‎src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33

44
using System;
55
using System.Collections.ObjectModel;
6-
using System.IO;
76
using System.Linq;
87
using System.Threading;
98
using System.Threading.Tasks;
@@ -28,12 +27,12 @@ public class WebJobsScriptHostService : IHostedService, IScriptHostManager, IDis
2827
private readonly IOptions<HostHealthMonitorOptions> _healthMonitorOptions;
2928
private readonly SlidingWindow<bool> _healthCheckWindow;
3029
private readonly Timer _hostHealthCheckTimer;
30+
private readonly SemaphoreSlim _hostRestartSemaphore = new SemaphoreSlim(1, 1);
3131

3232
private IHost _host;
3333
private CancellationTokenSource _startupLoopTokenSource;
3434
private int _hostStartCount;
3535
private bool _disposed = false;
36-
private int _restarting = 0;
3736

3837
public WebJobsScriptHostService(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, IScriptHostBuilder scriptHostBuilder, ILoggerFactory loggerFactory, IServiceProvider rootServiceProvider,
3938
IServiceScopeFactory rootScopeFactory, IScriptWebHostEnvironment scriptWebHostEnvironment, IEnvironment environment,
@@ -238,14 +237,9 @@ public async Task StopAsync(CancellationToken cancellationToken)
238237

239238
public async Task RestartHostAsync(CancellationToken cancellationToken)
240239
{
241-
if (Interlocked.CompareExchange(ref _restarting, 1, 0) != 0)
242-
{
243-
// only one restart operation at a time
244-
return;
245-
}
246-
247240
try
248241
{
242+
await _hostRestartSemaphore.WaitAsync();
249243
if (State == ScriptHostState.Stopping || State == ScriptHostState.Stopped)
250244
{
251245
return;
@@ -266,7 +260,7 @@ public async Task RestartHostAsync(CancellationToken cancellationToken)
266260
}
267261
finally
268262
{
269-
Interlocked.Exchange(ref _restarting, 0);
263+
_hostRestartSemaphore.Release();
270264
}
271265
}
272266

@@ -396,6 +390,7 @@ protected virtual void Dispose(bool disposing)
396390
if (disposing)
397391
{
398392
_startupLoopTokenSource?.Dispose();
393+
_hostRestartSemaphore.Dispose();
399394
}
400395
_disposed = true;
401396
}

‎test/WebJobs.Script.Tests/WebJobsScriptHostServiceTests.cs‎

Lines changed: 86 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,64 +2,127 @@
22
// Licensed under the MIT License. See License.txt in the project root for license information.
33

44
using System;
5+
using System.Linq;
56
using System.Threading;
67
using System.Threading.Tasks;
78
using Microsoft.Azure.WebJobs.Script.Config;
9+
using Microsoft.Azure.WebJobs.Script.Rpc;
810
using Microsoft.Azure.WebJobs.Script.Scale;
911
using Microsoft.Azure.WebJobs.Script.WebHost;
10-
using Microsoft.Azure.WebJobs.Script.WebHost.Management;
12+
using Microsoft.Extensions.Configuration;
1113
using Microsoft.Extensions.DependencyInjection;
1214
using Microsoft.Extensions.Hosting;
15+
using Microsoft.Extensions.Logging;
1316
using Microsoft.Extensions.Logging.Abstractions;
1417
using Microsoft.Extensions.Options;
18+
using Microsoft.WebJobs.Script.Tests;
1519
using Moq;
1620
using Xunit;
1721

1822
namespace Microsoft.Azure.WebJobs.Script.Tests
1923
{
2024
public class WebJobsScriptHostServiceTests
2125
{
22-
[Fact]
23-
public async Task HostInitialization_OnInitializationException_MaintainsErrorInformation()
26+
private WebJobsScriptHostService _hostService;
27+
private ScriptApplicationHostOptionsMonitor _monitor;
28+
private TestLoggerProvider _testLoggerProvider;
29+
private ILoggerFactory _loggerFactory;
30+
private Mock<IServiceProvider> _mockRootServiceProvider;
31+
private Mock<IServiceScopeFactory> _mockRootScopeFactory;
32+
private Mock<IScriptWebHostEnvironment> _mockScriptWebHostEnvironment;
33+
private Mock<IEnvironment> _mockEnvironment;
34+
private OptionsWrapper<HostHealthMonitorOptions> _healthMonitorOptions;
35+
private HostPerformanceManager _hostPerformanceManager;
36+
private Mock<IHost> _host;
37+
38+
public WebJobsScriptHostServiceTests()
2439
{
2540
var options = new ScriptApplicationHostOptions
2641
{
2742
ScriptPath = @"c:\tests",
2843
LogPath = @"c:\tests\logs",
2944
};
30-
31-
var monitor = new ScriptApplicationHostOptionsMonitor(options);
32-
45+
_monitor = new ScriptApplicationHostOptionsMonitor(options);
3346
var services = new ServiceCollection()
3447
.AddLogging()
3548
.BuildServiceProvider();
36-
37-
var host = new Mock<IHost>();
38-
host.Setup(h => h.Services)
49+
_host = new Mock<IHost>();
50+
_host.Setup(h => h.Services)
3951
.Returns(services);
40-
host.SetupSequence(h => h.StartAsync(It.IsAny<CancellationToken>()))
52+
53+
_mockRootServiceProvider = new Mock<IServiceProvider>();
54+
_mockRootScopeFactory = new Mock<IServiceScopeFactory>();
55+
_mockScriptWebHostEnvironment = new Mock<IScriptWebHostEnvironment>();
56+
_mockEnvironment = new Mock<IEnvironment>();
57+
_healthMonitorOptions = new OptionsWrapper<HostHealthMonitorOptions>(new HostHealthMonitorOptions());
58+
_hostPerformanceManager = new HostPerformanceManager(_mockEnvironment.Object, _healthMonitorOptions);
59+
}
60+
61+
[Fact]
62+
public async Task HostInitialization_OnInitializationException_MaintainsErrorInformation()
63+
{
64+
_host.SetupSequence(h => h.StartAsync(It.IsAny<CancellationToken>()))
4165
.Throws(new HostInitializationException("boom"))
4266
.Returns(Task.CompletedTask);
4367

4468
var hostBuilder = new Mock<IScriptHostBuilder>();
4569
hostBuilder.Setup(b => b.BuildHost(It.IsAny<bool>(), It.IsAny<bool>()))
46-
.Returns(host.Object);
70+
.Returns(_host.Object);
71+
_hostService = new WebJobsScriptHostService(
72+
_monitor, hostBuilder.Object, NullLoggerFactory.Instance, _mockRootServiceProvider.Object, _mockRootScopeFactory.Object,
73+
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions);
4774

48-
var mockRootServiceProvider = new Mock<IServiceProvider>();
49-
var mockRootScopeFactory = new Mock<IServiceScopeFactory>();
50-
var mockScriptWebHostEnvironment = new Mock<IScriptWebHostEnvironment>();
51-
var mockEnvironment = new Mock<IEnvironment>();
52-
var healthMonitorOptions = new OptionsWrapper<HostHealthMonitorOptions>(new HostHealthMonitorOptions());
53-
var hostPerformanceManager = new HostPerformanceManager(mockEnvironment.Object, healthMonitorOptions);
75+
await _hostService.StartAsync(CancellationToken.None);
5476

55-
var hostService = new WebJobsScriptHostService(
56-
monitor, hostBuilder.Object, NullLoggerFactory.Instance, mockRootServiceProvider.Object, mockRootScopeFactory.Object,
57-
mockScriptWebHostEnvironment.Object, mockEnvironment.Object, hostPerformanceManager, healthMonitorOptions);
77+
Assert.Equal(ScriptHostState.Error, _hostService.State);
78+
Assert.IsType<HostInitializationException>(_hostService.LastError);
79+
}
5880

59-
await hostService.StartAsync(CancellationToken.None);
81+
[Fact]
82+
public async Task HostRestart_Specialization_Succeeds()
83+
{
84+
_host.Setup(h => h.StartAsync(It.IsAny<CancellationToken>()))
85+
.Returns(Task.CompletedTask);
86+
87+
var hostBuilder = new Mock<IScriptHostBuilder>();
88+
hostBuilder.Setup(b => b.BuildHost(It.IsAny<bool>(), It.IsAny<bool>()))
89+
.Returns(_host.Object);
90+
91+
_testLoggerProvider = new TestLoggerProvider();
92+
_loggerFactory = new LoggerFactory();
93+
_loggerFactory.AddProvider(_testLoggerProvider);
94+
95+
_hostService = new WebJobsScriptHostService(
96+
_monitor, hostBuilder.Object, _loggerFactory, _mockRootServiceProvider.Object, _mockRootScopeFactory.Object,
97+
_mockScriptWebHostEnvironment.Object, _mockEnvironment.Object, _hostPerformanceManager, _healthMonitorOptions);
98+
99+
await _hostService.StartAsync(CancellationToken.None);
100+
101+
Thread restartHostThread = new Thread(new ThreadStart(RestartHost));
102+
Thread specializeHostThread = new Thread(new ThreadStart(SpecializeHost));
103+
restartHostThread.Start();
104+
specializeHostThread.Start();
105+
restartHostThread.Join();
106+
specializeHostThread.Join();
60107

61-
Assert.Equal(ScriptHostState.Error, hostService.State);
62-
Assert.IsType<HostInitializationException>(hostService.LastError);
108+
var logMessages = _testLoggerProvider.GetAllLogMessages().Where(m => m.FormattedMessage.Contains("Restarting host."));
109+
Assert.Equal(2, logMessages.Count());
110+
}
111+
112+
public void RestartHost()
113+
{
114+
_hostService.RestartHostAsync(CancellationToken.None).Wait();
115+
}
116+
117+
public void SpecializeHost()
118+
{
119+
var mockScriptWebHostEnvironment = new Mock<IScriptWebHostEnvironment>();
120+
Mock<IConfigurationRoot> mockConfiguration = new Mock<IConfigurationRoot>();
121+
var mockEnvironment = new Mock<IEnvironment>();
122+
Mock<ILanguageWorkerChannelManager> mockLanguageWorkerChannelManager = new Mock<ILanguageWorkerChannelManager>();
123+
ILogger<StandbyManager> testLogger = new Logger<StandbyManager>(_loggerFactory);
124+
var manager = new StandbyManager(_hostService, mockLanguageWorkerChannelManager.Object, mockConfiguration.Object, mockScriptWebHostEnvironment.Object, mockEnvironment.Object, _monitor, testLogger);
125+
manager.SpecializeHostAsync().Wait();
63126
}
64127
}
65128
}

0 commit comments

Comments
 (0)