Skip to content

Commit 546b249

Browse files
mauroservientiramonsmits
mauroservienti
and
ramonsmits
authored
Allow setting shutdown timeout for all instances (#4822)
* Introduce a setting to control the primary instance shutdown timeout * Introduce a setting to control the audit instance shutdown timeout * Approved API * Set the default shutdown timeout to the max allowed by the most restrictive hosting platform * When running the installer engine (via SCMU or PowerShell) we can assume it is Windows, and thus we can set the shutdown timeout to 2 minutes * Ensure the ShutdownTimeout is set to 2 minutes when installing/updating audit instances * Update src/ServiceControl/Infrastructure/Settings/Settings.cs Co-authored-by: Ramon Smits <[email protected]> * Set the default shutdown timeout to 5 seconds * Add the required version * Fix approved files adding ShutdownTimeout * Add the ShutdownTimeout to the audit instance settings * Add the ShutdownTimeout to the Monitoring instance * Fix the settings namespace * Approval files * Set the SemanticVersion for the ShutdownTimeout to 6.4.1 * Surround AddWindowsService() with a check to validate it's running as a service * Add the WindowsServiceCustomLifetime to request additional time on stop to the audit instance * Suppress CA1416 * Extracted into separate project, support OnShutdown, using `SupportedOSPlatform` instead of suppressing CA1416 * fixup! Extracted into separate project, support OnShutdown, using `SupportedOSPlatform` instead of suppressing CA1416 * Forgot MaintenanceModeCommand * Add the monitoring instance custom lifecycle to request additional time on stop * Primary instance custom lifecycle * Add missing using directive * Reword TODO * Remove not needed lifecycles and using directive * Do not try to request additional time on shutdown * Add logging to the WindowsServiceWithRequestTimeout class * Use the correct package * Using structured logging * Also logging OnShutdown * Remove TODO, tested lifetime without constructor on stop and on shutdown * Add the SetupProjectFake project back --------- Co-authored-by: Ramon Smits <[email protected]>
1 parent 337b6c0 commit 546b249

26 files changed

+203
-24
lines changed

‎src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,6 @@
2828
"MaximumConcurrencyLevel": null,
2929
"ServiceControlQueueAddress": "Particular.ServiceControl",
3030
"TimeToRestartAuditIngestionAfterFailure": "00:01:00",
31-
"EnableFullTextSearchOnBodies": true
31+
"EnableFullTextSearchOnBodies": true,
32+
"ShutdownTimeout": "00:00:05"
3233
}

‎src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ namespace ServiceControl.Audit;
55
using System.Threading;
66
using System.Threading.Tasks;
77
using Auditing;
8+
using Hosting;
89
using Infrastructure;
910
using Infrastructure.Settings;
1011
using Microsoft.AspNetCore.HttpLogging;
1112
using Microsoft.Extensions.DependencyInjection;
1213
using Microsoft.Extensions.Hosting;
14+
using Microsoft.Extensions.Hosting.WindowsServices;
1315
using Microsoft.Extensions.Logging;
1416
using Monitoring;
1517
using NLog.Extensions.Logging;
@@ -45,7 +47,7 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder,
4547
var transportCustomization = TransportFactory.Create(transportSettings);
4648
transportCustomization.AddTransportForAudit(services, transportSettings);
4749

48-
services.Configure<HostOptions>(options => options.ShutdownTimeout = TimeSpan.FromSeconds(30));
50+
services.Configure<HostOptions>(options => options.ShutdownTimeout = settings.ShutdownTimeout);
4951

5052
services.AddSingleton(settings);
5153
services.AddSingleton<EndpointInstanceMonitoring>();
@@ -100,7 +102,11 @@ public static void AddServiceControlAudit(this IHostApplicationBuilder builder,
100102
services.AddHostedService<AuditIngestion>();
101103
}
102104

103-
builder.Services.AddWindowsService();
105+
if (WindowsServiceHelpers.IsWindowsService())
106+
{
107+
// The if is added for clarity, internally AddWindowsService has a similar logic
108+
builder.AddWindowsServiceWithRequestTimeout();
109+
}
104110
}
105111

106112
public static void AddServiceControlAuditInstallers(this IHostApplicationBuilder builder, Settings settings)

‎src/ServiceControl.Audit/Infrastructure/Hosting/Commands/MaintenanceModeCommand.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
{
33
using System.Threading.Tasks;
44
using Microsoft.Extensions.Hosting;
5+
using Microsoft.Extensions.Hosting.WindowsServices;
56
using Persistence;
7+
using ServiceControl.Hosting;
68
using Settings;
79

810
class MaintenanceModeCommand : AbstractCommand
@@ -17,7 +19,11 @@ public override async Task Execute(HostArguments args, Settings settings)
1719
var hostBuilder = Host.CreateApplicationBuilder();
1820
hostBuilder.Services.AddPersistence(persistenceSettings, persistenceConfiguration);
1921

20-
hostBuilder.Services.AddWindowsService();
22+
if (WindowsServiceHelpers.IsWindowsService())
23+
{
24+
// The if is added for clarity, internally AddWindowsService has a similar logic
25+
hostBuilder.AddWindowsServiceWithRequestTimeout();
26+
}
2127

2228
var host = hostBuilder.Build();
2329
await host.RunAsync();

‎src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ public Settings(string transportType = null, string persisterType = null, Loggin
4949
ServiceControlQueueAddress = SettingsReader.Read<string>(SettingsRootNamespace, "ServiceControlQueueAddress");
5050
TimeToRestartAuditIngestionAfterFailure = GetTimeToRestartAuditIngestionAfterFailure();
5151
EnableFullTextSearchOnBodies = SettingsReader.Read(SettingsRootNamespace, "EnableFullTextSearchOnBodies", true);
52+
ShutdownTimeout = SettingsReader.Read(SettingsRootNamespace, "ShutdownTimeout", ShutdownTimeout);
5253

5354
AssemblyLoadContextResolver = static assemblyPath => new PluginAssemblyLoadContext(assemblyPath);
5455
}
@@ -152,6 +153,12 @@ public int MaxBodySizeToStore
152153

153154
public bool EnableFullTextSearchOnBodies { get; set; }
154155

156+
// The default value is set to the maximum allowed time by the most
157+
// restrictive hosting platform, which is Linux containers. Linux
158+
// containers allow for a maximum of 10 seconds. We set it to 5 to
159+
// allow for cancellation and logging to take place
160+
public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(5);
161+
155162
public TransportSettings ToTransportSettings()
156163
{
157164
var transportSettings = new TransportSettings

‎src/ServiceControl.Audit/ServiceControl.Audit.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
<ProjectReference Include="..\ServiceControl.Infrastructure\ServiceControl.Infrastructure.csproj" />
2121
<ProjectReference Include="..\ServiceControl.LicenseManagement\ServiceControl.LicenseManagement.csproj" />
2222
<ProjectReference Include="..\ServiceControl.Transports\ServiceControl.Transports.csproj" />
23+
<ProjectReference Include="..\ServiceControl.Hosting\ServiceControl.Hosting.csproj" />
2324
</ItemGroup>
2425

2526
<ItemGroup>

‎src/ServiceControl.Hosting/IHostApplicationBuilderExtensions.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
namespace ServiceControl.Hosting;
2+
3+
using Microsoft.Extensions.DependencyInjection;
4+
using Microsoft.Extensions.Hosting;
5+
using Microsoft.Extensions.Hosting.WindowsServices;
6+
7+
public static class IHostApplicationBuilderExtensions
8+
{
9+
public static void AddWindowsServiceWithRequestTimeout(this IHostApplicationBuilder builder)
10+
{
11+
if (WindowsServiceHelpers.IsWindowsService())
12+
{
13+
builder.Services.AddWindowsService();
14+
builder.Services.AddSingleton<IHostLifetime, WindowsServiceWithRequestTimeout>();
15+
}
16+
}
17+
}

‎src/ServiceControl.Hosting/ServiceControl.Hosting.csproj‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net8.0</TargetFramework>
5+
</PropertyGroup>
6+
7+
<ItemGroup>
8+
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" />
9+
<PackageReference Include="NLog.Extensions.Logging" />
10+
</ItemGroup>
11+
12+
</Project>

‎src/ServiceControl.Hosting/WindowsServiceWithRequestTimeout.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
namespace ServiceControl.Hosting;
2+
3+
using System;
4+
using System.Runtime.Versioning;
5+
using Microsoft.Extensions.Hosting;
6+
using Microsoft.Extensions.Hosting.WindowsServices;
7+
using Microsoft.Extensions.Logging;
8+
using Microsoft.Extensions.Options;
9+
10+
[SupportedOSPlatform("windows")]
11+
sealed class WindowsServiceWithRequestTimeout : WindowsServiceLifetime
12+
{
13+
static readonly TimeSpan CancellationDuration = TimeSpan.FromSeconds(5);
14+
readonly HostOptions hostOptions;
15+
16+
public WindowsServiceWithRequestTimeout(IHostEnvironment environment, IHostApplicationLifetime applicationLifetime, ILoggerFactory loggerFactory, IOptions<HostOptions> optionsAccessor, IOptions<WindowsServiceLifetimeOptions> windowsServiceOptionsAccessor)
17+
: base(environment, applicationLifetime, loggerFactory, optionsAccessor, windowsServiceOptionsAccessor)
18+
{
19+
hostOptions = optionsAccessor.Value;
20+
}
21+
22+
protected override void OnStop()
23+
{
24+
var logger = NLog.LogManager.GetCurrentClassLogger();
25+
var additionalTime = hostOptions.ShutdownTimeout + CancellationDuration;
26+
27+
logger.Info("OnStop invoked, going to ask for additional time: {additionalTime}", additionalTime);
28+
RequestAdditionalTime(additionalTime);
29+
logger.Info("Additional time requested");
30+
31+
base.OnStop();
32+
}
33+
34+
protected override void OnShutdown()
35+
{
36+
var logger = NLog.LogManager.GetCurrentClassLogger();
37+
logger.Info("OnShutdown invoked, process may exit ungracefully");
38+
base.OnShutdown();
39+
}
40+
}

‎src/ServiceControl.Monitoring.UnitTests/ApprovalFiles/SettingsTests.PlatformSampleSettings.approved.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,6 @@
1515
"EndpointUptimeGracePeriod": "00:00:40",
1616
"RootUrl": "http://localhost:9999/",
1717
"MaximumConcurrencyLevel": null,
18-
"ServiceControlThroughputDataQueue": "ServiceControl.ThroughputData"
18+
"ServiceControlThroughputDataQueue": "ServiceControl.ThroughputData",
19+
"ShutdownTimeout": "00:00:05"
1920
}

‎src/ServiceControl.Monitoring/HostApplicationBuilderExtensions.cs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ namespace ServiceControl.Monitoring;
55
using System.Threading;
66
using System.Threading.Tasks;
77
using Configuration;
8+
using Hosting;
89
using Infrastructure;
910
using Infrastructure.BackgroundTasks;
1011
using Infrastructure.Extensions;
@@ -13,6 +14,7 @@ namespace ServiceControl.Monitoring;
1314
using Microsoft.AspNetCore.HttpLogging;
1415
using Microsoft.Extensions.DependencyInjection;
1516
using Microsoft.Extensions.Hosting;
17+
using Microsoft.Extensions.Hosting.WindowsServices;
1618
using Microsoft.Extensions.Logging;
1719
using NLog.Extensions.Logging;
1820
using NServiceBus;
@@ -39,7 +41,13 @@ public static void AddServiceControlMonitoring(this IHostApplicationBuilder host
3941
var transportCustomization = TransportFactory.Create(transportSettings);
4042
transportCustomization.AddTransportForMonitoring(services, transportSettings);
4143

42-
services.AddWindowsService();
44+
services.Configure<HostOptions>(options => options.ShutdownTimeout = settings.ShutdownTimeout);
45+
46+
if (WindowsServiceHelpers.IsWindowsService())
47+
{
48+
// The if is added for clarity, internally AddWindowsService has a similar logic
49+
hostBuilder.AddWindowsServiceWithRequestTimeout();
50+
}
4351

4452
services.AddSingleton(settings);
4553
services.AddSingleton<EndpointRegistry>();

0 commit comments

Comments
 (0)