diff --git a/seqcli.sln.DotSettings b/seqcli.sln.DotSettings index 93bc904..4474106 100644 --- a/seqcli.sln.DotSettings +++ b/seqcli.sln.DotSettings @@ -34,8 +34,11 @@ True True True + True True True True True + True + True True \ No newline at end of file diff --git a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs index d3452bf..cbdcfde 100644 --- a/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs +++ b/src/SeqCli/Cli/Commands/Forwarder/InstallCommand.cs @@ -17,9 +17,7 @@ using System; using System.Diagnostics.CodeAnalysis; using System.IO; -using System.Runtime.InteropServices; using System.Security.AccessControl; -using System.ServiceProcess; using System.Threading.Tasks; using SeqCli.Cli.Features; using SeqCli.Config; @@ -27,236 +25,128 @@ using SeqCli.Forwarder.ServiceProcess; using SeqCli.Forwarder.Util; +namespace SeqCli.Cli.Commands.Forwarder; + // ReSharper disable once ClassNeverInstantiated.Global -namespace SeqCli.Cli.Commands.Forwarder +[Command("forwarder", "install", "Install the forwarder as a Windows service", Visibility = FeatureVisibility.Preview)] +[SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] +class InstallCommand : Command { - [Command("forwarder", "install", "Install the forwarder as a Windows service", Visibility = FeatureVisibility.Preview)] - [SuppressMessage("Interoperability", "CA1416:Validate platform compatibility")] - class InstallCommand : Command + readonly StoragePathFeature _storagePath; + readonly ServiceCredentialsFeature _serviceCredentials; + readonly ListenUriFeature _listenUri; + public InstallCommand() { - readonly StoragePathFeature _storagePath; - readonly ServiceCredentialsFeature _serviceCredentials; - readonly ListenUriFeature _listenUri; - - bool _setup; - - public InstallCommand() - { - _storagePath = Enable(); - _listenUri = Enable(); - _serviceCredentials = Enable(); - - Options.Add( - "setup", - "Install and start the service only if it does not exist; otherwise reconfigure the binary location", - _ => _setup = true); - } + _storagePath = Enable(); + _listenUri = Enable(); + _serviceCredentials = Enable(); + } - string ServiceUsername => _serviceCredentials.IsUsernameSpecified ? _serviceCredentials.Username : "NT AUTHORITY\\LocalService"; + string ServiceUsername => _serviceCredentials.IsUsernameSpecified ? _serviceCredentials.Username : "NT AUTHORITY\\LocalService"; - protected override Task Run() + protected override Task Run() + { + try { - try - { - if (!_setup) - { - Install(); - return Task.FromResult(0); - } - - var exit = Setup(); - if (exit == 0) - { - Console.ForegroundColor = ConsoleColor.Green; - Console.WriteLine("Setup completed successfully."); - Console.ResetColor(); - } - return Task.FromResult(exit); - } - catch (DirectoryNotFoundException dex) - { - Console.WriteLine("Could not install the service, directory not found: " + dex.Message); - return Task.FromResult(-1); - } - catch (Exception ex) - { - Console.WriteLine("Could not install the service: " + ex.Message); - return Task.FromResult(-1); - } + Install(); + return Task.FromResult(0); } - - int Setup() + catch (DirectoryNotFoundException dex) { - ServiceController controller; - try - { - Console.WriteLine($"Checking the status of the {SeqCliForwarderWindowsService.WindowsServiceName} service..."); - - controller = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); - Console.WriteLine("Status is {0}", controller.Status); - } - catch (InvalidOperationException) - { - Install(); - var controller2 = new ServiceController(SeqCliForwarderWindowsService.WindowsServiceName); - return Start(controller2); - } - - Console.WriteLine("Service is installed; checking path and dependency configuration..."); - Reconfigure(controller); - - return controller.Status != ServiceControllerStatus.Running ? Start(controller) : 0; + Console.WriteLine("Could not install the service, directory not found: " + dex.Message); + return Task.FromResult(-1); } - - static void Reconfigure(ServiceController controller) + catch (Exception ex) { - var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); - if (0 != CaptiveProcess.Run(sc, "config \"" + controller.ServiceName + "\" depend= Winmgmt/Tcpip/CryptSvc", Console.WriteLine, Console.WriteLine)) - Console.WriteLine("Could not reconfigure service dependencies; ignoring."); - - if (!ServiceConfiguration.GetServiceBinaryPath(controller, out var path)) - return; - - var current = "\"" + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, Program.BinaryName) + "\""; - if (path.StartsWith(current)) - return; - - var seqRun = path.IndexOf(Program.BinaryName + "\" run", StringComparison.OrdinalIgnoreCase); - if (seqRun == -1) - { - Console.WriteLine("Current binary path is an unrecognized format."); - return; - } - - Console.WriteLine("Existing service binary path is: {0}", path); - - var trimmed = path.Substring((seqRun + Program.BinaryName + " ").Length); - var newPath = current + trimmed; - Console.WriteLine("Updating service binary path configuration to: {0}", newPath); - - var escaped = newPath.Replace("\"", "\\\""); - if (0 != CaptiveProcess.Run(sc, "config \"" + controller.ServiceName + "\" binPath= \"" + escaped + "\"", Console.WriteLine, Console.WriteLine)) - { - Console.WriteLine("Could not reconfigure service path; ignoring."); - return; - } - - Console.WriteLine("Service binary path reconfigured successfully."); + Console.WriteLine("Could not install the service: " + ex.Message); + return Task.FromResult(-1); } + } - static int Start(ServiceController controller) - { - controller.Start(); - - if (controller.Status != ServiceControllerStatus.Running) - { - Console.WriteLine("Waiting up to 60 seconds for the service to start (currently: " + controller.Status + ")..."); - controller.WaitForStatus(ServiceControllerStatus.Running, TimeSpan.FromSeconds(60)); - } - - if (controller.Status == ServiceControllerStatus.Running) - { - Console.WriteLine("Started."); - return 0; - } + void Install() + { + Console.WriteLine("Installing service..."); - Console.WriteLine("The service hasn't started successfully."); - return -1; + Console.WriteLine($"Updating the configuration in {_storagePath.ConfigFilePath}..."); + var config = SeqCliConfig.ReadFromFile(_storagePath.ConfigFilePath); + + if (!string.IsNullOrEmpty(_listenUri.ListenUri)) + { + config.Forwarder.Api.ListenUri = _listenUri.ListenUri; + SeqCliConfig.WriteToFile(config, _storagePath.ConfigFilePath); } - [DllImport("shlwapi.dll")] - static extern bool PathIsNetworkPath(string pszPath); - - void Install() + if (_serviceCredentials.IsUsernameSpecified) { - Console.WriteLine("Installing service..."); - - if (PathIsNetworkPath(_storagePath.StorageRootPath)) - throw new ArgumentException("Seq requires a local (or SAN) storage location; network shares are not supported."); - - Console.WriteLine($"Updating the configuration in {_storagePath.ConfigFilePath}..."); - var config = SeqCliConfig.ReadFromFile(_storagePath.ConfigFilePath); - - if (!string.IsNullOrEmpty(_listenUri.ListenUri)) - { - config.Forwarder.Api.ListenUri = _listenUri.ListenUri; - SeqCliConfig.WriteToFile(config, _storagePath.ConfigFilePath); - } - - if (_serviceCredentials.IsUsernameSpecified) - { - if (!_serviceCredentials.IsPasswordSpecified) - throw new ArgumentException( - "If a service user account is specified, a password for the account must also be specified."); - - // https://technet.microsoft.com/en-us/library/cc794944(v=ws.10).aspx - Console.WriteLine($"Ensuring {_serviceCredentials.Username} is granted 'Log on as a Service' rights..."); - AccountRightsHelper.EnsureServiceLogOnRights(_serviceCredentials.Username); - } + if (!_serviceCredentials.IsPasswordSpecified) + throw new ArgumentException( + "If a service user account is specified, a password for the account must also be specified."); - Console.WriteLine($"Granting {ServiceUsername} rights to {_storagePath.StorageRootPath}..."); - GiveFullControl(_storagePath.StorageRootPath); - - Console.WriteLine($"Granting {ServiceUsername} rights to {_storagePath.InternalLogPath}..."); - GiveFullControl(_storagePath.InternalLogPath); + // https://technet.microsoft.com/en-us/library/cc794944(v=ws.10).aspx + Console.WriteLine($"Ensuring {_serviceCredentials.Username} is granted 'Log on as a Service' rights..."); + AccountRightsHelper.EnsureServiceLogOnRights(_serviceCredentials.Username); + } - var listenUri = MakeListenUriReservationPattern(config.Forwarder.Api.ListenUri); - Console.WriteLine($"Adding URL reservation at {listenUri} for {ServiceUsername}..."); - var netshResult = CaptiveProcess.Run("netsh", $"http add urlacl url={listenUri} user=\"{ServiceUsername}\"", Console.WriteLine, Console.WriteLine); - if (netshResult != 0) - Console.WriteLine($"Could not add URL reservation for {listenUri}: `netsh` returned {netshResult}; ignoring"); + Console.WriteLine($"Granting {ServiceUsername} rights to {_storagePath.StorageRootPath}..."); + GiveFullControl(_storagePath.StorageRootPath); - var exePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory!, Program.BinaryName); - var forwarderRunCmdline = $"\"{exePath}\" run --storage=\"{_storagePath.StorageRootPath}\""; + Console.WriteLine($"Granting {ServiceUsername} rights to {_storagePath.InternalLogPath}..."); + GiveFullControl(_storagePath.InternalLogPath); - var binPath = forwarderRunCmdline.Replace("\"", "\\\""); + var listenUri = MakeListenUriReservationPattern(config.Forwarder.Api.ListenUri); + Console.WriteLine($"Adding URL reservation at {listenUri} for {ServiceUsername}..."); + var netshResult = CaptiveProcess.Run("netsh", $"http add urlacl url={listenUri} user=\"{ServiceUsername}\"", Console.WriteLine, Console.WriteLine); + if (netshResult != 0) + Console.WriteLine($"Could not add URL reservation for {listenUri}: `netsh` returned {netshResult}; ignoring"); - var scCmdline = "create \"" + SeqCliForwarderWindowsService.WindowsServiceName + "\"" + - " binPath= \"" + binPath + "\"" + - " start= auto" + - " depend= Winmgmt/Tcpip/CryptSvc"; + var exePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, Program.BinaryName); + var forwarderRunCmdline = $"\"{exePath}\" forwarder run --pre --storage=\"{_storagePath.StorageRootPath}\""; - if (_serviceCredentials.IsUsernameSpecified) - scCmdline += $" obj= {_serviceCredentials.Username} password= {_serviceCredentials.Password}"; + var binPath = forwarderRunCmdline.Replace("\"", "\\\""); - var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); - if (0 != CaptiveProcess.Run(sc, scCmdline, Console.WriteLine, Console.WriteLine)) - { - throw new ArgumentException("Service setup failed"); - } + var scCmdline = "create \"" + SeqCliForwarderWindowsService.WindowsServiceName + "\"" + + " binPath= \"" + binPath + "\"" + + " start= auto" + + " depend= Winmgmt/Tcpip/CryptSvc"; - Console.WriteLine("Setting service restart policy..."); - if (0 != CaptiveProcess.Run(sc, $"failure \"{SeqCliForwarderWindowsService.WindowsServiceName}\" actions= restart/60000/restart/60000/restart/60000// reset= 600000", Console.WriteLine, Console.WriteLine)) - Console.WriteLine("Could not set service restart policy; ignoring"); - Console.WriteLine("Setting service description..."); - if (0 != CaptiveProcess.Run(sc, $"description \"{SeqCliForwarderWindowsService.WindowsServiceName}\" \"Durable storage and forwarding of application log events\"", Console.WriteLine, Console.WriteLine)) - Console.WriteLine("Could not set service description; ignoring"); + if (_serviceCredentials.IsUsernameSpecified) + scCmdline += $" obj= {_serviceCredentials.Username} password= {_serviceCredentials.Password}"; - Console.WriteLine("Service installed successfully."); + var sc = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "sc.exe"); + if (0 != CaptiveProcess.Run(sc, scCmdline, Console.WriteLine, Console.WriteLine)) + { + throw new ArgumentException("Service setup failed."); } - void GiveFullControl(string target) - { - if (target == null) throw new ArgumentNullException(nameof(target)); + Console.WriteLine("Setting service restart policy..."); + if (0 != CaptiveProcess.Run(sc, $"failure \"{SeqCliForwarderWindowsService.WindowsServiceName}\" actions= restart/60000/restart/60000/restart/60000// reset= 600000", Console.WriteLine, Console.WriteLine)) + Console.WriteLine("Could not set service restart policy; ignoring"); + Console.WriteLine("Setting service description..."); + if (0 != CaptiveProcess.Run(sc, $"description \"{SeqCliForwarderWindowsService.WindowsServiceName}\" \"Durable storage and forwarding of application log events\"", Console.WriteLine, Console.WriteLine)) + Console.WriteLine("Could not set service description; ignoring"); - if (!Directory.Exists(target)) - Directory.CreateDirectory(target); + Console.WriteLine("Service installed successfully."); + } - var storageInfo = new DirectoryInfo(target); - var storageAccessControl = storageInfo.GetAccessControl(); - storageAccessControl.AddAccessRule(new FileSystemAccessRule(ServiceUsername, - FileSystemRights.FullControl, AccessControlType.Allow)); - storageInfo.SetAccessControl(storageAccessControl); - } + void GiveFullControl(string target) + { + if (!Directory.Exists(target)) + Directory.CreateDirectory(target); + + var storageInfo = new DirectoryInfo(target); + var storageAccessControl = storageInfo.GetAccessControl(); + storageAccessControl.AddAccessRule(new FileSystemAccessRule(ServiceUsername, + FileSystemRights.FullControl, AccessControlType.Allow)); + storageInfo.SetAccessControl(storageAccessControl); + } - static string MakeListenUriReservationPattern(string uri) - { - var listenUri = uri.Replace("localhost", "+"); - if (!listenUri.EndsWith("/")) - listenUri += "/"; - return listenUri; - } + static string MakeListenUriReservationPattern(string uri) + { + var listenUri = uri.Replace("localhost", "+"); + if (!listenUri.EndsWith('/')) + listenUri += "/"; + return listenUri; } } diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannel.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannel.cs index 15c6e90..5dffbb3 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannel.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannel.cs @@ -7,6 +7,7 @@ using SeqCli.Forwarder.Diagnostics; using SeqCli.Forwarder.Storage; using SeqCli.Ingestion; +using Serilog; namespace SeqCli.Forwarder.Channel; @@ -32,59 +33,79 @@ public ForwardingChannel(BufferAppender appender, BufferReader reader, Bookmark _writer = channel.Writer; _writeWorker = Task.Run(async () => { - await foreach (var entry in channel.Reader.ReadAllAsync(hardCancel)) + try { - try + await foreach (var entry in channel.Reader.ReadAllAsync(hardCancel)) { - const int maxTries = 3; - for (var retry = 0; retry < maxTries; ++retry) + try { - if (appender.TryAppend(entry.Data.AsSpan(), targetChunkSizeBytes, maxChunks)) - { - entry.CompletionSource.SetResult(); - break; - } - - if (retry == maxTries - 1) - { - IngestionLog.Log.Error("Buffering failed due to an I/O error; the incoming chunk was rejected"); - entry.CompletionSource.TrySetException(new IOException("Buffering failed due to an I/O error.")); - } + const int maxTries = 3; + for (var retry = 0; retry < maxTries; ++retry) + { + if (appender.TryAppend(entry.Data.AsSpan(), targetChunkSizeBytes, maxChunks)) + { + entry.CompletionSource.SetResult(); + break; + } + + if (retry == maxTries - 1) + { + IngestionLog.Log.Error("Buffering failed due to an I/O error; the incoming chunk was rejected"); + entry.CompletionSource.TrySetException(new IOException("Buffering failed due to an I/O error.")); + } + } + } + catch (Exception e) + { + entry.CompletionSource.TrySetException(e); } - } - catch (Exception e) - { - entry.CompletionSource.TrySetException(e); } } + catch (Exception ex) + { + // We don't loop here; the exception was unexpected, so it's either hard cancellation or an + // unknown condition that could cause CPU-burning hot looping. + Log.ForContext().Fatal(ex, "Forwarding ingest reader failed and exited"); + } }, cancellationToken: hardCancel); _readWorker = Task.Run(async () => { - if (bookmark.TryGet(out var bookmarkValue)) - { - reader.AdvanceTo(bookmarkValue.Value); - } - - while (true) + try { - if (_hardCancel.IsCancellationRequested) return; - - if (!reader.TryFillBatch(batchSizeLimitBytes, out var batch)) + if (bookmark.TryGet(out var bookmarkValue)) { - await Task.Delay(100, hardCancel); - continue; + reader.AdvanceTo(bookmarkValue.Value); } + + // Stopping shipping is a priority during shut-down, the work represented by the persistent buffer is unbounded + // so leaving it un-shipped avoids messier hard cancellation if we can't complete the work in time. + while (!_stop.IsCancellationRequested) + { + if (_hardCancel.IsCancellationRequested) return; - await LogShipper.ShipBufferAsync(connection, apiKey, batch.Value.AsArraySegment(), IngestionLog.Log, hardCancel); + if (!reader.TryFillBatch(batchSizeLimitBytes, out var batch)) + { + await Task.Delay(100, hardCancel); + continue; + } - if (bookmark.TrySet(new BufferPosition(batch.Value.ReaderHead.ChunkId, - batch.Value.ReaderHead.Offset))) - { - reader.AdvanceTo(batch.Value.ReaderHead); - } + await LogShipper.ShipBufferAsync(connection, apiKey, batch.Value.AsArraySegment(), IngestionLog.Log, hardCancel); + + if (bookmark.TrySet(new BufferPosition(batch.Value.ReaderHead.ChunkId, + batch.Value.ReaderHead.Offset))) + { + reader.AdvanceTo(batch.Value.ReaderHead); + } - batch.Value.Return(); + batch.Value.Return(); + } + } + catch (Exception ex) + { + // We don't loop here; the exception was unexpected, so it's either hard cancellation or an + // unknown condition that could cause CPU-burning hot looping. + Log.ForContext().Fatal(ex, "Forwarding log shipper failed and exited"); } }, cancellationToken: hardCancel); } diff --git a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs index 9dc9cd3..553efcc 100644 --- a/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs +++ b/src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs @@ -38,7 +38,7 @@ ForwardingChannel OpenOrCreateChannel(string? apiKey, string name) var storePath = Path.Combine(_bufferPath, name); var store = new SystemStoreDirectory(storePath); - Log.Information("Opening local buffer in {StorePath}", storePath); + Log.ForContext().Information("Opening local buffer in {StorePath}", storePath); return new ForwardingChannel( BufferAppender.Open(store), @@ -77,7 +77,7 @@ public ForwardingChannel Get(string? apiKey) public async Task StopAsync() { - Log.Information("Flushing log buffers"); + Log.ForContext().Information("Flushing log buffers"); _shutdownTokenSource.CancelAfter(TimeSpan.FromSeconds(30)); diff --git a/src/SeqCli/Forwarder/ForwarderModule.cs b/src/SeqCli/Forwarder/ForwarderModule.cs index ce000a2..4199124 100644 --- a/src/SeqCli/Forwarder/ForwarderModule.cs +++ b/src/SeqCli/Forwarder/ForwarderModule.cs @@ -23,7 +23,6 @@ using SeqCli.Forwarder.Web.Host; using Serilog; using Serilog.Formatting; -using Serilog.Formatting.Display; using Serilog.Templates; namespace SeqCli.Forwarder; @@ -52,13 +51,13 @@ protected override void Load(ContainerBuilder builder) if (_config.Forwarder.Diagnostics.ExposeIngestionLog) { - Log.Warning("Configured to expose ingestion log via HTTP API"); + Log.ForContext().Warning("Configured to expose ingestion log via HTTP API"); builder.RegisterType().As(); var ingestionLogTemplate = "[{@t:o} {@l:u3}] {@m}\n"; if (_config.Forwarder.Diagnostics.IngestionLogShowDetail) { - Log.Warning("Including full client, payload, and error detail in the ingestion log"); + Log.ForContext().Warning("Including full client, payload, and error detail in the ingestion log"); ingestionLogTemplate += "{#if ClientHostIP is not null}Client IP address: {ClientHostIP}\n{#end}" + "{#if DocumentStart is not null}First {StartToLog} characters of payload: {DocumentStart:l}\n{#end}" +