Merge branch 'dev' into review-shutdown

Author: nblumhardt

README.md

@@ -1648,3 +1648,101 @@ PS > (echo $warnings | ConvertTo-Json) | seqcli signal update --json-stdin
 PS > seqcli signal list -i signal-m33302 --json                                 
 {"Title": "Alarms", "Description": "Automatically created", "Filters": [{"De...
 ```
+
+## Store-and-forward ingestion proxy (preview)
+
+The `seqcli forwarder` family of commands provide simple, durable ingestion buffering for occasionally-connected and
+intermittently-disconnected systems. The forwarder implements the Seq ingestion API, so applications that write
+directly to Seq can instead write to the forwarder, which will persist data locally until it can be sent to the
+destination Seq server.
+
+> [!NOTE]
+> 
+> Forwarder is designed for local use in isolated environments — for example, locally on a firewalled machine, or
+> within a secured container network.
+> 
+> The forwarder HTTP API does not authenticate incoming requests. By default, only the ingestion endpoint is exposed,
+> but if you opt into additional APIs such as the ingestion log, ensure the API is not reachable externally. Even in the
+> default configuration, be aware that clients may trigger disk space exhaustion and other issues by sending
+> excessive or maliciously-crafted ingestion traffic.
+
+### Running the forwarder
+
+To start a forwarder instance at the terminal, listening on port 5341 and forwarding to `seq.example.com`, run:
+
+```shell
+seqcli forwarder run --pre --listen http://127.0.0.1:5341 -s https://seq.example.com
+```
+
+> While the `forwarder` command group is in preview, all `forwarder` commands require the `--pre` switch; you'll
+> also need to supply `--pre` when requesting help, e.g. `seqcli help forwarder run --pre`.
+
+You can test your forwarder using the `seqcli log` command:
+
+```shell
+seqcli log -m Test -s http://127.0.0.1:5341
+```
+
+If forwarding is successful, the event will appear in the target Seq instance within a few seconds. If it isn't,
+and `seqcli log` didn't report an error, run through these troubleshooting steps:
+
+ 1. Read the output of the `seqcli forwarder run` command: are the paths and URLs in there the ones you expect?
+ 2. Check the destination Seq server's ingestion log; if the payload is being rejected by Seq, its likely you need to configure an API key for the forwarder's outbound connections, or API key forwarding (see below).
+ 3. Check the forwarder's log file; this will be in the `SeqCli/Logs` subdirectory of the forwarder's storage path, which will be your user's home directory unless you overrode it with `--storage` or `SEQCLI_STORAGE_PATH`.
+ 4. Enable the forwarder's ingestion log temporarily, ensuring this isn't accessible to untrusted callers.
+
+### The forwarder's ingestion log
+
+The ingestion log records ingestion issues in a short in-memory buffer, so repetitive ingestion issues don't chew up
+disk space. The ingestion log may contain fragments of event data so it's not enabled by default; you can turn it on
+with:
+
+```shell
+SEQCLI_FORWARDER_DIAGNOSTICS_EXPOSEINGESTIONLOG=True seqcli forwarder run <other args>
+```
+
+The log can be retrieved by pointing `seqcli` at the forwarder:
+
+```shell
+seqcli diagnostics ingestionlog -s http://127.0.0.1:5341
+```
+
+If the ingestion log contains relevant entries but doesn't include enough information to track down the issue, you
+can opt in to showing even more information with `SEQCLI_FORWARDER_DIAGNOSTICS_INGESTIONLOGSHOWDETAIL=True`.
+
+### Configuring the forwarder
+
+The forwarder reads its configuration from the environment and the `SeqCli.json` file. Run `seqcli config` to view
+all of the supported `forwarder.*` (`SEQCLI_FORWARDER_*`) settings.
+
+Locally buffered events are stored in the `SeqCli/Buffer` subdirectory of the directory containing `SeqCli.json`. By 
+default, this will be your user account's home folder.
+
+To change the location of the `SeqCli.json` config file and local buffers, specify the `--storage` argument
+or set the `SEQCLI_STORAGE_PATH` environment variable when configuring and running the forwarder.
+
+### Connection authentication and API key forwarding
+
+When connecting to Seq, `seqcli forwarder` will ignore any API keys attached to incoming payloads, and instead use the server URL 
+and API key stored in `SeqCli.json`, in the `SEQCLI_CONNECTION_*` environment variables, or passed on 
+the `forwarder run` command line.
+
+To use the API keys from incoming payloads instead, set `forwarder.useApiKeyForwarding` or `SEQCLI_FORWARDER_USEAPIKEYFORWARDING`
+to `True`.
+
+> [!WARNING]
+> 
+> In order for durable buffering to work, `seqcli forwarder` needs to store the incoming API keys in its local buffer
+> storage.
+> 
+> On Windows, these will be encrypted using machine-scoped DPAPI by default, and on other platforms they'll be saved
+> as plain text. To perform encryption of stored API keys, supply shell scripts in the `encryption.encryptor` and
+> `encryption.decryptor` (`SEQCLI_ENCRYPTION_ENCRYPTOR` and `SEQCLI_ENCRYPTION_DECRYPTOR`) configuration entries. The scripts
+> will need to process data on `STDIN` and `STDOUT`: you can verify their effects by examining the `api.key` files stored
+> in subdirectories of `SeqCli/Buffer`.
+
+### Windows service installation
+
+On Windows it's possible to install the forwarder as a service, using the `seqcli forwarder install` command. Pass
+a directory to make readable by the service in the `--storage` argument when installing the it. Make sure this directory
+is also passed in `--storage` when configuring the forwarder.

src/SeqCli/Cli/CommandAttribute.cs

@@ -23,12 +23,13 @@ public class CommandAttribute : Attribute, ICommandMetadata
     public string? SubCommand { get; }
     public string HelpText { get; }
     public string? Example { get; set; }
-    public bool IsPreview { get; set; }
+    public FeatureVisibility Visibility { get; set; }
 
     public CommandAttribute(string name, string helpText)
     {
         Name = name;
         HelpText = helpText;
+        Visibility = FeatureVisibility.Visible;
     }
 
     public CommandAttribute(string name, string subCommand, string helpText) : this(name, helpText)

src/SeqCli/Cli/CommandFeature.cs

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-using System;
 using System.Collections.Generic;
 
 namespace SeqCli.Cli;
@@ -21,5 +20,5 @@ abstract class CommandFeature
 {
     public abstract void Enable(OptionSet options);
 
-    public virtual IEnumerable<string> GetUsageErrors() => Array.Empty<string>();
+    public virtual IEnumerable<string> GetUsageErrors() => [];
 }

src/SeqCli/Cli/CommandLineHost.cs

@@ -41,21 +41,30 @@ public async Task<int> Run(string[] args, LoggingLevelSwitch levelSwitch)
         {
             const string prereleaseArg = "--pre", verboseArg = "--verbose";
 
-            var norm = args[0].ToLowerInvariant();
-            var subCommandNorm = args.Length > 1 && !args[1].Contains('-') ? args[1].ToLowerInvariant() : null;
+            var commandName = args[0].ToLowerInvariant();
+            var subCommandName = args.Length > 1 && !args[1].Contains('-') ? args[1].ToLowerInvariant() : null;
 
-            var pre = args.Any(a => a == prereleaseArg);
+            var hiddenLegacyCommand = false;
+            if (subCommandName == null && commandName == "config")
+            {
+                hiddenLegacyCommand = true;
+                subCommandName = "legacy";
+            }
+            
+            var featureVisibility = FeatureVisibility.Visible | FeatureVisibility.Hidden;
+            if (args.Any(a => a.Trim() is prereleaseArg))
+                featureVisibility |= FeatureVisibility.Preview;
 
             var cmd = _availableCommands.SingleOrDefault(c =>
-                (!c.Metadata.IsPreview || pre) &&
-                c.Metadata.Name == norm &&
-                (c.Metadata.SubCommand == subCommandNorm || c.Metadata.SubCommand == null));
+                featureVisibility.HasFlag(c.Metadata.Visibility) &&
+                c.Metadata.Name == commandName &&
+                (c.Metadata.SubCommand == subCommandName || c.Metadata.SubCommand == null));
 
             if (cmd != null)
             {
-                var amountToSkip = cmd.Metadata.SubCommand == null ? 1 : 2;
-                var commandSpecificArgs = args.Skip(amountToSkip).Where(arg => cmd.Metadata.Name == "help" || arg != prereleaseArg).ToArray();
-                    
+                var amountToSkip = cmd.Metadata.SubCommand == null || hiddenLegacyCommand ? 1 : 2;
+                var commandSpecificArgs = args.Skip(amountToSkip).Where(arg => cmd.Metadata.Name == "help" || arg is not prereleaseArg).ToArray();
+                
                 var verbose = commandSpecificArgs.Any(arg => arg == verboseArg);
                 if (verbose)
                 {

src/SeqCli/Cli/CommandMetadata.cs

@@ -20,5 +20,5 @@ public class CommandMetadata : ICommandMetadata
     public string? SubCommand { get; set; }
     public required string HelpText { get; set; }
     public string? Example { get; set; }
-    public bool IsPreview { get; set; }
+    public FeatureVisibility Visibility { get; set; }
 }

src/SeqCli/Cli/Commands/Config/ClearCommand.cs

@@ -0,0 +1,43 @@
+// Copyright © Datalust Pty Ltd and Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System.Threading.Tasks;
+using SeqCli.Cli.Features;
+using SeqCli.Config;
+
+// ReSharper disable once UnusedType.Global
+
+namespace SeqCli.Cli.Commands.Config;
+
+[Command("config", "clear", "Clear fields in the `SeqCli.json` file")]
+class ClearCommand : Command
+{
+    readonly StoragePathFeature _storagePath;
+    readonly ConfigKeyFeature _key;
+
+    public ClearCommand()
+    {
+        _storagePath = Enable<StoragePathFeature>();
+        _key = Enable<ConfigKeyFeature>();
+    }
+
+    protected override Task<int> Run()
+    {
+        var config = SeqCliConfig.ReadFromFile(_storagePath.ConfigFilePath);
+            
+        KeyValueSettings.Clear(config, _key.GetKey());
+        SeqCliConfig.WriteToFile(config, _storagePath.ConfigFilePath);
+        return Task.FromResult(0);
+    }
+}

src/SeqCli/Cli/Commands/Config/GetCommand.cs

@@ -0,0 +1,45 @@
+// Copyright © Datalust Pty Ltd and Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Threading.Tasks;
+using SeqCli.Cli.Features;
+using SeqCli.Config;
+
+// ReSharper disable once UnusedType.Global
+
+namespace SeqCli.Cli.Commands.Config;
+
+[Command("config", "get", "View a field from the `SeqCli.json` file")]
+class GetCommand : Command
+{
+    readonly StoragePathFeature _storagePath;
+    readonly ConfigKeyFeature _key;
+
+    public GetCommand()
+    {
+        _storagePath = Enable<StoragePathFeature>();
+        _key = Enable<ConfigKeyFeature>();
+    }
+
+    protected override Task<int> Run()
+    {
+        var config = SeqCliConfig.ReadFromFile(_storagePath.ConfigFilePath);
+        if (!KeyValueSettings.TryGetValue(config, _key.GetKey(), out var value, out _))
+            throw new ArgumentException($"Field `{_key.GetKey()}` not found.");
+        
+        Console.WriteLine(value);
+        return Task.FromResult(0);
+    }
+}

src/SeqCli/Cli/Commands/ConfigCommand.cs …Cli/Cli/Commands/Config/LegacyCommand.cssrc/SeqCli/Cli/Commands/ConfigCommand.cs renamed to src/SeqCli/Cli/Commands/Config/LegacyCommand.cs src/SeqCli/Cli/Commands/ConfigCommand.cs renamed to src/SeqCli/Cli/Commands/Config/LegacyCommand.cs

@@ -19,16 +19,16 @@
 using SeqCli.Util;
 using Serilog;
 
-namespace SeqCli.Cli.Commands;
+namespace SeqCli.Cli.Commands.Config;
 
-[Command("config", "View and set fields in `SeqCli.json`; run with no arguments to list all fields")]
-class ConfigCommand : Command
+[Command("config", "legacy", "View and set fields in `SeqCli.json`; run with no arguments to list all fields", Visibility = FeatureVisibility.Hidden)]
+class LegacyCommand : Command
 {
     string? _key, _value;
     bool _clear;
     readonly StoragePathFeature _storagePath;
 
-    public ConfigCommand()
+    public LegacyCommand()
     {
         Options.Add("k|key=", "The field, for example `connection.serverUrl`", k => _key = k);
         Options.Add("v|value=", "The field value; if not specified, the command will print the current value", v => _value = v);

src/SeqCli/Cli/Commands/Config/ListCommand.cs

@@ -0,0 +1,43 @@
+// Copyright © Datalust Pty Ltd and Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Threading.Tasks;
+using SeqCli.Cli.Features;
+using SeqCli.Config;
+
+// ReSharper disable once UnusedType.Global
+
+namespace SeqCli.Cli.Commands.Config;
+
+[Command("config", "list", "View all fields in the `SeqCli.json` file")]
+class ListCommand : Command
+{
+    readonly StoragePathFeature _storagePath;
+
+    public ListCommand()
+    {
+        _storagePath = Enable<StoragePathFeature>();
+    }
+
+    protected override Task<int> Run()
+    {
+        var config = SeqCliConfig.ReadFromFile(_storagePath.ConfigFilePath);
+        foreach (var (key, value, _) in KeyValueSettings.Inspect(config))
+        {
+            Console.WriteLine($"{key}={value}");
+        }
+        return Task.FromResult(0);
+    }
+}

src/SeqCli/Cli/Commands/Config/SetCommand.cs

@@ -0,0 +1,48 @@
+// Copyright © Datalust Pty Ltd and Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using SeqCli.Cli.Features;
+using SeqCli.Config;
+using Serilog;
+
+// ReSharper disable once UnusedType.Global
+
+namespace SeqCli.Cli.Commands.Config;
+
+[Command("config", "set", "Set a field in the `SeqCli.json` file")]
+class SetCommand : Command
+{
+    readonly StoragePathFeature _storagePath;
+    readonly ConfigKeyFeature _key;
+    readonly ConfigValueFeature _value;
+        
+    public SetCommand()
+    {
+        _storagePath = Enable<StoragePathFeature>();
+        _key = Enable<ConfigKeyFeature>();
+        _value = Enable<ConfigValueFeature>();
+    }
+
+    protected override Task<int> Run()
+    {
+        var config = SeqCliConfig.ReadFromFile(_storagePath.ConfigFilePath);
+            
+        KeyValueSettings.Set(config, _key.GetKey(), _value.ReadValue());
+        SeqCliConfig.WriteToFile(config, _storagePath.ConfigFilePath);
+        return Task.FromResult(0);
+    }
+}