Merge branch 'master' into otel-metrics

andreasohlund · web-flow · commit 09facb9e123e · 2025-02-10T08:38:05.000+01:00
diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml
@@ -38,7 +38,7 @@ jobs:
         with:
           version: ${{ env.MinVerVersion }}
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.7.1
+        uses: docker/setup-buildx-action@v3.8.0
       - name: Log in to GitHub container registry
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
       - name: Build & inspect image
diff --git a/.github/workflows/push-container-images.yml b/.github/workflows/push-container-images.yml
@@ -53,6 +53,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
           repository: particular/servicecontrol
           readme-filepath: ./src/ServiceControl/Container-README.md
+          short-description: Gather status, performance and monitoring data for multiple endpoints from a single location.
       - name: Update Docker Hub Description - Audit
         if: ${{ steps.validate.outputs.latest == 'true' }}
         uses: peter-evans/dockerhub-description@v4.0.0
@@ -61,6 +62,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
           repository: particular/servicecontrol-audit
           readme-filepath: ./src/ServiceControl.Audit/Container-README.md
+          short-description: Provide valuable information about the message flow through a system.
       - name: Update Docker Hub Description - Monitoring
         if: ${{ steps.validate.outputs.latest == 'true' }}
         uses: peter-evans/dockerhub-description@v4.0.0
@@ -69,6 +71,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
           repository: particular/servicecontrol-monitoring
           readme-filepath: ./src/ServiceControl.Monitoring/Container-README.md
+          short-description: Track the health of a distributed system.
       - name: Update Docker Hub Description - RavenDB
         if: ${{ steps.validate.outputs.latest == 'true' }}
         uses: peter-evans/dockerhub-description@v4.0.0
@@ -77,3 +80,4 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
           repository: particular/servicecontrol-ravendb
           readme-filepath: ./src/ServiceControl.RavenDB/Container-README.md
+          short-description: The default storage for ServiceControl instances.
diff --git a/README.md b/README.md
@@ -10,6 +10,8 @@ ServiceControl, ServiceControl.Audit, and ServiceControl.Monitoring can be run/d
 
 - Edit the `app.config` file of the instance type that needs to be run/debugged to select which transport and persistence to use.
   - The configuration file contains commented settings for each supported transport and persistence. It also provides some guidance on additional required settings for specific persisters.
+  - ServiceControl works with a RavenDB persistence
+  - ServiceControl.Audit can work with RavenDB or an InMemory persistence
 - Run or debug the project as usual
 
 A video demo showing how to set it up is available on the Particular YouTube channel:
@@ -27,10 +29,6 @@ It's also possible to [locally test containers built from PRs in GitHub Containe
 If the instance is executed for the first time, it must set up the required infrastructure. To do so, once the instance is configured to use the selected transport and persister, run it in setup mode. This can be done by using the `Setup {instance name}` launch profile that is defined in 
 the `launchSettings.json` file of each instance. When started in setup mode, the instance will start as usual, execute the setup process, and exit. At this point the instance can be run normally by using the non-setup launch profile. 
 
-### Run Instances on Learning transport
-
-To help with local testing, the Learning transport has been added to the list of available transports when setting up a new instance in SCMU. For it to become available, an environment variable `ServiceControl_IncludeLearningTransport` needs to be created with a value of `true`.
-
 ## Secrets
 
 Testing using the [CI workflow](/.github/workflows/ci.yml) depends on the following secrets, which must be defined for both Actions and Dependabot secrets. The Particular values for these secrets are stored in the secure note named **ServiceControl Repo Secrets**.
@@ -83,4 +81,4 @@ Steps:
      -ErrorQueue errormq `
      -ErrorRetentionPeriod 10:00:00:00 `
      -Acknowledgements RabbitMQBrokerVersion310
-  ```
+  ```
diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props
@@ -27,9 +27,9 @@
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
     <PackageVersion Include="Microsoft-WindowsAPICodePack-Shell" Version="1.1.5" />
     <PackageVersion Include="Mindscape.Raygun4Net.NetCore" Version="11.2.1" />
-    <PackageVersion Include="NLog.Extensions.Logging" Version="5.3.15" />
-    <PackageVersion Include="NServiceBus" Version="9.2.3" />
-    <PackageVersion Include="NServiceBus.AcceptanceTesting" Version="9.2.3" />
+    <PackageVersion Include="NLog.Extensions.Logging" Version="5.4.0" />
+    <PackageVersion Include="NServiceBus" Version="9.2.4" />
+    <PackageVersion Include="NServiceBus.AcceptanceTesting" Version="9.2.4" />
     <PackageVersion Include="NServiceBus.AmazonSQS" Version="7.1.1" />
     <PackageVersion Include="NServiceBus.CustomChecks" Version="5.0.1" />
     <PackageVersion Include="NServiceBus.Extensions.Hosting" Version="3.0.1" />
@@ -49,7 +49,7 @@
     <PackageVersion Include="NuGet.Versioning" Version="6.12.1" />
     <PackageVersion Include="NUnit" Version="4.3.2" />
     <PackageVersion Include="NUnit.Analyzers" Version="4.6.0" />
-    <PackageVersion Include="NUnit3TestAdapter" Version="4.6.0" />
+    <PackageVersion Include="NUnit3TestAdapter" Version="5.0.0" />
     <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
     <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
     <PackageVersion Include="Particular.Approvals" Version="2.0.1" />
@@ -87,6 +87,6 @@
   <ItemGroup>
     <GlobalPackageReference Include="Microsoft.Build.Artifacts" Version="6.1.48" />
     <GlobalPackageReference Include="Microsoft.Build.CopyOnWrite" Version="1.0.334" />
-    <GlobalPackageReference Include="Particular.Packaging" Version="4.2.0" />
+    <GlobalPackageReference Include="Particular.Packaging" Version="4.2.2" />
   </ItemGroup>
 </Project>
diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs
@@ -3,9 +3,6 @@
 namespace ServiceControl.Audit.Persistence.RavenDB
 {
     using System;
-    using System.IO;
-    using System.Reflection;
-    using System.Security.Cryptography.X509Certificates;
     using System.Threading;
     using System.Threading.Tasks;
     using Raven.Client.Documents;
@@ -38,21 +35,11 @@ public async Task Initialize(CancellationToken cancellationToken = default)
             {
                 await initializeSemaphore.WaitAsync(cancellationToken);
 
-                // Look for raven-client-certificate.pfx in same directory as application code
-                var applicationDirectory = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location) ?? string.Empty;
-                var certificatePath = Path.Combine(applicationDirectory, "raven-client-certificate.pfx");
-                X509Certificate2? certificate = null;
-
-                if (File.Exists(certificatePath))
-                {
-                    certificate = new X509Certificate2(certificatePath);
-                }
-
                 var store = new DocumentStore
                 {
                     Database = configuration.Name,
                     Urls = [configuration.ServerConfiguration.ConnectionString],
-                    Certificate = certificate,
+                    Certificate = RavenClientCertificate.FindClientCertificate(configuration.ServerConfiguration),
                     Conventions = new DocumentConventions
                     {
                         SaveEnumsAsIntegers = true
diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs
@@ -12,6 +12,9 @@ public class RavenPersistenceConfiguration : IPersistenceConfiguration
         public const string DatabaseNameKey = "RavenDB/DatabaseName";
         public const string DatabasePathKey = "DbPath";
         public const string ConnectionStringKey = "RavenDB/ConnectionString";
+        public const string ClientCertificatePathKey = "RavenDB/ClientCertificatePath";
+        public const string ClientCertificateBase64Key = "RavenDB/ClientCertificateBase64";
+        public const string ClientCertificatePasswordKey = "RavenDB/ClientCertificatePassword";
         public const string DatabaseMaintenancePortKey = "DatabaseMaintenancePort";
         public const string ExpirationProcessTimerInSecondsKey = "ExpirationProcessTimerInSeconds";
         public const string LogPathKey = "LogPath";
@@ -24,6 +27,9 @@ public class RavenPersistenceConfiguration : IPersistenceConfiguration
             DatabaseNameKey,
             DatabasePathKey,
             ConnectionStringKey,
+            ClientCertificatePathKey,
+            ClientCertificateBase64Key,
+            ClientCertificatePasswordKey,
             DatabaseMaintenancePortKey,
             ExpirationProcessTimerInSecondsKey,
             LogPathKey,
@@ -59,6 +65,19 @@ internal static DatabaseConfiguration GetDatabaseConfiguration(PersistenceSettin
                 }
 
                 serverConfiguration = new ServerConfiguration(connectionString);
+
+                if (settings.PersisterSpecificSettings.TryGetValue(ClientCertificatePathKey, out var clientCertificatePath))
+                {
+                    serverConfiguration.ClientCertificatePath = clientCertificatePath;
+                }
+                if (settings.PersisterSpecificSettings.TryGetValue(ClientCertificateBase64Key, out var clientCertificateBase64))
+                {
+                    serverConfiguration.ClientCertificateBase64 = clientCertificateBase64;
+                }
+                if (settings.PersisterSpecificSettings.TryGetValue(ClientCertificatePasswordKey, out var clientCertificatePassword))
+                {
+                    serverConfiguration.ClientCertificatePassword = clientCertificatePassword;
+                }
             }
             else
             {
diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/ServerConfiguration.cs b/src/ServiceControl.Audit.Persistence.RavenDB/ServerConfiguration.cs
@@ -1,6 +1,8 @@
 ﻿namespace ServiceControl.Audit.Persistence.RavenDB
 {
-    public class ServerConfiguration
+    using ServiceControl.RavenDB;
+
+    public class ServerConfiguration : IRavenClientCertificateInfo
     {
         public ServerConfiguration(string connectionString)
         {
@@ -18,6 +20,9 @@ public ServerConfiguration(string dbPath, string serverUrl, string logPath, stri
         }
 
         public string ConnectionString { get; }
+        public string ClientCertificatePath { get; set; }
+        public string ClientCertificateBase64 { get; set; }
+        public string ClientCertificatePassword { get; set; }
         public bool UseEmbeddedServer { get; }
         public string DbPath { get; internal set; } //Setter for ATT only
         public string ServerUrl { get; }
diff --git a/src/ServiceControl.Infrastructure/LoggingConfigurator.cs b/src/ServiceControl.Infrastructure/LoggingConfigurator.cs
@@ -22,7 +22,7 @@ public static void ConfigureLogging(LoggingSettings loggingSettings)
             }
 
             var nlogConfig = new LoggingConfiguration();
-            var simpleLayout = new SimpleLayout("${longdate}|${threadid}|${level}|${logger}|${message}${onexception:|${exception:format=tostring}}");
+            var simpleLayout = new SimpleLayout("${longdate}|${processtime}|${threadid}|${level}|${logger}|${message}${onexception:|${exception:format=tostring}}");
 
             var fileTarget = new FileTarget
             {
diff --git a/src/ServiceControl.Persistence.RavenDB/RavenBootstrapper.cs b/src/ServiceControl.Persistence.RavenDB/RavenBootstrapper.cs
@@ -7,6 +7,9 @@ static class RavenBootstrapper
         public const string DatabaseMaintenancePortKey = "DatabaseMaintenancePort";
         public const string ExpirationProcessTimerInSecondsKey = "ExpirationProcessTimerInSeconds";
         public const string ConnectionStringKey = "RavenDB/ConnectionString";
+        public const string ClientCertificatePathKey = "RavenDB/ClientCertificatePath";
+        public const string ClientCertificateBase64Key = "RavenDB/ClientCertificateBase64";
+        public const string ClientCertificatePasswordKey = "RavenDB/ClientCertificatePassword";
         public const string MinimumStorageLeftRequiredForIngestionKey = "MinimumStorageLeftRequiredForIngestion";
         public const string DatabaseNameKey = "RavenDB/DatabaseName";
         public const string LogsPathKey = "LogPath";
diff --git a/src/ServiceControl.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs
@@ -3,9 +3,6 @@
 namespace ServiceControl.Persistence.RavenDB
 {
     using System;
-    using System.IO;
-    using System.Reflection;
-    using System.Security.Cryptography.X509Certificates;
     using System.Threading;
     using System.Threading.Tasks;
     using Raven.Client.Documents;
@@ -38,21 +35,11 @@ public async Task Initialize(CancellationToken cancellationToken)
             {
                 await initializeSemaphore.WaitAsync(cancellationToken);
 
-                // Look for raven-client-certificate.pfx in same directory as application code
-                var applicationDirectory = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location) ?? string.Empty;
-                var certificatePath = Path.Combine(applicationDirectory, "raven-client-certificate.pfx");
-                X509Certificate2? certificate = null;
-
-                if (File.Exists(certificatePath))
-                {
-                    certificate = new X509Certificate2(certificatePath);
-                }
-
                 var store = new DocumentStore
                 {
                     Database = settings.DatabaseName,
                     Urls = [settings.ConnectionString],
-                    Certificate = certificate,
+                    Certificate = RavenClientCertificate.FindClientCertificate(settings),
                     Conventions = new DocumentConventions
                     {
                         SaveEnumsAsIntegers = true
diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs
@@ -34,6 +34,9 @@ static T GetRequiredSetting<T>(SettingsRootNamespace settingsRootNamespace, stri
             var settings = new RavenPersisterSettings
             {
                 ConnectionString = SettingsReader.Read<string>(settingsRootNamespace, RavenBootstrapper.ConnectionStringKey),
+                ClientCertificatePath = SettingsReader.Read<string>(settingsRootNamespace, RavenBootstrapper.ClientCertificatePathKey),
+                ClientCertificateBase64 = SettingsReader.Read<string>(settingsRootNamespace, RavenBootstrapper.ClientCertificateBase64Key),
+                ClientCertificatePassword = SettingsReader.Read<string>(settingsRootNamespace, RavenBootstrapper.ClientCertificatePasswordKey),
                 DatabaseName = SettingsReader.Read(settingsRootNamespace, RavenBootstrapper.DatabaseNameKey, RavenPersisterSettings.DatabaseNameDefault),
                 DatabasePath = SettingsReader.Read(settingsRootNamespace, RavenBootstrapper.DatabasePathKey, DefaultDatabaseLocation()),
                 DatabaseMaintenancePort = SettingsReader.Read(settingsRootNamespace, RavenBootstrapper.DatabaseMaintenancePortKey, RavenPersisterSettings.DatabaseMaintenancePortDefault),
diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersisterSettings.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersisterSettings.cs
@@ -2,8 +2,9 @@
 using Particular.LicensingComponent.Contracts;
 using ServiceControl.Persistence;
 using ServiceControl.Persistence.RavenDB.CustomChecks;
+using ServiceControl.RavenDB;
 
-class RavenPersisterSettings : PersistenceSettings
+class RavenPersisterSettings : PersistenceSettings, IRavenClientCertificateInfo
 {
     public int DatabaseMaintenancePort { get; set; } = DatabaseMaintenancePortDefault;
     public int ExpirationProcessTimerInSeconds { get; set; } = ExpirationProcessTimerInSecondsDefault;
@@ -23,6 +24,9 @@ class RavenPersisterSettings : PersistenceSettings
     /// User provided external RavenDB instance connection string
     /// </summary>
     public string ConnectionString { get; set; }
+    public string ClientCertificatePath { get; set; }
+    public string ClientCertificateBase64 { get; set; }
+    public string ClientCertificatePassword { get; set; }
     public bool UseEmbeddedServer => string.IsNullOrWhiteSpace(ConnectionString);
     public string LogPath { get; set; }
     public string LogsMode { get; set; } = LogsModeDefault;
diff --git a/src/ServiceControl.RavenDB/RavenClientCertificate.cs b/src/ServiceControl.RavenDB/RavenClientCertificate.cs
@@ -0,0 +1,51 @@
+#nullable enable
+
+namespace ServiceControl.RavenDB;
+
+using System.Reflection;
+using System.Security.Cryptography;
+using System.Security.Cryptography.X509Certificates;
+
+public static class RavenClientCertificate
+{
+    public static X509Certificate2? FindClientCertificate(IRavenClientCertificateInfo certInfo)
+    {
+        if (certInfo.ClientCertificateBase64 is not null)
+        {
+            try
+            {
+                var bytes = Convert.FromBase64String(certInfo.ClientCertificateBase64);
+                return new X509Certificate2(bytes, certInfo.ClientCertificatePassword);
+            }
+            catch (Exception x) when (x is FormatException or CryptographicException)
+            {
+                throw new Exception("Could not read the RavenDB client certificate from the configured Base64 value.", x);
+            }
+        }
+
+        if (certInfo.ClientCertificatePath is not null)
+        {
+            if (!File.Exists(certInfo.ClientCertificatePath))
+            {
+                throw new Exception("Could not read the RavenDB client certificate from the supplied path because no file was found.");
+            }
+            return new X509Certificate2(certInfo.ClientCertificatePath, certInfo.ClientCertificatePassword);
+        }
+
+        var applicationDirectory = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location) ?? string.Empty;
+        var certificatePath = Path.Combine(applicationDirectory, "raven-client-certificate.pfx");
+
+        if (File.Exists(certificatePath))
+        {
+            return new X509Certificate2(certificatePath, certInfo.ClientCertificatePassword);
+        }
+        return null;
+    }
+}
+
+public interface IRavenClientCertificateInfo
+{
+    string? ClientCertificatePath { get; }
+    string? ClientCertificateBase64 { get; }
+    string? ClientCertificatePassword { get; }
+}

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,8 @@`
`1`	`1`	`namespace ServiceControl.Audit.Persistence.RavenDB`
`2`	`2`	`{`
`3`		`- public class ServerConfiguration`
	`3`	`+ using ServiceControl.RavenDB;`
	`4`	`+`
	`5`	`+ public class ServerConfiguration : IRavenClientCertificateInfo`
`4`	`6`	`{`
`5`	`7`	`public ServerConfiguration(string connectionString)`
`6`	`8`	`{`
`@@ -18,6 +20,9 @@ public ServerConfiguration(string dbPath, string serverUrl, string logPath, stri`
`18`	`20`	`}`
`19`	`21`
`20`	`22`	`public string ConnectionString { get; }`
	`23`	`+ public string ClientCertificatePath { get; set; }`
	`24`	`+ public string ClientCertificateBase64 { get; set; }`
	`25`	`+ public string ClientCertificatePassword { get; set; }`
`21`	`26`	`public bool UseEmbeddedServer { get; }`
`22`	`27`	`public string DbPath { get; internal set; } //Setter for ATT only`
`23`	`28`	`public string ServerUrl { get; }`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ public static void ConfigureLogging(LoggingSettings loggingSettings)`
`22`	`22`	`}`
`23`	`23`
`24`	`24`	`var nlogConfig = new LoggingConfiguration();`
`25`		`- var simpleLayout = new SimpleLayout("${longdate}\|${threadid}\|${level}\|${logger}\|${message}${onexception:\|${exception:format=tostring}}");`
	`25`	`+ var simpleLayout = new SimpleLayout("${longdate}\|${processtime}\|${threadid}\|${level}\|${logger}\|${message}${onexception:\|${exception:format=tostring}}");`
`26`	`26`
`27`	`27`	`var fileTarget = new FileTarget`
`28`	`28`	`{`