Fix euid enrichment in logs and refactor folder structure (#2240)

Author: reakaleek

src/api/Elastic.Documentation.Api.Infrastructure/Middleware/EuidLoggingMiddleware.cs

@@ -0,0 +1,42 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Diagnostics;
+using Elastic.Documentation.Api.Core;
+using OpenTelemetry;
+using OpenTelemetry.Logs;
+
+namespace Elastic.Documentation.Api.Infrastructure.OpenTelemetry;
+
+/// <summary>
+/// OpenTelemetry log processor that automatically adds user.euid attribute to log records
+/// when it exists in the current activity's baggage.
+/// This ensures the euid is present on all log records when set by the ASP.NET Core instrumentation.
+/// </summary>
+public class EuidLogProcessor : BaseProcessor<LogRecord>
+{
+	public override void OnEnd(LogRecord logRecord)
+	{
+		// Check if euid already exists as an attribute
+		var hasEuidAttribute = logRecord.Attributes?.Any(a =>
+			a.Key == TelemetryConstants.UserEuidAttributeName) ?? false;
+
+		if (hasEuidAttribute)
+		{
+			return;
+		}
+
+		// Read euid from current activity baggage (set by ASP.NET Core request enrichment)
+		var euid = Activity.Current?.GetBaggageItem(TelemetryConstants.UserEuidAttributeName);
+		if (!string.IsNullOrEmpty(euid))
+		{
+			// Add euid as an attribute to this log record
+			var newAttributes = new List<KeyValuePair<string, object?>>(logRecord.Attributes ?? [])
+			{
+				new(TelemetryConstants.UserEuidAttributeName, euid)
+			};
+			logRecord.Attributes = newAttributes;
+		}
+	}
+}

src/api/Elastic.Documentation.Api.Infrastructure/OpenTelemetry/EuidLogProcessor.cs

@@ -6,7 +6,7 @@
 using Elastic.Documentation.Api.Core;
 using OpenTelemetry;
 
-namespace Elastic.Documentation.Api.Infrastructure;
+namespace Elastic.Documentation.Api.Infrastructure.OpenTelemetry;
 
 /// <summary>
 /// OpenTelemetry span processor that automatically adds user.euid tag to all spans

src/api/Elastic.Documentation.Api.Infrastructure/EuidSpanProcessor.cs renamed to src/api/Elastic.Documentation.Api.Infrastructure/OpenTelemetry/EuidSpanProcessor.cs

@@ -7,14 +7,26 @@
 using Elastic.OpenTelemetry;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
 using OpenTelemetry;
+using OpenTelemetry.Logs;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Trace;
 
-namespace Elastic.Documentation.Api.Infrastructure;
+namespace Elastic.Documentation.Api.Infrastructure.OpenTelemetry;
 
 public static class OpenTelemetryExtensions
 {
+	/// <summary>
+	/// Configures logging for the Docs API with euid enrichment.
+	/// This is the shared configuration used in both production and tests.
+	/// </summary>
+	public static LoggerProviderBuilder AddDocsApiLogging(this LoggerProviderBuilder builder)
+	{
+		_ = builder.AddProcessor<EuidLogProcessor>();
+		return builder;
+	}
+
 	/// <summary>
 	/// Configures tracing for the Docs API with sources, instrumentation, and enrichment.
 	/// This is the shared configuration used in both production and tests.
@@ -62,7 +74,7 @@ public static TBuilder AddDocsApiOpenTelemetry<TBuilder>(
 		_ = builder.AddElasticOpenTelemetry(options, edotBuilder =>
 		{
 			_ = edotBuilder
-				.WithLogging()
+				.WithLogging(logging => logging.AddDocsApiLogging())
 				.WithTracing(tracing => tracing.AddDocsApiTracing())
 				.WithMetrics(metrics =>
 				{

src/api/Elastic.Documentation.Api.Infrastructure/OpenTelemetryExtensions.cs renamed to src/api/Elastic.Documentation.Api.Infrastructure/OpenTelemetry/OpenTelemetryExtensions.cs

@@ -8,7 +8,7 @@
 using Elastic.Documentation.Api.Core.AskAi;
 using Elastic.Documentation.Api.Core.Search;
 using Elastic.Documentation.Api.Infrastructure;
-using Elastic.Documentation.Api.Infrastructure.Middleware;
+using Elastic.Documentation.Api.Infrastructure.OpenTelemetry;
 
 try
 {
@@ -39,9 +39,6 @@
 	builder.Services.AddElasticDocsApiUsecases(environment);
 	var app = builder.Build();
 
-	// Add middleware to enrich logs with euid cookie
-	_ = app.UseEuidLogging();
-
 	var v1 = app.MapGroup("/docs/_api/v1");
 	v1.MapElasticDocsApiEndpoints();
 	Console.WriteLine("API endpoints mapped");

src/api/Elastic.Documentation.Api.Lambda/Program.cs

@@ -5,6 +5,7 @@
 using System.Diagnostics;
 using System.Text;
 using Elastic.Documentation.Api.Core;
+using Elastic.Documentation.Api.Core.AskAi;
 using Elastic.Documentation.Api.IntegrationTests.Fixtures;
 using FluentAssertions;
 
@@ -64,22 +65,19 @@ public async Task AskAiEndpointPropagatatesEuidToAllSpansAndLogs()
 		askAiEuidTag.Should().NotBeNull("AskAi span should have user.euid tag from baggage");
 		askAiEuidTag.Value.Should().Be(expectedEuid, "AskAi span euid should match cookie value");
 
-		// Assert - Verify logs have euid in scope
-		var logEntries = _factory.LogEntries;
-		logEntries.Should().NotBeEmpty("Should have captured log entries");
+		// Assert - Verify logs have euid in attributes
+		var logRecords = _factory.ExportedLogRecords;
+		logRecords.Should().NotBeEmpty("Should have captured log records");
 
 		// Find a log entry from AskAiUsecase
-		var askAiLog = logEntries.FirstOrDefault(e =>
-			e.CategoryName.Contains("AskAiUsecase") &&
-			e.Message.Contains("Starting AskAI"));
-		askAiLog.Should().NotBeNull("Should have logged from AskAiUsecase");
+		var askAiLogRecord = logRecords.FirstOrDefault(r =>
+			string.Equals(r.CategoryName, typeof(AskAiUsecase).FullName, StringComparison.OrdinalIgnoreCase) &&
+			r.FormattedMessage?.Contains("Starting AskAI", StringComparison.OrdinalIgnoreCase) == true);
+		askAiLogRecord.Should().NotBeNull("Should have logged from AskAiUsecase");
 
-		// Verify euid is in the logging scope
-		var hasEuidInScope = askAiLog!.Scopes
-			.Any(scope => scope is IDictionary<string, object> dict &&
-						  dict.TryGetValue(TelemetryConstants.UserEuidAttributeName, out var value) &&
-						  value?.ToString() == expectedEuid);
-
-		hasEuidInScope.Should().BeTrue("Log entry should have user.euid in scope from middleware");
+		// Verify euid is present in OTEL log attributes (mirrors production exporter behavior)
+		var euidAttribute = askAiLogRecord!.Attributes?.FirstOrDefault(a => a.Key == TelemetryConstants.UserEuidAttributeName) ?? default;
+		euidAttribute.Should().NotBe(default(KeyValuePair<string, object?>), "Log record should include user.euid attribute");
+		(euidAttribute.Value?.ToString() ?? string.Empty).Should().Be(expectedEuid, "Log record euid should match cookie value");
 	}
 }

tests-integration/Elastic.Documentation.Api.IntegrationTests/EuidEnrichmentIntegrationTests.cs

@@ -7,12 +7,14 @@
 using Elastic.Documentation.Api.Core.AskAi;
 using Elastic.Documentation.Api.Infrastructure;
 using Elastic.Documentation.Api.Infrastructure.Aws;
+using Elastic.Documentation.Api.Infrastructure.OpenTelemetry;
 using FakeItEasy;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Mvc.Testing;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using OpenTelemetry;
+using OpenTelemetry.Logs;
 using OpenTelemetry.Trace;
 
 namespace Elastic.Documentation.Api.IntegrationTests.Fixtures;
@@ -24,25 +26,23 @@ namespace Elastic.Documentation.Api.IntegrationTests.Fixtures;
 public class ApiWebApplicationFactory : WebApplicationFactory<Program>
 {
 	public List<Activity> ExportedActivities { get; } = [];
-	public List<TestLogEntry> LogEntries { get; } = [];
-	private readonly List<MemoryStream> MockMemoryStreams = new();
+	public List<LogRecord> ExportedLogRecords { get; } = [];
+	private readonly List<MemoryStream> _mockMemoryStreams = [];
 	protected override void ConfigureWebHost(IWebHostBuilder builder) =>
 		builder.ConfigureServices(services =>
 		{
-			// Configure OpenTelemetry with in-memory exporter for testing
-			// Uses the same production configuration via AddDocsApiTracing()
-			_ = services.AddOpenTelemetry()
-				.WithTracing(tracing =>
-				{
-					_ = tracing
-						.AddDocsApiTracing() // Reuses production configuration
-						.AddInMemoryExporter(ExportedActivities);
-				});
-
-			// Configure logging to capture log entries
-			_ = services.AddLogging(logging =>
+			var otelBuilder = services.AddOpenTelemetry();
+			_ = otelBuilder.WithTracing(tracing =>
+			{
+				_ = tracing
+					.AddDocsApiTracing() // Reuses production configuration
+					.AddInMemoryExporter(ExportedActivities);
+			});
+			_ = otelBuilder.WithLogging(logging =>
 			{
-				_ = logging.AddProvider(new TestLoggerProvider(LogEntries));
+				_ = logging
+					.AddDocsApiLogging() // Reuses production configuration
+					.AddInMemoryExporter(ExportedLogRecords);
 			});
 
 			// Mock IParameterProvider to avoid AWS dependencies
@@ -88,58 +88,3 @@ protected override void Dispose(bool disposing)
 		base.Dispose(disposing);
 	}
 }
-
-/// <summary>
-/// Test logger provider for capturing log entries with scopes.
-/// </summary>
-internal sealed class TestLoggerProvider(List<TestLogEntry> logEntries) : ILoggerProvider
-{
-	private readonly List<object> _sharedScopes = [];
-
-	public ILogger CreateLogger(string categoryName) => new TestLogger(categoryName, logEntries, _sharedScopes);
-	public void Dispose() { }
-}
-
-/// <summary>
-/// Test logger that captures log entries with their scopes.
-/// </summary>
-internal sealed class TestLogger(string categoryName, List<TestLogEntry> logEntries, List<object> sharedScopes) : ILogger
-{
-	public IDisposable BeginScope<TState>(TState state) where TState : notnull
-	{
-		sharedScopes.Add(state);
-		return new ScopeDisposable(sharedScopes, state);
-	}
-
-	public bool IsEnabled(LogLevel logLevel) => true;
-
-	public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
-	{
-		var entry = new TestLogEntry
-		{
-			CategoryName = categoryName,
-			LogLevel = logLevel,
-			Message = formatter(state, exception),
-			Exception = exception,
-			Scopes = [.. sharedScopes]
-		};
-		logEntries.Add(entry);
-	}
-
-	private sealed class ScopeDisposable(List<object> scopes, object state) : IDisposable
-	{
-		public void Dispose() => scopes.Remove(state);
-	}
-}
-
-/// <summary>
-/// Represents a captured log entry with its scopes.
-/// </summary>
-public sealed class TestLogEntry
-{
-	public required string CategoryName { get; init; }
-	public LogLevel LogLevel { get; init; }
-	public required string Message { get; init; }
-	public Exception? Exception { get; init; }
-	public List<object> Scopes { get; init; } = [];
-}