From e369add72f27bed7e93d7ec8217af58e2f6166df Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Fri, 14 Nov 2025 12:27:17 -0800 Subject: [PATCH 01/20] implement resource attributes for geneva logs --- .../CHANGELOG.md | 1 + .../GenevaExporterOptions.cs | 2 + .../GenevaLogExporter.cs | 7 +- .../Internal/MsgPack/MsgPackLogExporter.cs | 140 ++++++++--- .../Exporter/LogExporterBenchmarks.cs | 23 +- .../Exporter/TLDLogExporterBenchmarks.cs | 19 +- .../GenevaLogExporterTests.cs | 226 ++++++++++++++++-- .../LogSerializationTests.cs | 3 +- .../MsgPackLogExporterTests.cs | 5 +- 9 files changed, 345 insertions(+), 81 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md index 5cda05213a..8a29ef41e1 100644 --- a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md @@ -4,6 +4,7 @@ * Add ResourceFieldNames to filter resource attributes to send to Geneva ([#3552](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/3552)) + ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/TODO)) ## 1.14.0 diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs index 9e83063502..8730a6b716 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs @@ -107,6 +107,8 @@ public IReadOnlyDictionary? TableNameMappings /// /// Gets or sets prepopulated fields. + /// + /// Pre-populated fields are fields that are added as dedicated fields to every record, unless it would conflict with a log or trace field that is marked as a custom field. /// public IReadOnlyDictionary PrepopulatedFields { diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs index 2faab4661a..be75191637 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs @@ -88,9 +88,14 @@ public GenevaLogExporter(GenevaExporterOptions options) throw new NotSupportedException($"Protocol '{connectionStringBuilder.Protocol}' is not supported"); } + Resources.Resource ResourceProvider() + { + return connectionStringBuilder.HonorResourceAttributes ? this.ParentProvider.GetResource() : Resources.Resource.Empty; + } + if (useMsgPackExporter) { - var msgPackLogExporter = new MsgPackLogExporter(options); + var msgPackLogExporter = new MsgPackLogExporter(options, ResourceProvider); this.IsUsingUnixDomainSocket = msgPackLogExporter.IsUsingUnixDomainSocket; this.exportLogRecord = msgPackLogExporter.Export; this.exporter = msgPackLogExporter; diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs index 7a616628f8..10a12e9c8d 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs @@ -12,6 +12,7 @@ using OpenTelemetry.Exporter.Geneva.Transports; using OpenTelemetry.Internal; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; namespace OpenTelemetry.Exporter.Geneva.MsgPack; @@ -33,16 +34,20 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable #if NET private readonly FrozenSet? customFields; - private readonly FrozenDictionary? prepopulatedFields; #else private readonly HashSet? customFields; - private readonly Dictionary? prepopulatedFields; #endif private readonly ExceptionStackExportMode exportExceptionStack; - private readonly List? prepopulatedFieldKeys; private readonly byte[] bufferEpilogue; private readonly IDataTransport dataTransport; + private readonly Func resourceProvider; + + // These are values that are always added to the body as dedicated fields + private readonly Dictionary prepopulatedFields; + + // These are values that are always added to env_properties + private readonly Dictionary propertiesEntries; private readonly int stringFieldSizeLimitCharCount; // the maximum string size limit for MsgPack strings // This is used for Scopes @@ -50,9 +55,12 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable private bool isDisposed; - public MsgPackLogExporter(GenevaExporterOptions options) + public MsgPackLogExporter(GenevaExporterOptions options, Func resourceProvider) { Guard.ThrowIfNull(options); + Guard.ThrowIfNull(resourceProvider); + + this.resourceProvider = resourceProvider; this.tableNameSerializer = new(options, defaultTableName: "Log"); this.exportExceptionStack = options.ExceptionStackExportMode; @@ -88,21 +96,17 @@ public MsgPackLogExporter(GenevaExporterOptions options) } this.stringFieldSizeLimitCharCount = connectionStringBuilder.PrivatePreviewLogMessagePackStringSizeLimit; + + this.propertiesEntries = []; + + this.prepopulatedFields = new Dictionary(options.PrepopulatedFields.Count, StringComparer.Ordinal); + if (options.PrepopulatedFields != null) { - this.prepopulatedFieldKeys = []; - var tempPrepopulatedFields = new Dictionary(options.PrepopulatedFields.Count, StringComparer.Ordinal); foreach (var kv in options.PrepopulatedFields) { - tempPrepopulatedFields[kv.Key] = kv.Value; - this.prepopulatedFieldKeys.Add(kv.Key); + this.prepopulatedFields[kv.Key] = kv.Value; } - -#if NET - this.prepopulatedFields = tempPrepopulatedFields.ToFrozenDictionary(StringComparer.Ordinal); -#else - this.prepopulatedFields = tempPrepopulatedFields; -#endif } // TODO: Validate custom fields (reserved name? etc). @@ -174,19 +178,59 @@ public void Dispose() this.isDisposed = true; } + internal void AddResourceAttributesToPrepopulated() + { + // This function needs to be idempotent + + foreach (var entry in this.resourceProvider().Attributes) + { + string key = entry.Key; + bool isDedicatedField = false; + if (entry.Value is string) + { + switch (key) + { + case "service.name": + key = Schema.V40.PartA.Extensions.Cloud.Role; + isDedicatedField = true; + break; + case "service.instanceId": + key = Schema.V40.PartA.Extensions.Cloud.RoleInstance; + isDedicatedField = true; + break; + } + } + + if (isDedicatedField || this.customFields == null || this.customFields.Contains(key)) + { + if (!this.prepopulatedFields.ContainsKey(key)) + { + this.prepopulatedFields.Add(key, entry.Value); + } + } + else + { + if (!this.propertiesEntries.ContainsKey(key)) + { + this.propertiesEntries.Add(key, entry.Value); + } + } + } + } + internal ArraySegment SerializeLogRecord(LogRecord logRecord) { // `LogRecord.State` and `LogRecord.StateValues` were marked Obsolete in https://github.com/open-telemetry/opentelemetry-dotnet/pull/4334 #pragma warning disable 0618 - IReadOnlyList>? listKvp; + IReadOnlyList>? logFields; if (logRecord.StateValues != null) { - listKvp = logRecord.StateValues; + logFields = logRecord.StateValues; } else { // Attempt to see if State could be ROL_KVP. - listKvp = logRecord.State as IReadOnlyList>; + logFields = logRecord.State as IReadOnlyList> ?? []; } #pragma warning restore 0618 @@ -194,7 +238,7 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) /* Fluentd Forward Mode: [ - "Log", + "Log", // (or category name) [ [ , { "env_ver": "4.0", ... } ] ], @@ -227,15 +271,20 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) ushort cntFields = 0; var idxMapSizePatch = cursor - 2; - if (this.prepopulatedFieldKeys != null) + this.AddResourceAttributesToPrepopulated(); + + foreach (var entry in this.prepopulatedFields) { - for (var i = 0; i < this.prepopulatedFieldKeys.Count; i++) + // A prepopulated entry should not be added if the same key exists in the log, + // and customFields configuration would make it a dedicated field. + if ((this.customFields == null || this.customFields.Contains(entry.Key)) + && logFields.Any(kvp => kvp.Key == entry.Key)) { - var key = this.prepopulatedFieldKeys[i]; - var value = this.prepopulatedFields![key]; - cursor = AddPartAField(buffer, cursor, key, value); - cntFields += 1; + continue; } + + cursor = AddPartAField(buffer, cursor, entry.Key, entry.Value); + cntFields += 1; } // Part A - core envelope @@ -298,10 +347,8 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) var hasEnvProperties = false; var bodyPopulated = false; var namePopulated = false; - for (var i = 0; i < listKvp?.Count; i++) + foreach (var entry in logFields) { - var entry = listKvp[i]; - // Iteration #1 - Get those fields which become dedicated columns // i.e all Part B fields and opt-in Part C fields. if (entry.Key == "{OriginalFormat}") @@ -369,27 +416,44 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) cursor = dataForScopes.Cursor; cntFields = dataForScopes.FieldsCount; - if (hasEnvProperties) + if (hasEnvProperties || this.propertiesEntries.Count > 0) { - // Iteration #2 - Get all "other" fields and collapse them into single field - // named "env_properties". + // Anything that's not a dedicated field gets put into a part C field called "env_properties". ushort envPropertiesCount = 0; cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_properties"); cursor = MessagePackSerializer.WriteMapHeader(buffer, cursor, ushort.MaxValue); var idxMapSizeEnvPropertiesPatch = cursor - 2; - for (var i = 0; i < listKvp!.Count; i++) + + if (hasEnvProperties) { - var entry = listKvp[i]; - if (entry.Key == "{OriginalFormat}" || this.customFields!.Contains(entry.Key)) + foreach (var entry in logFields) { - continue; + if (entry.Key == "{OriginalFormat}" || this.customFields!.Contains(entry.Key)) + { + continue; + } + else + { + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key, this.stringFieldSizeLimitCharCount); + cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); + envPropertiesCount += 1; + } } - else + } + + foreach (var entry in this.propertiesEntries) + { + // A prepopulated env_properties entry should not be added if the same key exists in the log, + // and lack of customFields configuration would place it in env_properties. + if (this.customFields != null && !this.customFields.Contains(entry.Key) + && logFields.Any(kvp => kvp.Key == entry.Key)) { - cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key, this.stringFieldSizeLimitCharCount); - cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); - envPropertiesCount += 1; + continue; } + + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key); + cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); + envPropertiesCount += 1; } // Prepare state for scopes diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs index 1c1bef453d..477d1109d8 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using OpenTelemetry.Exporter.Geneva.MsgPack; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; /* BenchmarkDotNet v0.13.10, Windows 11 (10.0.23424.1000) @@ -74,16 +75,18 @@ public LogExporterBenchmarks() // For msgpack serialization + export this.logRecord = GenerateTestLogRecord(); this.batch = GenerateTestLogRecordBatch(); - this.exporter = new MsgPackLogExporter(new GenevaExporterOptions - { - ConnectionString = "EtwSession=OpenTelemetry", - PrepopulatedFields = new Dictionary - { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", - ["cloud.roleVer"] = "9.0.15289.2", - }, - }); + this.exporter = new MsgPackLogExporter( + new GenevaExporterOptions + { + ConnectionString = "EtwSession=OpenTelemetry", + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, + }, + () => Resource.Empty); } [Benchmark] diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/TLDLogExporterBenchmarks.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/TLDLogExporterBenchmarks.cs index 4256741e44..8808c1d016 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/TLDLogExporterBenchmarks.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/TLDLogExporterBenchmarks.cs @@ -6,6 +6,7 @@ using OpenTelemetry.Exporter.Geneva.MsgPack; using OpenTelemetry.Exporter.Geneva.Tld; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; using OpenTelemetry.Trace; /* @@ -37,16 +38,18 @@ public class TLDLogExporterBenchmarks public TLDLogExporterBenchmarks() { - this.msgPackExporter = new MsgPackLogExporter(new GenevaExporterOptions - { - ConnectionString = "EtwSession=OpenTelemetry", - PrepopulatedFields = new Dictionary + this.msgPackExporter = new MsgPackLogExporter( + new GenevaExporterOptions { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", - ["cloud.roleVer"] = "9.0.15289.2", + ConnectionString = "EtwSession=OpenTelemetry", + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, }, - }); + () => Resource.Empty); this.tldExporter = new TldLogExporter(new GenevaExporterOptions() { diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index f1e07b7138..d367b98aa2 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging; using OpenTelemetry.Exporter.Geneva.MsgPack; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; using OpenTelemetry.Tests; using Xunit; @@ -179,7 +180,7 @@ public void TableNameMappingTest(params string[] category) .AddFilter("*", LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); ILogger logger; object fluentdData; @@ -298,7 +299,7 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() .AddFilter("*", LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); ILogger passThruTableMappingsLogger, userInitializedTableMappingsLogger; var m_buffer = MsgPackLogExporter.Buffer; @@ -473,7 +474,7 @@ public void SerializeILoggerScopes(bool hasCustomFields) Assert.Single(exportedItems); var logRecord = exportedItems[0]; - this.AssertFluentdForwardModeForLogRecord(exporterOptions, fluentdData, logRecord); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, Resource.Empty, fluentdData, logRecord); } finally { @@ -532,7 +533,7 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -666,9 +667,12 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin ["cloud.role"] = "BusyWorker", ["cloud.roleInstance"] = "CY1SCH030021417", ["cloud.roleVer"] = "9.0.15289.2", + ["prepopulated"] = "prepopulated field", }, }; + var resource = ResourceBuilder.CreateEmpty().AddAttributes([new KeyValuePair("resourceAttr", "from resource")]).Build(); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; @@ -713,7 +717,7 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => resource); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -764,7 +768,154 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin { _ = exporter.SerializeLogRecord(logRecord); var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); - this.AssertFluentdForwardModeForLogRecord(exporterOptions, fluentdData, logRecord); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resource, fluentdData, logRecord); + } + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Theory] + [InlineData(false, true, true, false)] + [InlineData(false, true, true, true)] + [InlineData(true, false, true, false)] + [InlineData(true, false, true, true)] + [InlineData(true, true, false, false)] + [InlineData(true, true, false, true)] + [InlineData(true, true, true, false)] + [InlineData(true, true, true, true)] + public void SerializationTestWithDuplicateFields(bool conflictingPrepopulatedField, bool conflictingResourceAttribute, bool conflictingLogField, bool isCustomField) + { + var path = string.Empty; + Socket server = null; + var logRecordList = new List(); + try + { + var exporterOptions = new GenevaExporterOptions(); + if (conflictingPrepopulatedField) + { + exporterOptions.PrepopulatedFields = new Dictionary + { + ["Conflict"] = "prepopulated field", + }; + } + + var resource = Resource.Empty; + if (conflictingResourceAttribute) + { + resource = ResourceBuilder.CreateEmpty().AddAttributes([new KeyValuePair("Conflict", "resource attribute")]).Build(); + } + + exporterOptions.CustomFields = []; + if (isCustomField) + { + exporterOptions.CustomFields = ["Conflict"]; + } + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => options.AddInMemoryExporter(logRecordList)) + .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels + var logger = loggerFactory.CreateLogger(); + + if (conflictingLogField) + { + logger.Log(LogLevel.Trace, 101, "Log a message with a {Conflict} with other fields", "log field"); + } + else + { + logger.Log(LogLevel.Trace, 101, "Log a normal message without conflict"); + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + path = GenerateTempFilePath(); + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + using var exporter = new MsgPackLogExporter(exporterOptions, () => resource); + var m_buffer = MsgPackLogExporter.Buffer; + Assert.Single(logRecordList); + var serializedLog = exporter.SerializeLogRecord(logRecordList[0]); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var signal = (fluentdData as object[])[0] as string; + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var timeStamp = (DateTime)(TimeStampAndMappings as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + var env_properties = mapping.GetValueOrDefault("env_properties") as Dictionary ?? []; + + void AssertField(bool isDedicated, string fieldValue) + { + if (isDedicated) + { + Assert.Contains("Conflict", mapping); + Assert.Equal(fieldValue, mapping["Conflict"]); + } + else + { + Assert.Contains("Conflict", env_properties); + Assert.Equal(fieldValue, env_properties["Conflict"]); + } + } + + if (isCustomField) + { + // If Conflict is marked as a custom field, it should never appear in env_properties + Assert.DoesNotContain("Conflict", env_properties); + + // If Conflict is marked as a custom field, + // then the conflict will occur in a dedicated field. + // Log field has highest precedence, followed by prepopulated field + if (conflictingLogField) + { + AssertField(true, "log field"); + } + else if (conflictingPrepopulatedField) + { + AssertField(true, "prepopulated field"); + } + + // no need to check resource attribute because it will never win a conflict + } + else + { + // Prepopulated fields are unaffected by not being marked as custom fields + if (conflictingPrepopulatedField) + { + AssertField(true, "prepopulated field"); + } + else + { + Assert.DoesNotContain("Conflict", mapping); + } + + // If Conflict is not marked as a custom field, + // log fields and resource attributes conflict in env_properties, + // and log fields have precedence + if (conflictingLogField) + { + AssertField(false, "log field"); + } + else if (conflictingResourceAttribute) + { + AssertField(false, "resource attribute"); + } } } finally @@ -844,16 +995,18 @@ public void SuccessfulExport_Linux() serverSocket.ReceiveTimeout = 10000; // Create a test exporter to get MessagePack byte data for validation of the data received via Socket. - using var exporter = new MsgPackLogExporter(new GenevaExporterOptions - { - ConnectionString = "Endpoint=unix:" + path, - PrepopulatedFields = new Dictionary + using var exporter = new MsgPackLogExporter( + new GenevaExporterOptions { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", - ["cloud.roleVer"] = "9.0.15289.2", + ConnectionString = "Endpoint=unix:" + path, + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, }, - }); + () => Resource.Empty); // Emit a LogRecord and grab a copy of internal buffer for validation. var logger = loggerFactory.CreateLogger(); @@ -938,7 +1091,7 @@ public void SerializationTestForException() .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1031,7 +1184,7 @@ public void SerializationTestForEventName(EventNameExportMode eventNameExportMod })); // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1158,7 +1311,7 @@ public void SerializationTestForPartBName(bool hasCustomFields, bool hasNameInCu })); // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1273,7 +1426,7 @@ public void SerializationTestForEventId() .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1508,7 +1661,7 @@ private static object GetField(object fluentdData, string key) return mapping.TryGetValue(key, out var value) ? value : null; } - private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporterOptions, object fluentdData, LogRecord logRecord) + private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporterOptions, Resource resource, object fluentdData, LogRecord logRecord) { /* Fluentd Forward Mode: [ @@ -1559,7 +1712,7 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter foreach (var item in exporterOptions.PrepopulatedFields) { var partAValue = item.Value as string; - var partAKey = MsgPackExporter.V40_PART_A_MAPPING[item.Key]; + var partAKey = MsgPackExporter.V40_PART_A_MAPPING.GetValueOrDefault(item.Key, item.Key); Assert.Equal(partAValue, mapping[partAKey]); } @@ -1636,14 +1789,45 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter } else if (exporterOptions.CustomFields == null || exporterOptions.CustomFields.Contains(item.Key)) { + // It should be found as a custom field + if (item.Value != null) + { + Assert.Equal(item.Value, mapping[item.Key]); + } + + if (envPropertiesMapping != null) + { + Assert.DoesNotContain(item.Key, envPropertiesMapping.Keys); + } + } + else + { + // It should be found in env_properties + Assert.Equal(item.Value, envPropertiesMapping[item.Key]); + Assert.DoesNotContain(item.Key, mapping); + } + } + + foreach (var item in resource.Attributes) + { + if (exporterOptions.CustomFields == null || exporterOptions.CustomFields.Contains(item.Key)) + { + // It should be found as a custom field if (item.Value != null) { Assert.Equal(item.Value, mapping[item.Key]); } + + if (envPropertiesMapping != null) + { + Assert.DoesNotContain(item.Key, envPropertiesMapping.Keys); + } } else { + // It should be found in env_properties Assert.Equal(item.Value, envPropertiesMapping[item.Key]); + Assert.DoesNotContain(item.Key, mapping); } } } @@ -1653,7 +1837,7 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter Assert.Equal(logRecord.EventId.Id, int.Parse(mapping["eventId"].ToString(), CultureInfo.InvariantCulture)); } - // Epilouge + // Epilogue Assert.Equal("DateTime", timeFormat["TimeFormat"]); } diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs index 0c9758db87..ca2ecd2868 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging; using OpenTelemetry.Exporter.Geneva.MsgPack; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; using Xunit; namespace OpenTelemetry.Exporter.Geneva.Tests; @@ -110,7 +111,7 @@ private static Dictionary GetExportedFieldsAfterLogging(Action Resource.Empty); _ = exporter.SerializeLogRecord(logRecordList[0]); var fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/MsgPackLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/MsgPackLogExporterTests.cs index e42e4427e3..3d2a63f95a 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/MsgPackLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/MsgPackLogExporterTests.cs @@ -4,6 +4,7 @@ using System.Net.Sockets; using System.Runtime.InteropServices; using OpenTelemetry.Exporter.Geneva.MsgPack; +using OpenTelemetry.Resources; using Xunit; namespace OpenTelemetry.Exporter.Geneva.Tests; @@ -55,7 +56,7 @@ public void StringSizeLimit_Default_Success() { ConnectionString = this.connectionString, }; - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); } [Fact] @@ -65,7 +66,7 @@ public void StringSizeLimit_Valid_Success() { ConnectionString = this.connectionString + ";PrivatePreviewLogMessagePackStringSizeLimit=65360", }; - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); } private static string GenerateTempFilePath() From d6c3940510f466448c35843b9de993ae2e84f2b3 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Fri, 14 Nov 2025 12:37:24 -0800 Subject: [PATCH 02/20] fix indentation --- .../Exporter/LogExporterBenchmarks.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs index 477d1109d8..7c539c56ed 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs @@ -77,16 +77,16 @@ public LogExporterBenchmarks() this.batch = GenerateTestLogRecordBatch(); this.exporter = new MsgPackLogExporter( new GenevaExporterOptions + { + ConnectionString = "EtwSession=OpenTelemetry", + PrepopulatedFields = new Dictionary { - ConnectionString = "EtwSession=OpenTelemetry", - PrepopulatedFields = new Dictionary - { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", - ["cloud.roleVer"] = "9.0.15289.2", - }, + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", }, - () => Resource.Empty); + }, + () => Resource.Empty); } [Benchmark] From 1b8bec5bfc06999e2e4705703443c0093eaed4c7 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Fri, 14 Nov 2025 13:09:51 -0800 Subject: [PATCH 03/20] add tests for cloud extension --- .../GenevaLogExporterTests.cs | 42 ++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index d367b98aa2..beffbbb9d3 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -664,14 +664,16 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin { PrepopulatedFields = new Dictionary { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", ["cloud.roleVer"] = "9.0.15289.2", ["prepopulated"] = "prepopulated field", }, }; - var resource = ResourceBuilder.CreateEmpty().AddAttributes([new KeyValuePair("resourceAttr", "from resource")]).Build(); + var resource = ResourceBuilder.CreateEmpty().AddAttributes([ + new KeyValuePair("resourceAttr", "from resource"), + new KeyValuePair("service.name", "BusyWorker"), + new KeyValuePair("service.instanceId", "CY1SCH030021417")]) + .Build(); if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { @@ -858,7 +860,11 @@ public void SerializationTestWithDuplicateFields(bool conflictingPrepopulatedFie var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; var timeStamp = (DateTime)(TimeStampAndMappings as object[])[0]; var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; - var env_properties = mapping.GetValueOrDefault("env_properties") as Dictionary ?? []; + var env_properties = new Dictionary(); + if (mapping.ContainsKey("env_properties")) + { + env_properties = mapping["env_properties"] as Dictionary ?? []; + } void AssertField(bool isDedicated, string fieldValue) { @@ -1712,7 +1718,12 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter foreach (var item in exporterOptions.PrepopulatedFields) { var partAValue = item.Value as string; - var partAKey = MsgPackExporter.V40_PART_A_MAPPING.GetValueOrDefault(item.Key, item.Key); + var partAKey = item.Key; + if (MsgPackExporter.V40_PART_A_MAPPING.ContainsKey(item.Key)) + { + partAKey = MsgPackExporter.V40_PART_A_MAPPING[item.Key]; + } + Assert.Equal(partAValue, mapping[partAKey]); } @@ -1737,6 +1748,21 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter Assert.Equal(logRecord.Exception.Message, mapping["env_ex_msg"]); } + // Part A cloud extensions + var serviceNameField = resource.Attributes.FirstOrDefault(attr => attr.Key == "service.name"); + if (serviceNameField.Key == "service.name" && !exporterOptions.PrepopulatedFields.ContainsKey("cloud.role")) + { + Assert.Contains("env_cloud_role", mapping); + Assert.Equal(serviceNameField.Value, mapping["env_cloud_role"]); + } + + var serviceInstanceField = resource.Attributes.FirstOrDefault(attr => attr.Key == "service.instanceId"); + if (serviceInstanceField.Key == "service.instanceId" && !exporterOptions.PrepopulatedFields.ContainsKey("cloud.roleInstance")) + { + Assert.Contains("env_cloud_roleInstance", mapping); + Assert.Equal(serviceInstanceField.Value, mapping["env_cloud_roleInstance"]); + } + // Part B fields // `LogRecord.LogLevel` was marked Obsolete in https://github.com/open-telemetry/opentelemetry-dotnet/pull/4568 @@ -1810,6 +1836,12 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter foreach (var item in resource.Attributes) { + if (item.Key == "service.name" || item.Key == "service.instanceId") + { + // these ones are already checked. + continue; + } + if (exporterOptions.CustomFields == null || exporterOptions.CustomFields.Contains(item.Key)) { // It should be found as a custom field From 604fd2bb0d7df9d58be28ecbc133123396170522 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Wed, 17 Dec 2025 15:01:34 -0800 Subject: [PATCH 04/20] a bunch of changes to re-up the branch --- .../GenevaLogExporter.cs | 17 +- .../Internal/MsgPack/MsgPackLogExporter.cs | 189 +++++++------ .../Internal/MsgPack/MsgPackTraceExporter.cs | 24 +- .../GenevaLogExporterAFDCorrelationTests.cs | 12 +- .../GenevaLogExporterTests.cs | 251 +++++------------- .../GenevaTraceExporterTests.cs | 78 ++++-- .../LogSerializationTests.cs | 2 +- 7 files changed, 266 insertions(+), 307 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs index be75191637..cbb83e8f23 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs @@ -17,10 +17,10 @@ namespace OpenTelemetry.Exporter.Geneva; /// public class GenevaLogExporter : GenevaBaseExporter { + internal readonly IDisposable Exporter; internal bool IsUsingUnixDomainSocket; private readonly ExportLogRecordFunc exportLogRecord; - private readonly IDisposable exporter; private bool isDisposed; @@ -46,7 +46,7 @@ public GenevaLogExporter(GenevaExporterOptions options) var eventHeaderLogExporter = new EventHeaderLogExporter(options); this.IsUsingUnixDomainSocket = false; this.exportLogRecord = eventHeaderLogExporter.Export; - this.exporter = eventHeaderLogExporter; + this.Exporter = eventHeaderLogExporter; return; #else throw new ArgumentException("Exporting data in user_events is only supported for .NET 8 or later on Linux."); @@ -88,24 +88,19 @@ public GenevaLogExporter(GenevaExporterOptions options) throw new NotSupportedException($"Protocol '{connectionStringBuilder.Protocol}' is not supported"); } - Resources.Resource ResourceProvider() - { - return connectionStringBuilder.HonorResourceAttributes ? this.ParentProvider.GetResource() : Resources.Resource.Empty; - } - if (useMsgPackExporter) { - var msgPackLogExporter = new MsgPackLogExporter(options, ResourceProvider); + var msgPackLogExporter = new MsgPackLogExporter(options, this.ParentProvider.GetResource); this.IsUsingUnixDomainSocket = msgPackLogExporter.IsUsingUnixDomainSocket; this.exportLogRecord = msgPackLogExporter.Export; - this.exporter = msgPackLogExporter; + this.Exporter = msgPackLogExporter; } else { var tldLogExporter = new TldLogExporter(options); this.IsUsingUnixDomainSocket = false; this.exportLogRecord = tldLogExporter.Export; - this.exporter = tldLogExporter; + this.Exporter = tldLogExporter; } } @@ -129,7 +124,7 @@ protected override void Dispose(bool disposing) { try { - this.exporter.Dispose(); + this.Exporter.Dispose(); } catch (Exception ex) { diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs index 10a12e9c8d..e7f62c4ec1 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs @@ -20,7 +20,7 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable { public const int BUFFER_SIZE = 65360; // the maximum ETW payload (inclusive) - internal static readonly ThreadLocal Buffer = new(); + internal readonly ThreadLocal Buffer = new(); private static readonly Action ProcessScopeForIndividualColumnsAction = OnProcessScopeForIndividualColumns; private static readonly Action ProcessScopeForEnvPropertiesAction = OnProcessScopeForEnvProperties; @@ -39,15 +39,11 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable #endif private readonly ExceptionStackExportMode exportExceptionStack; + private readonly Dictionary? prepopulatedFields; + private readonly IEnumerable? resourceFieldNames; private readonly byte[] bufferEpilogue; private readonly IDataTransport dataTransport; private readonly Func resourceProvider; - - // These are values that are always added to the body as dedicated fields - private readonly Dictionary prepopulatedFields; - - // These are values that are always added to env_properties - private readonly Dictionary propertiesEntries; private readonly int stringFieldSizeLimitCharCount; // the maximum string size limit for MsgPack strings // This is used for Scopes @@ -97,12 +93,28 @@ public MsgPackLogExporter(GenevaExporterOptions options, Func resource this.stringFieldSizeLimitCharCount = connectionStringBuilder.PrivatePreviewLogMessagePackStringSizeLimit; - this.propertiesEntries = []; + if (options.PrepopulatedFields != null && options.PrepopulatedFields.Count > 0 && options.ResourceFieldNames != null) + { + throw new ArgumentException("PrepopulatedFields and ResourceFieldNames are mutually exclusive options"); + } + + if (options.ResourceFieldNames != null) + { + foreach (var wantedResourceAttribute in options.ResourceFieldNames) + { + if (PART_A_MAPPING_DICTIONARY.Values.Contains(wantedResourceAttribute)) + { + throw new ArgumentException($"'{wantedResourceAttribute}' cannot be specified through a resource attribute. Remove it from ResourceFieldNames"); + } + } - this.prepopulatedFields = new Dictionary(options.PrepopulatedFields.Count, StringComparer.Ordinal); + this.prepopulatedFields = new Dictionary(0, StringComparer.Ordinal); + this.resourceFieldNames = options.ResourceFieldNames; + } if (options.PrepopulatedFields != null) { + this.prepopulatedFields = new Dictionary(options.PrepopulatedFields.Count, StringComparer.Ordinal); foreach (var kv in options.PrepopulatedFields) { this.prepopulatedFields[kv.Key] = kv.Value; @@ -164,11 +176,11 @@ public void Dispose() return; } - // DO NOT Dispose m_buffer as it is a static type try { (this.dataTransport as IDisposable)?.Dispose(); this.serializationData.Dispose(); + this.Buffer.Dispose(); } catch (Exception ex) { @@ -178,42 +190,85 @@ public void Dispose() this.isDisposed = true; } + /// + /// Updates the prepopulatedFields field to include resource attributes only available at runtime. + /// This function needs to be idempotent in case it's accidentally called twice. + /// internal void AddResourceAttributesToPrepopulated() { - // This function needs to be idempotent + Guard.ThrowIfNull(this.prepopulatedFields); + + var resourceAttributes = this.resourceProvider().Attributes; - foreach (var entry in this.resourceProvider().Attributes) + foreach (var resourceAttribute in resourceAttributes) { - string key = entry.Key; - bool isDedicatedField = false; - if (entry.Value is string) + var key = resourceAttribute.Key; + var value = resourceAttribute.Value; + + var isWantedAttribute = false; + if (this.resourceFieldNames != null) { - switch (key) + // this might seem inefficient, but it's only run once and I don't expect there to be many resource attributes + foreach (var wantedAttribute in this.resourceFieldNames!) { - case "service.name": - key = Schema.V40.PartA.Extensions.Cloud.Role; - isDedicatedField = true; - break; - case "service.instanceId": - key = Schema.V40.PartA.Extensions.Cloud.RoleInstance; - isDedicatedField = true; + if (wantedAttribute == key) + { + switch (value) + { + case bool: + case byte: + case sbyte: + case short: + case ushort: + case int: + case uint: + case long: + case ulong: + case float: + case double: + case string: + break; + case null: + // This should be impossible because Resource attributes cannot have null values. + // But just in case, turn it into something serializable to avoid crashing. + value = ""; + break; + default: + // Try to construct a value that communicates that the type is not supported. + try + { + var stringValue = Convert.ToString(value, CultureInfo.InvariantCulture); + value = stringValue == null ? "" : $""; + } + catch + { + value = ""; + } + + break; + } + + isWantedAttribute = true; break; + } } } - if (isDedicatedField || this.customFields == null || this.customFields.Contains(key)) + if (key == "service.name") { - if (!this.prepopulatedFields.ContainsKey(key)) - { - this.prepopulatedFields.Add(key, entry.Value); - } + key = Schema.V40.PartA.Extensions.Cloud.Role; + isWantedAttribute = true; } - else + + if (key == "service.instanceId") { - if (!this.propertiesEntries.ContainsKey(key)) - { - this.propertiesEntries.Add(key, entry.Value); - } + key = Schema.V40.PartA.Extensions.Cloud.RoleInstance; + isWantedAttribute = true; + } + + if (isWantedAttribute) + { + this.prepopulatedFields.Add(key, value); } } } @@ -230,11 +285,17 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) else { // Attempt to see if State could be ROL_KVP. - logFields = logRecord.State as IReadOnlyList> ?? []; + logFields = logRecord.State as IReadOnlyList>; } #pragma warning restore 0618 - var buffer = Buffer.Value ??= new byte[BUFFER_SIZE]; // TODO: handle OOM + var buffer = this.Buffer.Value; + if (buffer == null) + { + this.AddResourceAttributesToPrepopulated(); + buffer = new byte[BUFFER_SIZE]; // TODO: handle OOM + this.Buffer.Value = buffer; + } /* Fluentd Forward Mode: [ @@ -271,20 +332,13 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) ushort cntFields = 0; var idxMapSizePatch = cursor - 2; - this.AddResourceAttributesToPrepopulated(); - - foreach (var entry in this.prepopulatedFields) + if (this.prepopulatedFields != null) { - // A prepopulated entry should not be added if the same key exists in the log, - // and customFields configuration would make it a dedicated field. - if ((this.customFields == null || this.customFields.Contains(entry.Key)) - && logFields.Any(kvp => kvp.Key == entry.Key)) + foreach (var field in this.prepopulatedFields) { - continue; + cursor = AddPartAField(buffer, cursor, field.Key, field.Value); + cntFields += 1; } - - cursor = AddPartAField(buffer, cursor, entry.Key, entry.Value); - cntFields += 1; } // Part A - core envelope @@ -347,8 +401,10 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) var hasEnvProperties = false; var bodyPopulated = false; var namePopulated = false; - foreach (var entry in logFields) + for (var i = 0; i < logFields?.Count; i++) { + var entry = logFields[i]; + // Iteration #1 - Get those fields which become dedicated columns // i.e all Part B fields and opt-in Part C fields. if (entry.Key == "{OriginalFormat}") @@ -416,44 +472,27 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) cursor = dataForScopes.Cursor; cntFields = dataForScopes.FieldsCount; - if (hasEnvProperties || this.propertiesEntries.Count > 0) + if (hasEnvProperties) { - // Anything that's not a dedicated field gets put into a part C field called "env_properties". + // Iteration #2 - Get all "other" fields and collapse them into single field + // named "env_properties". ushort envPropertiesCount = 0; cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_properties"); cursor = MessagePackSerializer.WriteMapHeader(buffer, cursor, ushort.MaxValue); var idxMapSizeEnvPropertiesPatch = cursor - 2; - - if (hasEnvProperties) + for (var i = 0; i < logFields!.Count; i++) { - foreach (var entry in logFields) + var entry = logFields[i]; + if (entry.Key == "{OriginalFormat}" || this.customFields!.Contains(entry.Key)) { - if (entry.Key == "{OriginalFormat}" || this.customFields!.Contains(entry.Key)) - { - continue; - } - else - { - cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key, this.stringFieldSizeLimitCharCount); - cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); - envPropertiesCount += 1; - } + continue; } - } - - foreach (var entry in this.propertiesEntries) - { - // A prepopulated env_properties entry should not be added if the same key exists in the log, - // and lack of customFields configuration would place it in env_properties. - if (this.customFields != null && !this.customFields.Contains(entry.Key) - && logFields.Any(kvp => kvp.Key == entry.Key)) + else { - continue; + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key, this.stringFieldSizeLimitCharCount); + cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); + envPropertiesCount += 1; } - - cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key); - cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); - envPropertiesCount += 1; } // Prepare state for scopes diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs index 13628d5a7e..606f267f60 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs @@ -133,6 +133,11 @@ public MsgPackTraceExporter(GenevaExporterOptions options, Func resour throw new NotSupportedException($"Protocol '{connectionStringBuilder.Protocol}' is not supported"); } + if (options.PrepopulatedFields != null && options.PrepopulatedFields.Count > 0 && options.ResourceFieldNames != null) + { + throw new ArgumentException("PrepopulatedFields and ResourceFieldNames are mutually exclusive options"); + } + if (options.ResourceFieldNames != null) { foreach (var wantedResourceAttribute in options.ResourceFieldNames) @@ -144,6 +149,15 @@ public MsgPackTraceExporter(GenevaExporterOptions options, Func resour } } + this.prepopulatedFields = new Dictionary(0, StringComparer.Ordinal); + if (options.PrepopulatedFields != null) + { + foreach (var entry in options.PrepopulatedFields) + { + this.prepopulatedFields.Add(entry.Key, entry.Value); + } + } + // TODO: Validate custom fields (reserved name? etc). if (options.CustomFields != null) { @@ -187,12 +201,6 @@ public MsgPackTraceExporter(GenevaExporterOptions options, Func resour #endif } - this.prepopulatedFields = []; - foreach (var entry in options.PrepopulatedFields) - { - this.prepopulatedFields.Add(entry.Key, entry.Value); - } - this.resourceFieldNames = options.ResourceFieldNames; this.shouldIncludeTraceState = options.IncludeTraceStateForSpan; } @@ -338,8 +346,8 @@ internal void CreateFraming() if (this.resourceFieldNames != null) { - // if ResourceFieldNames is set, it overrides the existing prepopulated fields setting. - this.prepopulatedFields = []; + // if ResourceFieldNames is set, we use resource attributes rather than PrepopulatedFields + this.prepopulatedFields = new Dictionary(0, StringComparer.Ordinal); } // this is guaranteed to not be null because it's set in the constructor diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs index 4202aff167..ffc6ef74f4 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs @@ -93,7 +93,8 @@ public void AFDCorrelationIdLogProcessor_MultithreadedAccess_HandlesGracefully() byte[] serializedData; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - serializedData = MsgPackLogExporter.Buffer.Value; + var msgPackExporter = exporter.Exporter as MsgPackLogExporter; + serializedData = msgPackExporter.Buffer.Value; } else { @@ -123,7 +124,8 @@ public void AFDCorrelationIdLogProcessor_MultithreadedAccess_HandlesGracefully() byte[] serializedData; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - serializedData = MsgPackLogExporter.Buffer.Value; + var msgPackExporter = exporter.Exporter as MsgPackLogExporter; + serializedData = msgPackExporter.Buffer.Value; } else { @@ -232,7 +234,8 @@ public void AFDCorrelationIdLogProcessor_WithoutCorrelationId_HandlesGracefully( byte[] serializedData; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - serializedData = MsgPackLogExporter.Buffer.Value; + var msgPackExporter = exporter.Exporter as MsgPackLogExporter; + serializedData = msgPackExporter.Buffer.Value; } else { @@ -323,7 +326,8 @@ public void GenevaExporter_WithAFDCorrelationId_IncludesCorrelationId() byte[] serializedData; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - serializedData = MsgPackLogExporter.Buffer.Value; + var msgPackExporter = exporter.Exporter as MsgPackLogExporter; + serializedData = msgPackExporter.Buffer.Value; } else { diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index beffbbb9d3..137dd8912a 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -21,13 +21,60 @@ namespace OpenTelemetry.Exporter.Geneva.Tests; public class GenevaLogExporterTests { + private static string GetRandomFilePath() + { + while (true) + { + var path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + if (!File.Exists(path)) + { + return path; + } + } + } + [Fact] public void BadArgs() { - GenevaExporterOptions exporterOptions = null; + string connectionString; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + connectionString = "EtwSession=OpenTelemetry"; + } + else + { + var path = GetRandomFilePath(); + connectionString = "Endpoint=unix:" + path; + } + + // should reject null exporter options Assert.Throws(() => { - using var exporter = new GenevaLogExporter(exporterOptions); + using var exporter = new GenevaLogExporter(null); + }); + + // reserved field in ResourceFieldNames + Assert.Throws(() => + { + using var exporter = new GenevaTraceExporter(new GenevaExporterOptions + { + ConnectionString = connectionString, + ResourceFieldNames = ["env_cloud_role"], + }); + }); + + // should reject mutually exclusive ResourceFieldNames and PrepopulatedFields + Assert.Throws(() => + { + using var exporter = new GenevaTraceExporter(new GenevaExporterOptions + { + ConnectionString = connectionString, + ResourceFieldNames = ["resource"], + PrepopulatedFields = new Dictionary + { + ["prepopulated"] = "hello", + }, + }); }); } @@ -197,7 +244,7 @@ public void TableNameMappingTest(params string[] category) Assert.Single(logRecordList); _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); actualTableName = (fluentdData as object[])[0] as string; Assert.Equal(mapping.Value, actualTableName); logRecordList.Clear(); @@ -214,7 +261,7 @@ public void TableNameMappingTest(params string[] category) Assert.Single(logRecordList); _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); actualTableName = (fluentdData as object[])[0] as string; Assert.Equal(defaultLogTable, actualTableName); logRecordList.Clear(); @@ -302,7 +349,7 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); ILogger passThruTableMappingsLogger, userInitializedTableMappingsLogger; - var m_buffer = MsgPackLogExporter.Buffer; + var m_buffer = exporter.Buffer; object fluentdData; string actualTableName; @@ -423,7 +470,8 @@ public void SerializeILoggerScopes(bool hasCustomFields) byte[] serializedData; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - serializedData = MsgPackLogExporter.Buffer.Value; + var msgPackExporter = exporter.Exporter as MsgPackLogExporter; + serializedData = msgPackExporter.Buffer.Value; } else { @@ -554,7 +602,7 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) // VALIDATE Assert.Single(logRecordList); _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); var body = GetField(fluentdData, "body"); // Body gets populated as "Formatted Message" regardless of the value of `IncludeFormattedMessage` @@ -578,7 +626,7 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) // VALIDATE Assert.Single(logRecordList); _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); body = GetField(fluentdData, "body"); // Body gets populated as "Formatted Message" regardless of the value of `IncludeFormattedMessage` @@ -599,7 +647,7 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) // VALIDATE Assert.Single(logRecordList); _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); body = GetField(fluentdData, "body"); // Even though Formatter is null, body is populated with the state @@ -623,7 +671,7 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) // VALIDATE Assert.Single(logRecordList); _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); Assert.Equal("Value1", GetField(fluentdData, "Key1")); body = GetField(fluentdData, "body"); @@ -666,6 +714,7 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin { ["cloud.roleVer"] = "9.0.15289.2", ["prepopulated"] = "prepopulated field", + ["test"] = $"{hasTableNameMapping}{hasCustomFields}{parseStateValues}", }, }; @@ -708,11 +757,6 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddGenevaLogExporter(options => - { - options.ConnectionString = exporterOptions.ConnectionString; - options.PrepopulatedFields = exporterOptions.PrepopulatedFields; - }); options.AddInMemoryExporter(logRecordList); options.ParseStateValues = parseStateValues; }) @@ -764,7 +808,7 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin // logRecordList should have 14 logRecord entries as there were 14 Log calls Assert.Equal(14, logRecordList.Count); - var m_buffer = MsgPackLogExporter.Buffer; + var m_buffer = exporter.Buffer; foreach (var logRecord in logRecordList) { @@ -786,157 +830,6 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin } } - [Theory] - [InlineData(false, true, true, false)] - [InlineData(false, true, true, true)] - [InlineData(true, false, true, false)] - [InlineData(true, false, true, true)] - [InlineData(true, true, false, false)] - [InlineData(true, true, false, true)] - [InlineData(true, true, true, false)] - [InlineData(true, true, true, true)] - public void SerializationTestWithDuplicateFields(bool conflictingPrepopulatedField, bool conflictingResourceAttribute, bool conflictingLogField, bool isCustomField) - { - var path = string.Empty; - Socket server = null; - var logRecordList = new List(); - try - { - var exporterOptions = new GenevaExporterOptions(); - if (conflictingPrepopulatedField) - { - exporterOptions.PrepopulatedFields = new Dictionary - { - ["Conflict"] = "prepopulated field", - }; - } - - var resource = Resource.Empty; - if (conflictingResourceAttribute) - { - resource = ResourceBuilder.CreateEmpty().AddAttributes([new KeyValuePair("Conflict", "resource attribute")]).Build(); - } - - exporterOptions.CustomFields = []; - if (isCustomField) - { - exporterOptions.CustomFields = ["Conflict"]; - } - - using var loggerFactory = LoggerFactory.Create(builder => builder - .AddOpenTelemetry(options => options.AddInMemoryExporter(logRecordList)) - .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels - var logger = loggerFactory.CreateLogger(); - - if (conflictingLogField) - { - logger.Log(LogLevel.Trace, 101, "Log a message with a {Conflict} with other fields", "log field"); - } - else - { - logger.Log(LogLevel.Trace, 101, "Log a normal message without conflict"); - } - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; - } - else - { - path = GenerateTempFilePath(); - exporterOptions.ConnectionString = "Endpoint=unix:" + path; - var endpoint = new UnixDomainSocketEndPoint(path); - server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); - server.Bind(endpoint); - server.Listen(1); - } - - using var exporter = new MsgPackLogExporter(exporterOptions, () => resource); - var m_buffer = MsgPackLogExporter.Buffer; - Assert.Single(logRecordList); - var serializedLog = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); - var signal = (fluentdData as object[])[0] as string; - var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; - var timeStamp = (DateTime)(TimeStampAndMappings as object[])[0]; - var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; - var env_properties = new Dictionary(); - if (mapping.ContainsKey("env_properties")) - { - env_properties = mapping["env_properties"] as Dictionary ?? []; - } - - void AssertField(bool isDedicated, string fieldValue) - { - if (isDedicated) - { - Assert.Contains("Conflict", mapping); - Assert.Equal(fieldValue, mapping["Conflict"]); - } - else - { - Assert.Contains("Conflict", env_properties); - Assert.Equal(fieldValue, env_properties["Conflict"]); - } - } - - if (isCustomField) - { - // If Conflict is marked as a custom field, it should never appear in env_properties - Assert.DoesNotContain("Conflict", env_properties); - - // If Conflict is marked as a custom field, - // then the conflict will occur in a dedicated field. - // Log field has highest precedence, followed by prepopulated field - if (conflictingLogField) - { - AssertField(true, "log field"); - } - else if (conflictingPrepopulatedField) - { - AssertField(true, "prepopulated field"); - } - - // no need to check resource attribute because it will never win a conflict - } - else - { - // Prepopulated fields are unaffected by not being marked as custom fields - if (conflictingPrepopulatedField) - { - AssertField(true, "prepopulated field"); - } - else - { - Assert.DoesNotContain("Conflict", mapping); - } - - // If Conflict is not marked as a custom field, - // log fields and resource attributes conflict in env_properties, - // and log fields have precedence - if (conflictingLogField) - { - AssertField(false, "log field"); - } - else if (conflictingResourceAttribute) - { - AssertField(false, "resource attribute"); - } - } - } - finally - { - server?.Dispose(); - try - { - File.Delete(path); - } - catch - { - } - } - } - [SkipUnlessPlatformMatchesFact(TestPlatform.Windows)] public void SuccessfulExport_Windows() { @@ -1114,7 +1007,7 @@ public void SerializationTestForException() // VALIDATE Assert.Single(logRecordList); _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); var exceptionType = GetField(fluentdData, "env_ex_type"); var exceptionMessage = GetField(fluentdData, "env_ex_msg"); Assert.Equal("System.Exception", exceptionType); @@ -1209,7 +1102,7 @@ public void SerializationTestForEventName(EventNameExportMode eventNameExportMod // VALIDATE Assert.Single(logRecordList); _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); var eventName = GetField(fluentdData, "env_name"); if (eventNameExportMode.HasFlag(EventNameExportMode.ExportAsPartAName)) @@ -1229,7 +1122,7 @@ public void SerializationTestForEventName(EventNameExportMode eventNameExportMod logger.LogInformation(eventId: new EventId(1, "TestEventNameWithLogExtensionMethod"), "Hello from {Name} {Price}.", "tomato", 2.99); _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); eventName = GetField(fluentdData, "env_name"); if (eventNameExportMode.HasFlag(EventNameExportMode.ExportAsPartAName)) @@ -1248,7 +1141,7 @@ public void SerializationTestForEventName(EventNameExportMode eventNameExportMod logger.LogInformation(eventId: 1, "Hello from {Name} {Price}.", "tomato", 2.99); _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); eventName = GetField(fluentdData, "env_name"); Assert.Equal(hasTableNameMapping ? "CustomTableName" : "Log", eventName); #endregion @@ -1346,7 +1239,7 @@ public void SerializationTestForPartBName(bool hasCustomFields, bool hasNameInCu // VALIDATE Assert.Single(logRecordList); _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); var signal = (fluentdData as object[])[0] as string; var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; @@ -1449,7 +1342,7 @@ public void SerializationTestForEventId() // VALIDATE Assert.Single(logRecordList); _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; @@ -1750,14 +1643,14 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter // Part A cloud extensions var serviceNameField = resource.Attributes.FirstOrDefault(attr => attr.Key == "service.name"); - if (serviceNameField.Key == "service.name" && !exporterOptions.PrepopulatedFields.ContainsKey("cloud.role")) + if (serviceNameField.Key == "service.name") { Assert.Contains("env_cloud_role", mapping); Assert.Equal(serviceNameField.Value, mapping["env_cloud_role"]); } var serviceInstanceField = resource.Attributes.FirstOrDefault(attr => attr.Key == "service.instanceId"); - if (serviceInstanceField.Key == "service.instanceId" && !exporterOptions.PrepopulatedFields.ContainsKey("cloud.roleInstance")) + if (serviceInstanceField.Key == "service.instanceId") { Assert.Contains("env_cloud_roleInstance", mapping); Assert.Equal(serviceInstanceField.Value, mapping["env_cloud_roleInstance"]); @@ -1842,7 +1735,7 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter continue; } - if (exporterOptions.CustomFields == null || exporterOptions.CustomFields.Contains(item.Key)) + if (exporterOptions.ResourceFieldNames != null && exporterOptions.ResourceFieldNames.Contains(item.Key)) { // It should be found as a custom field if (item.Value != null) @@ -1855,12 +1748,6 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter Assert.DoesNotContain(item.Key, envPropertiesMapping.Keys); } } - else - { - // It should be found in env_properties - Assert.Equal(item.Value, envPropertiesMapping[item.Key]); - Assert.DoesNotContain(item.Key, mapping); - } } } diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs index e780fc8a30..a5b77576b2 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs @@ -77,6 +77,20 @@ public void GenevaTraceExporter_constructor_Invalid_Input() }); }); + // mutually exclusive ResourceFieldNames and PrepopulatedFields + Assert.Throws(() => + { + using var exporter = new GenevaTraceExporter(new GenevaExporterOptions + { + ConnectionString = connectionString, + ResourceFieldNames = ["resource"], + PrepopulatedFields = new Dictionary + { + ["prepopulated"] = "hello", + }, + }); + }); + // unsupported types(char) for PrepopulatedFields Assert.Throws(() => { @@ -204,30 +218,44 @@ public void GenevaTraceExporter_Success_Windows() } [Theory] - [InlineData(false, false, false)] - [InlineData(false, true, false)] - [InlineData(true, false, false)] - [InlineData(true, true, false)] - [InlineData(false, false, true)] - [InlineData(false, true, true)] - [InlineData(true, false, true)] - [InlineData(true, true, true)] - public void GenevaTraceExporter_Serialization_Success(bool hasTableNameMapping, bool hasCustomFields, bool includeTraceState) + [InlineData(false, false, false, false, true)] + [InlineData(false, true, false, false, true)] + [InlineData(true, false, false, false, true)] + [InlineData(true, true, false, false, true)] + [InlineData(false, false, true, false, true)] + [InlineData(false, true, true, false, true)] + [InlineData(true, false, true, false, true)] + [InlineData(true, true, true, false, true)] + [InlineData(false, false, false, true, false)] + [InlineData(false, true, false, true, false)] + [InlineData(true, false, false, true, false)] + [InlineData(true, true, false, true, false)] + [InlineData(false, false, true, true, false)] + [InlineData(false, true, true, true, false)] + [InlineData(true, false, true, true, false)] + [InlineData(true, true, true, true, false)] + public void GenevaTraceExporter_Serialization_Success(bool hasTableNameMapping, bool hasCustomFields, bool includeTraceState, bool hasPrepopulatedFields, bool hasResourceAttributes) { var path = string.Empty; Socket server = null; try { var invocationCount = 0; - var exporterOptions = new GenevaExporterOptions + var exporterOptions = new GenevaExporterOptions(); + if (hasPrepopulatedFields) { - PrepopulatedFields = new Dictionary + exporterOptions.PrepopulatedFields = new Dictionary { ["cloud.roleVer"] = "9.0.15289.2", ["resourceAndPrepopulated"] = "comes from prepopulated", - }, - ResourceFieldNames = ["resourceAttribute", "resourceAndPrepopulated"], - }; + }; + } + + if (hasResourceAttributes) + { + exporterOptions.ResourceFieldNames = ["resourceAttribute", "resourceAndPrepopulated"]; + } + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; @@ -347,8 +375,16 @@ public void GenevaTraceExporter_Serialization_Success(bool hasTableNameMapping, this.AssertMappingEntry(userFieldsLocation, "foo", 1); this.AssertMappingEntry(userFieldsLocation, "bar", 2); - this.AssertMappingEntry(mapping, "resourceAttribute", "resourceValue"); - this.AssertMappingEntry(mapping, "resourceAndPrepopulated", "comes from resource"); + + if (hasResourceAttributes) + { + this.AssertMappingEntry(mapping, "resourceAttribute", "resourceValue"); + this.AssertMappingEntry(mapping, "resourceAndPrepopulated", "comes from resource"); + } + else if (hasPrepopulatedFields) + { + this.AssertMappingEntry(mapping, "resourceAndPrepopulated", "comes from prepopulated"); + } // Linked spans are checked in CheckSpanForActivity, so no need to do a custom check here }); @@ -637,11 +673,6 @@ public void GenevaTraceExporter_WithEmptyResourceAttributes() { var exporterOptions = new GenevaExporterOptions { - PrepopulatedFields = new Dictionary - { - ["unaffected prepopulated"] = "should be present", - }, - ResourceFieldNames = [], // ResourceFieldNames empty }; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -724,10 +755,6 @@ public void GenevaTraceExporter_ResourceFieldNames() { var exporterOptions = new GenevaExporterOptions { - PrepopulatedFields = new Dictionary - { - ["overridden prepopulated"] = "should not be present", - }, ResourceFieldNames = new HashSet { "wanted", @@ -780,7 +807,6 @@ public void GenevaTraceExporter_ResourceFieldNames() this.ExpectSpanFromActivity(activity, (mapping) => { this.AssertMappingEntry(mapping, "wanted", "should be present"); - Assert.DoesNotContain("overridden prepopulated", mapping); Assert.DoesNotContain("unwanted", mapping); }); } diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs index ca2ecd2868..a20d7c15fb 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs @@ -113,7 +113,7 @@ private static Dictionary GetExportedFieldsAfterLogging(Action Resource.Empty); _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); return GetFields(fluentdData); } From 70d310c1f0ac287590290ea26012060ba6c52d85 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Wed, 17 Dec 2025 15:19:16 -0800 Subject: [PATCH 05/20] more tests --- .../GenevaLogExporterTests.cs | 312 +++++++++++++++++- 1 file changed, 298 insertions(+), 14 deletions(-) diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index 137dd8912a..17ad654b55 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -21,18 +21,6 @@ namespace OpenTelemetry.Exporter.Geneva.Tests; public class GenevaLogExporterTests { - private static string GetRandomFilePath() - { - while (true) - { - var path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); - if (!File.Exists(path)) - { - return path; - } - } - } - [Fact] public void BadArgs() { @@ -43,7 +31,7 @@ public void BadArgs() } else { - var path = GetRandomFilePath(); + var path = GenerateTempFilePath(); connectionString = "Endpoint=unix:" + path; } @@ -714,7 +702,6 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin { ["cloud.roleVer"] = "9.0.15289.2", ["prepopulated"] = "prepopulated field", - ["test"] = $"{hasTableNameMapping}{hasCustomFields}{parseStateValues}", }, }; @@ -1484,6 +1471,303 @@ public void AddGenevaBatchExportProcessorOptions() } } + [Fact] + public void InvalidResourceAttrType_PlaceholderMessage() + { + var path = string.Empty; + Socket server = null; + var logRecordList = new List(); + try + { + var exporterOptions = new GenevaExporterOptions + { + ResourceFieldNames = ["badresource"], + }; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + path = GenerateTempFilePath(); + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + var resourceAttributes = new Dictionary + { + { "badresource", new int[1] }, // the exporter does not accept complex types like an array + }; + var resource = new Resource(resourceAttributes); + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => + { + options.AddInMemoryExporter(logRecordList); + }) + .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels + + // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. + using var exporter = new MsgPackLogExporter(exporterOptions, () => resource); + + // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter + var logger = loggerFactory.CreateLogger(); + + // Set the ActivitySourceName to the unique value of the test method name to avoid interference with + // the ActivitySource used by other unit tests. + var sourceName = GetTestMethodName(); + + using var listener = new ActivityListener(); + listener.ShouldListenTo = (activitySource) => activitySource.Name == sourceName; + listener.Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllDataAndRecorded; + ActivitySource.AddActivityListener(listener); + + using var source = new ActivitySource(sourceName); + + using (var activity = source.StartActivity("Activity")) + { + // Log inside an activity to set LogRecord.TraceId and LogRecord.SpanId + logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); // structured logging + } + + // logRecordList should have a logRecord entry + Assert.Single(logRecordList); + + var m_buffer = exporter.Buffer; + + foreach (var logRecord in logRecordList) + { + _ = exporter.SerializeLogRecord(logRecord); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + + Assert.Contains("badresource", mapping.Keys); + Assert.Equal("", mapping["badresource"]); + } + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Fact] + public void WithoutResourceAttributes() + { + var path = string.Empty; + Socket server = null; + var logRecordList = new List(); + try + { + var exporterOptions = new GenevaExporterOptions + { + PrepopulatedFields = new Dictionary + { + ["unaffected prepopulated"] = "should be present", + }, + + // ResourceFieldNames not set + }; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + path = GenerateTempFilePath(); + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + var resourceAttributes = new Dictionary + { + { "resourceAttributes", "should not be present" }, + }; + var resource = new Resource(resourceAttributes); + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => + { + options.AddInMemoryExporter(logRecordList); + }) + .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels + + // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. + using var exporter = new MsgPackLogExporter(exporterOptions, () => resource); + + // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter + var logger = loggerFactory.CreateLogger(); + + // Set the ActivitySourceName to the unique value of the test method name to avoid interference with + // the ActivitySource used by other unit tests. + var sourceName = GetTestMethodName(); + + using var listener = new ActivityListener(); + listener.ShouldListenTo = (activitySource) => activitySource.Name == sourceName; + listener.Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllDataAndRecorded; + ActivitySource.AddActivityListener(listener); + + using var source = new ActivitySource(sourceName); + + using (var activity = source.StartActivity("Activity")) + { + // Log inside an activity to set LogRecord.TraceId and LogRecord.SpanId + logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); // structured logging + } + + // logRecordList should have a logRecord entry + Assert.Single(logRecordList); + + var m_buffer = exporter.Buffer; + + foreach (var logRecord in logRecordList) + { + _ = exporter.SerializeLogRecord(logRecord); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + + Assert.DoesNotContain("resourceAttributes", mapping.Keys); + + if (mapping.ContainsKey("env_properties")) + { + var env_properties = mapping["env_properties"] as Dictionary ?? []; + Assert.DoesNotContain("resourceAttributes", env_properties); + } + + Assert.Contains("unaffected prepopulated", mapping.Keys); + } + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + /// + /// The purpose of this test is to make sure that when ResourceFieldNames is set to empty, + /// that no resource attributes make it to Geneva. + /// + [Fact] + public void WithEmptyResourceAttributes() + { + var path = string.Empty; + Socket server = null; + var logRecordList = new List(); + try + { + var exporterOptions = new GenevaExporterOptions + { + ResourceFieldNames = [], // ResourceFieldNames empty + }; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + path = GenerateTempFilePath(); + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + var resourceAttributes = new Dictionary + { + { "resourceAttributes", "should not be present" }, + }; + var resource = new Resource(resourceAttributes); + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => + { + options.AddInMemoryExporter(logRecordList); + }) + .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels + + // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. + using var exporter = new MsgPackLogExporter(exporterOptions, () => resource); + + // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter + var logger = loggerFactory.CreateLogger(); + + // Set the ActivitySourceName to the unique value of the test method name to avoid interference with + // the ActivitySource used by other unit tests. + var sourceName = GetTestMethodName(); + + using var listener = new ActivityListener(); + listener.ShouldListenTo = (activitySource) => activitySource.Name == sourceName; + listener.Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllDataAndRecorded; + ActivitySource.AddActivityListener(listener); + + using var source = new ActivitySource(sourceName); + + using (var activity = source.StartActivity("Activity")) + { + // Log inside an activity to set LogRecord.TraceId and LogRecord.SpanId + logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); // structured logging + } + + // logRecordList should have a logRecord entry + Assert.Single(logRecordList); + + var m_buffer = exporter.Buffer; + + foreach (var logRecord in logRecordList) + { + _ = exporter.SerializeLogRecord(logRecord); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + + Assert.DoesNotContain("resourceAttributes", mapping.Keys); + + if (mapping.ContainsKey("env_properties")) + { + var env_properties = mapping["env_properties"] as Dictionary ?? []; + Assert.DoesNotContain("resourceAttributes", env_properties); + } + } + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + [SkipUnlessPlatformMatchesFact(TestPlatform.Linux)] public void SuccessfulUserEventsExport_Linux() { From 58056691082ddf28f07c4b24585c1509d323d5e7 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Tue, 30 Dec 2025 10:13:38 -0800 Subject: [PATCH 06/20] fixed all tests --- .../GenevaLogExporter.cs | 6 +- .../GenevaTraceExporter.cs | 6 +- .../Internal/MsgPack/MsgPackLogExporter.cs | 12 +- .../GenevaLogExporterAFDCorrelationTests.cs | 136 +++--- .../GenevaLogExporterTests.cs | 418 +++++++----------- .../LogSerializationTests.cs | 40 +- 6 files changed, 254 insertions(+), 364 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs index cbb83e8f23..0838ab4cce 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs @@ -90,7 +90,11 @@ public GenevaLogExporter(GenevaExporterOptions options) if (useMsgPackExporter) { - var msgPackLogExporter = new MsgPackLogExporter(options, this.ParentProvider.GetResource); + var msgPackLogExporter = new MsgPackLogExporter(options, () => + { + // this is not equivalent to passing a method reference, because the ParentProvider could change after the constructor. + return this.ParentProvider.GetResource(); + }); this.IsUsingUnixDomainSocket = msgPackLogExporter.IsUsingUnixDomainSocket; this.exportLogRecord = msgPackLogExporter.Export; this.Exporter = msgPackLogExporter; diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs index 17319575db..5238dd0595 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs @@ -68,7 +68,11 @@ public GenevaTraceExporter(GenevaExporterOptions options) if (useMsgPackExporter) { - var msgPackTraceExporter = new MsgPackTraceExporter(options, this.ParentProvider.GetResource); + var msgPackTraceExporter = new MsgPackTraceExporter(options, () => + { + // this is not equivalent to passing a method reference, because the ParentProvider could change after the constructor. + return this.ParentProvider.GetResource(); + }); this.IsUsingUnixDomainSocket = msgPackTraceExporter.IsUsingUnixDomainSocket; this.exportActivity = msgPackTraceExporter.Export; this.exporter = msgPackTraceExporter; diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs index e7f62c4ec1..edccd43ae2 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs @@ -20,7 +20,8 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable { public const int BUFFER_SIZE = 65360; // the maximum ETW payload (inclusive) - internal readonly ThreadLocal Buffer = new(); + // This helps tests subscribe to the output of this class + internal Action>? DataTransportListener; private static readonly Action ProcessScopeForIndividualColumnsAction = OnProcessScopeForIndividualColumns; private static readonly Action ProcessScopeForEnvPropertiesAction = OnProcessScopeForEnvProperties; @@ -29,6 +30,7 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable "Trace", "Debug", "Information", "Warning", "Error", "Critical", "None" ]; + private readonly ThreadLocal buffer = new(); private readonly bool shouldExportEventName; private readonly TableNameSerializer tableNameSerializer; @@ -155,6 +157,8 @@ public ExportResult Export(in Batch batch) { var data = this.SerializeLogRecord(logRecord); + this.DataTransportListener?.Invoke(data); + this.dataTransport.Send(data.Array!, data.Count); } catch (Exception ex) @@ -180,7 +184,7 @@ public void Dispose() { (this.dataTransport as IDisposable)?.Dispose(); this.serializationData.Dispose(); - this.Buffer.Dispose(); + this.buffer.Dispose(); } catch (Exception ex) { @@ -289,12 +293,12 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) } #pragma warning restore 0618 - var buffer = this.Buffer.Value; + var buffer = this.buffer.Value; if (buffer == null) { this.AddResourceAttributesToPrepopulated(); buffer = new byte[BUFFER_SIZE]; // TODO: handle OOM - this.Buffer.Value = buffer; + this.buffer.Value = buffer; } /* Fluentd Forward Mode: diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs index ffc6ef74f4..6f940cb8a5 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs @@ -65,11 +65,32 @@ public void AFDCorrelationIdLogProcessor_MultithreadedAccess_HandlesGracefully() // Create a test exporter using var exporter = new GenevaLogExporter(exporterOptions); + List exportedCorrelationIds = []; + int foundWithoutCorrelationIds = 0; + var msgPackExporter = exporter.Exporter as MsgPackLogExporter; + msgPackExporter.DataTransportListener = (data) => + { + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(data, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var signal = (fluentdData as object[])[0] as string; + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + + if (mapping.ContainsKey("AFDCorrelationId")) + { + exportedCorrelationIds.Add(mapping["AFDCorrelationId"] as string); + } + else + { + foundWithoutCorrelationIds++; + } + }; + // Now create multiple threads to simulate concurrent access var logger = loggerFactory.CreateLogger(); const int threadCount = 10; var threads = new Thread[threadCount]; var countWithCorrelationId = 0; + List expectedCorrelationIds = []; var countWithoutCorrelationId = 0; for (int i = 0; i < threadCount; i++) @@ -80,36 +101,16 @@ public void AFDCorrelationIdLogProcessor_MultithreadedAccess_HandlesGracefully() if (threadIndex % 2 == 0) { // This thread sets AFDCorrelationId before logging - var actualCorrelationId = $"CorrelationId-{threadIndex}"; - OpenTelemetryContext.SetAFDCorrelationId(actualCorrelationId); + var expectedCorrelationId = $"CorrelationId-{threadIndex}"; + OpenTelemetryContext.SetAFDCorrelationId(expectedCorrelationId); #pragma warning disable CA2254 // Template should be a static expression logger.LogInformation($"Thread {threadIndex} with correlation ID"); #pragma warning restore CA2254 // Template should be a static expression lock (syncObj) { countWithCorrelationId++; + expectedCorrelationIds.Add(expectedCorrelationId); } - - byte[] serializedData; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - var msgPackExporter = exporter.Exporter as MsgPackLogExporter; - serializedData = msgPackExporter.Buffer.Value; - } - else - { - // Read the data sent via socket. - serializedData = new byte[65360]; - _ = receiverSocket.Receive(serializedData); - } - - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(serializedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); - var signal = (fluentdData as object[])[0] as string; - var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; - var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; - - var expectedCorrelationId = mapping["AFDCorrelationId"] as string; - Assert.Equal(actualCorrelationId, expectedCorrelationId); } else { @@ -120,27 +121,6 @@ public void AFDCorrelationIdLogProcessor_MultithreadedAccess_HandlesGracefully() { countWithoutCorrelationId++; } - - byte[] serializedData; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - var msgPackExporter = exporter.Exporter as MsgPackLogExporter; - serializedData = msgPackExporter.Buffer.Value; - } - else - { - // Read the data sent via socket - serializedData = new byte[65360]; - _ = receiverSocket.Receive(serializedData); - } - - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(serializedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); - var signal = (fluentdData as object[])[0] as string; - var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; - var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; - - // Verify that AFDCorrelationId is not present in the serialized data - Assert.False(mapping.ContainsKey("AFDCorrelationId")); } }); } @@ -162,6 +142,9 @@ public void AFDCorrelationIdLogProcessor_MultithreadedAccess_HandlesGracefully() Assert.Equal(threadCount / 2, countWithCorrelationId); Assert.Equal(threadCount / 2, countWithoutCorrelationId); + Assert.Equal(expectedCorrelationIds, exportedCorrelationIds); + Assert.Equal(countWithoutCorrelationId, foundWithoutCorrelationIds); + // Check that no exceptions were thrown // If our implementation is correct, logs from threads without correlation ID // should have been processed without exceptions @@ -207,15 +190,19 @@ public void AFDCorrelationIdLogProcessor_WithoutCorrelationId_HandlesGracefully( var exportedItems = new List(); + using var exporter = new GenevaLogExporter(exporterOptions); + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { options.IncludeScopes = true; options.AddInMemoryExporter(exportedItems); - options.AddGenevaLogExporter(options => - { - options.ConnectionString = exporterOptions.ConnectionString; - }); + options.AddProcessor(sp => + new CompositeProcessor( + [ + new AFDCorrelationIdLogProcessor(), + new ReentrantExportProcessor(exporter), + ])); })); if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -224,25 +211,16 @@ public void AFDCorrelationIdLogProcessor_WithoutCorrelationId_HandlesGracefully( receiverSocket.ReceiveTimeout = 10000; } - // Create a test exporter to get MessagePack byte data for validation - using var exporter = new GenevaLogExporter(exporterOptions); + byte[] serializedData = null; + var msgPackExporter = exporter.Exporter as MsgPackLogExporter; + msgPackExporter.DataTransportListener = (data) => serializedData = [.. data]; // In this test, AFDCorrelationId is not set in RuntimeContext var logger = loggerFactory.CreateLogger(); logger.LogInformation("No correlation ID should be present"); + loggerFactory.Dispose(); - byte[] serializedData; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - var msgPackExporter = exporter.Exporter as MsgPackLogExporter; - serializedData = msgPackExporter.Buffer.Value; - } - else - { - // Read the data sent via socket - serializedData = new byte[65360]; - _ = receiverSocket.Receive(serializedData); - } + Assert.NotNull(serializedData); var fluentdData = MessagePack.MessagePackSerializer.Deserialize(serializedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); var signal = (fluentdData as object[])[0] as string; @@ -299,15 +277,19 @@ public void GenevaExporter_WithAFDCorrelationId_IncludesCorrelationId() var exportedItems = new List(); + using var exporter = new GenevaLogExporter(exporterOptions); + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { options.IncludeScopes = true; options.AddInMemoryExporter(exportedItems); - options.AddGenevaLogExporter(options => - { - options.ConnectionString = exporterOptions.ConnectionString; - }); + options.AddProcessor(sp => + new CompositeProcessor( + [ + new AFDCorrelationIdLogProcessor(), + new ReentrantExportProcessor(exporter), + ])); })); if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -316,25 +298,19 @@ public void GenevaExporter_WithAFDCorrelationId_IncludesCorrelationId() receiverSocket.ReceiveTimeout = 10000; } - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new GenevaLogExporter(exporterOptions); + byte[] serializedData = null; + var msgPackExporter = exporter.Exporter as MsgPackLogExporter; + msgPackExporter.DataTransportListener = (data) => + { + serializedData = [.. data]; + }; // Emit a LogRecord and grab a copy of internal buffer for validation. var logger = loggerFactory.CreateLogger(); logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); + loggerFactory.Dispose(); - byte[] serializedData; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - var msgPackExporter = exporter.Exporter as MsgPackLogExporter; - serializedData = msgPackExporter.Buffer.Value; - } - else - { - // Read the data sent via socket. - serializedData = new byte[65360]; - _ = receiverSocket.Receive(serializedData); - } + Assert.NotNull(serializedData); // data should have been sent by now var fluentdData = MessagePack.MessagePackSerializer.Deserialize(serializedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); var signal = (fluentdData as object[])[0] as string; diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index 17ad654b55..e6453cd21f 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -176,8 +176,6 @@ public void TableNameMappingTest(params string[] category) { // ARRANGE var path = string.Empty; - Socket server = null; - var logRecordList = new List(); Dictionary mappingsDict = null; try { @@ -202,20 +200,19 @@ public void TableNameMappingTest(params string[] category) path = GenerateTempFilePath(); exporterOptions.ConnectionString = "Endpoint=unix:" + path; var endpoint = new UnixDomainSocketEndPoint(path); - server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); - server.Bind(endpoint); - server.Listen(1); } + using var exporter = new GenevaLogExporter(exporterOptions); + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddInMemoryExporter(logRecordList); + options.AddProcessor(new ReentrantExportProcessor(exporter)); }) .AddFilter("*", LogLevel.Trace)); // Enable all LogLevels - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); ILogger logger; object fluentdData; @@ -230,12 +227,11 @@ public void TableNameMappingTest(params string[] category) logger = loggerFactory.CreateLogger(mapping.Key); logger.LogError("this does not matter"); - Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.Single(exportedData); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); actualTableName = (fluentdData as object[])[0] as string; Assert.Equal(mapping.Value, actualTableName); - logRecordList.Clear(); + exportedData.Clear(); } else { @@ -247,17 +243,15 @@ public void TableNameMappingTest(params string[] category) logger = loggerFactory.CreateLogger("random category"); logger.LogError("this does not matter"); - Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.Single(exportedData); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); actualTableName = (fluentdData as object[])[0] as string; Assert.Equal(defaultLogTable, actualTableName); - logRecordList.Clear(); + exportedData.Clear(); } } finally { - server?.Dispose(); try { File.Delete(path); @@ -336,22 +330,25 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); + ArraySegment exportedData = null; + exporter.DataTransportListener = (data) => exportedData = data; + ILogger passThruTableMappingsLogger, userInitializedTableMappingsLogger; - var m_buffer = exporter.Buffer; object fluentdData; string actualTableName; // Verify that the category table mappings specified by the users in the Geneva Configuration are mapped correctly. foreach (var mapping in userInitializedCategoryToTableNameMappings) { + exportedData = null; if (mapping.Key != "*") { userInitializedTableMappingsLogger = loggerFactory.CreateLogger(mapping.Key); userInitializedTableMappingsLogger.LogInformation("This information does not matter."); Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + _ = exporter.Export(new Batch(logRecordList[0])); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); actualTableName = (fluentdData as object[])[0] as string; userInitializedCategoryToTableNameMappings.TryGetValue(mapping.Key, out var expectedTableNme); Assert.Equal(expectedTableNme, actualTableName); @@ -363,12 +360,13 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() // Verify that when the "*" = "*" were enabled, the correct table names were being deduced following the set of rules. foreach (var mapping in expectedCategoryToTableNameList) { + exportedData = null; passThruTableMappingsLogger = loggerFactory.CreateLogger(mapping.Key); passThruTableMappingsLogger.LogInformation("This information does not matter."); Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + _ = exporter.Export(new Batch(logRecordList[0])); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); actualTableName = (fluentdData as object[])[0] as string; var expectedTableName = string.Empty; expectedTableName = mapping.Value; @@ -396,8 +394,6 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() public void SerializeILoggerScopes(bool hasCustomFields) { var path = string.Empty; - Socket senderSocket = null; - Socket receiverSocket = null; try { var exporterOptions = new GenevaExporterOptions(); @@ -410,10 +406,6 @@ public void SerializeILoggerScopes(bool hasCustomFields) { path = GenerateTempFilePath(); exporterOptions.ConnectionString = "Endpoint=unix:" + path; - var endpoint = new UnixDomainSocketEndPoint(path); - senderSocket = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); - senderSocket.Bind(endpoint); - senderSocket.Listen(1); } if (hasCustomFields) @@ -423,26 +415,26 @@ public void SerializeILoggerScopes(bool hasCustomFields) var exportedItems = new List(); + var resourceBuilder = ResourceBuilder.CreateEmpty().AddAttributes([ + new KeyValuePair("resourceAttr", "from resource"), + new KeyValuePair("service.name", "BusyWorker"), + new KeyValuePair("service.instanceId", "CY1SCH030021417") + ]); + + using var exporter = new GenevaLogExporter(exporterOptions); + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { options.IncludeScopes = true; + options.SetResourceBuilder(resourceBuilder); options.AddInMemoryExporter(exportedItems); - options.AddGenevaLogExporter(options => - { - options.ConnectionString = exporterOptions.ConnectionString; - options.CustomFields = exporterOptions.CustomFields; - }); + options.AddProcessor(new ReentrantExportProcessor(exporter)); })); - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - receiverSocket = senderSocket.Accept(); - receiverSocket.ReceiveTimeout = 10000; - } + List> exportedData = []; - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new GenevaLogExporter(exporterOptions); + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); // Emit a LogRecord and grab a copy of internal buffer for validation. var logger = loggerFactory.CreateLogger(); @@ -455,20 +447,11 @@ public void SerializeILoggerScopes(bool hasCustomFields) logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); } - byte[] serializedData; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - var msgPackExporter = exporter.Exporter as MsgPackLogExporter; - serializedData = msgPackExporter.Buffer.Value; - } - else - { - // Read the data sent via socket. - serializedData = new byte[65360]; - _ = receiverSocket.Receive(serializedData); - } + Assert.Single(exportedItems); + Assert.Single(exportedData); + var logRecord = exportedItems[0]; - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(serializedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); var signal = (fluentdData as object[])[0] as string; var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; @@ -507,15 +490,10 @@ public void SerializeILoggerScopes(bool hasCustomFields) } // Check other fields - Assert.Single(exportedItems); - var logRecord = exportedItems[0]; - this.AssertFluentdForwardModeForLogRecord(exporterOptions, Resource.Empty, fluentdData, logRecord); } finally { - senderSocket?.Dispose(); - receiverSocket?.Dispose(); try { File.Delete(path); @@ -527,21 +505,26 @@ public void SerializeILoggerScopes(bool hasCustomFields) } [Theory] - [InlineData(true)] - [InlineData(false)] - public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) + [InlineData(true, true)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(false, false)] + public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage, bool includeResourceAttributes) { // Dedicated test for the raw ILogger.Log method // https://docs.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger.log // ARRANGE var path = string.Empty; - Socket server = null; - var logRecordList = new List(); try { var exporterOptions = new GenevaExporterOptions(); + if (includeResourceAttributes) + { + exporterOptions.ResourceFieldNames = ["resourceAttr"]; + } + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; @@ -551,25 +534,29 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) path = GenerateTempFilePath(); exporterOptions.ConnectionString = "Endpoint=unix:" + path; var endpoint = new UnixDomainSocketEndPoint(path); - server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); - server.Bind(endpoint); - server.Listen(1); } + var resourceBuilder = ResourceBuilder.CreateEmpty().AddAttributes([ + new KeyValuePair("resourceAttr", "from resource"), + new KeyValuePair("service.name", "BusyWorker"), + new KeyValuePair("service.instanceId", "CY1SCH030021417") + ]); + + using var exporter = new GenevaLogExporter(exporterOptions); + List exportedItems = []; + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddGenevaLogExporter(options => - { - options.ConnectionString = exporterOptions.ConnectionString; - }); - options.AddInMemoryExporter(logRecordList); + options.SetResourceBuilder(resourceBuilder); + options.AddProcessor(new ReentrantExportProcessor(exporter)); + options.AddInMemoryExporter(exportedItems); options.IncludeFormattedMessage = includeFormattedMessage; }) .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -588,19 +575,13 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) (state, ex) => "Formatted Message"); // VALIDATE - Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); - var body = GetField(fluentdData, "body"); - - // Body gets populated as "Formatted Message" regardless of the value of `IncludeFormattedMessage` - Assert.Equal("Formatted Message", body); - - Assert.Equal("Value1", GetField(fluentdData, "Key1")); - Assert.Equal("Value2", GetField(fluentdData, "Key2")); + Assert.Single(exportedData); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedItems[0]); // ARRANGE - logRecordList.Clear(); + exportedData.Clear(); + exportedItems.Clear(); // ACT // This is treated as Un-structured logging as the state cannot be converted to IReadOnlyList> @@ -612,16 +593,13 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) formatter: (state, ex) => "Formatted Message"); // VALIDATE - Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); - body = GetField(fluentdData, "body"); - - // Body gets populated as "Formatted Message" regardless of the value of `IncludeFormattedMessage` - Assert.Equal("Formatted Message", body); + Assert.Single(exportedData); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedItems[0]); // ARRANGE - logRecordList.Clear(); + exportedData.Clear(); + exportedItems.Clear(); // ACT // This is treated as Un-structured logging as the state cannot be converted to IReadOnlyList> @@ -633,16 +611,13 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) formatter: null); // VALIDATE - Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); - body = GetField(fluentdData, "body"); - - // Even though Formatter is null, body is populated with the state - Assert.Equal("somestringasdata", body); + Assert.Single(exportedData); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedItems[0]); // ARRANGE - logRecordList.Clear(); + exportedData.Clear(); + exportedItems.Clear(); // ACT // This is treated as Structured logging as the state can be converted to IReadOnlyList> @@ -657,19 +632,12 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) formatter: (state, ex) => "Example formatted message."); // VALIDATE - Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); - Assert.Equal("Value1", GetField(fluentdData, "Key1")); - - body = GetField(fluentdData, "body"); - - // Body gets populated as "Formatted Message" regardless of the value of `IncludeFormattedMessage` - Assert.Equal("Example formatted message.", body); + Assert.Single(exportedData); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedItems[0]); } finally { - server?.Dispose(); try { File.Delete(path); @@ -795,12 +763,10 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin // logRecordList should have 14 logRecord entries as there were 14 Log calls Assert.Equal(14, logRecordList.Count); - var m_buffer = exporter.Buffer; - foreach (var logRecord in logRecordList) { - _ = exporter.SerializeLogRecord(logRecord); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var serializedLog = exporter.SerializeLogRecord(logRecord); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(serializedLog, MessagePack.Resolvers.ContractlessStandardResolver.Options); this.AssertFluentdForwardModeForLogRecord(exporterOptions, resource, fluentdData, logRecord); } } @@ -946,7 +912,6 @@ public void SerializationTestForException() // ARRANGE var path = string.Empty; Socket server = null; - var logRecordList = new List(); try { var exporterOptions = new GenevaExporterOptions(); @@ -965,19 +930,17 @@ public void SerializationTestForException() server.Listen(1); } + using var exporter = new GenevaLogExporter(exporterOptions); + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddGenevaLogExporter(options => - { - options.ConnectionString = exporterOptions.ConnectionString; - }); - options.AddInMemoryExporter(logRecordList); + options.AddProcessor(sp => new ReentrantExportProcessor(exporter)); }) .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -992,9 +955,8 @@ public void SerializationTestForException() formatter: null); // VALIDATE - Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.Single(exportedData); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); var exceptionType = GetField(fluentdData, "env_ex_type"); var exceptionMessage = GetField(fluentdData, "env_ex_msg"); Assert.Equal("System.Exception", exceptionType); @@ -1023,7 +985,6 @@ public void SerializationTestForEventName(EventNameExportMode eventNameExportMod // ARRANGE var path = string.Empty; Socket server = null; - var logRecordList = new List(); try { var exporterOptions = new GenevaExporterOptions @@ -1053,24 +1014,16 @@ public void SerializationTestForEventName(EventNameExportMode eventNameExportMod server.Listen(1); } + using var exporter = new GenevaLogExporter(exporterOptions); + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddGenevaLogExporter(options => - { - options.ConnectionString = exporterOptions.ConnectionString; - options.EventNameExportMode = exporterOptions.EventNameExportMode; - - if (hasTableNameMapping) - { - options.TableNameMappings = exporterOptions.TableNameMappings; - } - }); - options.AddInMemoryExporter(logRecordList); + options.AddProcessor(sp => new ReentrantExportProcessor(exporter)); })); - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1087,9 +1040,8 @@ public void SerializationTestForEventName(EventNameExportMode eventNameExportMod formatter: null); // VALIDATE - Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.Single(exportedData); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); var eventName = GetField(fluentdData, "env_name"); if (eventNameExportMode.HasFlag(EventNameExportMode.ExportAsPartAName)) @@ -1103,13 +1055,13 @@ public void SerializationTestForEventName(EventNameExportMode eventNameExportMod #endregion - logRecordList.Clear(); + exportedData.Clear(); #region Test for extension method logger.LogInformation(eventId: new EventId(1, "TestEventNameWithLogExtensionMethod"), "Hello from {Name} {Price}.", "tomato", 2.99); - _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.Single(exportedData); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); eventName = GetField(fluentdData, "env_name"); if (eventNameExportMode.HasFlag(EventNameExportMode.ExportAsPartAName)) @@ -1122,13 +1074,13 @@ public void SerializationTestForEventName(EventNameExportMode eventNameExportMod } #endregion - logRecordList.Clear(); + exportedData.Clear(); #region Test with eventName as null logger.LogInformation(eventId: 1, "Hello from {Name} {Price}.", "tomato", 2.99); - _ = exporter.SerializeLogRecord(logRecordList[0]); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.Single(exportedData); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); eventName = GetField(fluentdData, "env_name"); Assert.Equal(hasTableNameMapping ? "CustomTableName" : "Log", eventName); #endregion @@ -1160,8 +1112,6 @@ public void SerializationTestForPartBName(bool hasCustomFields, bool hasNameInCu { // ARRANGE var path = string.Empty; - Socket server = null; - var logRecordList = new List(); try { var exporterOptions = new GenevaExporterOptions(); @@ -1175,9 +1125,6 @@ public void SerializationTestForPartBName(bool hasCustomFields, bool hasNameInCu path = GenerateTempFilePath(); exporterOptions.ConnectionString = "Endpoint=unix:" + path; var endpoint = new UnixDomainSocketEndPoint(path); - server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); - server.Bind(endpoint); - server.Listen(1); } if (hasCustomFields) @@ -1185,19 +1132,16 @@ public void SerializationTestForPartBName(bool hasCustomFields, bool hasNameInCu exporterOptions.CustomFields = hasNameInCustomFields ? ["name", "Key1"] : ["Key1"]; } + using var exporter = new GenevaLogExporter(exporterOptions); + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddGenevaLogExporter(options => - { - options.ConnectionString = exporterOptions.ConnectionString; - options.CustomFields = exporterOptions.CustomFields; - }); - options.AddInMemoryExporter(logRecordList); + options.AddProcessor(new ReentrantExportProcessor(exporter)); })); - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1224,9 +1168,8 @@ public void SerializationTestForPartBName(bool hasCustomFields, bool hasNameInCu null); // VALIDATE - Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.Single(exportedData); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); var signal = (fluentdData as object[])[0] as string; var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; @@ -1259,12 +1202,9 @@ public void SerializationTestForPartBName(bool hasCustomFields, bool hasNameInCu } } } - - logRecordList.Clear(); } finally { - server?.Dispose(); try { File.Delete(path); @@ -1281,7 +1221,6 @@ public void SerializationTestForEventId() // ARRANGE var path = string.Empty; Socket server = null; - var logRecordList = new List(); try { var exporterOptions = new GenevaExporterOptions(); @@ -1300,19 +1239,17 @@ public void SerializationTestForEventId() server.Listen(1); } + using var exporter = new GenevaLogExporter(exporterOptions); + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddGenevaLogExporter(options => - { - options.ConnectionString = exporterOptions.ConnectionString; - }); - options.AddInMemoryExporter(logRecordList); + options.AddProcessor(sp => new ReentrantExportProcessor(exporter)); }) .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1327,9 +1264,8 @@ public void SerializationTestForEventId() formatter: null); // VALIDATE - Assert.Single(logRecordList); - _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.Single(exportedData); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; @@ -1475,8 +1411,6 @@ public void AddGenevaBatchExportProcessorOptions() public void InvalidResourceAttrType_PlaceholderMessage() { var path = string.Empty; - Socket server = null; - var logRecordList = new List(); try { var exporterOptions = new GenevaExporterOptions @@ -1493,26 +1427,26 @@ public void InvalidResourceAttrType_PlaceholderMessage() path = GenerateTempFilePath(); exporterOptions.ConnectionString = "Endpoint=unix:" + path; var endpoint = new UnixDomainSocketEndPoint(path); - server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); - server.Bind(endpoint); - server.Listen(1); } var resourceAttributes = new Dictionary { { "badresource", new int[1] }, // the exporter does not accept complex types like an array }; - var resource = new Resource(resourceAttributes); + var resourceBuilder = ResourceBuilder.CreateEmpty().AddAttributes(resourceAttributes); + var resource = resourceBuilder.Build(); + using var exporter = new GenevaLogExporter(exporterOptions); using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddInMemoryExporter(logRecordList); + options.AddProcessor(new ReentrantExportProcessor(exporter)); + options.SetResourceBuilder(resourceBuilder); }) .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions, () => resource); + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1534,25 +1468,17 @@ public void InvalidResourceAttrType_PlaceholderMessage() logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); // structured logging } - // logRecordList should have a logRecord entry - Assert.Single(logRecordList); - - var m_buffer = exporter.Buffer; + Assert.Single(exportedData); - foreach (var logRecord in logRecordList) - { - _ = exporter.SerializeLogRecord(logRecord); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); - var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; - var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; - Assert.Contains("badresource", mapping.Keys); - Assert.Equal("", mapping["badresource"]); - } + Assert.Contains("badresource", mapping.Keys); + Assert.Equal("", mapping["badresource"]); } finally { - server?.Dispose(); try { File.Delete(path); @@ -1567,8 +1493,6 @@ public void InvalidResourceAttrType_PlaceholderMessage() public void WithoutResourceAttributes() { var path = string.Empty; - Socket server = null; - var logRecordList = new List(); try { var exporterOptions = new GenevaExporterOptions @@ -1590,26 +1514,27 @@ public void WithoutResourceAttributes() path = GenerateTempFilePath(); exporterOptions.ConnectionString = "Endpoint=unix:" + path; var endpoint = new UnixDomainSocketEndPoint(path); - server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); - server.Bind(endpoint); - server.Listen(1); } var resourceAttributes = new Dictionary { { "resourceAttributes", "should not be present" }, }; - var resource = new Resource(resourceAttributes); + var resourceBuilder = ResourceBuilder.CreateEmpty().AddAttributes(resourceAttributes); + var resource = resourceBuilder.Build(); + + using var exporter = new GenevaLogExporter(exporterOptions); using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddInMemoryExporter(logRecordList); + options.SetResourceBuilder(resourceBuilder); + options.AddProcessor(new ReentrantExportProcessor(exporter)); }) .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions, () => resource); + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1631,32 +1556,22 @@ public void WithoutResourceAttributes() logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); // structured logging } - // logRecordList should have a logRecord entry - Assert.Single(logRecordList); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; - var m_buffer = exporter.Buffer; + Assert.DoesNotContain("resourceAttributes", mapping.Keys); - foreach (var logRecord in logRecordList) + if (mapping.ContainsKey("env_properties")) { - _ = exporter.SerializeLogRecord(logRecord); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); - var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; - var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; - - Assert.DoesNotContain("resourceAttributes", mapping.Keys); - - if (mapping.ContainsKey("env_properties")) - { - var env_properties = mapping["env_properties"] as Dictionary ?? []; - Assert.DoesNotContain("resourceAttributes", env_properties); - } - - Assert.Contains("unaffected prepopulated", mapping.Keys); + var env_properties = mapping["env_properties"] as Dictionary ?? []; + Assert.DoesNotContain("resourceAttributes", env_properties); } + + Assert.Contains("unaffected prepopulated", mapping.Keys); } finally { - server?.Dispose(); try { File.Delete(path); @@ -1675,8 +1590,6 @@ public void WithoutResourceAttributes() public void WithEmptyResourceAttributes() { var path = string.Empty; - Socket server = null; - var logRecordList = new List(); try { var exporterOptions = new GenevaExporterOptions @@ -1693,9 +1606,6 @@ public void WithEmptyResourceAttributes() path = GenerateTempFilePath(); exporterOptions.ConnectionString = "Endpoint=unix:" + path; var endpoint = new UnixDomainSocketEndPoint(path); - server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); - server.Bind(endpoint); - server.Listen(1); } var resourceAttributes = new Dictionary @@ -1704,15 +1614,17 @@ public void WithEmptyResourceAttributes() }; var resource = new Resource(resourceAttributes); + using var exporter = new GenevaLogExporter(exporterOptions); + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddInMemoryExporter(logRecordList); + options.AddProcessor(new ReentrantExportProcessor(exporter)); }) .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions, () => resource); + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1734,30 +1646,22 @@ public void WithEmptyResourceAttributes() logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); // structured logging } - // logRecordList should have a logRecord entry - Assert.Single(logRecordList); + Assert.Single(exportedData); - var m_buffer = exporter.Buffer; - - foreach (var logRecord in logRecordList) - { - _ = exporter.SerializeLogRecord(logRecord); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); - var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; - var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; - Assert.DoesNotContain("resourceAttributes", mapping.Keys); + Assert.DoesNotContain("resourceAttributes", mapping.Keys); - if (mapping.ContainsKey("env_properties")) - { - var env_properties = mapping["env_properties"] as Dictionary ?? []; - Assert.DoesNotContain("resourceAttributes", env_properties); - } + if (mapping.ContainsKey("env_properties")) + { + var env_properties = mapping["env_properties"] as Dictionary ?? []; + Assert.DoesNotContain("resourceAttributes", env_properties); } } finally { - server?.Dispose(); try { File.Delete(path); @@ -1969,7 +1873,11 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter { // `LogRecord.State` and `LogRecord.StateValues` were marked Obsolete in https://github.com/open-telemetry/opentelemetry-dotnet/pull/4334 #pragma warning disable 0618 - if (logRecord.State != null) + if (logRecord.FormattedMessage != null) + { + Assert.Equal(logRecord.FormattedMessage!, mapping["body"]); + } + else if (logRecord.State != null) { Assert.Equal(logRecord.State.ToString(), mapping["body"]); } diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs index a20d7c15fb..4fdd00eab8 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs @@ -3,12 +3,10 @@ #nullable disable -using System.Net.Sockets; using System.Runtime.InteropServices; using Microsoft.Extensions.Logging; using OpenTelemetry.Exporter.Geneva.MsgPack; using OpenTelemetry.Logs; -using OpenTelemetry.Resources; using Xunit; namespace OpenTelemetry.Exporter.Geneva.Tests; @@ -78,22 +76,9 @@ public void SerializationTestForExceptionTrim() private static Dictionary GetExportedFieldsAfterLogging(Action doLog, Action configureGeneva = null) { - Socket server = null; var path = string.Empty; try { - var logRecordList = new List(); - using var loggerFactory = LoggerFactory.Create(builder => builder - .AddOpenTelemetry(options => - { - options.AddInMemoryExporter(logRecordList); - }) - .AddFilter(typeof(LogSerializationTests).FullName, LogLevel.Trace)); // Enable all LogLevels - - var logger = loggerFactory.CreateLogger(); - doLog(logger); - - Assert.Single(logRecordList); var exporterOptions = new GenevaExporterOptions(); if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { @@ -103,23 +88,32 @@ private static Dictionary GetExportedFieldsAfterLogging(Action Resource.Empty); - _ = exporter.SerializeLogRecord(logRecordList[0]); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + using var exporter = new GenevaLogExporter(exporterOptions); + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => + { + options.AddProcessor(new ReentrantExportProcessor(exporter)); + }) + .AddFilter(typeof(LogSerializationTests).FullName, LogLevel.Trace)); // Enable all LogLevels + + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); + + var logger = loggerFactory.CreateLogger(); + doLog(logger); + + Assert.Single(exportedData); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); return GetFields(fluentdData); } finally { - server?.Dispose(); try { File.Delete(path); From a53b3bf24d567348cc6e5077508af1402e37ff18 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Tue, 30 Dec 2025 13:51:04 -0800 Subject: [PATCH 07/20] fix test --- .../GenevaLogExporterTests.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index e6453cd21f..d5d7985825 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -330,7 +330,7 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); - ArraySegment exportedData = null; + ArraySegment? exportedData = null; exporter.DataTransportListener = (data) => exportedData = data; ILogger passThruTableMappingsLogger, userInitializedTableMappingsLogger; @@ -348,7 +348,8 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() Assert.Single(logRecordList); _ = exporter.Export(new Batch(logRecordList[0])); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.True(exportedData.HasValue); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); actualTableName = (fluentdData as object[])[0] as string; userInitializedCategoryToTableNameMappings.TryGetValue(mapping.Key, out var expectedTableNme); Assert.Equal(expectedTableNme, actualTableName); @@ -366,7 +367,8 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() Assert.Single(logRecordList); _ = exporter.Export(new Batch(logRecordList[0])); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.True(exportedData.HasValue); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); actualTableName = (fluentdData as object[])[0] as string; var expectedTableName = string.Empty; expectedTableName = mapping.Value; From 6149cb10aa3cfc9e75502cd71985e5f5b0695042 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Tue, 30 Dec 2025 13:41:58 -0800 Subject: [PATCH 08/20] proof reading --- .../GenevaLogExporterAFDCorrelationTests.cs | 24 +++----- .../GenevaLogExporterTests.cs | 61 +++++++++---------- .../GenevaTraceExporterTests.cs | 9 +++ 3 files changed, 48 insertions(+), 46 deletions(-) diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs index 6f940cb8a5..cecf3adaf8 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterAFDCorrelationTests.cs @@ -67,8 +67,7 @@ public void AFDCorrelationIdLogProcessor_MultithreadedAccess_HandlesGracefully() List exportedCorrelationIds = []; int foundWithoutCorrelationIds = 0; - var msgPackExporter = exporter.Exporter as MsgPackLogExporter; - msgPackExporter.DataTransportListener = (data) => + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => { var fluentdData = MessagePack.MessagePackSerializer.Deserialize(data, MessagePack.Resolvers.ContractlessStandardResolver.Options); var signal = (fluentdData as object[])[0] as string; @@ -211,18 +210,17 @@ public void AFDCorrelationIdLogProcessor_WithoutCorrelationId_HandlesGracefully( receiverSocket.ReceiveTimeout = 10000; } - byte[] serializedData = null; - var msgPackExporter = exporter.Exporter as MsgPackLogExporter; - msgPackExporter.DataTransportListener = (data) => serializedData = [.. data]; + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); // In this test, AFDCorrelationId is not set in RuntimeContext var logger = loggerFactory.CreateLogger(); logger.LogInformation("No correlation ID should be present"); loggerFactory.Dispose(); - Assert.NotNull(serializedData); + Assert.Single(exportedData); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(serializedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); var signal = (fluentdData as object[])[0] as string; var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; @@ -298,21 +296,17 @@ public void GenevaExporter_WithAFDCorrelationId_IncludesCorrelationId() receiverSocket.ReceiveTimeout = 10000; } - byte[] serializedData = null; - var msgPackExporter = exporter.Exporter as MsgPackLogExporter; - msgPackExporter.DataTransportListener = (data) => - { - serializedData = [.. data]; - }; + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); // Emit a LogRecord and grab a copy of internal buffer for validation. var logger = loggerFactory.CreateLogger(); logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); loggerFactory.Dispose(); - Assert.NotNull(serializedData); // data should have been sent by now + Assert.Single(exportedData); - var fluentdData = MessagePack.MessagePackSerializer.Deserialize(serializedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); var signal = (fluentdData as object[])[0] as string; var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index d5d7985825..f709f2306d 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -320,18 +320,17 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() server.Listen(1); } + using var exporter = new GenevaLogExporter(exporterOptions); + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddInMemoryExporter(logRecordList); + options.AddProcessor(new ReentrantExportProcessor(exporter)); }) .AddFilter("*", LogLevel.Trace)); // Enable all LogLevels - // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); - - ArraySegment? exportedData = null; - exporter.DataTransportListener = (data) => exportedData = data; + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); ILogger passThruTableMappingsLogger, userInitializedTableMappingsLogger; object fluentdData; @@ -340,35 +339,31 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() // Verify that the category table mappings specified by the users in the Geneva Configuration are mapped correctly. foreach (var mapping in userInitializedCategoryToTableNameMappings) { - exportedData = null; if (mapping.Key != "*") { userInitializedTableMappingsLogger = loggerFactory.CreateLogger(mapping.Key); userInitializedTableMappingsLogger.LogInformation("This information does not matter."); Assert.Single(logRecordList); + Assert.Single(exportedData); - _ = exporter.Export(new Batch(logRecordList[0])); - Assert.True(exportedData.HasValue); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); actualTableName = (fluentdData as object[])[0] as string; userInitializedCategoryToTableNameMappings.TryGetValue(mapping.Key, out var expectedTableNme); Assert.Equal(expectedTableNme, actualTableName); - - logRecordList.Clear(); } + logRecordList.Clear(); + exportedData.Clear(); } // Verify that when the "*" = "*" were enabled, the correct table names were being deduced following the set of rules. foreach (var mapping in expectedCategoryToTableNameList) { - exportedData = null; passThruTableMappingsLogger = loggerFactory.CreateLogger(mapping.Key); passThruTableMappingsLogger.LogInformation("This information does not matter."); - Assert.Single(logRecordList); - _ = exporter.Export(new Batch(logRecordList[0])); - Assert.True(exportedData.HasValue); - fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.Single(logRecordList); + Assert.Single(exportedData); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); actualTableName = (fluentdData as object[])[0] as string; var expectedTableName = string.Empty; expectedTableName = mapping.Value; @@ -415,7 +410,7 @@ public void SerializeILoggerScopes(bool hasCustomFields) exporterOptions.CustomFields = ["Food", "Name", "Key1"]; } - var exportedItems = new List(); + var exportedLogs = new List(); var resourceBuilder = ResourceBuilder.CreateEmpty().AddAttributes([ new KeyValuePair("resourceAttr", "from resource"), @@ -430,7 +425,7 @@ public void SerializeILoggerScopes(bool hasCustomFields) { options.IncludeScopes = true; options.SetResourceBuilder(resourceBuilder); - options.AddInMemoryExporter(exportedItems); + options.AddInMemoryExporter(exportedLogs); options.AddProcessor(new ReentrantExportProcessor(exporter)); })); @@ -449,9 +444,9 @@ public void SerializeILoggerScopes(bool hasCustomFields) logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); } - Assert.Single(exportedItems); + Assert.Single(exportedLogs); Assert.Single(exportedData); - var logRecord = exportedItems[0]; + var logRecord = exportedLogs[0]; var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); var signal = (fluentdData as object[])[0] as string; @@ -545,14 +540,14 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage, ]); using var exporter = new GenevaLogExporter(exporterOptions); - List exportedItems = []; + List exportedLogs = []; using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { options.SetResourceBuilder(resourceBuilder); options.AddProcessor(new ReentrantExportProcessor(exporter)); - options.AddInMemoryExporter(exportedItems); + options.AddInMemoryExporter(exportedLogs); options.IncludeFormattedMessage = includeFormattedMessage; }) .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels @@ -577,13 +572,14 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage, (state, ex) => "Formatted Message"); // VALIDATE + Assert.Single(exportedLogs); Assert.Single(exportedData); var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); - this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedItems[0]); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedLogs[0]); // ARRANGE + exportedLogs.Clear(); exportedData.Clear(); - exportedItems.Clear(); // ACT // This is treated as Un-structured logging as the state cannot be converted to IReadOnlyList> @@ -595,13 +591,14 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage, formatter: (state, ex) => "Formatted Message"); // VALIDATE + Assert.Single(exportedLogs); Assert.Single(exportedData); fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); - this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedItems[0]); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedLogs[0]); // ARRANGE + exportedLogs.Clear(); exportedData.Clear(); - exportedItems.Clear(); // ACT // This is treated as Un-structured logging as the state cannot be converted to IReadOnlyList> @@ -613,13 +610,14 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage, formatter: null); // VALIDATE + Assert.Single(exportedLogs); Assert.Single(exportedData); fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); - this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedItems[0]); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedLogs[0]); // ARRANGE + exportedLogs.Clear(); exportedData.Clear(); - exportedItems.Clear(); // ACT // This is treated as Structured logging as the state can be converted to IReadOnlyList> @@ -634,9 +632,10 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage, formatter: (state, ex) => "Example formatted message."); // VALIDATE + Assert.Single(exportedLogs); Assert.Single(exportedData); fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); - this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedItems[0]); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedLogs[0]); } finally { @@ -1931,7 +1930,7 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter if (exporterOptions.ResourceFieldNames != null && exporterOptions.ResourceFieldNames.Contains(item.Key)) { - // It should be found as a custom field + // It should always be found as a custom field if (item.Value != null) { Assert.Equal(item.Value, mapping[item.Key]); diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs index a5b77576b2..d5a3cbf29b 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs @@ -217,6 +217,7 @@ public void GenevaTraceExporter_Success_Windows() } } + // hasResourceAttributes and hasPrepopulatedFields are mutually exclusive [Theory] [InlineData(false, false, false, false, true)] [InlineData(false, true, false, false, true)] @@ -234,6 +235,14 @@ public void GenevaTraceExporter_Success_Windows() [InlineData(false, true, true, true, false)] [InlineData(true, false, true, true, false)] [InlineData(true, true, true, true, false)] + [InlineData(false, false, false, false, false)] + [InlineData(false, true, false, false, false)] + [InlineData(true, false, false, false, false)] + [InlineData(true, true, false, false, false)] + [InlineData(false, false, true, false, false)] + [InlineData(false, true, true, false, false)] + [InlineData(true, false, true, false, false)] + [InlineData(true, true, true, false, false)] public void GenevaTraceExporter_Serialization_Success(bool hasTableNameMapping, bool hasCustomFields, bool includeTraceState, bool hasPrepopulatedFields, bool hasResourceAttributes) { var path = string.Empty; From 988a75766cbdeb8deffce4a11d752f71ee7c7d80 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Tue, 30 Dec 2025 13:45:11 -0800 Subject: [PATCH 09/20] .net format --- .../Internal/MsgPack/MsgPackLogExporter.cs | 4 ++-- .../GenevaLogExporterTests.cs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs index edccd43ae2..c31b3dbbf1 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs @@ -142,7 +142,7 @@ public MsgPackLogExporter(GenevaExporterOptions options, Func resource var buffer = new byte[BUFFER_SIZE]; var cursor = MessagePackSerializer.Serialize(buffer, 0, new Dictionary { { "TimeFormat", "DateTime" } }); this.bufferEpilogue = new byte[cursor - 0]; - System.Buffer.BlockCopy(buffer, 0, this.bufferEpilogue, 0, cursor - 0); + Buffer.BlockCopy(buffer, 0, this.bufferEpilogue, 0, cursor - 0); } internal bool IsUsingUnixDomainSocket => this.dataTransport is UnixDomainSocketDataTransport; @@ -558,7 +558,7 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) } MessagePackSerializer.WriteUInt16(buffer, idxMapSizePatch, cntFields); - System.Buffer.BlockCopy(this.bufferEpilogue, 0, buffer, cursor, this.bufferEpilogue.Length); + Buffer.BlockCopy(this.bufferEpilogue, 0, buffer, cursor, this.bufferEpilogue.Length); cursor += this.bufferEpilogue.Length; return new(buffer, 0, cursor); } diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index f709f2306d..b264a97ec4 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -351,6 +351,7 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() userInitializedCategoryToTableNameMappings.TryGetValue(mapping.Key, out var expectedTableNme); Assert.Equal(expectedTableNme, actualTableName); } + logRecordList.Clear(); exportedData.Clear(); } From ccfb2fd7ba252593a4b6838606bf5c0c40c5745a Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Tue, 30 Dec 2025 13:49:31 -0800 Subject: [PATCH 10/20] changelog --- src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md index 8a29ef41e1..2cc23c5854 100644 --- a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md @@ -4,7 +4,7 @@ * Add ResourceFieldNames to filter resource attributes to send to Geneva ([#3552](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/3552)) - ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/TODO)) + ([#3646](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/3646)) ## 1.14.0 From 0538e927df7493d596e7016810c82df9e87c16fc Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Tue, 30 Dec 2025 13:58:50 -0800 Subject: [PATCH 11/20] fix test --- .../GenevaLogExporterTests.cs | 7 +------ .../GenevaTraceExporterTests.cs | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index b264a97ec4..5834710845 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -300,7 +300,6 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() new("1.2", null), }; - var logRecordList = new List(); var exporterOptions = new GenevaExporterOptions { TableNameMappings = userInitializedCategoryToTableNameMappings, @@ -343,7 +342,7 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() { userInitializedTableMappingsLogger = loggerFactory.CreateLogger(mapping.Key); userInitializedTableMappingsLogger.LogInformation("This information does not matter."); - Assert.Single(logRecordList); + Assert.Single(exportedData); fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); @@ -352,7 +351,6 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() Assert.Equal(expectedTableNme, actualTableName); } - logRecordList.Clear(); exportedData.Clear(); } @@ -362,15 +360,12 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() passThruTableMappingsLogger = loggerFactory.CreateLogger(mapping.Key); passThruTableMappingsLogger.LogInformation("This information does not matter."); - Assert.Single(logRecordList); Assert.Single(exportedData); fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); actualTableName = (fluentdData as object[])[0] as string; var expectedTableName = string.Empty; expectedTableName = mapping.Value; Assert.Equal(expectedTableName, actualTableName); - - logRecordList.Clear(); } } finally diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs index d5a3cbf29b..33d5d9662e 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs @@ -286,8 +286,8 @@ public void GenevaTraceExporter_Serialization_Success(bool hasTableNameMapping, if (hasCustomFields) { - // The tag "clientRequestId" should be present in the mapping as a separate key. Other tags which are not present - // in the m_dedicatedFields should be added in the mapping under "env_properties" + // The tag "clientRequestId" should be present in the exported data as a separate key. Other tags which are not present + // in DedicatedFields should be added in the mapping under "env_properties" exporterOptions.CustomFields = ["clientRequestId"]; } From 9987054088ff4ed0ea051e3d2359ebdb0681be88 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Tue, 30 Dec 2025 14:03:32 -0800 Subject: [PATCH 12/20] fix test --- .../GenevaLogExporterTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index 5834710845..f6c49ad6ea 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -366,6 +366,8 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() var expectedTableName = string.Empty; expectedTableName = mapping.Value; Assert.Equal(expectedTableName, actualTableName); + + exportedData.Clear(); } } finally From 2d6fc0625601b38293b290532fa6e8fadf8b9da4 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Tue, 30 Dec 2025 14:31:41 -0800 Subject: [PATCH 13/20] fix linux test --- .../Internal/MsgPack/MsgPackLogExporter.cs | 2 +- .../Internal/MsgPack/MsgPackTraceExporter.cs | 2 +- .../GenevaLogExporterTests.cs | 71 ++++++++++--------- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs index c31b3dbbf1..92284cb436 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs @@ -272,7 +272,7 @@ internal void AddResourceAttributesToPrepopulated() if (isWantedAttribute) { - this.prepopulatedFields.Add(key, value); + this.prepopulatedFields[key] = value; } } } diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs index 606f267f60..6fe4fe2a8e 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs @@ -421,7 +421,7 @@ internal void CreateFraming() if (isWantedAttribute) { - this.prepopulatedFields.Add(key, value); + this.prepopulatedFields[key] = value; } } diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index f6c49ad6ea..1da2eb16b2 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -819,7 +819,7 @@ public void SuccessfulExport_Windows() public void SuccessfulExport_Linux() { var path = GenerateTempFilePath(); - var logRecordList = new List(); + var exportedLogs = new List(); try { var endpoint = new UnixDomainSocketEndPoint(path); @@ -827,56 +827,51 @@ public void SuccessfulExport_Linux() server.Bind(endpoint); server.Listen(1); + // Create a test exporter to get MessagePack byte data for validation of the data received via Socket. + var exporterOptions = new GenevaExporterOptions + { + ConnectionString = "Endpoint=unix:" + path, + }; + + using var exporter = new GenevaLogExporter(exporterOptions); + + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); + + var resourceBuilder = ResourceBuilder.CreateDefault(); + using var loggerFactory = LoggerFactory.Create(builder => builder .AddOpenTelemetry(options => { - options.AddGenevaLogExporter(options => - { - options.ConnectionString = "Endpoint=unix:" + path; - options.PrepopulatedFields = new Dictionary - { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", - ["cloud.roleVer"] = "9.0.15289.2", - }; - }); - options.AddInMemoryExporter(logRecordList); + options.SetResourceBuilder(resourceBuilder); + options.AddInMemoryExporter(exportedLogs); + options.AddProcessor(new ReentrantExportProcessor(exporter)); })); + using var serverSocket = server.Accept(); serverSocket.ReceiveTimeout = 10000; - // Create a test exporter to get MessagePack byte data for validation of the data received via Socket. - using var exporter = new MsgPackLogExporter( - new GenevaExporterOptions - { - ConnectionString = "Endpoint=unix:" + path, - PrepopulatedFields = new Dictionary - { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", - ["cloud.roleVer"] = "9.0.15289.2", - }, - }, - () => Resource.Empty); - // Emit a LogRecord and grab a copy of internal buffer for validation. var logger = loggerFactory.CreateLogger(); logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); // logRecordList should have a singleLogRecord entry after the logger.LogInformation call - Assert.Single(logRecordList); - - var messagePackDataSize = exporter.SerializeLogRecord(logRecordList[0]).Count; + Assert.Single(exportedLogs); + Assert.Single(exportedData); // Read the data sent via socket. var receivedData = new byte[1024]; var receivedDataSize = serverSocket.Receive(receivedData); - // Validation - Assert.Equal(messagePackDataSize, receivedDataSize); + // the number of bytes received over the socket should match the number of bytes of data exported + Assert.Equal(exportedData[0].Count, receivedDataSize); - logRecordList.Clear(); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedLogs[0]); + + exportedLogs.Clear(); + exportedData.Clear(); // Emit log on a different thread to test for multithreading scenarios var thread = new Thread(() => @@ -887,11 +882,17 @@ public void SuccessfulExport_Linux() thread.Join(); // logRecordList should have a singleLogRecord entry after the logger.LogInformation call - Assert.Single(logRecordList); + Assert.Single(exportedLogs); + Assert.Single(exportedData); - messagePackDataSize = exporter.SerializeLogRecord(logRecordList[0]).Count; + // Read the data sent via socket. receivedDataSize = serverSocket.Receive(receivedData); - Assert.Equal(messagePackDataSize, receivedDataSize); + + // the number of bytes received over the socket should match the number of bytes of data exported + Assert.Equal(exportedData[0].Count, receivedDataSize); + + fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resourceBuilder.Build(), fluentdData, exportedLogs[0]); } finally { From 98a159f01c61eb332e38d1ca255971416a6f523a Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Wed, 31 Dec 2025 09:57:00 -0800 Subject: [PATCH 14/20] add test for auto-mapped resource attrs --- .../GenevaLogExporterTests.cs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index 1da2eb16b2..78d7d9c3e6 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -815,6 +815,58 @@ public void SuccessfulExport_Windows() logger.LogInformation("Hello from {Food} {Price}.", "artichoke", 3.99); } + [Fact] + public void AutoMappedResourceAttrReplacesPrepopulated() + { + var path = GenerateTempFilePath(); + try + { + var endpoint = new UnixDomainSocketEndPoint(path); + + var exporterOptions = new GenevaExporterOptions + { + ConnectionString = "Endpoint=unix:" + path, + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "cloud.role from prepopulated", + }, + }; + + using var exporter = new GenevaLogExporter(exporterOptions); + + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); + + var resourceBuilder = ResourceBuilder.CreateDefault().AddService("cloud.role from resource"); + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => + { + options.SetResourceBuilder(resourceBuilder); + options.AddProcessor(new ReentrantExportProcessor(exporter)); + })); + + var logger = loggerFactory.CreateLogger(); + + logger.LogInformation("Hello"); + + Assert.Single(exportedData); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + + Assert.Equal("cloud.role from resource", GetField(fluentdData, "env_cloud_role")); + } + finally + { + try + { + File.Delete(path); + } + catch + { + } + } + } + [SkipUnlessPlatformMatchesFact(TestPlatform.Linux)] public void SuccessfulExport_Linux() { From 50b68ddac7ad957d40189f333fdac180f2febe09 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Wed, 31 Dec 2025 10:17:29 -0800 Subject: [PATCH 15/20] fix tests --- .../GenevaTraceExporter.cs | 9 +-- .../GenevaLogExporterTests.cs | 10 ++- .../GenevaTraceExporterTests.cs | 70 +++++++++++++++++-- 3 files changed, 79 insertions(+), 10 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs index 5238dd0595..b46b9ac87d 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs @@ -16,8 +16,9 @@ public class GenevaTraceExporter : GenevaBaseExporter { internal readonly bool IsUsingUnixDomainSocket; + internal readonly IDisposable Exporter; + private readonly ExportActivityFunc exportActivity; - private readonly IDisposable exporter; private bool isDisposed; @@ -75,14 +76,14 @@ public GenevaTraceExporter(GenevaExporterOptions options) }); this.IsUsingUnixDomainSocket = msgPackTraceExporter.IsUsingUnixDomainSocket; this.exportActivity = msgPackTraceExporter.Export; - this.exporter = msgPackTraceExporter; + this.Exporter = msgPackTraceExporter; } else { var tldTraceExporter = new TldTraceExporter(options); this.IsUsingUnixDomainSocket = false; this.exportActivity = tldTraceExporter.Export; - this.exporter = tldTraceExporter; + this.Exporter = tldTraceExporter; } } @@ -106,7 +107,7 @@ protected override void Dispose(bool disposing) { try { - this.exporter.Dispose(); + this.Exporter.Dispose(); } catch (Exception ex) { diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index 78d7d9c3e6..275938050d 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -825,13 +825,21 @@ public void AutoMappedResourceAttrReplacesPrepopulated() var exporterOptions = new GenevaExporterOptions { - ConnectionString = "Endpoint=unix:" + path, PrepopulatedFields = new Dictionary { ["cloud.role"] = "cloud.role from prepopulated", }, }; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + } + using var exporter = new GenevaLogExporter(exporterOptions); List> exportedData = []; diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs index 33d5d9662e..bdb8a1759e 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs @@ -439,7 +439,6 @@ public void GenevaTraceExporter_Serialization_Success(bool hasTableNameMapping, public void GenevaTraceExporter_Resource_Overwrites_Prepopulated() { var path = string.Empty; - Socket server = null; try { var exporterOptions = new GenevaExporterOptions @@ -458,9 +457,6 @@ public void GenevaTraceExporter_Resource_Overwrites_Prepopulated() path = GetRandomFilePath(); exporterOptions.ConnectionString = "Endpoint=unix:" + path; var endpoint = new UnixDomainSocketEndPoint(path); - server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); - server.Bind(endpoint); - server.Listen(1); } Dictionary resourceAttributes = new Dictionary @@ -497,7 +493,71 @@ public void GenevaTraceExporter_Resource_Overwrites_Prepopulated() } finally { - server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Fact] + public void AutoMappedResourceAttrReplacesPrepopulated() + { + var path = string.Empty; + try + { + var exporterOptions = new GenevaExporterOptions + { + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "cloud.role from prepopulated", + }, + }; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + path = GetRandomFilePath(); + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + var endpoint = new UnixDomainSocketEndPoint(path); + } + + using var exporter = new GenevaTraceExporter(exporterOptions); + + // Set the ActivitySourceName to the unique value of the test method name to avoid interference with + // the ActivitySource used by other unit tests. + var sourceName = GetTestMethodName(); + + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .SetSampler(new AlwaysOnSampler()) + .ConfigureResource(resourceBuilder => resourceBuilder.AddService("cloud.role from resource")) + .AddSource(sourceName) + .AddProcessor(new ReentrantExportProcessor(exporter)) + .Build(); + + var source = new ActivitySource(sourceName); + using (var activity = source.StartActivity("test")) + { + } + + var exportedData = (exporter.Exporter as MsgPackTraceExporter).Buffer.Value; + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); + + var signal = (fluentdData as object[])[0] as string; + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + + Assert.Contains("env_cloud_role", mapping.Keys); + Assert.Equal("cloud.role from resource", mapping["env_cloud_role"]); + } + finally + { try { File.Delete(path); From 630b3abb23de22fb72d4bdecf9844fbbcc2e1cdd Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Fri, 2 Jan 2026 09:13:07 -0800 Subject: [PATCH 16/20] gate cloud extension mapping by prepopulated fields --- .../Internal/MsgPack/MsgPackLogExporter.cs | 22 +++-- .../Internal/MsgPack/MsgPackTraceExporter.cs | 23 +++-- .../GenevaLogExporterTests.cs | 87 ++++++++++++++++--- .../GenevaTraceExporterTests.cs | 71 ++++++++++++++- 4 files changed, 173 insertions(+), 30 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs index 92284cb436..1f0896a38f 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs @@ -41,6 +41,7 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable #endif private readonly ExceptionStackExportMode exportExceptionStack; + private readonly bool userProvidedPrepopulatedFields; private readonly Dictionary? prepopulatedFields; private readonly IEnumerable? resourceFieldNames; private readonly byte[] bufferEpilogue; @@ -114,6 +115,7 @@ public MsgPackLogExporter(GenevaExporterOptions options, Func resource this.resourceFieldNames = options.ResourceFieldNames; } + this.userProvidedPrepopulatedFields = options.PrepopulatedFields != null && options.PrepopulatedFields.Count > 0; if (options.PrepopulatedFields != null) { this.prepopulatedFields = new Dictionary(options.PrepopulatedFields.Count, StringComparer.Ordinal); @@ -258,16 +260,20 @@ internal void AddResourceAttributesToPrepopulated() } } - if (key == "service.name") + if (!this.userProvidedPrepopulatedFields) { - key = Schema.V40.PartA.Extensions.Cloud.Role; - isWantedAttribute = true; - } + // it's only safe to add these special resource fields if we are sure the user didn't provide them as a PrepopulatedField already + if (key == "service.name") + { + key = Schema.V40.PartA.Extensions.Cloud.Role; + isWantedAttribute = true; + } - if (key == "service.instanceId") - { - key = Schema.V40.PartA.Extensions.Cloud.RoleInstance; - isWantedAttribute = true; + if (key == "service.instanceId") + { + key = Schema.V40.PartA.Extensions.Cloud.RoleInstance; + isWantedAttribute = true; + } } if (isWantedAttribute) diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs index 6fe4fe2a8e..c7d310454b 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackTraceExporter.cs @@ -75,6 +75,7 @@ internal sealed class MsgPackTraceExporter : MsgPackExporter, IDisposable // so constructing a whole new data structure for it is overkill. private readonly IEnumerable? resourceFieldNames; private readonly bool shouldIncludeTraceState; + private readonly bool userProvidedPrepopulatedFields; private readonly string partAName; private readonly Func resourceProvider; @@ -149,6 +150,8 @@ public MsgPackTraceExporter(GenevaExporterOptions options, Func resour } } + this.userProvidedPrepopulatedFields = options.PrepopulatedFields != null && options.PrepopulatedFields.Count > 0; + this.prepopulatedFields = new Dictionary(0, StringComparer.Ordinal); if (options.PrepopulatedFields != null) { @@ -407,16 +410,20 @@ internal void CreateFraming() } } - if (key == "service.name") + if (!this.userProvidedPrepopulatedFields) { - key = Schema.V40.PartA.Extensions.Cloud.Role; - isWantedAttribute = true; - } + // it's only safe to add these special resource fields if we are sure the user didn't provide them as a PrepopulatedField already + if (key == "service.name") + { + key = Schema.V40.PartA.Extensions.Cloud.Role; + isWantedAttribute = true; + } - if (key == "service.instanceId") - { - key = Schema.V40.PartA.Extensions.Cloud.RoleInstance; - isWantedAttribute = true; + if (key == "service.instanceId") + { + key = Schema.V40.PartA.Extensions.Cloud.RoleInstance; + isWantedAttribute = true; + } } if (isWantedAttribute) diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index 275938050d..72ddb6c420 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -816,7 +816,7 @@ public void SuccessfulExport_Windows() } [Fact] - public void AutoMappedResourceAttrReplacesPrepopulated() + public void AutoMappedResourceAttrDoesNotOverridePrepopulated() { var path = GenerateTempFilePath(); try @@ -861,6 +861,66 @@ public void AutoMappedResourceAttrReplacesPrepopulated() Assert.Single(exportedData); var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.Equal("cloud.role from prepopulated", GetField(fluentdData, "env_cloud_role")); + } + finally + { + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Fact] + public void AutoMappedResourceAttr() + { + var path = GenerateTempFilePath(); + try + { + var endpoint = new UnixDomainSocketEndPoint(path); + + var exporterOptions = new GenevaExporterOptions + { + PrepopulatedFields = new Dictionary + { + // no prepopulated fields + }, + }; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + } + + using var exporter = new GenevaLogExporter(exporterOptions); + + List> exportedData = []; + (exporter.Exporter as MsgPackLogExporter).DataTransportListener = (data) => exportedData.Add(data); + + var resourceBuilder = ResourceBuilder.CreateDefault().AddService("cloud.role from resource"); + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => + { + options.SetResourceBuilder(resourceBuilder); + options.AddProcessor(new ReentrantExportProcessor(exporter)); + })); + + var logger = loggerFactory.CreateLogger(); + + logger.LogInformation("Hello"); + + Assert.Single(exportedData); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData[0], MessagePack.Resolvers.ContractlessStandardResolver.Options); + Assert.Equal("cloud.role from resource", GetField(fluentdData, "env_cloud_role")); } finally @@ -1890,18 +1950,23 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter } // Part A cloud extensions - var serviceNameField = resource.Attributes.FirstOrDefault(attr => attr.Key == "service.name"); - if (serviceNameField.Key == "service.name") + if (exporterOptions.PrepopulatedFields != null && exporterOptions.PrepopulatedFields.Count == 0) { - Assert.Contains("env_cloud_role", mapping); - Assert.Equal(serviceNameField.Value, mapping["env_cloud_role"]); - } + // this mapping only happens when there are no prepopulated fields, to avoid conflicts - var serviceInstanceField = resource.Attributes.FirstOrDefault(attr => attr.Key == "service.instanceId"); - if (serviceInstanceField.Key == "service.instanceId") - { - Assert.Contains("env_cloud_roleInstance", mapping); - Assert.Equal(serviceInstanceField.Value, mapping["env_cloud_roleInstance"]); + var serviceNameField = resource.Attributes.FirstOrDefault(attr => attr.Key == "service.name"); + if (serviceNameField.Key == "service.name") + { + Assert.Contains("env_cloud_role", mapping); + Assert.Equal(serviceNameField.Value, mapping["env_cloud_role"]); + } + + var serviceInstanceField = resource.Attributes.FirstOrDefault(attr => attr.Key == "service.instanceId"); + if (serviceInstanceField.Key == "service.instanceId") + { + Assert.Contains("env_cloud_roleInstance", mapping); + Assert.Equal(serviceInstanceField.Value, mapping["env_cloud_roleInstance"]); + } } // Part B fields diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs index bdb8a1759e..043d9d1fd3 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaTraceExporterTests.cs @@ -504,7 +504,7 @@ public void GenevaTraceExporter_Resource_Overwrites_Prepopulated() } [Fact] - public void AutoMappedResourceAttrReplacesPrepopulated() + public void AutoMappedResourceAttrDoesNotOverridePrepopulated() { var path = string.Empty; try @@ -553,6 +553,71 @@ public void AutoMappedResourceAttrReplacesPrepopulated() var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + Assert.Contains("env_cloud_role", mapping.Keys); + Assert.Equal("cloud.role from prepopulated", mapping["env_cloud_role"]); + } + finally + { + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Fact] + public void AutoMappedResourceAttr() + { + var path = string.Empty; + try + { + var exporterOptions = new GenevaExporterOptions + { + PrepopulatedFields = new Dictionary + { + // no prepopulated fields + }, + }; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + path = GetRandomFilePath(); + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + var endpoint = new UnixDomainSocketEndPoint(path); + } + + using var exporter = new GenevaTraceExporter(exporterOptions); + + // Set the ActivitySourceName to the unique value of the test method name to avoid interference with + // the ActivitySource used by other unit tests. + var sourceName = GetTestMethodName(); + + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .SetSampler(new AlwaysOnSampler()) + .ConfigureResource(resourceBuilder => resourceBuilder.AddService("cloud.role from resource")) + .AddSource(sourceName) + .AddProcessor(new ReentrantExportProcessor(exporter)) + .Build(); + + var source = new ActivitySource(sourceName); + using (var activity = source.StartActivity("test")) + { + } + + var exportedData = (exporter.Exporter as MsgPackTraceExporter).Buffer.Value; + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(exportedData, MessagePack.Resolvers.ContractlessStandardResolver.Options); + + var signal = (fluentdData as object[])[0] as string; + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + Assert.Contains("env_cloud_role", mapping.Keys); Assert.Equal("cloud.role from resource", mapping["env_cloud_role"]); } @@ -1374,13 +1439,13 @@ private void CheckSpanForActivity(GenevaExporterOptions exporterOptions, object // Part A cloud extensions if (resourceAttributes.TryGetValue("service.name", out var expectedServiceName) - && !exporterOptions.PrepopulatedFields.ContainsKey("cloud.role")) + && exporterOptions.PrepopulatedFields != null && exporterOptions.PrepopulatedFields.Count == 0) { this.AssertMappingEntry(mapping, "env_cloud_role", expectedServiceName); } if (resourceAttributes.TryGetValue("service.instanceId", out var expectedInstanceId) - && !exporterOptions.PrepopulatedFields.ContainsKey("cloud.roleInstance")) + && exporterOptions.PrepopulatedFields != null && exporterOptions.PrepopulatedFields.Count == 0) { this.AssertMappingEntry(mapping, "env_cloud_roleInstance", expectedInstanceId); } From 4fe80cbff5934ce88f9e692e2a21474e07294762 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Wed, 7 Jan 2026 11:41:57 -0800 Subject: [PATCH 17/20] Update src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md Co-authored-by: Reiley Yang --- src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md index 2cc23c5854..091eb1da70 100644 --- a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md @@ -3,8 +3,8 @@ ## Unreleased * Add ResourceFieldNames to filter resource attributes to send to Geneva - ([#3552](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/3552)) - ([#3646](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/3646)) + ([#3552](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/3552), + [#3646](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/3646)) ## 1.14.0 From 0393845319709ce9e797360654a6819b761e7da4 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Wed, 7 Jan 2026 12:03:04 -0800 Subject: [PATCH 18/20] keep using frozen dictionary as much as practical --- .../Internal/MsgPack/MsgPackLogExporter.cs | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs index 1f0896a38f..0a3fba686b 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs @@ -42,7 +42,6 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable private readonly ExceptionStackExportMode exportExceptionStack; private readonly bool userProvidedPrepopulatedFields; - private readonly Dictionary? prepopulatedFields; private readonly IEnumerable? resourceFieldNames; private readonly byte[] bufferEpilogue; private readonly IDataTransport dataTransport; @@ -52,6 +51,11 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable // This is used for Scopes private readonly ThreadLocal serializationData = new(); +#if NET + private FrozenDictionary? prepopulatedFields; +#else + private Dictionary? prepopulatedFields; +#endif private bool isDisposed; public MsgPackLogExporter(GenevaExporterOptions options, Func resourceProvider) @@ -111,18 +115,23 @@ public MsgPackLogExporter(GenevaExporterOptions options, Func resource } } - this.prepopulatedFields = new Dictionary(0, StringComparer.Ordinal); this.resourceFieldNames = options.ResourceFieldNames; } this.userProvidedPrepopulatedFields = options.PrepopulatedFields != null && options.PrepopulatedFields.Count > 0; if (options.PrepopulatedFields != null) { - this.prepopulatedFields = new Dictionary(options.PrepopulatedFields.Count, StringComparer.Ordinal); + var tempPrepopulatedFields = new Dictionary(options.PrepopulatedFields.Count, StringComparer.Ordinal); foreach (var kv in options.PrepopulatedFields) { - this.prepopulatedFields[kv.Key] = kv.Value; + tempPrepopulatedFields[kv.Key] = kv.Value; } + +#if NET + this.prepopulatedFields = tempPrepopulatedFields.ToFrozenDictionary(StringComparer.Ordinal); +#else + this.prepopulatedFields = tempPrepopulatedFields; +#endif } // TODO: Validate custom fields (reserved name? etc). @@ -202,10 +211,11 @@ public void Dispose() /// internal void AddResourceAttributesToPrepopulated() { - Guard.ThrowIfNull(this.prepopulatedFields); - var resourceAttributes = this.resourceProvider().Attributes; + var tempPrepopulatedFields = this.prepopulatedFields != null + ? new Dictionary(this.prepopulatedFields, StringComparer.Ordinal) + : new Dictionary(StringComparer.Ordinal); foreach (var resourceAttribute in resourceAttributes) { var key = resourceAttribute.Key; @@ -278,9 +288,15 @@ internal void AddResourceAttributesToPrepopulated() if (isWantedAttribute) { - this.prepopulatedFields[key] = value; + tempPrepopulatedFields[key] = value; } } + +#if NET + this.prepopulatedFields = tempPrepopulatedFields.ToFrozenDictionary(StringComparer.Ordinal); +#else + this.prepopulatedFields = tempPrepopulatedFields +#endif } internal ArraySegment SerializeLogRecord(LogRecord logRecord) From 6c2a3ebc0620642541ac8e525f6ce108120a9e24 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Wed, 7 Jan 2026 12:28:52 -0800 Subject: [PATCH 19/20] fix semicolon --- .../Internal/MsgPack/MsgPackLogExporter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs index 0a3fba686b..04b3539c49 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs @@ -295,7 +295,7 @@ internal void AddResourceAttributesToPrepopulated() #if NET this.prepopulatedFields = tempPrepopulatedFields.ToFrozenDictionary(StringComparer.Ordinal); #else - this.prepopulatedFields = tempPrepopulatedFields + this.prepopulatedFields = tempPrepopulatedFields; #endif } From 14690191f1fae4e7a3d4fe640264076397bb00c2 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Tue, 13 Jan 2026 10:07:12 -0800 Subject: [PATCH 20/20] respond to PR comments --- src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs | 3 ++- src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs index 0838ab4cce..cc10468e1f 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs @@ -9,6 +9,7 @@ using OpenTelemetry.Exporter.Geneva.Tld; using OpenTelemetry.Internal; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; namespace OpenTelemetry.Exporter.Geneva; @@ -93,7 +94,7 @@ public GenevaLogExporter(GenevaExporterOptions options) var msgPackLogExporter = new MsgPackLogExporter(options, () => { // this is not equivalent to passing a method reference, because the ParentProvider could change after the constructor. - return this.ParentProvider.GetResource(); + return this.ParentProvider?.GetResource() ?? Resource.Empty; }); this.IsUsingUnixDomainSocket = msgPackLogExporter.IsUsingUnixDomainSocket; this.exportLogRecord = msgPackLogExporter.Export; diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs index b46b9ac87d..7e40829b34 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs @@ -6,6 +6,7 @@ using OpenTelemetry.Exporter.Geneva.MsgPack; using OpenTelemetry.Exporter.Geneva.Tld; using OpenTelemetry.Internal; +using OpenTelemetry.Resources; namespace OpenTelemetry.Exporter.Geneva; @@ -72,7 +73,7 @@ public GenevaTraceExporter(GenevaExporterOptions options) var msgPackTraceExporter = new MsgPackTraceExporter(options, () => { // this is not equivalent to passing a method reference, because the ParentProvider could change after the constructor. - return this.ParentProvider.GetResource(); + return this.ParentProvider?.GetResource() ?? Resource.Empty; }); this.IsUsingUnixDomainSocket = msgPackTraceExporter.IsUsingUnixDomainSocket; this.exportActivity = msgPackTraceExporter.Export;