[geneva] Add support for exporting otlp metrics via user_events on Linux (open-telemetry#2113)

Author: CodeBlanch

src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md

@@ -5,6 +5,15 @@
 * Drop support for .NET 6 as this target is no longer supported.
   ([#2117](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2117))
 
+* Added support for exporting metrics via
+  [user_events](https://docs.kernel.org/trace/user_events.html) on Linux when
+  OTLP protobuf encoding is enabled via the
+  `PrivatePreviewEnableOtlpProtobufEncoding=true` connection string switch. With
+  this, `PrivatePreviewEnableOtlpProtobufEncoding=true` is now supported on both
+  Widows and Linux. Windows uses ETW as transport, while Linux uses user_events
+  as transport.
+  ([#2113](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2113))
+
 ## 1.9.0
 
 Released 2024-Jun-21

src/OpenTelemetry.Exporter.Geneva/Internal/ExporterEventSource.cs

@@ -18,6 +18,9 @@ internal sealed class ExporterEventSource : EventSource
     private const int EVENT_ID_ERROR = 4; // Other common exporter exceptions
     private const int EVENT_ID_OTLP_PROTOBUF_METRIC = 5; // Failed to serialize metric
     private const int EVENT_ID_COMPLETED_EXPORT = 6; // Completed export
+    private const int EVENT_ID_TRANSPORT_ERROR = 7; // Transport error
+    private const int EVENT_ID_TRANSPORT_EXCEPTION = 8; // Transport exception
+    private const int EVENT_ID_TRANSPORT_INFO = 9; // Transport info
 
     [NonEvent]
     public void FailedToSendTraceData(Exception ex)
@@ -76,6 +79,16 @@ public void FailedToSerializeMetric(string metricName, Exception ex)
         }
     }
 
+    [NonEvent]
+    public void TransportException(string transportType, string message, Exception ex)
+    {
+        if (Log.IsEnabled(EventLevel.Error, EventKeywords.All))
+        {
+            // TODO: Do not hit ETW size limit even for external library exception stack.
+            this.TransportException(transportType, message, ex.ToInvariantString());
+        }
+    }
+
     [Event(EVENT_ID_TRACE, Message = "Exporter failed to send trace data. Exception: {0}", Level = EventLevel.Error)]
     public void FailedToSendTraceData(string error)
     {
@@ -111,4 +124,22 @@ public void ExportCompleted(string exporterName)
     {
         this.WriteEvent(EVENT_ID_COMPLETED_EXPORT, exporterName);
     }
+
+    [Event(EVENT_ID_TRANSPORT_ERROR, Message = "Transport '{0}' error. Message: {1}", Level = EventLevel.Error)]
+    public void TransportError(string transportType, string error)
+    {
+        this.WriteEvent(EVENT_ID_TRANSPORT_ERROR, transportType, error);
+    }
+
+    [Event(EVENT_ID_TRANSPORT_EXCEPTION, Message = "Transport '{0}' error. Message: {1}, Exception: {2}", Level = EventLevel.Error)]
+    public void TransportException(string transportType, string error, string ex)
+    {
+        this.WriteEvent(EVENT_ID_TRANSPORT_EXCEPTION, transportType, error, ex);
+    }
+
+    [Event(EVENT_ID_TRANSPORT_INFO, Message = "Transport '{0}' information. Message: {1}", Level = EventLevel.Informational)]
+    public void TransportInformation(string transportType, string error)
+    {
+        this.WriteEvent(EVENT_ID_TRANSPORT_INFO, transportType, error);
+    }
 }

src/OpenTelemetry.Exporter.Geneva/Internal/Transports/UnixDomainSocketDataTransport.cs

@@ -73,7 +73,7 @@ private bool Connect()
         }
         catch (Exception ex)
         {
-            ExporterEventSource.Log.ExporterException("UDS Connect failed.", ex);
+            ExporterEventSource.Log.TransportException(nameof(UnixDomainSocketDataTransport), "Attempt to connect to socket failed. Connection will be retried periodically until established.", ex);
 
             return false;
         }

src/OpenTelemetry.Exporter.Geneva/Metrics/OtlpProtobuf/OtlpProtobufMetricExporter.cs

@@ -1,6 +1,7 @@
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 
+using System.Diagnostics;
 using System.Runtime.InteropServices;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Resources;
@@ -15,17 +16,32 @@ internal sealed class OtlpProtobufMetricExporter : IDisposable
 
     private readonly Func<Resource> getResource;
 
-    public OtlpProtobufMetricExporter(Func<Resource> getResource, ConnectionStringBuilder connectionStringBuilder, IReadOnlyDictionary<string, object> prepopulatedMetricDimensions)
+    public OtlpProtobufMetricExporter(
+        Func<Resource> getResource,
+        ConnectionStringBuilder connectionStringBuilder,
+        IReadOnlyDictionary<string, object> prepopulatedMetricDimensions)
     {
+        Debug.Assert(getResource != null, "getResource was null");
+
+        this.getResource = getResource;
+
+#if NET6_0_OR_GREATER
+        IMetricDataTransport transport = !RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+            ? MetricUnixUserEventsDataTransport.Instance
+            : MetricWindowsEventTracingDataTransport.Instance;
+#else
         if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
         {
-            // Temporary until we add support for user_events.
             throw new NotSupportedException("Exporting data in protobuf format is not supported on Linux.");
         }
 
-        this.getResource = getResource;
+        var transport = MetricWindowsEventTracingDataTransport.Instance;
+#endif
 
-        this.otlpProtobufSerializer = new OtlpProtobufSerializer(MetricWindowsEventTracingDataTransport.Instance, connectionStringBuilder, prepopulatedMetricDimensions);
+        this.otlpProtobufSerializer = new OtlpProtobufSerializer(
+            transport,
+            connectionStringBuilder,
+            prepopulatedMetricDimensions);
     }
 
     public ExportResult Export(in Batch<Metric> batch)

src/OpenTelemetry.Exporter.Geneva/Metrics/Transport/MetricUnixUserEventsDataTransport.cs

@@ -0,0 +1,70 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+#if NET
+
+#nullable enable
+
+using System.Text;
+using Microsoft.LinuxTracepoints.Provider;
+
+namespace OpenTelemetry.Exporter.Geneva;
+
+internal sealed class MetricUnixUserEventsDataTransport : IMetricDataTransport
+{
+    public const uint MetricsProtocol = 0U;
+    public const string MetricsVersion = "v0.19.00";
+    public const string MetricsTracepointName = "otlp_metrics";
+    public const string MetricsTracepointNameArgs = $"{MetricsTracepointName} u32 protocol;char[8] version;__rel_loc u8[] buffer";
+
+    private static readonly ReadOnlyMemory<byte> MetricsVersionUtf8 = Encoding.UTF8.GetBytes(MetricsVersion);
+    private readonly PerfTracepoint metricsTracepoint;
+
+    private MetricUnixUserEventsDataTransport()
+    {
+        this.metricsTracepoint = new PerfTracepoint(MetricsTracepointNameArgs);
+        if (this.metricsTracepoint.RegisterResult != 0)
+        {
+            // ENOENT (2): No such file or directory
+            if (this.metricsTracepoint.RegisterResult == 2)
+            {
+                throw new NotSupportedException(
+                    $"Tracepoint registration for 'otlp_metrics' failed with result: '{this.metricsTracepoint.RegisterResult}'. Verify your distribution/kernel supports user_events: https://docs.kernel.org/trace/user_events.html.");
+            }
+
+            ExporterEventSource.Log.TransportInformation(
+                nameof(MetricUnixUserEventsDataTransport),
+                $"Tracepoint registration operation for 'otlp_metrics' returned result '{this.metricsTracepoint.RegisterResult}' which is considered recoverable. Entering running state.");
+        }
+    }
+
+    public static MetricUnixUserEventsDataTransport Instance { get; } = new();
+
+    public void Send(MetricEventType eventType, byte[] body, int size)
+    {
+        throw new NotSupportedException();
+    }
+
+    public void SendOtlpProtobufEvent(byte[] body, int size)
+    {
+        if (this.metricsTracepoint.IsEnabled)
+        {
+            var buffer = new ReadOnlySpan<byte>(body, 0, size);
+
+            var bufferRelLoc = 0u | ((uint)buffer.Length << 16);
+
+            this.metricsTracepoint.Write(
+                [MetricsProtocol],
+                MetricsVersionUtf8.Span,
+                [bufferRelLoc],
+                buffer);
+        }
+    }
+
+    public void Dispose()
+    {
+        this.metricsTracepoint.Dispose();
+    }
+}
+
+#endif

src/OpenTelemetry.Exporter.Geneva/README.md

@@ -268,14 +268,25 @@ On Linux provide an `Endpoint` in addition to the `Account` and `Namespace`.
 For example:
 `Endpoint=unix:{UDS Path};Account={MetricAccount};Namespace={MetricNamespace}`.
 
-Set `PrivatePreviewEnableOtlpProtobufEncoding=true` to opt-in to the
-experimental feature for changing the underlying serialization format to binary
-protobuf following the schema defined in [OTLP
+##### OtlpProtobufEncoding
+
+On Windows set `PrivatePreviewEnableOtlpProtobufEncoding=true` on the
+`ConnectionString` to opt-in to the experimental feature for changing the
+underlying serialization format to binary protobuf following the schema defined
+in [OTLP
 specification](https://github.com/open-telemetry/opentelemetry-proto/blob/v1.1.0/opentelemetry/proto/metrics/v1/metrics.proto).
 
-> [!NOTE]
- > `PrivatePreviewEnableOtlpProtobufEncoding` is currently
- > only supported in Windows environment.
+As of `1.10.0` `PrivatePreviewEnableOtlpProtobufEncoding=true` is also supported
+on Linux. On Linux when using `PrivatePreviewEnableOtlpProtobufEncoding=true` an
+`Endpoint` is **NOT** required to be provided on `ConnectionString`. For
+example: `Endpoint=unix:Account={MetricAccount};Namespace={MetricNamespace}`.
+
+> [!IMPORTANT]
+> When `PrivatePreviewEnableOtlpProtobufEncoding` is enabled on Linux metrics
+> are written using
+> [user_events](https://docs.kernel.org/trace/user_events.html). `user_events`
+> are a newer feature of the Linux kernel and require a distro with the feature
+> enabled.
 
 #### `MetricExportIntervalMilliseconds` (optional)