Add Application Signals runtime metrics with feature disabled (#148)

Author: bjrara

src/AWS.Distro.OpenTelemetry.AutoInstrumentation/AwsMetricAttributeGenerator.cs

@@ -51,10 +51,6 @@ internal class AwsMetricAttributeGenerator : IMetricAttributeGenerator
     // Special DEPENDENCY attribute value if GRAPHQL_OPERATION_TYPE attribute key is present.
     private static readonly string GraphQL = "graphql";
 
-    // As per https://opentelemetry.io/docs/specs/semconv/resource/#service
-    // If service name is not specified, SDK defaults the service name starting with unknown_service
-    private static readonly string OtelUnknownServicePrefix = "unknown_service";
-
     /// <inheritdoc/>
     public virtual Dictionary<string, ActivityTagsCollection> GenerateMetricAttributeMapFromSpan(Activity span, Resource resource)
     {
@@ -100,10 +96,10 @@ private ActivityTagsCollection GenerateDependencyMetricAttributes(Activity span,
     private static void SetService(Resource resource, Activity span, ActivityTagsCollection attributes)
 #pragma warning restore SA1204 // Static elements should appear before instance elements
     {
-        string? service = (string?)resource.Attributes.FirstOrDefault(attribute => attribute.Key == AttributeServiceName).Value;
+        string? service = (string?)resource.Attributes.FirstOrDefault(attribute => attribute.Key == AttributeAWSLocalService).Value;
 
         // In practice the service name is never null, but we can be defensive here.
-        if (service == null || service.StartsWith(OtelUnknownServicePrefix))
+        if (service == null)
         {
             LogUnknownAttribute(AttributeAWSLocalService, span);
             service = UnknownService;

src/AWS.Distro.OpenTelemetry.AutoInstrumentation/Plugin.cs

@@ -41,6 +41,7 @@ public class Plugin
     private static readonly ILoggerFactory Factory = LoggerFactory.Create(builder => builder.AddProvider(new ConsoleLoggerProvider()));
     private static readonly ILogger Logger = Factory.CreateLogger<Plugin>();
     private static readonly string ApplicationSignalsExporterEndpointConfig = "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT";
+    private static readonly string MetricExporterConfig = "OTEL_METRICS_EXPORTER";
     private static readonly string MetricExportIntervalConfig = "OTEL_METRIC_EXPORT_INTERVAL";
     private static readonly int DefaultMetricExportInterval = 60000;
     private static readonly string DefaultProtocolEnvVarName = "OTEL_EXPORTER_OTLP_PROTOCOL";
@@ -59,6 +60,11 @@ public class Plugin
 
     private static readonly string FormatOtelSampledTracesBinaryPrefix = "T1S";
     private static readonly string FormatOtelUnSampledTracesBinaryPrefix = "T1U";
+    private static readonly string RuntimeMetricMeterName = "OpenTelemetry.Instrumentation.Runtime";
+
+    // As per https://opentelemetry.io/docs/specs/semconv/resource/#service
+    // If service name is not specified, SDK defaults the service name starting with unknown_service
+    private static readonly string OtelUnknownServicePrefix = "unknown_service";
 
     private static readonly int LambdaSpanExportBatchSize = 10;
 
@@ -119,27 +125,9 @@ public void TracerProviderInitialized(TracerProvider tracerProvider)
             // Disable Application Metrics for Lambda environment
             if (!AwsSpanProcessingUtil.IsLambdaEnvironment())
             {
-                string? intervalConfigString = System.Environment.GetEnvironmentVariable(MetricExportIntervalConfig);
-                int exportInterval = DefaultMetricExportInterval;
-                try
-                {
-                    int parsedExportInterval = Convert.ToInt32(intervalConfigString);
-                    exportInterval = parsedExportInterval != 0 ? parsedExportInterval : DefaultMetricExportInterval;
-                }
-                catch (Exception)
-                {
-                    Logger.Log(LogLevel.Trace, "Could not convert OTEL_METRIC_EXPORT_INTERVAL to integer. Using default value 60000.");
-                }
-
-                if (exportInterval.CompareTo(DefaultMetricExportInterval) > 0)
-                {
-                    exportInterval = DefaultMetricExportInterval;
-                    Logger.Log(LogLevel.Information, "AWS Application Signals metrics export interval capped to {0}", exportInterval);
-                }
-
                 // https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/README.md#enable-metric-exporter
                 // for setting the temporatityPref.
-                var metricReader = new PeriodicExportingMetricReader(this.ApplicationSignalsExporterProvider(), exportInterval)
+                var metricReader = new PeriodicExportingMetricReader(this.CreateApplicationSignalsMetricExporter(), GetMetricExportInterval())
                 {
                     TemporalityPreference = MetricReaderTemporalityPreference.Delta,
                 };
@@ -229,6 +217,40 @@ public TracerProviderBuilder AfterConfigureTracerProvider(TracerProviderBuilder
         return builder;
     }
 
+    /// <summary>
+    /// // To configure metrics SDK after Auto Instrumentation configured SDK
+    /// </summary>
+    /// <param name="builder">The metric provider builder</param>
+    /// <returns>The configured metric provider builder</returns>
+    public MeterProviderBuilder AfterConfigureMeterProvider(MeterProviderBuilder builder)
+    {
+        if (!this.IsApplicationSignalsRuntimeEnabled())
+        {
+            return builder;
+        }
+
+        var exporters = System.Environment.GetEnvironmentVariable(MetricExporterConfig);
+        if (!string.IsNullOrEmpty(exporters) && exporters.Contains("none"))
+        {
+            Logger.Log(LogLevel.Information, "Install runtime metric filter in metrics collection.");
+            builder.AddView(instrument => instrument.Meter.Name == RuntimeMetricMeterName
+                ? null
+                : MetricStreamConfiguration.Drop);
+        }
+
+        var runtimeScopeName = new HashSet<string>() { RuntimeMetricMeterName };
+        var metricReader = new PeriodicExportingMetricReader(
+            this.CreateScopeBasedOtlpMetricExporter(runtimeScopeName), GetMetricExportInterval())
+        {
+            TemporalityPreference = MetricReaderTemporalityPreference.Delta,
+        };
+
+        builder.AddReader(metricReader);
+        Logger.Log(LogLevel.Information, "AWS Application Signals runtime metrics enabled.");
+
+        return builder;
+    }
+
     /// <summary>
     /// To configure Resource with resource detectors and <see cref="DistroAttributes"/>
     /// Check <see cref="ResourceBuilderCustomizer"/> for more information.
@@ -357,6 +379,58 @@ public void ConfigureTracesOptions(AspNetTraceInstrumentationOptions options)
     }
 #endif
 
+    private static int GetMetricExportInterval()
+    {
+        var intervalConfigString = System.Environment.GetEnvironmentVariable(MetricExportIntervalConfig);
+        var exportInterval = DefaultMetricExportInterval;
+        try
+        {
+            var parsedExportInterval = Convert.ToInt32(intervalConfigString);
+            exportInterval = parsedExportInterval != 0 ? parsedExportInterval : DefaultMetricExportInterval;
+        }
+        catch (Exception)
+        {
+            Logger.Log(LogLevel.Warning, "Could not convert OTEL_METRIC_EXPORT_INTERVAL to integer. Using default value 60000.");
+        }
+
+        if (exportInterval.CompareTo(DefaultMetricExportInterval) > 0)
+        {
+            exportInterval = DefaultMetricExportInterval;
+            Logger.Log(LogLevel.Information, "AWS Application Signals metrics export interval capped to {0}", exportInterval);
+        }
+
+        return exportInterval;
+    }
+
+    private static void ConfigureOtlpExporterOptions(OtlpExporterOptions options)
+    {
+        var applicationSignalsEndpoint = System.Environment.GetEnvironmentVariable(ApplicationSignalsExporterEndpointConfig);
+        var protocolString = System.Environment.GetEnvironmentVariable(DefaultProtocolEnvVarName) ?? "http/protobuf";
+        OtlpExportProtocol protocol;
+
+        switch (protocolString)
+        {
+            case "http/protobuf":
+                applicationSignalsEndpoint = applicationSignalsEndpoint ?? "http://localhost:4316/v1/metrics";
+                protocol = OtlpExportProtocol.HttpProtobuf;
+                break;
+            case "grpc":
+                applicationSignalsEndpoint = applicationSignalsEndpoint ?? "http://localhost:4315";
+                protocol = OtlpExportProtocol.Grpc;
+                break;
+            default:
+                throw new NotSupportedException("Unsupported AWS Application Signals export protocol: " + protocolString);
+        }
+
+        options.Endpoint = new Uri(applicationSignalsEndpoint);
+        options.Protocol = protocol;
+
+        Logger.Log(
+            LogLevel.Debug, "AWS Application Signals export protocol: %{0}", options.Protocol);
+        Logger.Log(
+            LogLevel.Debug, "AWS Application Signals export endpoint: %{0}", options.Endpoint);
+    }
+
     // This new function runs the sampler a second time after the needed attributes (such as UrlPath and HttpTarget)
     // are finally available from the http instrumentation libraries. The sampler hooked into the Opentelemetry SDK
     // runs right before any activity is started so for the purposes of our X-Ray sampler, that isn't work and breaks
@@ -401,9 +475,23 @@ private bool IsApplicationSignalsEnabled()
         return System.Environment.GetEnvironmentVariable(ApplicationSignalsEnabledConfig) == "true";
     }
 
+    private bool IsApplicationSignalsRuntimeEnabled()
+    {
+        return false;
+    }
+
     private ResourceBuilder ResourceBuilderCustomizer(ResourceBuilder builder)
     {
         builder.AddAttributes(DistroAttributes);
+        var resource = builder.Build();
+        var serviceName = (string?)resource.Attributes.FirstOrDefault(attr => attr.Key == ResourceSemanticConventions.AttributeServiceName).Value;
+        if (serviceName == null || serviceName.StartsWith(OtelUnknownServicePrefix))
+        {
+            Logger.Log(LogLevel.Warning, "No valid service name provided.");
+            serviceName = AwsSpanProcessingUtil.UnknownService;
+        }
+
+        builder.AddAttributes(new Dictionary<string, object> { { AwsAttributeKeys.AttributeAWSLocalService, serviceName } });
 
         // ResourceDetectors are enabled by default. Adding config to be able to disable during local testing
         var resourceDetectorsEnabled = System.Environment.GetEnvironmentVariable(ResourceDetectorEnableConfig) ?? "true";
@@ -428,39 +516,21 @@ private ResourceBuilder ResourceBuilderCustomizer(ResourceBuilder builder)
         return builder;
     }
 
-    private OtlpMetricExporter ApplicationSignalsExporterProvider()
+    private OtlpMetricExporter CreateApplicationSignalsMetricExporter()
     {
         var options = new OtlpExporterOptions();
-
-        string? applicationSignalsEndpoint = System.Environment.GetEnvironmentVariable(ApplicationSignalsExporterEndpointConfig);
-        string? protocolString = System.Environment.GetEnvironmentVariable(DefaultProtocolEnvVarName) ?? "http/protobuf";
-        OtlpExportProtocol protocol;
-        if (protocolString == "http/protobuf")
-        {
-            applicationSignalsEndpoint = applicationSignalsEndpoint ?? "http://localhost:4316/v1/metrics";
-            protocol = OtlpExportProtocol.HttpProtobuf;
-        }
-        else if (protocolString == "grpc")
-        {
-            applicationSignalsEndpoint = applicationSignalsEndpoint ?? "http://localhost:4315";
-            protocol = OtlpExportProtocol.Grpc;
-        }
-        else
-        {
-            throw new NotSupportedException("Unsupported AWS Application Signals export protocol: " + protocolString);
-        }
-
-        options.Endpoint = new Uri(applicationSignalsEndpoint);
-        options.Protocol = protocol;
-
-        Logger.Log(
-          LogLevel.Debug, "AWS Application Signals export protocol: %{0}", options.Protocol);
-        Logger.Log(
-          LogLevel.Debug, "AWS Application Signals export endpoint: %{0}", options.Endpoint);
-
+        ConfigureOtlpExporterOptions(options);
         return new OtlpMetricExporter(options);
     }
 
+    private ScopeBasedOtlpMetricExporter CreateScopeBasedOtlpMetricExporter(HashSet<string> registeredScopeNames)
+    {
+        var options = new ScopeBasedOtlpMetricExporter.ScopeBasedOtlpExporterOptions();
+        ConfigureOtlpExporterOptions(options);
+        options.RegisteredScopeNames = registeredScopeNames;
+        return new ScopeBasedOtlpMetricExporter(options);
+    }
+
     private bool HasCustomTracesEndpoint()
     {
         // detect if running in AWS Lambda environment

src/AWS.Distro.OpenTelemetry.AutoInstrumentation/ScopeBasedOtlpMetricExporter.cs

@@ -0,0 +1,77 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+using AWS.Distro.OpenTelemetry.AutoInstrumentation.Logging;
+using Microsoft.Extensions.Logging;
+using OpenTelemetry;
+using OpenTelemetry.Exporter;
+using OpenTelemetry.Metrics;
+
+namespace AWS.Distro.OpenTelemetry.AutoInstrumentation;
+
+/// <summary>
+/// A custom metric exporter that extends <see cref="OtlpMetricExporter"/>.
+/// Exports metrics only for a specific instrumentation scope, filtering based on
+/// the configured scope name. This allows selective metric exporting to the OTLP endpoint,
+/// limiting exports to metrics that match the specified scope.
+/// </summary>
+public class ScopeBasedOtlpMetricExporter : OtlpMetricExporter
+{
+    private static readonly ILoggerFactory Factory = LoggerFactory.Create(builder => builder.AddProvider(new ConsoleLoggerProvider()));
+    private static readonly ILogger Logger = Factory.CreateLogger<ScopeBasedOtlpMetricExporter>();
+
+    private readonly HashSet<string> registeredScopedNames;
+    private readonly Func<Batch<Metric>, ExportResult> exportHandler;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ScopeBasedOtlpMetricExporter"/> class.
+    /// </summary>
+    /// <param name="options">Configuration options for the exporter.</param>
+    public ScopeBasedOtlpMetricExporter(ScopeBasedOtlpExporterOptions options)
+        : base(options)
+    {
+        this.registeredScopedNames = options.RegisteredScopeNames ?? new HashSet<string>();
+        this.exportHandler = batch => base.Export(batch);
+    }
+
+    internal ScopeBasedOtlpMetricExporter(
+        ScopeBasedOtlpExporterOptions options,
+        Func<Batch<Metric>, ExportResult> exportHandler)
+        : base(options)
+    {
+        this.registeredScopedNames = options.RegisteredScopeNames ?? new HashSet<string>();
+        this.exportHandler = exportHandler;
+    }
+
+    /// <inheritdoc />
+    public override ExportResult Export(in Batch<Metric> metrics)
+    {
+        var exportingMetrics = new List<Metric>();
+        foreach (var metric in metrics)
+        {
+            if (this.registeredScopedNames.Contains(metric.MeterName))
+            {
+                exportingMetrics.Add(metric);
+            }
+        }
+
+        if (exportingMetrics.Count == 0)
+        {
+            Logger.Log(LogLevel.Debug, "No metrics to export.");
+            return ExportResult.Success;
+        }
+
+        return this.exportHandler.Invoke(new Batch<Metric>(exportingMetrics.ToArray(), exportingMetrics.Count));
+    }
+
+    /// <summary>
+    /// Scope based OTLP exporter options.
+    /// </summary>
+    public class ScopeBasedOtlpExporterOptions : OtlpExporterOptions
+    {
+        /// <summary>
+        /// Gets or sets registered meter names whose metrics will be reserved.
+        /// </summary>
+        public HashSet<string>? RegisteredScopeNames { get; set; }
+    }
+}

test/AWS.Distro.OpenTelemetry.AutoInstrumentation.Tests/AwsMetricAttributesGeneratorTest.cs

@@ -1181,6 +1181,7 @@ private void UpdateResourceWithServiceName()
         List<KeyValuePair<string, object?>> resourceAttributes = new List<KeyValuePair<string, object?>>
         {
             new (AutoInstrumentation.AwsMetricAttributeGenerator.AttributeServiceName, this.serviceNameValue),
+            new (AwsAttributeKeys.AttributeAWSLocalService, this.serviceNameValue),
         };
         this.resource = new Resource(resourceAttributes);
     }