Add instrumentor for Azure Kusto

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -41,6 +41,7 @@ body:
         - OpenTelemetry.Instrumentation.GrpcNetClient
         - OpenTelemetry.Instrumentation.Hangfire
         - OpenTelemetry.Instrumentation.Http
+        - OpenTelemetry.Instrumentation.Kusto
         - OpenTelemetry.Instrumentation.MassTransit
         - OpenTelemetry.Instrumentation.MySqlData
         - OpenTelemetry.Instrumentation.Owin

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -41,6 +41,7 @@ body:
         - OpenTelemetry.Instrumentation.GrpcNetClient
         - OpenTelemetry.Instrumentation.Hangfire
         - OpenTelemetry.Instrumentation.Http
+        - OpenTelemetry.Instrumentation.Kusto
         - OpenTelemetry.Instrumentation.MassTransit
         - OpenTelemetry.Instrumentation.MySqlData
         - OpenTelemetry.Instrumentation.Owin

.github/ISSUE_TEMPLATE/release_request.yml

@@ -38,6 +38,7 @@ body:
         - OpenTelemetry.Instrumentation.GrpcNetClient
         - OpenTelemetry.Instrumentation.Hangfire
         - OpenTelemetry.Instrumentation.Http
+        - OpenTelemetry.Instrumentation.Kusto
         - OpenTelemetry.Instrumentation.MassTransit
         - OpenTelemetry.Instrumentation.MySqlData
         - OpenTelemetry.Instrumentation.Owin

.github/codecov.yml

@@ -135,6 +135,11 @@ flags:
     paths:
       - src/OpenTelemetry.Instrumentation.Http
 
+  unittests-Instrumentation.Kusto:
+    carryforward: true
+    paths:
+      - src/OpenTelemetry.Instrumentation.Kusto
+
   unittests-Instrumentation.Owin:
     carryforward: true
     paths:

.github/workflows/ci.yml

@@ -62,6 +62,7 @@ jobs:
           instrumentation-grpcnetclient: ['*/OpenTelemetry.Instrumentation.GrpcNetClient*/**', '!**/*.md']
           instrumentation-hangfire: ['*/OpenTelemetry.Instrumentation.Hangfire*/**', '!**/*.md']
           instrumentation-http: ['*/OpenTelemetry.Instrumentation.Http*/**', '!**/*.md']
+          instrumentation-kusto: ['*/OpenTelemetry.Instrumentation.Kusto*/**', '!**/*.md']
           instrumentation-owin: ['*/OpenTelemetry.Instrumentation.Owin*/**', 'examples/owin/**', '!**/*.md']
           instrumentation-process: ['*/OpenTelemetry.Instrumentation.Process*/**', 'examples/process-instrumentation/**', '!**/*.md']
           instrumentation-quartz: ['*/OpenTelemetry.Instrumentation.Quartz*/**', '!**/*.md']
@@ -363,6 +364,17 @@ jobs:
       project-name: Component[OpenTelemetry.Instrumentation.Http]
       code-cov-name: Instrumentation.Http
 
+  build-test-instrumentation-kusto:
+    needs: detect-changes
+    if: |
+      contains(needs.detect-changes.outputs.changes, 'instrumentation-kusto')
+      || contains(needs.detect-changes.outputs.changes, 'build')
+      || contains(needs.detect-changes.outputs.changes, 'shared')
+    uses: ./.github/workflows/Component.BuildTest.yml
+    with:
+      project-name: Component[OpenTelemetry.Instrumentation.Kusto]
+      code-cov-name: Instrumentation.Kusto
+
   build-test-instrumentation-owin:
     needs: detect-changes
     if: |

.github/workflows/prepare-release.yml

@@ -29,6 +29,7 @@ on:
         - OpenTelemetry.Instrumentation.GrpcNetClient
         - OpenTelemetry.Instrumentation.Hangfire
         - OpenTelemetry.Instrumentation.Http
+        - OpenTelemetry.Instrumentation.Kusto
         - OpenTelemetry.Instrumentation.MassTransit
         - OpenTelemetry.Instrumentation.MySqlData
         - OpenTelemetry.Instrumentation.Owin

.gitignore

@@ -446,3 +446,7 @@ test/**/BenchmarkResults/**
 !test/**/BenchmarkResults/results/
 # Do NOT ignore files ending with -report-github.md anywhere under BenchmarkResults
 !test/**/BenchmarkResults/results/*-report-github.md
+
+# Ignore Verify received files
+*.received.*
+*.received/

Directory.Packages.props

@@ -83,6 +83,8 @@
     <PackageVersion Include="Confluent.Kafka" Version="[2.4.0,)" />
     <PackageVersion Include="Grpc.Core.Api" Version="[2.46.6,)" />
     <PackageVersion Include="InfluxDB.Client" Version="[4.18.0,)" />
+    <PackageVersion Include="Microsoft.Azure.Kusto.Cloud.Platform" Version="[14.0.3,)" />
+    <PackageVersion Include="Microsoft.Azure.Kusto.Language" Version="[12.3.1,)" />
     <PackageVersion Include="Microsoft.ServiceFabric.Actors" Version="[7.1.2448,)" />
     <PackageVersion Include="Microsoft.ServiceFabric.Services.Remoting" Version="[7.1.2448,)" />
     <PackageVersion Include="Quartz" Version="[3.6.3,)" />
@@ -117,12 +119,14 @@
     <PackageVersion Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="8.0.22" />
     <PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.22" />
     <PackageVersion Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.22" />
+    <PackageVersion Include="Microsoft.Azure.Kusto.Data" Version="14.0.3" />
     <PackageVersion Include="Microsoft.Data.SqlClient" Version="6.1.3" />
     <PackageVersion Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.22" />
     <PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.22" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="8.0.1" />
     <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.1" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
+    <PackageVersion Include="Microsoft.VisualStudio.DiagnosticsHub.BenchmarkDotNetDiagnosers" Version="18.3.36714.1" />
     <PackageVersion Include="Microsoft.Web.Xdt" Version="3.2.0" />
     <PackageVersion Include="MinVer" Version="6.0.0" />
     <PackageVersion Include="MessagePack" Version="3.1.4" />
@@ -137,9 +141,11 @@
     <!-- These WCF dependencies are pinned until we drop support for net8.0 and net9.0 -->
     <PackageVersion Include="System.ServiceModel.Http" Version="[8.1.2,)" />
     <PackageVersion Include="System.ServiceModel.NetTcp" Version="[8.1.2,)" />
+    <PackageVersion Include="Testcontainers.Kusto" Version="4.9.0" />
     <PackageVersion Include="Testcontainers.MsSql" Version="4.9.0" />
     <PackageVersion Include="Testcontainers.MySql" Version="4.9.0" />
     <PackageVersion Include="Testcontainers.PostgreSql" Version="4.9.0" />
+    <PackageVersion Include="Verify.Xunit" Version="31.4.3" />
     <PackageVersion Include="Wiremock.Net" Version="1.16.0" />
     <!-- Microsoft.AspNetCore.Server.Kestrel.Core is transitive dependency for Wiremock.Net. It is here to fix https://github.com/advisories/GHSA-5rrx-jjjq-q2r5 -->
     <PackageVersion Include="Microsoft.AspNetCore.Server.Kestrel.Core" Version="2.3.6" />

opentelemetry-dotnet-contrib.slnx

@@ -157,6 +157,7 @@
     <Project Path="src/OpenTelemetry.Instrumentation.GrpcNetClient/OpenTelemetry.Instrumentation.GrpcNetClient.csproj" />
     <Project Path="src/OpenTelemetry.Instrumentation.Hangfire/OpenTelemetry.Instrumentation.Hangfire.csproj" />
     <Project Path="src/OpenTelemetry.Instrumentation.Http/OpenTelemetry.Instrumentation.Http.csproj" />
+    <Project Path="src/OpenTelemetry.Instrumentation.Kusto/OpenTelemetry.Instrumentation.Kusto.csproj" />
     <Project Path="src/OpenTelemetry.Instrumentation.Owin/OpenTelemetry.Instrumentation.Owin.csproj" />
     <Project Path="src/OpenTelemetry.Instrumentation.Process/OpenTelemetry.Instrumentation.Process.csproj" />
     <Project Path="src/OpenTelemetry.Instrumentation.Quartz/OpenTelemetry.Instrumentation.Quartz.csproj" />
@@ -267,6 +268,8 @@
     <Project Path="test/OpenTelemetry.Instrumentation.Hangfire.Tests/OpenTelemetry.Instrumentation.Hangfire.Tests.csproj" />
     <Project Path="test/OpenTelemetry.Instrumentation.Http.Benchmarks/OpenTelemetry.Instrumentation.Http.Benchmarks.csproj" />
     <Project Path="test/OpenTelemetry.Instrumentation.Http.Tests/OpenTelemetry.Instrumentation.Http.Tests.csproj" />
+    <Project Path="test/OpenTelemetry.Instrumentation.Kusto.Benchmarks/OpenTelemetry.Instrumentation.Kusto.Benchmarks.csproj" />
+    <Project Path="test/OpenTelemetry.Instrumentation.Kusto.Tests/OpenTelemetry.Instrumentation.Kusto.Tests.csproj" />
     <Project Path="test/OpenTelemetry.Instrumentation.Owin.Tests/OpenTelemetry.Instrumentation.Owin.Tests.csproj" />
     <Project Path="test/OpenTelemetry.Instrumentation.Process.Tests/OpenTelemetry.Instrumentation.Process.Tests.csproj" />
     <Project Path="test/OpenTelemetry.Instrumentation.Quartz.Tests/OpenTelemetry.Instrumentation.Quartz.Tests.csproj" />

src/OpenTelemetry.Instrumentation.Kusto/.publicApi/PublicAPI.Shipped.txt

@@ -0,0 +1 @@
+#nullable enable

src/OpenTelemetry.Instrumentation.Kusto/.publicApi/PublicAPI.Unshipped.txt

@@ -0,0 +1,21 @@
+#nullable enable
+OpenTelemetry.Metrics.KustoMeterInstrumentationOptions
+OpenTelemetry.Metrics.KustoMeterInstrumentationOptions.KustoMeterInstrumentationOptions() -> void
+OpenTelemetry.Metrics.KustoMeterInstrumentationOptions.RecordQuerySummary.get -> bool
+OpenTelemetry.Metrics.KustoMeterInstrumentationOptions.RecordQuerySummary.set -> void
+OpenTelemetry.Metrics.KustoMeterInstrumentationOptions.RecordQueryText.get -> bool
+OpenTelemetry.Metrics.KustoMeterInstrumentationOptions.RecordQueryText.set -> void
+OpenTelemetry.Metrics.MeterProviderBuilderExtensions
+OpenTelemetry.Trace.KustoTraceInstrumentationOptions
+OpenTelemetry.Trace.KustoTraceInstrumentationOptions.Enrich.get -> System.Action<System.Diagnostics.Activity!, Kusto.Cloud.Platform.Utils.TraceRecord!>?
+OpenTelemetry.Trace.KustoTraceInstrumentationOptions.Enrich.set -> void
+OpenTelemetry.Trace.KustoTraceInstrumentationOptions.KustoTraceInstrumentationOptions() -> void
+OpenTelemetry.Trace.KustoTraceInstrumentationOptions.RecordQuerySummary.get -> bool
+OpenTelemetry.Trace.KustoTraceInstrumentationOptions.RecordQuerySummary.set -> void
+OpenTelemetry.Trace.KustoTraceInstrumentationOptions.RecordQueryText.get -> bool
+OpenTelemetry.Trace.KustoTraceInstrumentationOptions.RecordQueryText.set -> void
+OpenTelemetry.Trace.TracerProviderBuilderExtensions
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.AddKustoInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder) -> OpenTelemetry.Metrics.MeterProviderBuilder!
+static OpenTelemetry.Metrics.MeterProviderBuilderExtensions.AddKustoInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder, System.Action<OpenTelemetry.Metrics.KustoMeterInstrumentationOptions!>? configureKustoMeterInstrumentationOptions) -> OpenTelemetry.Metrics.MeterProviderBuilder!
+static OpenTelemetry.Trace.TracerProviderBuilderExtensions.AddKustoInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder) -> OpenTelemetry.Trace.TracerProviderBuilder!
+static OpenTelemetry.Trace.TracerProviderBuilderExtensions.AddKustoInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder, System.Action<OpenTelemetry.Trace.KustoTraceInstrumentationOptions!>? configureKustoTraceInstrumentationOptions) -> OpenTelemetry.Trace.TracerProviderBuilder!

src/OpenTelemetry.Instrumentation.Kusto/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## Unreleased
+
+* Initial implementation.

src/OpenTelemetry.Instrumentation.Kusto/Implementation/InstrumentationHandleManagerExtensions.cs

@@ -0,0 +1,28 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+namespace OpenTelemetry.Instrumentation.Kusto.Implementation;
+
+/// <summary>
+/// Provides extension methods for <see cref="InstrumentationHandleManager" />.
+/// </summary>
+internal static class InstrumentationHandleManagerExtensions
+{
+    /// <summary>
+    /// Returns <see langword="true"/> if tracing is active (i.e., there is at least one tracing handle); otherwise, <see langword="false"/>.
+    /// </summary>
+    /// <param name="handleManager">
+    /// The <see cref="InstrumentationHandleManager"/> to check for active tracing handles.
+    /// </param>
+    /// <returns><see langword="true"/> if tracing is active; otherwise, <see langword="false"/>.</returns>
+    public static bool IsTracingActive(this InstrumentationHandleManager handleManager) => handleManager.TracingHandles > 0;
+
+    /// <summary>
+    /// Returns <see langword="true"/> if metrics is active (i.e., there is at least one metrics handle); otherwise, <see langword="false"/>.
+    /// </summary>
+    /// <param name="handleManager">
+    /// The <see cref="InstrumentationHandleManager"/> to check for active metrics handles.
+    /// </param>
+    /// <returns><see langword="true"/> if metrics is active; otherwise, <see langword="false"/>.</returns>
+    public static bool IsMetricsActive(this InstrumentationHandleManager handleManager) => handleManager.MetricHandles > 0;
+}

src/OpenTelemetry.Instrumentation.Kusto/Implementation/KustoActivitySourceHelper.cs

@@ -0,0 +1,34 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using System.Diagnostics;
+using System.Diagnostics.Metrics;
+using System.Reflection;
+using OpenTelemetry.Internal;
+
+namespace OpenTelemetry.Instrumentation.Kusto.Implementation;
+
+/// <summary>
+/// Helper class to hold common properties used by Kusto instrumentation.
+/// </summary>
+internal static class KustoActivitySourceHelper
+{
+    public const string DbSystem = "azure.kusto";
+    public const string ClientRequestIdTagKey = $"{DbSystem}.client_request_id";
+
+    public static readonly Assembly Assembly = typeof(KustoActivitySourceHelper).Assembly;
+    public static readonly AssemblyName AssemblyName = Assembly.GetName();
+    public static readonly string PackageVersion = Assembly.GetPackageVersion();
+
+    public static readonly string ActivitySourceName = AssemblyName.Name!;
+    public static readonly ActivitySource ActivitySource = new(ActivitySourceName, PackageVersion);
+
+    public static readonly string MeterName = AssemblyName.Name!;
+    public static readonly Meter Meter = new(MeterName, PackageVersion);
+
+    public static readonly Histogram<double> OperationDurationHistogram = Meter.CreateHistogram(
+        "db.client.operation.duration",
+        unit: "s",
+        advice: new InstrumentAdvice<double>() { HistogramBucketBoundaries = [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10] },
+        description: "Duration of database client operations");
+}

src/OpenTelemetry.Instrumentation.Kusto/Implementation/KustoInstrumentation.cs

@@ -0,0 +1,44 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using Kusto.Cloud.Platform.Utils;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Trace;
+
+namespace OpenTelemetry.Instrumentation.Kusto.Implementation;
+
+/// <summary>
+/// Class to hold the singleton instances used for Kusto instrumentation.
+/// </summary>
+internal static class KustoInstrumentation
+{
+    private static readonly Lazy<ITraceListener> Listener = new(() =>
+    {
+        Environment.SetEnvironmentVariable("KUSTO_DATA_TRACE_REQUEST_BODY", "1");
+
+        var listener = new KustoTraceRecordListener();
+        TraceSourceManager.AddTraceListener(listener, startupDone: true);
+
+        return listener;
+    });
+
+    /// <summary>
+    /// Gets or sets the post-configured trace options for Kusto instrumentation.
+    /// </summary>
+    public static KustoTraceInstrumentationOptions TraceOptions { get; set; } = new KustoTraceInstrumentationOptions();
+
+    /// <summary>
+    /// Gets or sets the post-configured meter options for Kusto instrumentation.
+    /// </summary>
+    public static KustoMeterInstrumentationOptions MeterOptions { get; set; } = new KustoMeterInstrumentationOptions();
+
+    /// <summary>
+    /// Gets the <see cref="InstrumentationHandleManager"/> that tracks if there are any active listeners for <see cref="KustoTraceRecordListener"/>.
+    /// </summary>
+    public static InstrumentationHandleManager HandleManager { get; } = new InstrumentationHandleManager();
+
+    /// <summary>
+    /// Initializes the Kusto instrumentation by ensuring the listener is created and registered with the client library.
+    /// </summary>
+    public static void Initialize() => _ = Listener.Value;
+}

src/OpenTelemetry.Instrumentation.Kusto/Implementation/KustoInstrumentationEventSource.cs

@@ -0,0 +1,46 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+using System.Diagnostics.Tracing;
+using OpenTelemetry.Internal;
+
+namespace OpenTelemetry.Instrumentation.Kusto.Implementation;
+
+/// <summary>
+/// EventSource for Kusto instrumentation.
+/// </summary>
+[EventSource(Name = "OpenTelemetry-Instrumentation-Kusto")]
+internal sealed class KustoInstrumentationEventSource : EventSource
+{
+    public static KustoInstrumentationEventSource Log { get; } = new();
+
+    [NonEvent]
+    public void EnrichmentException(Exception ex)
+    {
+        if (this.IsEnabled(EventLevel.Error, EventKeywords.All))
+        {
+            this.EnrichmentException(ex.ToInvariantString());
+        }
+    }
+
+    [Event(1, Message = "Enrichment exception: {0}", Level = EventLevel.Error)]
+    public void EnrichmentException(string exception) => this.WriteEvent(1, exception);
+
+    [Event(2, Message = "Trace record payload is NULL or has NULL message, record will not be processed.", Level = EventLevel.Warning)]
+    public void NullPayload() => this.WriteEvent(2);
+
+    [Event(3, Message = "Failed to find context for activity ID '{0}', operation data will not be recorded.", Level = EventLevel.Warning)]
+    public void ContextNotFound(string activityId) => this.WriteEvent(3, activityId);
+
+    [NonEvent]
+    public void UnknownErrorProcessingTraceRecord(Exception ex)
+    {
+        if (this.IsEnabled(EventLevel.Error, EventKeywords.All))
+        {
+            this.UnknownErrorProcessingTraceRecord(ex.ToInvariantString());
+        }
+    }
+
+    [Event(4, Message = "Unknown error processing trace record, Exception: {0}", Level = EventLevel.Error)]
+    public void UnknownErrorProcessingTraceRecord(string exception) => this.WriteEvent(4, exception);
+}