[Instrumentation.SqlClient] Introducing context propagation to SQL Server (#2709)

Co-authored-by: Piotr Kiełkowicz <[email protected]>
Co-authored-by: Zach Montoya <[email protected]>
Co-authored-by: Will Li <[email protected]>
Co-authored-by: Alan West <[email protected]>

Author: 5 people

src/OpenTelemetry.Instrumentation.SqlClient/CHANGELOG.md

@@ -10,6 +10,17 @@
   the new conventions.
   ([#2811](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2811))
 
+* Added `OTEL_DOTNET_EXPERIMENTAL_SQLCLIENT_ENABLE_TRACE_CONTEXT_PROPAGATION`
+  environment variable to propagate trace context to SQL Server databases.
+  This will remain experimental while the [specification](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/database/sql-server.md#context-propagation)
+  remains in development.
+  It is now only available on .NET 8 and newer.
+  ([#2709](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/2709))
+
+  > Propagate `traceparent` information to SQL Server databases
+    (see [SET CONTEXT_INFO](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-context-info-transact-sql?view=sql-server-ver16)).
+    Note that this option incurs an additional round-trip to the database.
+
 ## 1.12.0-beta.1
 
 Released 2025-May-06

src/OpenTelemetry.Instrumentation.SqlClient/Implementation/SqlClientDiagnosticListener.cs

@@ -8,6 +8,9 @@
 using System.Diagnostics.CodeAnalysis;
 #endif
 using System.Globalization;
+#if NET
+using System.Text;
+#endif
 using OpenTelemetry.Trace;
 
 namespace OpenTelemetry.Instrumentation.SqlClient.Implementation;
@@ -26,6 +29,11 @@ internal sealed class SqlClientDiagnosticListener : ListenerHandler
     public const string SqlDataWriteCommandError = "System.Data.SqlClient.WriteCommandError";
     public const string SqlMicrosoftWriteCommandError = "Microsoft.Data.SqlClient.WriteCommandError";
 
+#if NET
+    private const string ContextInfoParameterName = "@opentelemetry_traceparent";
+    private const string SetContextSql = $"set context_info {ContextInfoParameterName}";
+#endif
+
     private readonly PropertyFetcher<object> commandFetcher = new("Command");
     private readonly PropertyFetcher<object> connectionFetcher = new("Connection");
     private readonly PropertyFetcher<string> dataSourceFetcher = new("DataSource");
@@ -64,6 +72,15 @@ public override void OnEventWritten(string name, object? payload)
                         return;
                     }
 
+#if NET
+                    // skip if this is an injected query
+                    if (options.EnableTraceContextPropagation &&
+                        command is IDbCommand { CommandType: CommandType.Text, CommandText: SetContextSql })
+                    {
+                        return;
+                    }
+#endif
+
                     _ = this.connectionFetcher.TryFetch(command, out var connection);
                     _ = this.databaseFetcher.TryFetch(connection, out var databaseName);
                     _ = this.dataSourceFetcher.TryFetch(connection, out var dataSource);
@@ -82,6 +99,28 @@ public override void OnEventWritten(string name, object? payload)
                         return;
                     }
 
+#if NET
+                    if (options.EnableTraceContextPropagation &&
+                        command is IDbCommand { CommandType: CommandType.Text, Connection.State: ConnectionState.Open } iDbCommand)
+                    {
+                        var setContextCommand = iDbCommand.Connection.CreateCommand();
+                        setContextCommand.Transaction = iDbCommand.Transaction;
+                        setContextCommand.CommandText = SetContextSql;
+                        setContextCommand.CommandType = CommandType.Text;
+                        var parameter = setContextCommand.CreateParameter();
+                        parameter.ParameterName = ContextInfoParameterName;
+
+                        var tracedflags = (activity.ActivityTraceFlags & ActivityTraceFlags.Recorded) != 0 ? "01" : "00";
+                        var traceparent = $"00-{activity.TraceId.ToHexString()}-{activity.SpanId.ToHexString()}-{tracedflags}";
+
+                        parameter.DbType = DbType.Binary;
+                        parameter.Value = Encoding.UTF8.GetBytes(traceparent);
+                        setContextCommand.Parameters.Add(parameter);
+
+                        setContextCommand.ExecuteNonQuery();
+                    }
+#endif
+
                     if (activity.IsAllDataRequested)
                     {
                         try
@@ -168,6 +207,17 @@ public override void OnEventWritten(string name, object? payload)
             case SqlDataAfterExecuteCommand:
             case SqlMicrosoftAfterExecuteCommand:
                 {
+                    _ = this.commandFetcher.TryFetch(payload, out var command);
+
+#if NET
+                    // skip if this is an injected query
+                    if (options.EnableTraceContextPropagation &&
+                        command is IDbCommand { CommandType: CommandType.Text, CommandText: SetContextSql })
+                    {
+                        return;
+                    }
+#endif
+
                     if (activity == null)
                     {
                         SqlClientInstrumentationEventSource.Log.NullActivity(name);
@@ -189,6 +239,17 @@ public override void OnEventWritten(string name, object? payload)
             case SqlDataWriteCommandError:
             case SqlMicrosoftWriteCommandError:
                 {
+                    _ = this.commandFetcher.TryFetch(payload, out var command);
+
+#if NET
+                    // skip if this is an injected query
+                    if (options.EnableTraceContextPropagation &&
+                        command is IDbCommand { CommandType: CommandType.Text, CommandText: SetContextSql })
+                    {
+                        return;
+                    }
+#endif
+
                     if (activity == null)
                     {
                         SqlClientInstrumentationEventSource.Log.NullActivity(name);

src/OpenTelemetry.Instrumentation.SqlClient/Implementation/SqlClientInstrumentationEventSource.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 using System.Diagnostics.Tracing;
+using Microsoft.Extensions.Configuration;
 using OpenTelemetry.Internal;
 
 namespace OpenTelemetry.Instrumentation.SqlClient.Implementation;
@@ -10,7 +11,7 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Implementation;
 /// EventSource events emitted from the project.
 /// </summary>
 [EventSource(Name = "OpenTelemetry-Instrumentation-SqlClient")]
-internal sealed class SqlClientInstrumentationEventSource : EventSource
+internal sealed class SqlClientInstrumentationEventSource : EventSource, IConfigurationExtensionsLogger
 {
     public static SqlClientInstrumentationEventSource Log = new();
 
@@ -82,4 +83,15 @@ public void CommandFilterException(string exception)
     {
         this.WriteEvent(7, exception);
     }
+
+    [Event(8, Message = "Configuration key '{0}' has an invalid value: '{1}'", Level = EventLevel.Warning)]
+    public void InvalidConfigurationValue(string key, string value)
+    {
+        this.WriteEvent(8, key, value);
+    }
+
+    void IConfigurationExtensionsLogger.LogInvalidConfigurationValue(string key, string value)
+    {
+        this.InvalidConfigurationValue(key, value);
+    }
 }

src/OpenTelemetry.Instrumentation.SqlClient/OpenTelemetry.Instrumentation.SqlClient.csproj

@@ -16,6 +16,7 @@
 
   <ItemGroup>
     <Compile Include="$(RepoRoot)\src\Shared\AssemblyVersionExtensions.cs" Link="Includes\AssemblyVersionExtensions.cs" />
+    <Compile Include="$(RepoRoot)\src\Shared\Configuration\*.cs" Link="Includes\Configuration\%(Filename).cs" />
     <Compile Include="$(RepoRoot)\src\Shared\DatabaseSemanticConventionHelper.cs" Link="Includes\DatabaseSemanticConventionHelper.cs" />
     <Compile Include="$(RepoRoot)\src\Shared\DiagnosticSourceListener.cs" Link="Includes\DiagnosticSourceListener.cs" />
     <Compile Include="$(RepoRoot)\src\Shared\DiagnosticSourceSubscriber.cs" Link="Includes\DiagnosticSourceSubscriber.cs" />

src/OpenTelemetry.Instrumentation.SqlClient/README.md

@@ -281,6 +281,20 @@ using var traceProvider = Sdk.CreateTracerProviderBuilder()
 {
 ```
 
+### Trace Context Propagation
+
+> [!NOTE]
+> Only `CommandType.Text` commands are supported for trace context propagation.
+> Only .NET runtimes are supported.
+
+Database trace context propagation can be enabled by setting
+`OTEL_DOTNET_EXPERIMENTAL_SQLCLIENT_ENABLE_TRACE_CONTEXT_PROPAGATION`
+environment variable to `true`.
+This uses the [SET CONTEXT_INFO](https://learn.microsoft.com/en-us/sql/t-sql/statements/set-context-info-transact-sql?view=sql-server-ver16)
+command to set [traceparent](https://www.w3.org/TR/trace-context/#traceparent-header)information
+for the current connection, which results in
+**an additional round-trip to the database**.
+
 ## References
 
 * [OpenTelemetry Project](https://opentelemetry.io/)

src/OpenTelemetry.Instrumentation.SqlClient/SqlClientTraceInstrumentationOptions.cs

@@ -4,6 +4,9 @@
 using System.Data;
 using System.Diagnostics;
 using Microsoft.Extensions.Configuration;
+#if NET
+using OpenTelemetry.Instrumentation.SqlClient.Implementation;
+#endif
 using OpenTelemetry.Trace;
 using static OpenTelemetry.Internal.DatabaseSemanticConventionHelper;
 
@@ -17,6 +20,8 @@ namespace OpenTelemetry.Instrumentation.SqlClient;
 /// </remarks>
 public class SqlClientTraceInstrumentationOptions
 {
+    internal const string ContextPropagationLevelEnvVar = "OTEL_DOTNET_EXPERIMENTAL_SQLCLIENT_ENABLE_TRACE_CONTEXT_PROPAGATION";
+
     /// <summary>
     /// Initializes a new instance of the <see cref="SqlClientTraceInstrumentationOptions"/> class.
     /// </summary>
@@ -30,6 +35,18 @@ internal SqlClientTraceInstrumentationOptions(IConfiguration configuration)
         var databaseSemanticConvention = GetSemanticConventionOptIn(configuration);
         this.EmitOldAttributes = databaseSemanticConvention.HasFlag(DatabaseSemanticConvention.Old);
         this.EmitNewAttributes = databaseSemanticConvention.HasFlag(DatabaseSemanticConvention.New);
+
+#if NET
+        Debug.Assert(configuration != null, "configuration was null");
+
+        if (configuration!.TryGetBoolValue(
+                SqlClientInstrumentationEventSource.Log,
+                ContextPropagationLevelEnvVar,
+                out var enableTraceContextPropagation))
+        {
+            this.EnableTraceContextPropagation = enableTraceContextPropagation;
+        }
+#endif
     }
 
     /// <summary>
@@ -126,4 +143,18 @@ internal SqlClientTraceInstrumentationOptions(IConfiguration configuration)
     /// Gets or sets a value indicating whether the new database attributes should be emitted.
     /// </summary>
     internal bool EmitNewAttributes { get; set; }
+
+#if NET
+    /// <summary>
+    /// Gets or sets a value indicating whether to send traceparent information to SQL Server database.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// <b>Only `CommandType.Text` commands are supported for trace context propagation.</b>
+    /// Note: This uses the SET CONTEXT_INFO command to set traceparent information
+    /// for the current connection, which results in an additional round-trip to the database.
+    /// </para>
+    /// </remarks>
+    internal bool EnableTraceContextPropagation { get; set; }
+#endif
 }

test/OpenTelemetry.Instrumentation.SqlClient.Tests/SqlClientIntegrationTests.cs

@@ -3,6 +3,7 @@
 
 using System.Data;
 using System.Diagnostics;
+using System.Text;
 using Microsoft.Data.SqlClient;
 using OpenTelemetry.Instrumentation.SqlClient.Implementation;
 using OpenTelemetry.Tests;
@@ -16,6 +17,8 @@ namespace OpenTelemetry.Instrumentation.SqlClient.Tests;
 [Trait("CategoryName", "SqlIntegrationTests")]
 public sealed class SqlClientIntegrationTests : IClassFixture<SqlClientIntegrationTestsFixture>
 {
+    private const string GetContextInfoQuery = "SELECT CONTEXT_INFO()";
+
     private readonly SqlClientIntegrationTestsFixture fixture;
 
     public SqlClientIntegrationTests(SqlClientIntegrationTestsFixture fixture)
@@ -28,6 +31,10 @@ public SqlClientIntegrationTests(SqlClientIntegrationTestsFixture fixture)
     [InlineData(CommandType.Text, "select 1/1", true, "select ?/?")]
     [InlineData(CommandType.Text, "select 1/0", false, null, true)]
     [InlineData(CommandType.Text, "select 1/0", false, null, true, true)]
+#if NET
+    [InlineData(CommandType.Text, GetContextInfoQuery, false, null, false, false, false)]
+    [InlineData(CommandType.Text, GetContextInfoQuery, false, null, false, false, true)]
+#endif
 #if NETFRAMEWORK
     [InlineData(CommandType.StoredProcedure, "sp_who", false, null)]
 #else
@@ -40,8 +47,14 @@ public void SuccessfulCommandTest(
         bool captureTextCommandContent = false,
         string? sanitizedCommandText = null,
         bool isFailure = false,
-        bool recordException = false)
+        bool recordException = false,
+        bool enableTransaction = false)
     {
+        if (commandText == GetContextInfoQuery)
+        {
+            Environment.SetEnvironmentVariable(SqlClientTraceInstrumentationOptions.ContextPropagationLevelEnvVar, "true");
+        }
+
 #if NETFRAMEWORK
         // Disable things not available on netfx
         recordException = false;
@@ -66,24 +79,35 @@ public void SuccessfulCommandTest(
         var dataSource = sqlConnection.DataSource;
 
         sqlConnection.ChangeDatabase("master");
+        SqlTransaction? transaction = null;
 #pragma warning disable CA2100
         using var sqlCommand = new SqlCommand(commandText, sqlConnection)
 #pragma warning restore CA2100
         {
             CommandType = commandType,
         };
 
+        if (enableTransaction)
+        {
+            transaction = sqlConnection.BeginTransaction();
+            sqlCommand.Transaction = transaction;
+        }
+
+        object commandResult = DBNull.Value;
         try
         {
-            sqlCommand.ExecuteNonQuery();
+            commandResult = sqlCommand.ExecuteScalar();
         }
         catch
         {
         }
 
+        transaction?.Commit();
+
         Assert.Single(activities);
         var activity = activities[0];
 
+        VerifyContextInfo(commandText, commandResult, activity);
         VerifyActivityData(commandType, sanitizedCommandText, captureTextCommandContent, isFailure, recordException, activity);
         VerifySamplingParameters(sampler.LatestSamplingParameters);
 
@@ -103,6 +127,20 @@ public void SuccessfulCommandTest(
         }
     }
 
+    private static void VerifyContextInfo(
+        string? commandText,
+        object commandResult,
+        Activity activity)
+    {
+        if (commandText == GetContextInfoQuery)
+        {
+            Assert.NotEqual(commandResult, DBNull.Value);
+            Assert.True(commandResult is byte[]);
+            var contextInfo = Encoding.ASCII.GetString((byte[])commandResult).TrimEnd('\0');
+            Assert.Equal(contextInfo, activity.Id);
+        }
+    }
+
     private static void VerifyActivityData(
         CommandType commandType,
         string? commandText,