Skip to content

Add support for baggage propagation #3319

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
satvu wants to merge 8 commits into
base: main
Choose a base branch
from
+188 −12
Open

Add support for baggage propagation #3319

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e4887a0
initial wip works e2e
satvu Feb 4, 2026
113cc9a
use baggage from optel
satvu Feb 4, 2026
75334c3
refactor to use otel extension
satvu Feb 7, 2026
0cabf7e
refactor trace constants usage
satvu Feb 7, 2026
1fb88f6
cleanup and resolve comments
satvu Feb 7, 2026
5fc57c7
fix tests
satvu Feb 9, 2026
5fe7534
add tests
satvu Feb 9, 2026
ed4032e
refactor where baggage is stored
satvu Feb 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions protos/azure-functions-language-worker-protobuf/src/proto/FunctionRpc.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,9 @@ message RpcTraceContext {

// This corresponds to Activity.Current?.Tags
map<string, string> attributes = 3;

// This corresponds to Activity.Current?.Baggage
map<string, string> baggage = 4;
Copy link
Member Author

@satvu satvu Feb 7, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TODO: pull in final protobuf changes later

Here's the PR: Azure/azure-functions-language-worker-protobuf#99

}

// Host sends retry context for a function invocation
Expand Down
5 changes: 4 additions & 1 deletion src/DotNetWorker.Core/Context/DefaultTraceContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,20 @@ namespace Microsoft.Azure.Functions.Worker
{
internal sealed class DefaultTraceContext : TraceContext
{
public DefaultTraceContext(string traceParent, string traceState, IReadOnlyDictionary<string, string> attributes)
public DefaultTraceContext(string traceParent, string traceState, IReadOnlyDictionary<string, string> attributes, IReadOnlyDictionary<string, string> baggage)
{
TraceParent = traceParent;
TraceState = traceState;
Attributes = attributes;
Baggage = baggage;
}

public override string TraceParent { get; }

public override string TraceState { get; }

public override IReadOnlyDictionary<string, string> Attributes { get; }

public override IReadOnlyDictionary<string, string> Baggage { get; }
}
}
5 changes: 5 additions & 0 deletions src/DotNetWorker.Core/Context/TraceContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,10 @@ public abstract class TraceContext
/// Gets the attributes associated with the trace.
/// </summary>
public virtual IReadOnlyDictionary<string, string> Attributes => ImmutableDictionary<string, string>.Empty;

/// <summary>
/// Gets the baggage associated with the trace.
/// </summary>
public virtual IReadOnlyDictionary<string, string> Baggage => ImmutableDictionary<string, string>.Empty;
}
}
3 changes: 2 additions & 1 deletion src/DotNetWorker.Grpc/GrpcFunctionInvocation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ internal sealed class GrpcFunctionInvocation : FunctionInvocation, IExecutionRet
public GrpcFunctionInvocation(InvocationRequest invocationRequest)
{
_invocationRequest = invocationRequest;
TraceContext = new DefaultTraceContext(_invocationRequest.TraceContext.TraceParent, _invocationRequest.TraceContext.TraceState, _invocationRequest.TraceContext.Attributes);
TraceContext = new DefaultTraceContext(_invocationRequest.TraceContext.TraceParent,
_invocationRequest.TraceContext.TraceState, _invocationRequest.TraceContext.Attributes, _invocationRequest.TraceContext.Baggage);
}

public override string Id => _invocationRequest.InvocationId;
Expand Down
3 changes: 2 additions & 1 deletion src/DotNetWorker.Grpc/Handlers/InvocationHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System;
Expand All @@ -7,6 +7,7 @@
using System.Threading.Tasks;
using Microsoft.Azure.Functions.Worker.Context.Features;
using Microsoft.Azure.Functions.Worker.Core;
using Microsoft.Azure.Functions.Worker.Diagnostics;
using Microsoft.Azure.Functions.Worker.Grpc;
using Microsoft.Azure.Functions.Worker.Grpc.Features;
using Microsoft.Azure.Functions.Worker.Grpc.Messages;
Expand Down
30 changes: 30 additions & 0 deletions src/DotNetWorker.OpenTelemetry/BaggageMiddleware.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System.Threading.Tasks;
using Microsoft.Azure.Functions.Worker.Middleware;
using OpenTelemetry;

namespace Microsoft.Azure.Functions.Worker.OpenTelemetry
{
internal class BaggageMiddleware : IFunctionsWorkerMiddleware
{
public async Task Invoke(FunctionContext context, FunctionExecutionDelegate next)
{
try
{
foreach (var kv in context.TraceContext.Baggage)
{
Baggage.SetBaggage(kv.Key, kv.Value);
}

await next(context);
}
finally
{
Baggage.ClearBaggage();
}

}
}
}
18 changes: 18 additions & 0 deletions src/DotNetWorker.OpenTelemetry/OpenTelemetryWorkerBuilderExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System;
using Microsoft.Extensions.Hosting;

namespace Microsoft.Azure.Functions.Worker.OpenTelemetry
{
public static class OpenTelemetryWorkerBuilderExtensions
{
public static IFunctionsWorkerApplicationBuilder EnableBaggagePropagation(this IFunctionsWorkerApplicationBuilder builder)
Copy link
Contributor

@jviau jviau Feb 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the expectation that customers call this manually? If so, there is going to be an ordering problem with it. Extension auto-registration will already have ran by the time a customer calls this, and so the baggage middleware will always be after any extension middleware. This means extension middleware misses out on all the baggage.

I think we will need a way to insert this at the start. Do we have middleware insert semantics? If no, we might need a way to supply this to the app builder so it gets registered before any extensions.

Copy link
Member Author

@satvu satvu Feb 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this would be called manually in this design - let me see if there's anything existing that we can leverage to make sure this goes before extensions.

Copy link
Contributor

@jviau jviau Feb 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we need this to be a middleware approach at all. What if this sets a capability which we flow back to the host, and that turns host flowing baggage on/off. And then our implementation worker side can always populate baggage.

Copy link
Member

@kshyju kshyju Feb 18, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if we set this in FunctionsApplication.InvokeFunctionAsync method , before calling _functionExecutionDelegate? That way baggage is available to all middlewares. We are doing similar thing to setup Activity for the invocation there.

I like the capability idea @jviau suggested above. With this, we can send this capability with a new version of worker package (with an opt-out feature if necessary) and host can send this or not based on it. (Yea, this requires another host update, but will provide flexibility for workers to control this behavior).

Baggage comes from OTEL package and we can abstract this behavior (to set baggage) as a Feature and DotnetWorker.Otel project/package can implement the code (to set it if present in the invocation request)

Copy link
Member Author

@satvu satvu Feb 19, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My first iteration used that approach (set baggage inside InvokeFunctionAsync) but it would require taking an otel dependency in core.

Moving the baggage propagation into middleware inside the otel extension was to avoid taking an otel dependency in core (not to control enablement/disablement). The host PR that was merged flows the baggage any time the conditions necessary are met (otel enabled, baggage exists) since we didn't have a strong case for disabling baggage (if those values are sent, why should they be blocked).

{
builder.UseMiddleware<BaggageMiddleware>();
return builder;
}

}
}
6 changes: 6 additions & 0 deletions src/DotNetWorker.OpenTelemetry/Properties/AssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System.Runtime.CompilerServices;

[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.OpenTelemetry.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")]
78 changes: 78 additions & 0 deletions test/DotNetWorker.OpenTelemetry.Tests/BaggageMiddlewareTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.Azure.Functions.Worker.Middleware;
using Moq;
using OpenTelemetry;
using Xunit;

namespace Microsoft.Azure.Functions.Worker.OpenTelemetry.Tests
{
public class BaggageMiddlewareTests
{
[Fact]
public async Task Invoke_SetsAndClearsBaggage_WhenBaggagePresent()
{
var baggageKey = "TestKey";
var baggageValue = "TestValue";
var baggageDict = new Dictionary<string, string>
{
{ baggageKey, baggageValue }
};

var traceContextMock = new Mock<TraceContext>();
traceContextMock.Setup(t => t.Baggage).Returns(baggageDict);

var contextMock = new Mock<FunctionContext>();
contextMock.Setup(c => c.TraceContext).Returns(traceContextMock.Object);

bool nextCalled = false;
var middleware = new BaggageMiddleware();

// baggage is set before next is called
FunctionExecutionDelegate next = async ctx =>
{
nextCalled = true;
Assert.Equal(baggageValue, Baggage.GetBaggage(baggageKey));
await Task.CompletedTask;
};

await middleware.Invoke(contextMock.Object, next);

// cleared after invoking
Assert.True(nextCalled);
Assert.Null(Baggage.GetBaggage(baggageKey));
}

[Fact]
public async Task Invoke_ClearsBaggage_OnException()
{
var baggageKey = "TestKey";
var baggageValue = "TestValue";
var baggageDict = new Dictionary<string, string>
{
{ baggageKey, baggageValue }
};

var traceContextMock = new Mock<TraceContext>();
traceContextMock.Setup(t => t.Baggage).Returns(baggageDict);

var contextMock = new Mock<FunctionContext>();
contextMock.Setup(c => c.TraceContext).Returns(traceContextMock.Object);

var middleware = new BaggageMiddleware();

FunctionExecutionDelegate next = ctx =>
{
Assert.Equal(baggageValue, Baggage.GetBaggage(baggageKey));
throw new System.Exception("Test exception");
};

await Assert.ThrowsAsync<System.Exception>(() => middleware.Invoke(contextMock.Object, next));
Assert.Null(Baggage.GetBaggage(baggageKey));
}
}

}
6 changes: 4 additions & 2 deletions test/DotNetWorker.OpenTelemetry.Tests/EndToEndTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
using OpenTelemetry.Trace;
using Xunit;

using CoreTraceConstants = Microsoft.Azure.Functions.Worker.Diagnostics.TraceConstants;

namespace Microsoft.Azure.Functions.Worker.OpenTelemetry.Tests;

public class EndToEndTests
Expand Down Expand Up @@ -89,7 +91,7 @@ public async Task ContextPropagation()
Assert.Equal(activity.TraceId, activityContext.TraceId);
Assert.Equal(activity.TraceStateString, activityContext.TraceState);
Assert.Equal(ActivityKind.Internal, activity.Kind);
Assert.Contains(activity.Tags, t => t.Key == TraceConstants.OTelAttributes_1_37_0.InvocationId && t.Value == context.InvocationId);
Assert.Contains(activity.Tags, t => t.Key == CoreTraceConstants.OTelAttributes_1_37_0.InvocationId && t.Value == context.InvocationId);
}
else
{
Expand All @@ -112,7 +114,7 @@ public async Task ContextPropagationV17()
Assert.Equal(activity.TraceId, activityContext.TraceId);
Assert.Equal(activity.TraceStateString, activityContext.TraceState);
Assert.Equal(ActivityKind.Server, activity.Kind);
Assert.Contains(activity.Tags, t => t.Key == TraceConstants.OTelAttributes_1_17_0.InvocationId && t.Value == context.InvocationId);
Assert.Contains(activity.Tags, t => t.Key == CoreTraceConstants.OTelAttributes_1_17_0.InvocationId && t.Value == context.InvocationId);
}
else
{
Expand Down
27 changes: 26 additions & 1 deletion test/DotNetWorker.Tests/Handlers/InvocationHandlerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System;
Expand All @@ -15,6 +15,8 @@
using Moq;
using Xunit;

using CoreTraceConstants = Microsoft.Azure.Functions.Worker.Diagnostics.TraceConstants;

namespace Microsoft.Azure.Functions.Worker.Tests
{
public class InvocationHandlerTests
Expand Down Expand Up @@ -261,6 +263,29 @@ public async Task SetRetryContextToNull()
Assert.Null(_context.RetryContext);
}

[Fact]
public async Task InvokeAsync_SetsBaggage_WhenPresentInInvocationRequest()
{
var invocationId = Guid.NewGuid().ToString();
var request = TestUtility.CreateInvocationRequest(invocationId);

var testKey = "testKey";
var testValue = "testValue";
request.TraceContext.Baggage[testKey] = testValue;

_mockApplication
.Setup(m => m.InvokeFunctionAsync(It.IsAny<FunctionContext>()))
.Returns(Task.CompletedTask)
.Callback<FunctionContext>(ctx =>
{
Assert.True(ctx.TraceContext.Baggage.ContainsKey(testKey));
Assert.Equal(testValue, ctx.TraceContext.Baggage[testKey]);
});

var handler = CreateInvocationHandler();
await handler.InvokeAsync(request);
}

private InvocationHandler CreateInvocationHandler(IFunctionsApplication application = null,
IOptions<WorkerOptions> workerOptions = null)
{
Expand Down
5 changes: 2 additions & 3 deletions test/DotNetWorker.Tests/Properties/AssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System.Runtime.CompilerServices;
Expand All @@ -7,5 +7,4 @@
[assembly: CollectionBehavior(DisableTestParallelization = true)]
[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.Extensions.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")]
[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")]


[assembly: InternalsVisibleTo("Microsoft.Azure.Functions.Worker.OpenTelemetry.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001005148be37ac1d9f58bd40a2e472c9d380d635b6048278f7d47480b08c928858f0f7fe17a6e4ce98da0e7a7f0b8c308aecd9e9b02d7e9680a5b5b75ac7773cec096fbbc64aebd429e77cb5f89a569a79b28e9c76426783f624b6b70327eb37341eb498a2c3918af97c4860db6cdca4732787150841e395a29cfacb959c1fd971c1")]
4 changes: 2 additions & 2 deletions test/TestUtility/TestFunctionContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System;
Expand Down Expand Up @@ -86,7 +86,7 @@ public TestFunctionContext(FunctionDefinition functionDefinition, FunctionInvoca

public override FunctionDefinition FunctionDefinition { get; }

public override IDictionary<object, object> Items { get; set; }
public override IDictionary<object, object> Items { get; set; } = new Dictionary<object, object>();

public override IInvocationFeatures Features { get; } = new InvocationFeatures(Enumerable.Empty<IInvocationFeatureProvider>());

Expand Down
7 changes: 6 additions & 1 deletion test/TestUtility/TestFunctionInvocation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,12 @@ public TestFunctionInvocation(string id = null, string functionId = null)
{ TraceConstants.InternalKeys.AzFuncLiveLogsSessionId, Guid.NewGuid().ToString() },
};

TraceContext = new DefaultTraceContext(activity.Id, Guid.NewGuid().ToString(), attributes);
Dictionary<string, string> baggage = new Dictionary<string, string>
{
{ "TestBaggageKey", "TestBaggageValue" }
};

TraceContext = new DefaultTraceContext(activity.Id, Guid.NewGuid().ToString(), attributes, baggage);
}

public override string Id { get; } = Guid.NewGuid().ToString();
Expand Down