Skip to content

Commit c191d10

Browse files
nick863nick863
authored
Add open telemetry to Azure.AI.Inferencing (Azure#45751)
* Add open telemetry to Azure AI Inference
1 parent ae95bb5 commit c191d10

File tree

52 files changed

+3098
-4034
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+3098
-4034
lines changed

‎eng/Packages.Data.props‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,10 @@
340340
<PackageReference Update="NUnit" Version="3.13.2" />
341341
<PackageReference Update="NUnit3TestAdapter" Version="4.4.2" />
342342
<PackageReference Update="OpenTelemetry" Version="1.9.0" />
343+
<PackageReference Update="OpenTelemetry.Exporter.Console" Version="1.9.0" />
343344
<PackageReference Update="OpenTelemetry.Exporter.InMemory" Version="1.9.0" />
345+
<PackageReference Update="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
346+
<PackageReference Update="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
344347
<PackageReference Update="OpenTelemetry.Instrumentation.SqlClient" Version="1.9.0-beta.1" />
345348
<PackageReference Update="Polly" Version="7.1.0" />
346349
<PackageReference Update="Polly.Contrib.WaitAndRetry" Version="1.1.1" />

‎sdk/ai/Azure.AI.Inference/Azure.AI.Inference.sln‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ VisualStudioVersion = 17.9.34723.18
55
MinimumVisualStudioVersion = 10.0.40219.1
66
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Core.TestFramework", "..\..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj", "{ECC730C1-4AEA-420C-916A-66B19B79E4DC}"
77
EndProject
8-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.AI.Inference", "src\Azure.AI.Inference.csproj", "{C3781FDB-93C7-4636-95B1-7A6F2C00E25F}"
8+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.AI.Inference", "src\Azure.AI.Inference.csproj", "{C3781FDB-93C7-4636-95B1-7A6F2C00E25F}"
99
EndProject
10-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.AI.Inference.Tests", "tests\Azure.AI.Inference.Tests.csproj", "{5FDDC1E7-3A88-4BAF-948B-909BC6B65E1A}"
10+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.AI.Inference.Tests", "tests\Azure.AI.Inference.Tests.csproj", "{5FDDC1E7-3A88-4BAF-948B-909BC6B65E1A}"
1111
EndProject
1212
Global
1313
GlobalSection(SolutionConfigurationPlatforms) = preSolution

‎sdk/ai/Azure.AI.Inference/README.md‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,73 @@ To generate embeddings for additional phrases, simply call `client.embed` multip
340340

341341
## Troubleshooting
342342

343+
### Observability with OpenTelemetry
344+
345+
Azure AI Inference client library supports tracing and metrics with OpenTelemetry. Refer to
346+
[Azure SDK Diagnostics](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/samples/Diagnostics.md#distributed-tracing)
347+
documentation for general information on OpenTelemetry support in Azure client libraries.
348+
349+
Distributed tracing and metrics with OpenTelemetry are supported in Azure AI Inference in experimental mode and could be enabled through either
350+
of these steps:
351+
352+
- Set the `AZURE_EXPERIMENTAL_ENABLE_ACTIVITY_SOURCE` environment variable to `true`.
353+
- Set the `Azure.Experimental.EnableActivitySource` context switch to `true` in your application code
354+
355+
Refer to [Azure Monitor documentation](https://learn.microsoft.com/azure/azure-monitor/app/opentelemetry-enable?tabs=aspnetcore) on how to use
356+
Azure Monitor OpenTelemetry Distro.
357+
358+
> [!NOTE]
359+
> With the Azure Monitor OpenTelemetry Distro, you only need to opt-into Azure SDK experimental telemetry features with one of the ways documented at
360+
> the beginning of this section.
361+
> The distro enables activity sources and meters for Azure AI Inference automatically.
362+
363+
The following section provides an example on how to configure OpenTelemetry and enable Azure AI Inference tracing and metrics if your
364+
OpenTelemetry distro does not include Azure AI Inference by default.
365+
366+
#### Generic OpenTelemetry configuration
367+
368+
In this example we're going to export traces and metrics to console, and to the local [OTLP](https://opentelemetry.io/docs/specs/otel/protocol/) destination.
369+
[Aspire dashboard](https://learn.microsoft.com/dotnet/aspire/fundamentals/dashboard/standalone) can be used for local testing and exploration.
370+
371+
To run this example, you'll need to install the following dependencies (HTTP tracing and metrics instrumentation
372+
as well as console and OTLP exporters):
373+
374+
```dotnetcli
375+
dotnet add package OpenTelemetry.Instrumentation.Http
376+
dotnet add package OpenTelemetry.Exporter.Console
377+
dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol
378+
```
379+
380+
These packages also bring [OpenTelemetry SDK](https://www.nuget.org/packages/OpenTelemetry) as a dependency.
381+
382+
```C# Snippet:Azure_AI_Inference_EnableOpenTelemetry
383+
// Enables experimental Azure SDK observability
384+
AppContext.SetSwitch("Azure.Experimental.EnableActivitySource", true);
385+
386+
// By default instrumentation captures chat messages without content
387+
// since content can be very verbose and have sensitive information.
388+
// The following AppContext switch enables content recording.
389+
AppContext.SetSwitch("Azure.Experimental.TraceGenAIMessageContent", true);
390+
391+
using var tracerProvider = Sdk.CreateTracerProviderBuilder()
392+
.AddHttpClientInstrumentation()
393+
.AddSource("Azure.AI.Inference.*")
394+
.ConfigureResource(r => r.AddService("sample"))
395+
.AddConsoleExporter()
396+
.AddOtlpExporter()
397+
.Build();
398+
399+
using var meterProvider = Sdk.CreateMeterProviderBuilder()
400+
.AddHttpClientInstrumentation()
401+
.AddMeter("Azure.AI.Inference.*")
402+
.ConfigureResource(r => r.AddService("sample"))
403+
.AddConsoleExporter()
404+
.AddOtlpExporter()
405+
.Build();
406+
```
407+
408+
Check out [OpenTelemetry .NET](https://opentelemetry.io/docs/languages/net/) and your observability provider documentation on how to configure OpenTelemetry.
409+
343410
### Exceptions
344411

345412
The `complete`, `get_model_info` methods raise a `RequestFailedException` for a non-success HTTP status code response from the service. The exception's `code` will hold the HTTP response status code. The exception's `message` contains a detailed message that may be helpful in diagnosing the issue:

‎sdk/ai/Azure.AI.Inference/assets.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"AssetsRepo": "Azure/azure-sdk-assets",
3+
"AssetsRepoPrefixPath": "net",
4+
"TagPrefix": "net/ai/Azure.AI.Inference",
5+
"Tag": "net/ai/Azure.AI.Inference_99a4be5ef1"
6+
}

‎sdk/ai/Azure.AI.Inference/samples/Sample2_ChatCompletionsWithAoai.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Simple Chat Completions Targetting Azure OpenAI
1+
# Simple chat completions targeting Azure OpenAI
22

33
This sample demonstrates how to get a chat completions response from the service using a synchronous call, targetting an Azure OpenAI (AOAI) endpoint.
44

‎sdk/ai/Azure.AI.Inference/samples/Sample8_ChatCompletionsWithOpenTelemetry.md‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
# Using telemetry with Azure.AI.Inference
2+
3+
In this example we will demonstrate how to enable [OpenTelemetry](https://opentelemetry.io/) to get observability into Chat Completions call.
4+
5+
## Project creation and dependency installation
6+
7+
First, we will create the console application project and add `Azure.AI.Inference` as a dependency. The first command will create the project called `TelemetryDemo.csproj`. The `dotnet add package <…>` command will modify project file and in future we will need to run `dotnet restore` to install all dependencies, if we will remove the installed ones.
8+
9+
```dotnetcli
10+
dotnet new console --name TelemetryDemo --output TelemetryDemo
11+
dotnet add package Azure.AI.Inference --prerelease
12+
```
13+
14+
Now we will need to add the dependencies for OpenTelemetry exporters and HTTP client instrumentation.
15+
16+
```dotnetcli
17+
dotnet add package OpenTelemetry.Exporter.Console
18+
dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol
19+
dotnet add package OpenTelemetry.Instrumentation.Http
20+
```
21+
22+
## Create the simple application with telemetry
23+
24+
The `dotnet new` created the project with the single file called Program.cs. Let us edit this file with the IDE of choice.
25+
26+
First we will import open telemetry and Azure.AI.Inference
27+
```C# Snippet:Azure_AI_Inference_EnableOpenTelemetry_import
28+
//Azure imports
29+
// Open telemetry imports
30+
using OpenTelemetry;
31+
using OpenTelemetry.Resources;
32+
using OpenTelemetry.Trace;
33+
using OpenTelemetry.Metrics;
34+
```
35+
36+
In this example we will configure OpenTelemetry to export distributed traces and metrics to console and local [OTLP](https://opentelemetry.io/docs/specs/otel/protocol/) endpoint such as [Aspire Dashboard](https://learn.microsoft.com/dotnet/aspire/fundamentals/dashboard/standalone).
37+
38+
Azure AI Inference reports distributed traces using [ActivitySources](https://learn.microsoft.com/dotnet/api/system.diagnostics.activitysource) and [Meters](https://learn.microsoft.com/dotnet/api/system.diagnostics.metrics.meter). We need to configure OpenTelemetry to listen to these sources.
39+
40+
Check out general [Azure SDK Diagnostics](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/core/Azure.Core/samples/Diagnostics.md) documentation for the details.
41+
42+
```C# Snippet:Azure_AI_Inference_EnableOpenTelemetry
43+
// Enables experimental Azure SDK observability
44+
AppContext.SetSwitch("Azure.Experimental.EnableActivitySource", true);
45+
46+
// By default instrumentation captures chat messages without content
47+
// since content can be very verbose and have sensitive information.
48+
// The following AppContext switch enables content recording.
49+
AppContext.SetSwitch("Azure.Experimental.TraceGenAIMessageContent", true);
50+
51+
using var tracerProvider = Sdk.CreateTracerProviderBuilder()
52+
.AddHttpClientInstrumentation()
53+
.AddSource("Azure.AI.Inference.*")
54+
.ConfigureResource(r => r.AddService("sample"))
55+
.AddConsoleExporter()
56+
.AddOtlpExporter()
57+
.Build();
58+
59+
using var meterProvider = Sdk.CreateMeterProviderBuilder()
60+
.AddHttpClientInstrumentation()
61+
.AddMeter("Azure.AI.Inference.*")
62+
.ConfigureResource(r => r.AddService("sample"))
63+
.AddConsoleExporter()
64+
.AddOtlpExporter()
65+
.Build();
66+
```
67+
68+
Now let's define the `endpoint`, `credential` and `model`.
69+
70+
```C# Snippet:Azure_AI_Inference_EnableOpenTelemetry_variables
71+
var endpoint = new Uri(System.Environment.GetEnvironmentVariable("MODEL_ENDPOINT"));
72+
var credential = new AzureKeyCredential(System.Environment.GetEnvironmentVariable("GITHUB_TOKEN"));
73+
var model = System.Environment.GetEnvironmentVariable("MODEL_NAME");
74+
```
75+
76+
Now we can create `ChatCompletionsClient` and make calls as usual - telemetry will be collected and exported without
77+
any additional code.
78+
79+
```C# Snippet:Azure_AI_Inference_EnableOpenTelemetry_inference
80+
var client = new ChatCompletionsClient(
81+
endpoint,
82+
credential,
83+
new ChatCompletionsClientOptions());
84+
85+
var requestOptions = new ChatCompletionsOptions()
86+
{
87+
Messages =
88+
{
89+
new ChatRequestSystemMessage("You are a helpful assistant."),
90+
new ChatRequestUserMessage("What is the capital of France?"),
91+
},
92+
Model = model,
93+
Temperature = 1,
94+
MaxTokens = 1000
95+
};
96+
// Call the endpoint and output the response.
97+
Response<ChatCompletions> response = client.Complete(requestOptions);
98+
Console.WriteLine(response.Value.Choices[0].Message.Content);
99+
```
100+
101+
## Running the application
102+
103+
Optionally run the Aspire dashboard
104+
105+
```bash
106+
docker run --rm -it \
107+
-p 18888:18888 \
108+
-p 4317:18889 -d \
109+
--name aspire-dashboard \
110+
mcr.microsoft.com/dotnet/aspire-dashboard:latest
111+
```
112+
113+
Now run the application from IDE, or use
114+
```dotnetcli
115+
dotnet run
116+
```
117+
118+
Check out telemetry in Aspire dashboard (at http://localhost:18888/traces):
119+
120+
![image](./images/Sample8_ChatCompletionsWithOpenTelemetry.png)
121+
122+
## Next steps
123+
124+
After we have run the application, we can list all the metrics and events on the Application Insights. Please refer to the [documentation](https://learn.microsoft.com/azure/azure-monitor/app/opentelemetry-enable?tabs=aspnetcore). The events will be written to the table `traces`, metrics can be found in `customMetrics`.

‎sdk/ai/Azure.AI.Inference/samples/images/Sample8_ChatCompletionsWithOpenTelemetry.png‎

101 KB
Loading

‎sdk/ai/Azure.AI.Inference/src/Azure.AI.Inference.csproj‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22
<PropertyGroup>
3-
<Description>This is the Inference client library for developing .NET applications with rich experience.</Description>
3+
<Description>This is the Microsoft Azure AI Inference Service client library</Description>
44
<AssemblyTitle>Microsoft Azure.AI.Inference client library</AssemblyTitle>
55
<Version>1.0.0-beta.2</Version>
66
<PackageTags>Azure Inference</PackageTags>

‎sdk/ai/Azure.AI.Inference/src/Customized/ChatCompletionsClient.cs‎

Lines changed: 60 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Diagnostics.CodeAnalysis;
66
using System.Threading;
77
using System.Threading.Tasks;
8+
using Azure.AI.Inference.Telemetry;
89
using Azure.Core;
910
using Azure.Core.Pipeline;
1011
using Azure.Core.Sse;
@@ -47,8 +48,21 @@ public virtual async Task<Response<ChatCompletions>> CompleteAsync(ChatCompletio
4748

4849
using RequestContent content = chatCompletionsOptions.ToRequestContent();
4950
RequestContext context = FromCancellationToken(cancellationToken);
50-
Response response = await CompleteAsync(content, extraParams?.ToString(), context).ConfigureAwait(false);
51-
return Response.FromValue(ChatCompletions.FromResponse(response), response);
51+
using OpenTelemetryScope otelScope = OpenTelemetryScope.Start(chatCompletionsOptions, _endpoint);
52+
Response response = null;
53+
ChatCompletions chatCompletions = null;
54+
try
55+
{
56+
response = await CompleteAsync(content, extraParams?.ToString(), context).ConfigureAwait(false);
57+
chatCompletions = ChatCompletions.FromResponse(response);
58+
otelScope?.RecordResponse(chatCompletions);
59+
}
60+
catch (Exception ex)
61+
{
62+
otelScope?.RecordError(ex);
63+
throw;
64+
}
65+
return Response.FromValue(chatCompletions, response);
5266
}
5367

5468
/// <summary>
@@ -81,8 +95,20 @@ public virtual Response<ChatCompletions> Complete(ChatCompletionsOptions chatCom
8195

8296
using RequestContent content = chatCompletionsOptions.ToRequestContent();
8397
RequestContext context = FromCancellationToken(cancellationToken);
84-
Response response = Complete(content, extraParams?.ToString(), context);
85-
return Response.FromValue(ChatCompletions.FromResponse(response), response);
98+
using OpenTelemetryScope otelScope = OpenTelemetryScope.Start(chatCompletionsOptions, _endpoint);
99+
Response response = null;
100+
ChatCompletions chatCompletions = null;
101+
try
102+
{
103+
response = Complete(content, extraParams?.ToString(), context);
104+
chatCompletions = ChatCompletions.FromResponse(response);
105+
otelScope?.RecordResponse(chatCompletions);
106+
}
107+
catch (Exception ex) {
108+
otelScope?.RecordError(ex);
109+
throw;
110+
}
111+
return Response.FromValue(chatCompletions, response);
86112
}
87113

88114
/// <summary>
@@ -112,36 +138,38 @@ public virtual async Task<StreamingResponse<StreamingChatCompletionsUpdate>> Com
112138
{
113139
Argument.AssertNotNull(chatCompletionsOptions, nameof(chatCompletionsOptions));
114140

115-
using DiagnosticScope scope = ClientDiagnostics.CreateScope("ChatCompletionsClient.CompleteStreaming");
116-
scope.Start();
117-
118141
chatCompletionsOptions.InternalShouldStreamResponse = true;
119142

120143
RequestContent content = chatCompletionsOptions.ToRequestContent();
121144
RequestContext context = FromCancellationToken(cancellationToken);
122145

146+
OpenTelemetryScope otelScope = OpenTelemetryScope.Start(chatCompletionsOptions, _endpoint);
147+
Response baseResponse = null;
123148
try
124149
{
125-
// Response value object takes IDisposable ownership of message
150+
// Response value object takes IDisposable ownership of message and scope.
126151
HttpMessage message = CreatePostRequestMessage(chatCompletionsOptions, content, context);
127152
message.BufferResponse = false;
128-
Response baseResponse = await _pipeline.ProcessMessageAsync(
153+
baseResponse = await _pipeline.ProcessMessageAsync(
129154
message,
130155
context,
131156
cancellationToken).ConfigureAwait(false);
132-
return StreamingResponse<StreamingChatCompletionsUpdate>.CreateFromResponse(
133-
baseResponse,
134-
(responseForEnumeration)
135-
=> SseAsyncEnumerator<StreamingChatCompletionsUpdate>.EnumerateFromSseStream(
136-
responseForEnumeration.ContentStream,
137-
StreamingChatCompletionsUpdate.DeserializeStreamingChatCompletionsUpdates,
138-
cancellationToken));
139157
}
140158
catch (Exception e)
141159
{
142-
scope.Failed(e);
160+
otelScope?.RecordError(e);
161+
otelScope?.Dispose();
143162
throw;
144163
}
164+
return StreamingResponse<StreamingChatCompletionsUpdate>.CreateFromResponse(
165+
baseResponse,
166+
(responseForEnumeration)
167+
=> SseAsyncEnumerator<StreamingChatCompletionsUpdate>.EnumerateFromSseStream(
168+
responseForEnumeration.ContentStream,
169+
StreamingChatCompletionsUpdate.DeserializeStreamingChatCompletionsUpdates,
170+
otelScope,
171+
cancellationToken
172+
));
145173
}
146174

147175
/// <summary>
@@ -168,33 +196,35 @@ public virtual StreamingResponse<StreamingChatCompletionsUpdate> CompleteStreami
168196
{
169197
Argument.AssertNotNull(chatCompletionsOptions, nameof(chatCompletionsOptions));
170198

171-
using DiagnosticScope scope = ClientDiagnostics.CreateScope("ChatCompletionsClient.CompleteStreaming");
172-
scope.Start();
173-
174199
chatCompletionsOptions.InternalShouldStreamResponse = true;
175200

176201
RequestContent content = chatCompletionsOptions.ToRequestContent();
177202
RequestContext context = FromCancellationToken(cancellationToken);
178203

204+
OpenTelemetryScope otelScope = OpenTelemetryScope.Start(chatCompletionsOptions, _endpoint);
205+
Response baseResponse;
179206
try
180207
{
181-
// Response value object takes IDisposable ownership of message
208+
// Response value object takes IDisposable ownership of message and scope.
182209
HttpMessage message = CreatePostRequestMessage(chatCompletionsOptions, content, context);
183210
message.BufferResponse = false;
184-
Response baseResponse = _pipeline.ProcessMessage(message, context, cancellationToken);
185-
return StreamingResponse<StreamingChatCompletionsUpdate>.CreateFromResponse(
186-
baseResponse,
187-
(responseForEnumeration)
188-
=> SseAsyncEnumerator<StreamingChatCompletionsUpdate>.EnumerateFromSseStream(
189-
responseForEnumeration.ContentStream,
190-
StreamingChatCompletionsUpdate.DeserializeStreamingChatCompletionsUpdates,
191-
cancellationToken));
211+
baseResponse = _pipeline.ProcessMessage(message, context, cancellationToken);
192212
}
193213
catch (Exception e)
194214
{
195-
scope.Failed(e);
215+
otelScope?.RecordError(e);
216+
otelScope?.Dispose();
196217
throw;
197218
}
219+
return StreamingResponse<StreamingChatCompletionsUpdate>.CreateFromResponse(
220+
baseResponse,
221+
(responseForEnumeration)
222+
=> SseAsyncEnumerator<StreamingChatCompletionsUpdate>.EnumerateFromSseStream(
223+
responseForEnumeration.ContentStream,
224+
StreamingChatCompletionsUpdate.DeserializeStreamingChatCompletionsUpdates,
225+
otelScope,
226+
cancellationToken
227+
));
198228
}
199229

200230
internal HttpMessage CreatePostRequestMessage(

0 commit comments

Comments
 (0)