Skip to content

Commit 7acc574

Browse files
keelerm84keelerm84
authored
feat: Add TrackError to mirror TrackSuccess (#64)
Additionally, emit new `$ld:ai:generation:(success|error)` events on success or failure.
1 parent ac29d46 commit 7acc574

File tree

3 files changed

+88
-16
lines changed

3 files changed

+88
-16
lines changed

‎pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ public interface ILdAiConfigTracker
4545
/// </summary>
4646
public void TrackSuccess();
4747

48+
/// <summary>
49+
/// Tracks an unsuccessful generation event related to this config.
50+
/// </summary>
51+
public void TrackError();
52+
4853
/// <summary>
4954
/// Tracks a request to a provider. The request is a task that returns a <see cref="Response"/>, which
5055
/// contains information about the request such as token usage and metrics.

‎pkgs/sdk/server-ai/src/LdAiConfigTracker.cs‎

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Diagnostics;
4+
using System.Runtime.CompilerServices;
45
using System.Threading.Tasks;
56
using LaunchDarkly.Sdk.Server.Ai.Config;
67
using LaunchDarkly.Sdk.Server.Ai.Interfaces;
@@ -21,6 +22,8 @@ public class LdAiConfigTracker : ILdAiConfigTracker
2122
private const string FeedbackPositive = "$ld:ai:feedback:user:positive";
2223
private const string FeedbackNegative = "$ld:ai:feedback:user:negative";
2324
private const string Generation = "$ld:ai:generation";
25+
private const string GenerationSuccess = "$ld:ai:generation:success";
26+
private const string GenerationError = "$ld:ai:generation:error";
2427
private const string TokenTotal = "$ld:ai:tokens:total";
2528
private const string TokenInput = "$ld:ai:tokens:input";
2629
private const string TokenOutput = "$ld:ai:tokens:output";
@@ -57,18 +60,14 @@ public void TrackDuration(float durationMs) =>
5760

5861
/// <inheritdoc/>
5962
public async Task<T> TrackDurationOfTask<T>(Task<T> task)
60-
{
61-
var result = await MeasureDurationOfTaskMs(task);
62-
TrackDuration(result.Item2);
63-
return result.Item1;
64-
}
65-
66-
private static async Task<Tuple<T, long>> MeasureDurationOfTaskMs<T>(Task<T> task)
6763
{
6864
var sw = Stopwatch.StartNew();
69-
var result = await task;
70-
sw.Stop();
71-
return Tuple.Create(result, sw.ElapsedMilliseconds);
65+
try {
66+
return await task;
67+
} finally {
68+
sw.Stop();
69+
TrackDuration(sw.ElapsedMilliseconds);
70+
}
7271
}
7372

7473
/// <inheritdoc/>
@@ -90,23 +89,44 @@ public void TrackFeedback(Feedback feedback)
9089
/// <inheritdoc/>
9190
public void TrackSuccess()
9291
{
92+
_client.Track(GenerationSuccess, _context, _trackData, 1);
93+
_client.Track(Generation, _context, _trackData, 1);
94+
}
95+
96+
/// <inheritdoc/>
97+
public void TrackError()
98+
{
99+
_client.Track(GenerationError, _context, _trackData, 1);
93100
_client.Track(Generation, _context, _trackData, 1);
94101
}
95102

96103
/// <inheritdoc/>
97104
public async Task<Response> TrackRequest(Task<Response> request)
98105
{
99-
var (result, durationMs) = await MeasureDurationOfTaskMs(request);
100-
TrackSuccess();
106+
var sw = Stopwatch.StartNew();
107+
try
108+
{
109+
var result = await request;
110+
TrackSuccess();
111+
112+
sw.Stop();
113+
TrackDuration(result.Metrics?.LatencyMs ?? sw.ElapsedMilliseconds);
101114

102-
TrackDuration(result.Metrics?.LatencyMs ?? durationMs);
115+
if (result.Usage != null)
116+
{
117+
TrackTokens(result.Usage.Value);
118+
}
103119

104-
if (result.Usage != null)
120+
return result;
121+
}
122+
catch (Exception)
105123
{
106-
TrackTokens(result.Usage.Value);
124+
sw.Stop();
125+
TrackDuration(sw.ElapsedMilliseconds);
126+
TrackError();
127+
throw;
107128
}
108129

109-
return result;
110130
}
111131

112132
/// <inheritdoc/>

‎pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,27 @@ public void CanTrackSuccess()
6868
var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context);
6969
tracker.TrackSuccess();
7070
mockClient.Verify(x => x.Track("$ld:ai:generation", context, data, 1.0f), Times.Once);
71+
mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, data, 1.0f), Times.Once);
72+
}
73+
74+
75+
[Fact]
76+
public void CanTrackError()
77+
{
78+
var mockClient = new Mock<ILaunchDarklyClient>();
79+
var context = Context.New("key");
80+
const string flagKey = "key";
81+
var config = LdAiConfig.Disabled;
82+
var data = LdValue.ObjectFrom(new Dictionary<string, LdValue>
83+
{
84+
{ "variationKey", LdValue.Of(config.VariationKey) },
85+
{ "configKey", LdValue.Of(flagKey) }
86+
});
87+
88+
var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context);
89+
tracker.TrackError();
90+
mockClient.Verify(x => x.Track("$ld:ai:generation", context, data, 1.0f), Times.Once);
91+
mockClient.Verify(x => x.Track("$ld:ai:generation:error", context, data, 1.0f), Times.Once);
7192
}
7293

7394

@@ -189,6 +210,8 @@ public void CanTrackResponseWithSpecificLatency()
189210

190211
var result = tracker.TrackRequest(Task.Run(() => givenResponse));
191212
Assert.Equal(givenResponse, result.Result);
213+
mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, data, 1.0f), Times.Once);
214+
mockClient.Verify(x => x.Track("$ld:ai:generation", context, data, 1.0f), Times.Once);
192215
mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, data, 1.0f), Times.Once);
193216
mockClient.Verify(x => x.Track("$ld:ai:tokens:input", context, data, 2.0f), Times.Once);
194217
mockClient.Verify(x => x.Track("$ld:ai:tokens:output", context, data, 3.0f), Times.Once);
@@ -228,5 +251,29 @@ public void CanTrackResponseWithPartialData()
228251
// if latency isn't provided via Statistics, then it is automatically measured.
229252
mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, data, It.IsAny<double>()), Times.Once);
230253
}
254+
255+
[Fact]
256+
public async Task CanTrackExceptionFromResponse()
257+
{
258+
var mockClient = new Mock<ILaunchDarklyClient>();
259+
var context = Context.New("key");
260+
const string flagKey = "key";
261+
var config = LdAiConfig.Disabled;
262+
var data = LdValue.ObjectFrom(new Dictionary<string, LdValue>
263+
{
264+
{ "variationKey", LdValue.Of(config.VariationKey) },
265+
{ "configKey", LdValue.Of(flagKey) }
266+
});
267+
268+
var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context);
269+
270+
await Assert.ThrowsAsync<System.Exception>(() => tracker.TrackRequest(Task.FromException<Response>(new System.Exception("I am an exception"))));
271+
272+
mockClient.Verify(x => x.Track("$ld:ai:generation", context, data, 1.0f), Times.Once);
273+
mockClient.Verify(x => x.Track("$ld:ai:generation:error", context, data, 1.0f), Times.Once);
274+
275+
// if latency isn't provided via Statistics, then it is automatically measured.
276+
mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, data, It.IsAny<double>()), Times.Once);
277+
}
231278
}
232279
}

0 commit comments

Comments
 (0)