Skip to content

Commit e4e8ec5

Browse files
ctawiahctawiah
committed
feat: Add support for versioned metrics for AI Configs
1 parent 6058249 commit e4e8ec5

File tree

3 files changed

+51
-45
lines changed

3 files changed

+51
-45
lines changed

‎packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts‎

Lines changed: 45 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -14,25 +14,26 @@ const mockLdClient: LDClientMin = {
1414
const testContext: LDContext = { kind: 'user', key: 'test-user' };
1515
const configKey = 'test-config';
1616
const variationKey = 'v1';
17+
const version = 1;
1718

1819
beforeEach(() => {
1920
jest.clearAllMocks();
2021
});
2122

2223
it('tracks duration', () => {
23-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
24+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
2425
tracker.trackDuration(1000);
2526

2627
expect(mockTrack).toHaveBeenCalledWith(
2728
'$ld:ai:duration:total',
2829
testContext,
29-
{ configKey, variationKey },
30+
{ configKey, variationKey, version },
3031
1000,
3132
);
3233
});
3334

3435
it('tracks duration of async function', async () => {
35-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
36+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
3637
jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000);
3738

3839
const result = await tracker.trackDurationOf(async () => 'test-result');
@@ -41,61 +42,61 @@ it('tracks duration of async function', async () => {
4142
expect(mockTrack).toHaveBeenCalledWith(
4243
'$ld:ai:duration:total',
4344
testContext,
44-
{ configKey, variationKey },
45+
{ configKey, variationKey, version },
4546
1000,
4647
);
4748
});
4849

4950
it('tracks time to first token', () => {
50-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
51+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
5152
tracker.trackTimeToFirstToken(1000);
5253

5354
expect(mockTrack).toHaveBeenCalledWith(
5455
'$ld:ai:tokens:ttf',
5556
testContext,
56-
{ configKey, variationKey },
57+
{ configKey, variationKey, version },
5758
1000,
5859
);
5960
});
6061

6162
it('tracks positive feedback', () => {
62-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
63+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
6364
tracker.trackFeedback({ kind: LDFeedbackKind.Positive });
6465

6566
expect(mockTrack).toHaveBeenCalledWith(
6667
'$ld:ai:feedback:user:positive',
6768
testContext,
68-
{ configKey, variationKey },
69+
{ configKey, variationKey, version },
6970
1,
7071
);
7172
});
7273

7374
it('tracks negative feedback', () => {
74-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
75+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
7576
tracker.trackFeedback({ kind: LDFeedbackKind.Negative });
7677

7778
expect(mockTrack).toHaveBeenCalledWith(
7879
'$ld:ai:feedback:user:negative',
7980
testContext,
80-
{ configKey, variationKey },
81+
{ configKey, variationKey, version },
8182
1,
8283
);
8384
});
8485

8586
it('tracks success', () => {
86-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
87+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
8788
tracker.trackSuccess();
8889

8990
expect(mockTrack).toHaveBeenCalledWith(
9091
'$ld:ai:generation',
9192
testContext,
92-
{ configKey, variationKey },
93+
{ configKey, variationKey, version },
9394
1,
9495
);
9596
});
9697

9798
it('tracks OpenAI usage', async () => {
98-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
99+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
99100
jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000);
100101

101102
const TOTAL_TOKENS = 100;
@@ -113,41 +114,41 @@ it('tracks OpenAI usage', async () => {
113114
expect(mockTrack).toHaveBeenCalledWith(
114115
'$ld:ai:duration:total',
115116
testContext,
116-
{ configKey, variationKey },
117+
{ configKey, variationKey, version },
117118
1000,
118119
);
119120

120121
expect(mockTrack).toHaveBeenCalledWith(
121122
'$ld:ai:generation',
122123
testContext,
123-
{ configKey, variationKey },
124+
{ configKey, variationKey, version },
124125
1,
125126
);
126127

127128
expect(mockTrack).toHaveBeenCalledWith(
128129
'$ld:ai:tokens:total',
129130
testContext,
130-
{ configKey, variationKey },
131+
{ configKey, variationKey, version },
131132
TOTAL_TOKENS,
132133
);
133134

134135
expect(mockTrack).toHaveBeenCalledWith(
135136
'$ld:ai:tokens:input',
136137
testContext,
137-
{ configKey, variationKey },
138+
{ configKey, variationKey, version },
138139
PROMPT_TOKENS,
139140
);
140141

141142
expect(mockTrack).toHaveBeenCalledWith(
142143
'$ld:ai:tokens:output',
143144
testContext,
144-
{ configKey, variationKey },
145+
{ configKey, variationKey, version },
145146
COMPLETION_TOKENS,
146147
);
147148
});
148149

149150
it('tracks error when OpenAI metrics function throws', async () => {
150-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
151+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
151152
jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000);
152153

153154
const error = new Error('OpenAI API error');
@@ -160,27 +161,27 @@ it('tracks error when OpenAI metrics function throws', async () => {
160161
expect(mockTrack).toHaveBeenCalledWith(
161162
'$ld:ai:duration:total',
162163
testContext,
163-
{ configKey, variationKey },
164+
{ configKey, variationKey, version },
164165
1000,
165166
);
166167

167168
expect(mockTrack).toHaveBeenCalledWith(
168169
'$ld:ai:generation',
169170
testContext,
170-
{ configKey, variationKey },
171+
{ configKey, variationKey, version },
171172
1,
172173
);
173174

174175
expect(mockTrack).toHaveBeenCalledWith(
175176
'$ld:ai:generation:error',
176177
testContext,
177-
{ configKey, variationKey },
178+
{ configKey, variationKey, version },
178179
1,
179180
);
180181
});
181182

182183
it('tracks Bedrock conversation with successful response', () => {
183-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
184+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
184185

185186
const TOTAL_TOKENS = 100;
186187
const PROMPT_TOKENS = 49;
@@ -201,41 +202,41 @@ it('tracks Bedrock conversation with successful response', () => {
201202
expect(mockTrack).toHaveBeenCalledWith(
202203
'$ld:ai:generation',
203204
testContext,
204-
{ configKey, variationKey },
205+
{ configKey, variationKey, version },
205206
1,
206207
);
207208

208209
expect(mockTrack).toHaveBeenCalledWith(
209210
'$ld:ai:duration:total',
210211
testContext,
211-
{ configKey, variationKey },
212+
{ configKey, variationKey, version },
212213
500,
213214
);
214215

215216
expect(mockTrack).toHaveBeenCalledWith(
216217
'$ld:ai:tokens:total',
217218
testContext,
218-
{ configKey, variationKey },
219+
{ configKey, variationKey, version },
219220
TOTAL_TOKENS,
220221
);
221222

222223
expect(mockTrack).toHaveBeenCalledWith(
223224
'$ld:ai:tokens:input',
224225
testContext,
225-
{ configKey, variationKey },
226+
{ configKey, variationKey, version },
226227
PROMPT_TOKENS,
227228
);
228229

229230
expect(mockTrack).toHaveBeenCalledWith(
230231
'$ld:ai:tokens:output',
231232
testContext,
232-
{ configKey, variationKey },
233+
{ configKey, variationKey, version },
233234
COMPLETION_TOKENS,
234235
);
235236
});
236237

237238
it('tracks Bedrock conversation with error response', () => {
238-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
239+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
239240

240241
const response = {
241242
$metadata: { httpStatusCode: 400 },
@@ -247,20 +248,20 @@ it('tracks Bedrock conversation with error response', () => {
247248
expect(mockTrack).toHaveBeenCalledWith(
248249
'$ld:ai:generation',
249250
testContext,
250-
{ configKey, variationKey },
251+
{ configKey, variationKey, version },
251252
1,
252253
);
253254

254255
expect(mockTrack).toHaveBeenCalledWith(
255256
'$ld:ai:generation:error',
256257
testContext,
257-
{ configKey, variationKey },
258+
{ configKey, variationKey, version },
258259
1,
259260
);
260261
});
261262

262263
it('tracks tokens', () => {
263-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
264+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
264265

265266
const TOTAL_TOKENS = 100;
266267
const PROMPT_TOKENS = 49;
@@ -275,27 +276,27 @@ it('tracks tokens', () => {
275276
expect(mockTrack).toHaveBeenCalledWith(
276277
'$ld:ai:tokens:total',
277278
testContext,
278-
{ configKey, variationKey },
279+
{ configKey, variationKey, version },
279280
TOTAL_TOKENS,
280281
);
281282

282283
expect(mockTrack).toHaveBeenCalledWith(
283284
'$ld:ai:tokens:input',
284285
testContext,
285-
{ configKey, variationKey },
286+
{ configKey, variationKey, version },
286287
PROMPT_TOKENS,
287288
);
288289

289290
expect(mockTrack).toHaveBeenCalledWith(
290291
'$ld:ai:tokens:output',
291292
testContext,
292-
{ configKey, variationKey },
293+
{ configKey, variationKey, version },
293294
COMPLETION_TOKENS,
294295
);
295296
});
296297

297298
it('only tracks non-zero token counts', () => {
298-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
299+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
299300

300301
tracker.trackTokens({
301302
total: 0,
@@ -313,7 +314,7 @@ it('only tracks non-zero token counts', () => {
313314
expect(mockTrack).toHaveBeenCalledWith(
314315
'$ld:ai:tokens:input',
315316
testContext,
316-
{ configKey, variationKey },
317+
{ configKey, variationKey, version },
317318
50,
318319
);
319320

@@ -326,15 +327,15 @@ it('only tracks non-zero token counts', () => {
326327
});
327328

328329
it('returns empty summary when no metrics tracked', () => {
329-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
330+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
330331

331332
const summary = tracker.getSummary();
332333

333334
expect(summary).toEqual({});
334335
});
335336

336337
it('summarizes tracked metrics', () => {
337-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
338+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
338339

339340
tracker.trackDuration(1000);
340341
tracker.trackTokens({
@@ -362,7 +363,7 @@ it('summarizes tracked metrics', () => {
362363
});
363364

364365
it('tracks duration when async function throws', async () => {
365-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
366+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
366367
jest.spyOn(global.Date, 'now').mockReturnValueOnce(1000).mockReturnValueOnce(2000);
367368

368369
const error = new Error('test error');
@@ -375,26 +376,26 @@ it('tracks duration when async function throws', async () => {
375376
expect(mockTrack).toHaveBeenCalledWith(
376377
'$ld:ai:duration:total',
377378
testContext,
378-
{ configKey, variationKey },
379+
{ configKey, variationKey, version },
379380
1000,
380381
);
381382
});
382383

383384
it('tracks error', () => {
384-
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, testContext);
385+
const tracker = new LDAIConfigTrackerImpl(mockLdClient, configKey, variationKey, version, testContext);
385386
tracker.trackError();
386387

387388
expect(mockTrack).toHaveBeenCalledWith(
388389
'$ld:ai:generation',
389390
testContext,
390-
{ configKey, variationKey },
391+
{ configKey, variationKey, version },
391392
1,
392393
);
393394

394395
expect(mockTrack).toHaveBeenCalledWith(
395396
'$ld:ai:generation:error',
396397
testContext,
397-
{ configKey, variationKey },
398+
{ configKey, variationKey, version },
398399
1,
399400
);
400401
});

‎packages/sdk/server-ai/src/LDAIClientImpl.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { LDClientMin } from './LDClientMin';
1313
interface LDMeta {
1414
variationKey: string;
1515
enabled: boolean;
16+
version: number;
1617
}
1718

1819
/**
@@ -45,6 +46,8 @@ export class LDAIClientImpl implements LDAIClient {
4546
key,
4647
// eslint-disable-next-line no-underscore-dangle
4748
value._ldMeta?.variationKey ?? '',
49+
// eslint-disable-next-line no-underscore-dangle
50+
value._ldMeta?.version ?? 1,
4851
context,
4952
);
5053
// eslint-disable-next-line no-underscore-dangle

‎packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,15 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker {
1313
private _ldClient: LDClientMin,
1414
private _configKey: string,
1515
private _variationKey: string,
16+
private _version: number,
1617
private _context: LDContext,
1718
) {}
1819

19-
private _getTrackData(): { variationKey: string; configKey: string } {
20+
private _getTrackData(): { variationKey: string; configKey: string; version: number } {
2021
return {
2122
variationKey: this._variationKey,
2223
configKey: this._configKey,
24+
version: this._version,
2325
};
2426
}
2527

0 commit comments

Comments
 (0)