Skip to content

Commit 9f78679

Browse files
author
Daniel OBrien
committed
Update tracking methods
Add Bedrock and OpenAI methods fix npm scripts
1 parent c88ad2d commit 9f78679

File tree

13 files changed

+207
-107
lines changed

13 files changed

+207
-107
lines changed

‎packages/sdk/ai/package.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@
55
"main": "dist/index.js",
66
"types": "dist/index.d.ts",
77
"scripts": {
8-
"build": "tsc",
9-
"test": "jest",
10-
"lint": "eslint . --ext .ts"
8+
"build": "npx tsc",
9+
"lint": "npx eslint . --ext .ts",
10+
"lint:fix": "yarn run lint --fix"
1111
},
1212
"keywords": [
1313
"launchdarkly",

‎packages/sdk/ai/src/LDAIConfigTracker.ts‎

Lines changed: 0 additions & 77 deletions
This file was deleted.

‎packages/sdk/ai/src/api/config/LDAIConfig.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,17 @@ import { LDAIConfigTracker } from './LDAIConfigTracker';
55
*/
66
export interface LDAIConfig {
77
/**
8-
* The result of the AI Config evaluation.
8+
* The result of the AI Config customization.
99
*/
1010
config: unknown;
1111

1212
/**
13-
* A tracker which can be used to generate analytics for the migration.
13+
* A tracker which can be used to generate analytics.
1414
*/
1515
tracker: LDAIConfigTracker;
16+
17+
/**
18+
* Whether the configuration is not found.
19+
*/
20+
noConfiguration: boolean;
1621
}

‎packages/sdk/ai/src/api/config/LDAIConfigTracker.ts‎

Lines changed: 110 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,111 @@
1-
import { BedrockTokenUsage, FeedbackKind, TokenUsage, UnderscoreTokenUsage } from '../metrics';
2-
3-
export interface LDAIConfigTracker {
4-
trackDuration: (duration: number) => void;
5-
trackTokens: (tokens: TokenUsage | UnderscoreTokenUsage | BedrockTokenUsage) => void;
6-
trackError: (error: number) => void;
7-
trackGeneration: (generation: number) => void;
8-
trackFeedback: (feedback: { kind: FeedbackKind }) => void;
1+
import { LDClient, LDContext } from '@launchdarkly/node-server-sdk';
2+
3+
import {
4+
BedrockTokenUsage,
5+
FeedbackKind,
6+
OpenAITokenUsage,
7+
TokenUsage,
8+
UnderscoreTokenUsage,
9+
} from '../metrics';
10+
11+
export class LDAIConfigTracker {
12+
private ldClient: LDClient;
13+
private variationId: string;
14+
private configKey: string;
15+
private context: LDContext;
16+
17+
constructor(ldClient: LDClient, configKey: string, variationId: string,context: LDContext) {
18+
this.ldClient = ldClient;
19+
this.variationId = variationId;
20+
this.configKey = configKey;
21+
this.context = context;
22+
}
23+
24+
private getTrackData() {
25+
return {
26+
variationId: this.variationId,
27+
configKey: this.configKey,
28+
};
29+
}
30+
31+
trackDuration(duration: number): void {
32+
this.ldClient.track('$ld:ai:duration:total', this.context, this.getTrackData(), duration);
33+
}
34+
35+
async trackDurationOf(func: Function, ...args: any[]): Promise<any> {
36+
const startTime = Date.now();
37+
const result = await func(...args);
38+
const endTime = Date.now();
39+
const duration = endTime - startTime; // duration in milliseconds
40+
this.trackDuration(duration);
41+
return result;
42+
}
43+
44+
trackError(error: number): void {
45+
this.ldClient.track('$ld:ai:error', this.context, this.getTrackData(), error);
46+
}
47+
48+
trackFeedback(feedback: { kind: FeedbackKind }): void {
49+
if (feedback.kind === FeedbackKind.Positive) {
50+
this.ldClient.track('$ld:ai:feedback:user:positive', this.context, this.getTrackData(), 1);
51+
} else if (feedback.kind === FeedbackKind.Negative) {
52+
this.ldClient.track('$ld:ai:feedback:user:negative', this.context, this.getTrackData(), 1);
53+
}
54+
}
55+
56+
trackGeneration(generation: number): void {
57+
this.ldClient.track('$ld:ai:generation', this.context, this.getTrackData(), generation);
58+
}
59+
60+
async trackOpenAI(func: Function, ...args: any[]): Promise<any> {
61+
const result = await this.trackDurationOf(func, ...args);
62+
this.trackGeneration(1);
63+
if (result.usage) {
64+
this.trackTokens(new OpenAITokenUsage(result.usage));
65+
}
66+
return result;
67+
}
68+
69+
async trackBedrockConverse(res: any): Promise<any> {
70+
if (res.$metadata?.httpStatusCode === 200) {
71+
this.trackGeneration(1);
72+
} else if (res.$metadata?.httpStatusCode >= 400) {
73+
this.trackError(res.$metadata.httpStatusCode);
74+
}
75+
if (res.metrics) {
76+
this.trackDuration(res.metrics.latencyMs);
77+
}
78+
if (res.usage) {
79+
this.trackTokens(new BedrockTokenUsage(res.usage));
80+
}
81+
return res;
82+
}
83+
84+
trackTokens(tokens: TokenUsage | UnderscoreTokenUsage | BedrockTokenUsage): void {
85+
const tokenMetrics = tokens.toMetrics();
86+
if (tokenMetrics.total > 0) {
87+
this.ldClient.track(
88+
'$ld:ai:tokens:total',
89+
this.context,
90+
this.getTrackData(),
91+
tokenMetrics.total,
92+
);
93+
}
94+
if (tokenMetrics.input > 0) {
95+
this.ldClient.track(
96+
'$ld:ai:tokens:input',
97+
this.context,
98+
this.getTrackData(),
99+
tokenMetrics.input,
100+
);
101+
}
102+
if (tokenMetrics.output > 0) {
103+
this.ldClient.track(
104+
'$ld:ai:tokens:output',
105+
this.context,
106+
this.getTrackData(),
107+
tokenMetrics.output,
108+
);
109+
}
110+
}
9111
}

‎packages/sdk/ai/src/api/config/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
export * from './LDAIConfig';
2-
export * from './LDAIConfigTracker';
2+
export { LDAIConfigTracker } from './LDAIConfigTracker';

‎packages/sdk/ai/src/api/metrics/BedrockTokenUsage.ts‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,19 @@
1-
export interface BedrockTokenUsage {
1+
export class BedrockTokenUsage {
2+
totalTokens: number;
23
inputTokens: number;
34
outputTokens: number;
4-
totalTokens: number;
5+
6+
constructor(data: any) {
7+
this.totalTokens = data.totalTokens || 0;
8+
this.inputTokens = data.inputTokens || 0;
9+
this.outputTokens = data.outputTokens || 0;
10+
}
11+
12+
toMetrics() {
13+
return {
14+
total: this.totalTokens,
15+
input: this.inputTokens,
16+
output: this.outputTokens,
17+
};
18+
}
519
}

‎packages/sdk/ai/src/api/metrics/OpenAITokenUsage.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
export class OpenAITokenUsage {
2+
total_tokens: number;
3+
prompt_tokens: number;
4+
completion_tokens: number;
5+
6+
constructor(data: any) {
7+
this.total_tokens = data.total_tokens;
8+
this.prompt_tokens = data.prompt_tokens;
9+
this.completion_tokens = data.completion_tokens;
10+
}
11+
12+
toMetrics() {
13+
return {
14+
total: this.total_tokens,
15+
input: this.prompt_tokens,
16+
output: this.completion_tokens,
17+
};
18+
}
19+
}

‎packages/sdk/ai/src/api/metrics/TokenUsage.ts‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,19 @@
1-
export interface TokenUsage {
2-
completionTokens?: number;
3-
promptTokens?: number;
4-
totalTokens?: number;
1+
export class TokenUsage {
2+
totalTokens: number;
3+
promptTokens: number;
4+
completionTokens: number;
5+
6+
constructor(data: any) {
7+
this.totalTokens = data.total_tokens || 0;
8+
this.promptTokens = data.prompt_tokens || 0;
9+
this.completionTokens = data.completion_tokens || 0;
10+
}
11+
12+
toMetrics() {
13+
return {
14+
total: this.totalTokens,
15+
input: this.promptTokens,
16+
output: this.completionTokens,
17+
};
18+
}
519
}

‎packages/sdk/ai/src/api/metrics/UnderscoreTokenUsage.ts‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,19 @@
1-
export interface UnderscoreTokenUsage {
2-
completion_tokens?: number;
3-
prompt_tokens?: number;
4-
total_tokens?: number;
1+
export class UnderscoreTokenUsage {
2+
total_tokens: number;
3+
prompt_tokens: number;
4+
completion_tokens: number;
5+
6+
constructor(data: any) {
7+
this.total_tokens = data.total_tokens || 0;
8+
this.prompt_tokens = data.prompt_tokens || 0;
9+
this.completion_tokens = data.completion_tokens || 0;
10+
}
11+
12+
toMetrics() {
13+
return {
14+
total: this.total_tokens,
15+
input: this.prompt_tokens,
16+
output: this.completion_tokens,
17+
};
18+
}
519
}

‎packages/sdk/ai/src/api/metrics/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
export * from './BedrockTokenUsage';
22
export * from './FeedbackKind';
3+
export * from './OpenAITokenUsage';
34
export * from './TokenMetrics';
45
export * from './TokenUsage';
56
export * from './UnderScoreTokenUsage';

0 commit comments

Comments
 (0)