Skip to content

Commit 90b0cee

Browse files
ctawiahctawiah
committed
Added variation version to SDK variation payload to track variation metrics per version
1 parent f065fd0 commit 90b0cee

File tree

4 files changed

+59
-56
lines changed

4 files changed

+59
-56
lines changed

‎ldai/client.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,7 @@ def config(
186186
tracker = LDAIConfigTracker(
187187
self._client,
188188
variation.get('_ldMeta', {}).get('variationKey', ''),
189+
variation.get('_ldMeta', {}).get('version', 1),
189190
key,
190191
context,
191192
)

‎ldai/testing/test_model_config.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ def td() -> TestData:
1515
'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.5, 'maxTokens': 4096}, 'custom': {'extra-attribute': 'value'}},
1616
'provider': {'name': 'fakeProvider'},
1717
'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}],
18-
'_ldMeta': {'enabled': True, 'variationKey': 'abcd'},
18+
'_ldMeta': {'enabled': True, 'variationKey': 'abcd', 'version': 1},
1919
},
2020
"green",
2121
)
@@ -31,7 +31,7 @@ def td() -> TestData:
3131
{'role': 'system', 'content': 'Hello, {{name}}!'},
3232
{'role': 'user', 'content': 'The day is, {{day}}!'},
3333
],
34-
'_ldMeta': {'enabled': True, 'variationKey': 'abcd'},
34+
'_ldMeta': {'enabled': True, 'variationKey': 'abcd', 'version': 1},
3535
},
3636
"green",
3737
)
@@ -44,7 +44,7 @@ def td() -> TestData:
4444
{
4545
'model': {'name': 'fakeModel', 'parameters': {'extra-attribute': 'I can be anything I set my mind/type to'}},
4646
'messages': [{'role': 'system', 'content': 'Hello, {{ldctx.name}}! Is your last name {{ldctx.last}}?'}],
47-
'_ldMeta': {'enabled': True, 'variationKey': 'abcd'},
47+
'_ldMeta': {'enabled': True, 'variationKey': 'abcd', 'version': 1},
4848
}
4949
)
5050
.variation_for_all(0)
@@ -56,7 +56,7 @@ def td() -> TestData:
5656
{
5757
'model': {'name': 'fakeModel', 'parameters': {'extra-attribute': 'I can be anything I set my mind/type to'}},
5858
'messages': [{'role': 'system', 'content': 'Hello, {{ldctx.user.name}}! Do you work for {{ldctx.org.shortname}}?'}],
59-
'_ldMeta': {'enabled': True, 'variationKey': 'abcd'},
59+
'_ldMeta': {'enabled': True, 'variationKey': 'abcd', 'version': 1},
6060
}
6161
)
6262
.variation_for_all(0)
@@ -68,7 +68,7 @@ def td() -> TestData:
6868
{
6969
'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.1}},
7070
'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}],
71-
'_ldMeta': {'enabled': False, 'variationKey': 'abcd'},
71+
'_ldMeta': {'enabled': False, 'variationKey': 'abcd', 'version': 1},
7272
}
7373
)
7474
.variation_for_all(0)

‎ldai/testing/test_tracker.py

Lines changed: 50 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ def td() -> TestData:
1818
'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.5, 'maxTokens': 4096}, 'custom': {'extra-attribute': 'value'}},
1919
'provider': {'name': 'fakeProvider'},
2020
'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}],
21-
'_ldMeta': {'enabled': True, 'variationKey': 'abcd'},
21+
'_ldMeta': {'enabled': True, 'variationKey': 'abcd', 'version': 1},
2222
},
2323
"green",
2424
)
@@ -38,7 +38,7 @@ def client(td: TestData) -> LDClient:
3838

3939
def test_summary_starts_empty(client: LDClient):
4040
context = Context.create('user-key')
41-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
41+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 1, context)
4242

4343
assert tracker.get_summary().duration is None
4444
assert tracker.get_summary().feedback is None
@@ -48,13 +48,13 @@ def test_summary_starts_empty(client: LDClient):
4848

4949
def test_tracks_duration(client: LDClient):
5050
context = Context.create('user-key')
51-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
51+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
5252
tracker.track_duration(100)
5353

5454
client.track.assert_called_with( # type: ignore
5555
'$ld:ai:duration:total',
5656
context,
57-
{'variationKey': 'variation-key', 'configKey': 'config-key'},
57+
{'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3},
5858
100
5959
)
6060

@@ -63,27 +63,27 @@ def test_tracks_duration(client: LDClient):
6363

6464
def test_tracks_duration_of(client: LDClient):
6565
context = Context.create('user-key')
66-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
66+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
6767
tracker.track_duration_of(lambda: sleep(0.01))
6868

6969
calls = client.track.mock_calls # type: ignore
7070

7171
assert len(calls) == 1
7272
assert calls[0].args[0] == '$ld:ai:duration:total'
7373
assert calls[0].args[1] == context
74-
assert calls[0].args[2] == {'variationKey': 'variation-key', 'configKey': 'config-key'}
74+
assert calls[0].args[2] == {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}
7575
assert calls[0].args[3] == pytest.approx(10, rel=10)
7676

7777

7878
def test_tracks_time_to_first_token(client: LDClient):
7979
context = Context.create('user-key')
80-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
80+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
8181
tracker.track_time_to_first_token(100)
8282

8383
client.track.assert_called_with( # type: ignore
8484
'$ld:ai:tokens:ttf',
8585
context,
86-
{'variationKey': 'variation-key', 'configKey': 'config-key'},
86+
{'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3},
8787
100
8888
)
8989

@@ -92,7 +92,7 @@ def test_tracks_time_to_first_token(client: LDClient):
9292

9393
def test_tracks_duration_of_with_exception(client: LDClient):
9494
context = Context.create('user-key')
95-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
95+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
9696

9797
def sleep_and_throw():
9898
sleep(0.01)
@@ -109,21 +109,21 @@ def sleep_and_throw():
109109
assert len(calls) == 1
110110
assert calls[0].args[0] == '$ld:ai:duration:total'
111111
assert calls[0].args[1] == context
112-
assert calls[0].args[2] == {'variationKey': 'variation-key', 'configKey': 'config-key'}
112+
assert calls[0].args[2] == {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}
113113
assert calls[0].args[3] == pytest.approx(10, rel=10)
114114

115115

116116
def test_tracks_token_usage(client: LDClient):
117117
context = Context.create('user-key')
118-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
118+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
119119

120120
tokens = TokenUsage(300, 200, 100)
121121
tracker.track_tokens(tokens)
122122

123123
calls = [
124-
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 300),
125-
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 200),
126-
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 100),
124+
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 300),
125+
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 200),
126+
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 100),
127127
]
128128

129129
client.track.assert_has_calls(calls) # type: ignore
@@ -133,7 +133,7 @@ def test_tracks_token_usage(client: LDClient):
133133

134134
def test_tracks_bedrock_metrics(client: LDClient):
135135
context = Context.create('user-key')
136-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
136+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
137137

138138
bedrock_result = {
139139
'$metadata': {'httpStatusCode': 200},
@@ -149,12 +149,12 @@ def test_tracks_bedrock_metrics(client: LDClient):
149149
tracker.track_bedrock_converse_metrics(bedrock_result)
150150

151151
calls = [
152-
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
153-
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
154-
call('$ld:ai:duration:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 50),
155-
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 330),
156-
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 220),
157-
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 110),
152+
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
153+
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
154+
call('$ld:ai:duration:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 50),
155+
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 330),
156+
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 220),
157+
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 110),
158158
]
159159

160160
client.track.assert_has_calls(calls) # type: ignore
@@ -166,7 +166,7 @@ def test_tracks_bedrock_metrics(client: LDClient):
166166

167167
def test_tracks_bedrock_metrics_with_error(client: LDClient):
168168
context = Context.create('user-key')
169-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
169+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
170170

171171
bedrock_result = {
172172
'$metadata': {'httpStatusCode': 500},
@@ -182,12 +182,12 @@ def test_tracks_bedrock_metrics_with_error(client: LDClient):
182182
tracker.track_bedrock_converse_metrics(bedrock_result)
183183

184184
calls = [
185-
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
186-
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
187-
call('$ld:ai:duration:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 50),
188-
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 330),
189-
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 220),
190-
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 110),
185+
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
186+
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
187+
call('$ld:ai:duration:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 50),
188+
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 330),
189+
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 220),
190+
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 110),
191191
]
192192

193193
client.track.assert_has_calls(calls) # type: ignore
@@ -199,7 +199,7 @@ def test_tracks_bedrock_metrics_with_error(client: LDClient):
199199

200200
def test_tracks_openai_metrics(client: LDClient):
201201
context = Context.create('user-key')
202-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
202+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
203203

204204
class Result:
205205
def __init__(self):
@@ -216,11 +216,11 @@ def to_dict(self):
216216
tracker.track_openai_metrics(lambda: Result())
217217

218218
calls = [
219-
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
220-
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
221-
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 330),
222-
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 220),
223-
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 110),
219+
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
220+
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
221+
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 330),
222+
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 220),
223+
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 110),
224224
]
225225

226226
client.track.assert_has_calls(calls, any_order=False) # type: ignore
@@ -230,7 +230,7 @@ def to_dict(self):
230230

231231
def test_tracks_openai_metrics_with_exception(client: LDClient):
232232
context = Context.create('user-key')
233-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
233+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
234234

235235
def raise_exception():
236236
raise ValueError("Something went wrong")
@@ -242,8 +242,8 @@ def raise_exception():
242242
pass
243243

244244
calls = [
245-
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
246-
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
245+
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
246+
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
247247
]
248248

249249
client.track.assert_has_calls(calls, any_order=False) # type: ignore
@@ -260,27 +260,27 @@ def raise_exception():
260260
)
261261
def test_tracks_feedback(client: LDClient, kind: FeedbackKind, label: str):
262262
context = Context.create('user-key')
263-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
263+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
264264

265265
tracker.track_feedback({'kind': kind})
266266

267267
client.track.assert_called_with( # type: ignore
268268
f'$ld:ai:feedback:user:{label}',
269269
context,
270-
{'variationKey': 'variation-key', 'configKey': 'config-key'},
270+
{'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3},
271271
1
272272
)
273273
assert tracker.get_summary().feedback == {'kind': kind}
274274

275275

276276
def test_tracks_success(client: LDClient):
277277
context = Context.create('user-key')
278-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
278+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
279279
tracker.track_success()
280280

281281
calls = [
282-
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
283-
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
282+
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
283+
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
284284
]
285285

286286
client.track.assert_has_calls(calls) # type: ignore
@@ -290,12 +290,12 @@ def test_tracks_success(client: LDClient):
290290

291291
def test_tracks_error(client: LDClient):
292292
context = Context.create('user-key')
293-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
293+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
294294
tracker.track_error()
295295

296296
calls = [
297-
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
298-
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
297+
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
298+
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
299299
]
300300

301301
client.track.assert_has_calls(calls) # type: ignore
@@ -305,15 +305,15 @@ def test_tracks_error(client: LDClient):
305305

306306
def test_error_overwrites_success(client: LDClient):
307307
context = Context.create('user-key')
308-
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
308+
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
309309
tracker.track_success()
310310
tracker.track_error()
311311

312312
calls = [
313-
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
314-
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
315-
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
316-
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
313+
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
314+
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
315+
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
316+
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
317317
]
318318

319319
client.track.assert_has_calls(calls) # type: ignore

‎ldai/tracker.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ class LDAIConfigTracker:
6969
"""
7070

7171
def __init__(
72-
self, ld_client: LDClient, variation_key: str, config_key: str, context: Context
72+
self, ld_client: LDClient, variation_key: str, config_key: str, variation_version: int, context: Context
7373
):
7474
"""
7575
Initialize an AI configuration tracker.
@@ -82,6 +82,7 @@ def __init__(
8282
self._ld_client = ld_client
8383
self._variation_key = variation_key
8484
self._config_key = config_key
85+
self._variation_version = variation_version
8586
self._context = context
8687
self._summary = LDAIMetricSummary()
8788

@@ -94,6 +95,7 @@ def __get_track_data(self):
9495
return {
9596
'variationKey': self._variation_key,
9697
'configKey': self._config_key,
98+
'version': self._variation_version,
9799
}
98100

99101
def track_duration(self, duration: int) -> None:

0 commit comments

Comments
 (0)