Skip to content

feat: Add variation version to metric data #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 7, 2025
Merged

feat: Add variation version to metric data #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
90b0cee
Added variation version to SDK variation payload to track variation m…
ctawiah Feb 4, 2025
8f3ec46
fixed ld config tracker param bug and linting
ctawiah Feb 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ldai/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ def config(
self._client,
variation.get('_ldMeta', {}).get('variationKey', ''),
key,
int(variation.get('_ldMeta', {}).get('version', 1)),
context,
)

Expand Down
10 changes: 5 additions & 5 deletions ldai/testing/test_model_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def td() -> TestData:
'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.5, 'maxTokens': 4096}, 'custom': {'extra-attribute': 'value'}},
'provider': {'name': 'fakeProvider'},
'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}],
'_ldMeta': {'enabled': True, 'variationKey': 'abcd'},
'_ldMeta': {'enabled': True, 'variationKey': 'abcd', 'version': 1},
},
"green",
)
Expand All @@ -31,7 +31,7 @@ def td() -> TestData:
{'role': 'system', 'content': 'Hello, {{name}}!'},
{'role': 'user', 'content': 'The day is, {{day}}!'},
],
'_ldMeta': {'enabled': True, 'variationKey': 'abcd'},
'_ldMeta': {'enabled': True, 'variationKey': 'abcd', 'version': 1},
},
"green",
)
Expand All @@ -44,7 +44,7 @@ def td() -> TestData:
{
'model': {'name': 'fakeModel', 'parameters': {'extra-attribute': 'I can be anything I set my mind/type to'}},
'messages': [{'role': 'system', 'content': 'Hello, {{ldctx.name}}! Is your last name {{ldctx.last}}?'}],
'_ldMeta': {'enabled': True, 'variationKey': 'abcd'},
'_ldMeta': {'enabled': True, 'variationKey': 'abcd', 'version': 1},
}
)
.variation_for_all(0)
Expand All @@ -56,7 +56,7 @@ def td() -> TestData:
{
'model': {'name': 'fakeModel', 'parameters': {'extra-attribute': 'I can be anything I set my mind/type to'}},
'messages': [{'role': 'system', 'content': 'Hello, {{ldctx.user.name}}! Do you work for {{ldctx.org.shortname}}?'}],
'_ldMeta': {'enabled': True, 'variationKey': 'abcd'},
'_ldMeta': {'enabled': True, 'variationKey': 'abcd', 'version': 1},
}
)
.variation_for_all(0)
Expand All @@ -68,7 +68,7 @@ def td() -> TestData:
{
'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.1}},
'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}],
'_ldMeta': {'enabled': False, 'variationKey': 'abcd'},
'_ldMeta': {'enabled': False, 'variationKey': 'abcd', 'version': 1},
}
)
.variation_for_all(0)
Expand Down
100 changes: 50 additions & 50 deletions ldai/testing/test_tracker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def td() -> TestData:
'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.5, 'maxTokens': 4096}, 'custom': {'extra-attribute': 'value'}},
'provider': {'name': 'fakeProvider'},
'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}],
'_ldMeta': {'enabled': True, 'variationKey': 'abcd'},
'_ldMeta': {'enabled': True, 'variationKey': 'abcd', 'version': 1},
},
"green",
)
Expand All @@ -38,7 +38,7 @@ def client(td: TestData) -> LDClient:

def test_summary_starts_empty(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 1, context)

assert tracker.get_summary().duration is None
assert tracker.get_summary().feedback is None
Expand All @@ -48,13 +48,13 @@ def test_summary_starts_empty(client: LDClient):

def test_tracks_duration(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
tracker.track_duration(100)

client.track.assert_called_with( # type: ignore
'$ld:ai:duration:total',
context,
{'variationKey': 'variation-key', 'configKey': 'config-key'},
{'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3},
100
)

Expand All @@ -63,27 +63,27 @@ def test_tracks_duration(client: LDClient):

def test_tracks_duration_of(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
tracker.track_duration_of(lambda: sleep(0.01))

calls = client.track.mock_calls # type: ignore

assert len(calls) == 1
assert calls[0].args[0] == '$ld:ai:duration:total'
assert calls[0].args[1] == context
assert calls[0].args[2] == {'variationKey': 'variation-key', 'configKey': 'config-key'}
assert calls[0].args[2] == {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}
assert calls[0].args[3] == pytest.approx(10, rel=10)


def test_tracks_time_to_first_token(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
tracker.track_time_to_first_token(100)

client.track.assert_called_with( # type: ignore
'$ld:ai:tokens:ttf',
context,
{'variationKey': 'variation-key', 'configKey': 'config-key'},
{'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3},
100
)

Expand All @@ -92,7 +92,7 @@ def test_tracks_time_to_first_token(client: LDClient):

def test_tracks_duration_of_with_exception(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)

def sleep_and_throw():
sleep(0.01)
Expand All @@ -109,21 +109,21 @@ def sleep_and_throw():
assert len(calls) == 1
assert calls[0].args[0] == '$ld:ai:duration:total'
assert calls[0].args[1] == context
assert calls[0].args[2] == {'variationKey': 'variation-key', 'configKey': 'config-key'}
assert calls[0].args[2] == {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}
assert calls[0].args[3] == pytest.approx(10, rel=10)


def test_tracks_token_usage(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)

tokens = TokenUsage(300, 200, 100)
tracker.track_tokens(tokens)

calls = [
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 300),
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 200),
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 100),
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 300),
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 200),
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 100),
]

client.track.assert_has_calls(calls) # type: ignore
Expand All @@ -133,7 +133,7 @@ def test_tracks_token_usage(client: LDClient):

def test_tracks_bedrock_metrics(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)

bedrock_result = {
'$metadata': {'httpStatusCode': 200},
Expand All @@ -149,12 +149,12 @@ def test_tracks_bedrock_metrics(client: LDClient):
tracker.track_bedrock_converse_metrics(bedrock_result)

calls = [
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:duration:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 50),
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 330),
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 220),
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 110),
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:duration:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 50),
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 330),
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 220),
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 110),
]

client.track.assert_has_calls(calls) # type: ignore
Expand All @@ -166,7 +166,7 @@ def test_tracks_bedrock_metrics(client: LDClient):

def test_tracks_bedrock_metrics_with_error(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)

bedrock_result = {
'$metadata': {'httpStatusCode': 500},
Expand All @@ -182,12 +182,12 @@ def test_tracks_bedrock_metrics_with_error(client: LDClient):
tracker.track_bedrock_converse_metrics(bedrock_result)

calls = [
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:duration:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 50),
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 330),
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 220),
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 110),
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:duration:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 50),
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 330),
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 220),
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 110),
]

client.track.assert_has_calls(calls) # type: ignore
Expand All @@ -199,7 +199,7 @@ def test_tracks_bedrock_metrics_with_error(client: LDClient):

def test_tracks_openai_metrics(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)

class Result:
def __init__(self):
Expand All @@ -216,11 +216,11 @@ def to_dict(self):
tracker.track_openai_metrics(lambda: Result())

calls = [
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 330),
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 220),
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 110),
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:tokens:total', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 330),
call('$ld:ai:tokens:input', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 220),
call('$ld:ai:tokens:output', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 110),
]

client.track.assert_has_calls(calls, any_order=False) # type: ignore
Expand All @@ -230,7 +230,7 @@ def to_dict(self):

def test_tracks_openai_metrics_with_exception(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)

def raise_exception():
raise ValueError("Something went wrong")
Expand All @@ -242,8 +242,8 @@ def raise_exception():
pass

calls = [
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
]

client.track.assert_has_calls(calls, any_order=False) # type: ignore
Expand All @@ -260,27 +260,27 @@ def raise_exception():
)
def test_tracks_feedback(client: LDClient, kind: FeedbackKind, label: str):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)

tracker.track_feedback({'kind': kind})

client.track.assert_called_with( # type: ignore
f'$ld:ai:feedback:user:{label}',
context,
{'variationKey': 'variation-key', 'configKey': 'config-key'},
{'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3},
1
)
assert tracker.get_summary().feedback == {'kind': kind}


def test_tracks_success(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
tracker.track_success()

calls = [
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
]

client.track.assert_has_calls(calls) # type: ignore
Expand All @@ -290,12 +290,12 @@ def test_tracks_success(client: LDClient):

def test_tracks_error(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
tracker.track_error()

calls = [
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
]

client.track.assert_has_calls(calls) # type: ignore
Expand All @@ -305,15 +305,15 @@ def test_tracks_error(client: LDClient):

def test_error_overwrites_success(client: LDClient):
context = Context.create('user-key')
tracker = LDAIConfigTracker(client, "variation-key", "config-key", context)
tracker = LDAIConfigTracker(client, "variation-key", "config-key", 3, context)
tracker.track_success()
tracker.track_error()

calls = [
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key'}, 1),
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:generation:success', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:generation', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
call('$ld:ai:generation:error', context, {'variationKey': 'variation-key', 'configKey': 'config-key', 'version': 3}, 1),
]

client.track.assert_has_calls(calls) # type: ignore
Expand Down
5 changes: 4 additions & 1 deletion ldai/tracker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,19 +69,21 @@ class LDAIConfigTracker:
"""

def __init__(
self, ld_client: LDClient, variation_key: str, config_key: str, context: Context
self, ld_client: LDClient, variation_key: str, config_key: str, version: int, context: Context
):
"""
Initialize an AI configuration tracker.

:param ld_client: LaunchDarkly client instance.
:param variation_key: Variation key for tracking.
:param config_key: Configuration key for tracking.
:param version: Version of the variation.
:param context: Context for evaluation.
"""
self._ld_client = ld_client
self._variation_key = variation_key
self._config_key = config_key
self._version = version
self._context = context
self._summary = LDAIMetricSummary()

Expand All @@ -94,6 +96,7 @@ def __get_track_data(self):
return {
'variationKey': self._variation_key,
'configKey': self._config_key,
'version': self._version,
}

def track_duration(self, duration: int) -> None:
Expand Down