Skip to content

Commit a49fe0b

Browse files
jjmachanjjmachan
authored
Merge pull request #14 from explodinggradients/fix/langfuse
 [merge after select PR] fix: langfuse experimental
2 parents e28f60c + d20ccad commit a49fe0b

File tree

2 files changed

+129
-71
lines changed

2 files changed

+129
-71
lines changed

‎nbs/project/experiments.ipynb‎

Lines changed: 113 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -319,18 +319,6 @@
319319
" async def run_async(self, name: str, dataset: Dataset): ..."
320320
]
321321
},
322-
{
323-
"cell_type": "code",
324-
"execution_count": null,
325-
"metadata": {},
326-
"outputs": [],
327-
"source": [
328-
"# | export\n",
329-
"\n",
330-
"# this one we have to clean up\n",
331-
"from langfuse.decorators import observe"
332-
]
333-
},
334322
{
335323
"cell_type": "code",
336324
"execution_count": null,
@@ -477,13 +465,13 @@
477465
"name": "stderr",
478466
"output_type": "stream",
479467
"text": [
480-
"Running experiment: 100%|██████████| 6/6 [00:01<00:00, 3.84it/s]\n"
468+
"Running experiment: 100%|██████████| 6/6 [00:01<00:00, 3.23it/s]\n"
481469
]
482470
},
483471
{
484472
"data": {
485473
"text/plain": [
486-
"Experiment(name=gallant_torvalds, model=TextExperimentModel)"
474+
"Experiment(name=dazzling_knuth, model=TextExperimentModel)"
487475
]
488476
},
489477
"execution_count": null,
@@ -496,6 +484,18 @@
496484
"await test_experiment.run_async(test_dataset)"
497485
]
498486
},
487+
{
488+
"cell_type": "code",
489+
"execution_count": null,
490+
"metadata": {},
491+
"outputs": [],
492+
"source": [
493+
"# | export\n",
494+
"\n",
495+
"# this one we have to clean up\n",
496+
"from langfuse.decorators import observe"
497+
]
498+
},
499499
{
500500
"cell_type": "code",
501501
"execution_count": null,
@@ -518,37 +518,111 @@
518518
" \"\"\"\n",
519519
"\n",
520520
" def decorator(func: t.Callable) -> ExperimentProtocol:\n",
521-
" # First, create a base experiment wrapper\n",
522-
" base_experiment = self.experiment(experiment_model, name_prefix)(func)\n",
523-
"\n",
524-
" # Override the wrapped function to add Langfuse observation\n",
525521
" @wraps(func)\n",
526-
" async def wrapped_with_langfuse(*args, **kwargs):\n",
527-
" # wrap the function with langfuse observation\n",
528-
" observed_func = observe(name=f\"{name_prefix}-{func.__name__}\")(func)\n",
522+
" async def langfuse_wrapped_func(*args, **kwargs):\n",
523+
" # Apply langfuse observation directly here\n",
524+
" trace_name = f\"{name_prefix}-{func.__name__}\" if name_prefix else func.__name__\n",
525+
" observed_func = observe(name=trace_name)(func)\n",
529526
" return await observed_func(*args, **kwargs)\n",
530-
"\n",
531-
" # Replace the async function to use Langfuse\n",
532-
" original_run_async = base_experiment.run_async\n",
533-
"\n",
534-
" # Use the original run_async but with the Langfuse-wrapped function\n",
535-
" async def run_async_with_langfuse(\n",
536-
" dataset: Dataset, name: t.Optional[str] = None\n",
537-
" ):\n",
538-
" # Override the internal wrapped_experiment with our Langfuse version\n",
539-
" base_experiment.__wrapped__ = wrapped_with_langfuse\n",
540-
"\n",
541-
" # Call the original run_async which will now use our Langfuse-wrapped function\n",
542-
" return await original_run_async(dataset, name)\n",
543-
"\n",
544-
" # Replace the run_async method\n",
545-
" base_experiment.__setattr__(\"run_async\", run_async_with_langfuse)\n",
546-
"\n",
547-
" return t.cast(ExperimentProtocol, base_experiment)\n",
527+
" \n",
528+
" # Now create the experiment wrapper with our already-observed function\n",
529+
" experiment_wrapper = self.experiment(experiment_model, name_prefix)(langfuse_wrapped_func)\n",
530+
" \n",
531+
" return t.cast(ExperimentProtocol, experiment_wrapper)\n",
548532
"\n",
549533
" return decorator"
550534
]
551535
},
536+
{
537+
"cell_type": "code",
538+
"execution_count": null,
539+
"metadata": {},
540+
"outputs": [],
541+
"source": [
542+
"import os\n",
543+
"# import langfuse\n",
544+
"from langfuse import Langfuse"
545+
]
546+
},
547+
{
548+
"cell_type": "code",
549+
"execution_count": null,
550+
"metadata": {},
551+
"outputs": [],
552+
"source": [
553+
"\n",
554+
"langfuse = Langfuse(\n",
555+
" secret_key=os.getenv(\"LANGFUSE_SECRET_KEY\"),\n",
556+
" public_key=os.getenv(\"LANGFUSE_PUBLIC_KEY\"),\n",
557+
" host=\"https://us.cloud.langfuse.com\"\n",
558+
")"
559+
]
560+
},
561+
{
562+
"cell_type": "code",
563+
"execution_count": null,
564+
"metadata": {},
565+
"outputs": [],
566+
"source": [
567+
"@p.langfuse_experiment(TextExperimentModel)\n",
568+
"async def test_experiment(item: TestModel):\n",
569+
" return TextExperimentModel(**item.model_dump(), response=\"test response\", is_correct=\"yes\")"
570+
]
571+
},
572+
{
573+
"cell_type": "code",
574+
"execution_count": null,
575+
"metadata": {},
576+
"outputs": [
577+
{
578+
"data": {
579+
"text/plain": [
580+
"TextExperimentModel(name='test item 1', description='test item 1 description', price=100.0, url='https://www.google.com', tags='test', response='test response', is_correct='yes')"
581+
]
582+
},
583+
"execution_count": null,
584+
"metadata": {},
585+
"output_type": "execute_result"
586+
}
587+
],
588+
"source": [
589+
"await test_experiment(test_dataset[0])"
590+
]
591+
},
592+
{
593+
"cell_type": "code",
594+
"execution_count": null,
595+
"metadata": {},
596+
"outputs": [
597+
{
598+
"name": "stderr",
599+
"output_type": "stream",
600+
"text": [
601+
"Running experiment: 100%|██████████| 6/6 [00:01<00:00, 4.01it/s]\n"
602+
]
603+
},
604+
{
605+
"data": {
606+
"text/plain": [
607+
"Experiment(name=cool_matsumoto, model=TextExperimentModel)"
608+
]
609+
},
610+
"execution_count": null,
611+
"metadata": {},
612+
"output_type": "execute_result"
613+
}
614+
],
615+
"source": [
616+
"await test_experiment.run_async(test_dataset)"
617+
]
618+
},
619+
{
620+
"cell_type": "markdown",
621+
"metadata": {},
622+
"source": [
623+
"## Compare and Plot"
624+
]
625+
},
552626
{
553627
"cell_type": "code",
554628
"execution_count": null,

‎ragas_experimental/project/experiments.py‎

Lines changed: 16 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -124,16 +124,12 @@ async def __call__(self, *args, **kwargs): ...
124124
async def run_async(self, name: str, dataset: Dataset): ...
125125

126126
# %% ../../nbs/project/experiments.ipynb 16
127-
# this one we have to clean up
128-
from langfuse.decorators import observe
129-
130-
# %% ../../nbs/project/experiments.ipynb 17
131127
from .naming import MemorableNames
132128

133-
# %% ../../nbs/project/experiments.ipynb 18
129+
# %% ../../nbs/project/experiments.ipynb 17
134130
memorable_names = MemorableNames()
135131

136-
# %% ../../nbs/project/experiments.ipynb 19
132+
# %% ../../nbs/project/experiments.ipynb 18
137133
@patch
138134
def experiment(
139135
self: Project, experiment_model, name_prefix: str = ""
@@ -214,6 +210,10 @@ async def run_async(dataset: Dataset, name: t.Optional[str] = None):
214210

215211

216212

213+
# %% ../../nbs/project/experiments.ipynb 22
214+
# this one we have to clean up
215+
from langfuse.decorators import observe
216+
217217
# %% ../../nbs/project/experiments.ipynb 23
218218
@patch
219219
def langfuse_experiment(
@@ -230,37 +230,21 @@ def langfuse_experiment(
230230
"""
231231

232232
def decorator(func: t.Callable) -> ExperimentProtocol:
233-
# First, create a base experiment wrapper
234-
base_experiment = self.experiment(experiment_model, name_prefix)(func)
235-
236-
# Override the wrapped function to add Langfuse observation
237233
@wraps(func)
238-
async def wrapped_with_langfuse(*args, **kwargs):
239-
# wrap the function with langfuse observation
240-
observed_func = observe(name=f"{name_prefix}-{func.__name__}")(func)
234+
async def langfuse_wrapped_func(*args, **kwargs):
235+
# Apply langfuse observation directly here
236+
trace_name = f"{name_prefix}-{func.__name__}" if name_prefix else func.__name__
237+
observed_func = observe(name=trace_name)(func)
241238
return await observed_func(*args, **kwargs)
242-
243-
# Replace the async function to use Langfuse
244-
original_run_async = base_experiment.run_async
245-
246-
# Use the original run_async but with the Langfuse-wrapped function
247-
async def run_async_with_langfuse(
248-
dataset: Dataset, name: t.Optional[str] = None
249-
):
250-
# Override the internal wrapped_experiment with our Langfuse version
251-
base_experiment.__wrapped__ = wrapped_with_langfuse
252-
253-
# Call the original run_async which will now use our Langfuse-wrapped function
254-
return await original_run_async(dataset, name)
255-
256-
# Replace the run_async method
257-
base_experiment.__setattr__("run_async", run_async_with_langfuse)
258-
259-
return t.cast(ExperimentProtocol, base_experiment)
239+
240+
# Now create the experiment wrapper with our already-observed function
241+
experiment_wrapper = self.experiment(experiment_model, name_prefix)(langfuse_wrapped_func)
242+
243+
return t.cast(ExperimentProtocol, experiment_wrapper)
260244

261245
return decorator
262246

263-
# %% ../../nbs/project/experiments.ipynb 24
247+
# %% ../../nbs/project/experiments.ipynb 30
264248
import logging
265249
from ..utils import plot_experiments_as_subplots
266250

0 commit comments

Comments
 (0)