Skip to content

Commit 6ecbe84

Browse files
jjmachanjjmachan
committed
added experiment without langfuse
1 parent c14f806 commit 6ecbe84

File tree

5 files changed

+262
-22
lines changed

5 files changed

+262
-22
lines changed

‎README.md‎

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,64 @@ $ pip install ragas_annotator
2222

2323
## Getting Started
2424

25-
Fill me in please! Don’t forget code examples:
25+
First lets init a
26+
[`Project`](https://explodinggradients.github.io/ragas_annotator/project/core.html#project)
27+
from notion.
2628

2729
``` python
28-
1 + 1
30+
from ragas_annotator import Project
31+
32+
project = Project(
33+
name="Ragas Dashboard",
34+
notion_root_page_id="1b35d9bf94ff801792bfd1824fac0c96"
35+
)
36+
project
37+
```
38+
39+
Project(name='Ragas Dashboard', root_page_id=1b35d9bf94ff801792bfd1824fac0c96)
40+
41+
NOTE: ideally this should be optional - just send a dict
42+
43+
``` python
44+
from ragas_annotator import NotionModel, nmt
45+
46+
# define the model
47+
class RAGDataset(NotionModel):
48+
id: int = nmt.ID()
49+
query: str = nmt.Title()
50+
ground_truth: str = nmt.Text()
2951
```
3052

31-
2
53+
Lets check the datasets it has
54+
55+
``` python
56+
dataset = project.get_dataset("RAG Dataset", RAGDataset)
57+
dataset
58+
```
59+
60+
Dataset(name=RAG Dataset, model=RAGDataset, len=0)
61+
62+
``` python
63+
def example_llm_app(query: str):
64+
return "This is a test"
65+
66+
67+
68+
@project.experiment(RAGDataset)
69+
def random_experiment(row):
70+
print(row)
71+
return row
72+
```
73+
74+
AttributeError: 'Project' object has no attribute 'experiment'
75+
---------------------------------------------------------------------------
76+
AttributeError Traceback (most recent call last)
77+
Cell In[5], line 6
78+
 1 def example_llm_app(query: str):
79+
 2 return "This is a test"
80+
----> 6 @project.experiment(RAGDataset)
81+
 7 def random_experiment(row):
82+
 8 print(row)
83+
 9 return row
84+
85+
AttributeError: 'Project' object has no attribute 'experiment'

‎nbs/index.ipynb‎

Lines changed: 93 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
"cell_type": "markdown",
5858
"metadata": {},
5959
"source": [
60-
"Fill me in please! Don't forget code examples:"
60+
"First lets init a `Project` from notion."
6161
]
6262
},
6363
{
@@ -68,7 +68,7 @@
6868
{
6969
"data": {
7070
"text/plain": [
71-
"2"
71+
"Project(name='Ragas Dashboard', root_page_id=1b35d9bf94ff801792bfd1824fac0c96)"
7272
]
7373
},
7474
"execution_count": null,
@@ -77,7 +77,97 @@
7777
}
7878
],
7979
"source": [
80-
"1 + 1"
80+
"from ragas_annotator import Project\n",
81+
"\n",
82+
"project = Project(\n",
83+
" name=\"Ragas Dashboard\",\n",
84+
" notion_root_page_id=\"1b35d9bf94ff801792bfd1824fac0c96\"\n",
85+
")\n",
86+
"project"
87+
]
88+
},
89+
{
90+
"cell_type": "markdown",
91+
"metadata": {},
92+
"source": [
93+
"NOTE: ideally this should be optional - just send a dict"
94+
]
95+
},
96+
{
97+
"cell_type": "code",
98+
"execution_count": null,
99+
"metadata": {},
100+
"outputs": [],
101+
"source": [
102+
"from ragas_annotator import NotionModel, nmt\n",
103+
"\n",
104+
"# define the model\n",
105+
"class RAGDataset(NotionModel):\n",
106+
" id: int = nmt.ID()\n",
107+
" query: str = nmt.Title()\n",
108+
" ground_truth: str = nmt.Text()"
109+
]
110+
},
111+
{
112+
"cell_type": "markdown",
113+
"metadata": {},
114+
"source": [
115+
"Lets check the datasets it has"
116+
]
117+
},
118+
{
119+
"cell_type": "code",
120+
"execution_count": null,
121+
"metadata": {},
122+
"outputs": [
123+
{
124+
"data": {
125+
"text/plain": [
126+
"Dataset(name=RAG Dataset, model=RAGDataset, len=0)"
127+
]
128+
},
129+
"execution_count": null,
130+
"metadata": {},
131+
"output_type": "execute_result"
132+
}
133+
],
134+
"source": [
135+
"dataset = project.get_dataset(\"RAG Dataset\", RAGDataset)\n",
136+
"dataset\n"
137+
]
138+
},
139+
{
140+
"cell_type": "code",
141+
"execution_count": null,
142+
"metadata": {},
143+
"outputs": [
144+
{
145+
"ename": "AttributeError",
146+
"evalue": "'Project' object has no attribute 'experiment'",
147+
"output_type": "error",
148+
"traceback": [
149+
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
150+
"\u001b[31mAttributeError\u001b[39m Traceback (most recent call last)",
151+
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[5]\u001b[39m\u001b[32m, line 6\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mexample_llm_app\u001b[39m(query: \u001b[38;5;28mstr\u001b[39m):\n\u001b[32m 2\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mThis is a test\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m \u001b[38;5;129m@project\u001b[39m\u001b[43m.\u001b[49m\u001b[43mexperiment\u001b[49m(RAGDataset)\n\u001b[32m 7\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mrandom_experiment\u001b[39m(row):\n\u001b[32m 8\u001b[39m \u001b[38;5;28mprint\u001b[39m(row)\n\u001b[32m 9\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m row\n",
152+
"\u001b[31mAttributeError\u001b[39m: 'Project' object has no attribute 'experiment'"
153+
]
154+
}
155+
],
156+
"source": [
157+
"def example_llm_app(query: str):\n",
158+
" return \"This is a test\"\n",
159+
"\n",
160+
"\n",
161+
"\n",
162+
"@project.experiment(RAGDataset)\n",
163+
"def random_experiment(row):\n",
164+
" print(row)\n",
165+
" return row\n",
166+
"\n",
167+
"\n",
168+
"\n",
169+
"\n",
170+
"\n"
81171
]
82172
},
83173
{

‎nbs/project/experiments.ipynb‎

Lines changed: 58 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -159,12 +159,13 @@
159159
"source": [
160160
"# | export\n",
161161
"@patch\n",
162-
"def langfuse_experiment(\n",
162+
"def experiment(\n",
163163
" self: Project, experiment_model: t.Type[NotionModel], name_prefix: str = \"\"\n",
164164
"):\n",
165-
" \"\"\"Decorator for creating experiment functions.\n",
165+
" \"\"\"Decorator for creating experiment functions without Langfuse integration.\n",
166166
"\n",
167167
" Args:\n",
168+
" experiment_model: The NotionModel type to use for experiment results\n",
168169
" name_prefix: Optional prefix for experiment names\n",
169170
"\n",
170171
" Returns:\n",
@@ -174,11 +175,8 @@
174175
" def decorator(func: t.Callable) -> ExperimentProtocol:\n",
175176
" @wraps(func)\n",
176177
" async def wrapped_experiment(*args, **kwargs):\n",
177-
" # wrap the function with langfuse observation so that it can be traced\n",
178-
" # and spans inside the function can be retrieved with sync_trace()\n",
179-
" observed_func = observe(name=f\"{name_prefix}-{func.__name__}\")(func)\n",
180-
"\n",
181-
" return await observed_func(*args, **kwargs)\n",
178+
" # Simply call the function without Langfuse observation\n",
179+
" return await func(*args, **kwargs)\n",
182180
"\n",
183181
" # Add run method to the wrapped function\n",
184182
" async def run_async(dataset: Dataset, name: t.Optional[str] = None):\n",
@@ -196,7 +194,8 @@
196194
" for future in tqdm(asyncio.as_completed(tasks), total=len(tasks)):\n",
197195
" result = await future\n",
198196
" # Add each result to experiment view as it completes\n",
199-
" results.append(result) if result is not None else None\n",
197+
" if result is not None:\n",
198+
" results.append(result)\n",
200199
"\n",
201200
" # upload results to experiment view\n",
202201
" experiment_view = self.create_experiment(name=name, model=experiment_model)\n",
@@ -208,6 +207,57 @@
208207
" wrapped_experiment.__setattr__(\"run_async\", run_async)\n",
209208
" return t.cast(ExperimentProtocol, wrapped_experiment)\n",
210209
"\n",
210+
" return decorator\n"
211+
]
212+
},
213+
{
214+
"cell_type": "code",
215+
"execution_count": null,
216+
"metadata": {},
217+
"outputs": [],
218+
"source": [
219+
"# | export\n",
220+
"@patch\n",
221+
"def langfuse_experiment(\n",
222+
" self: Project, experiment_model: t.Type[NotionModel], name_prefix: str = \"\"\n",
223+
"):\n",
224+
" \"\"\"Decorator for creating experiment functions with Langfuse integration.\n",
225+
"\n",
226+
" Args:\n",
227+
" experiment_model: The NotionModel type to use for experiment results\n",
228+
" name_prefix: Optional prefix for experiment names\n",
229+
"\n",
230+
" Returns:\n",
231+
" Decorator function that wraps experiment functions with Langfuse observation\n",
232+
" \"\"\"\n",
233+
"\n",
234+
" def decorator(func: t.Callable) -> ExperimentProtocol:\n",
235+
" # First, create a base experiment wrapper\n",
236+
" base_experiment = self.experiment(experiment_model, name_prefix)(func)\n",
237+
" \n",
238+
" # Override the wrapped function to add Langfuse observation\n",
239+
" @wraps(func)\n",
240+
" async def wrapped_with_langfuse(*args, **kwargs):\n",
241+
" # wrap the function with langfuse observation\n",
242+
" observed_func = observe(name=f\"{name_prefix}-{func.__name__}\")(func)\n",
243+
" return await observed_func(*args, **kwargs)\n",
244+
" \n",
245+
" # Replace the async function to use Langfuse\n",
246+
" original_run_async = base_experiment.run_async\n",
247+
" \n",
248+
" # Use the original run_async but with the Langfuse-wrapped function\n",
249+
" async def run_async_with_langfuse(dataset: Dataset, name: t.Optional[str] = None):\n",
250+
" # Override the internal wrapped_experiment with our Langfuse version\n",
251+
" base_experiment.__wrapped__ = wrapped_with_langfuse\n",
252+
" \n",
253+
" # Call the original run_async which will now use our Langfuse-wrapped function\n",
254+
" return await original_run_async(dataset, name)\n",
255+
" \n",
256+
" # Replace the run_async method\n",
257+
" base_experiment.__setattr__(\"run_async\", run_async_with_langfuse)\n",
258+
" \n",
259+
" return t.cast(ExperimentProtocol, base_experiment)\n",
260+
"\n",
211261
" return decorator"
212262
]
213263
},

‎ragas_annotator/_modidx.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,8 @@
376376
'ragas_annotator/project/experiments.py'),
377377
'ragas_annotator.project.experiments.Project.create_experiment': ( 'project/experiments.html#project.create_experiment',
378378
'ragas_annotator/project/experiments.py'),
379+
'ragas_annotator.project.experiments.Project.experiment': ( 'project/experiments.html#project.experiment',
380+
'ragas_annotator/project/experiments.py'),
379381
'ragas_annotator.project.experiments.Project.get_experiment': ( 'project/experiments.html#project.get_experiment',
380382
'ragas_annotator/project/experiments.py'),
381383
'ragas_annotator.project.experiments.Project.langfuse_experiment': ( 'project/experiments.html#project.langfuse_experiment',

‎ragas_annotator/project/experiments.py‎

Lines changed: 52 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -90,12 +90,13 @@ async def run_async(self, name: str, dataset: Dataset): ...
9090

9191
# %% ../../nbs/project/experiments.ipynb 9
9292
@patch
93-
def langfuse_experiment(
93+
def experiment(
9494
self: Project, experiment_model: t.Type[NotionModel], name_prefix: str = ""
9595
):
96-
"""Decorator for creating experiment functions.
96+
"""Decorator for creating experiment functions without Langfuse integration.
9797
9898
Args:
99+
experiment_model: The NotionModel type to use for experiment results
99100
name_prefix: Optional prefix for experiment names
100101
101102
Returns:
@@ -105,11 +106,8 @@ def langfuse_experiment(
105106
def decorator(func: t.Callable) -> ExperimentProtocol:
106107
@wraps(func)
107108
async def wrapped_experiment(*args, **kwargs):
108-
# wrap the function with langfuse observation so that it can be traced
109-
# and spans inside the function can be retrieved with sync_trace()
110-
observed_func = observe(name=f"{name_prefix}-{func.__name__}")(func)
111-
112-
return await observed_func(*args, **kwargs)
109+
# Simply call the function without Langfuse observation
110+
return await func(*args, **kwargs)
113111

114112
# Add run method to the wrapped function
115113
async def run_async(dataset: Dataset, name: t.Optional[str] = None):
@@ -127,7 +125,8 @@ async def run_async(dataset: Dataset, name: t.Optional[str] = None):
127125
for future in tqdm(asyncio.as_completed(tasks), total=len(tasks)):
128126
result = await future
129127
# Add each result to experiment view as it completes
130-
results.append(result) if result is not None else None
128+
if result is not None:
129+
results.append(result)
131130

132131
# upload results to experiment view
133132
experiment_view = self.create_experiment(name=name, model=experiment_model)
@@ -140,3 +139,48 @@ async def run_async(dataset: Dataset, name: t.Optional[str] = None):
140139
return t.cast(ExperimentProtocol, wrapped_experiment)
141140

142141
return decorator
142+
143+
144+
# %% ../../nbs/project/experiments.ipynb 10
145+
@patch
146+
def langfuse_experiment(
147+
self: Project, experiment_model: t.Type[NotionModel], name_prefix: str = ""
148+
):
149+
"""Decorator for creating experiment functions with Langfuse integration.
150+
151+
Args:
152+
experiment_model: The NotionModel type to use for experiment results
153+
name_prefix: Optional prefix for experiment names
154+
155+
Returns:
156+
Decorator function that wraps experiment functions with Langfuse observation
157+
"""
158+
159+
def decorator(func: t.Callable) -> ExperimentProtocol:
160+
# First, create a base experiment wrapper
161+
base_experiment = self.experiment(experiment_model, name_prefix)(func)
162+
163+
# Override the wrapped function to add Langfuse observation
164+
@wraps(func)
165+
async def wrapped_with_langfuse(*args, **kwargs):
166+
# wrap the function with langfuse observation
167+
observed_func = observe(name=f"{name_prefix}-{func.__name__}")(func)
168+
return await observed_func(*args, **kwargs)
169+
170+
# Replace the async function to use Langfuse
171+
original_run_async = base_experiment.run_async
172+
173+
# Use the original run_async but with the Langfuse-wrapped function
174+
async def run_async_with_langfuse(dataset: Dataset, name: t.Optional[str] = None):
175+
# Override the internal wrapped_experiment with our Langfuse version
176+
base_experiment.__wrapped__ = wrapped_with_langfuse
177+
178+
# Call the original run_async which will now use our Langfuse-wrapped function
179+
return await original_run_async(dataset, name)
180+
181+
# Replace the run_async method
182+
base_experiment.__setattr__("run_async", run_async_with_langfuse)
183+
184+
return t.cast(ExperimentProtocol, base_experiment)
185+
186+
return decorator

0 commit comments

Comments
 (0)