Skip to content

Commit 9c77367

Browse files
jjmachanjjmachan
committed
added support for experiment
1 parent 74f4de2 commit 9c77367

File tree

7 files changed

+288
-99
lines changed

7 files changed

+288
-99
lines changed

‎nbs/experiment.ipynb‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,11 @@
4747
" name: str,\n",
4848
" model: t.Type[BaseModel],\n",
4949
" project_id: str,\n",
50-
" dataset_id: str,\n",
50+
" experiment_id: str,\n",
5151
" ragas_api_client: RagasApiClient,\n",
5252
" ):\n",
53-
" super().__init__(name, model, project_id, dataset_id, ragas_api_client)\n",
53+
" self.experiment_id = experiment_id\n",
54+
" super().__init__(name, model, project_id, experiment_id, ragas_api_client)\n",
5455
"\n",
5556
" def __str__(self):\n",
5657
" return f\"Experiment(name={self.name}, model={self.model.__name__})\"\n",

‎nbs/project/core.ipynb‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -55,18 +55,11 @@
5555
"from ragas_annotator.backends.factory import RagasApiClientFactory\n",
5656
"from ragas_annotator.backends.ragas_api_client import RagasApiClient\n",
5757
"import ragas_annotator.typing as rt\n",
58-
"from ragas_annotator.utils import async_to_sync\n",
58+
"from ragas_annotator.utils import async_to_sync, create_nano_id\n",
5959
"from ragas_annotator.dataset import Dataset\n",
6060
"from ragas_annotator.experiment import Experiment"
6161
]
6262
},
63-
{
64-
"cell_type": "code",
65-
"execution_count": null,
66-
"metadata": {},
67-
"outputs": [],
68-
"source": []
69-
},
7063
{
7164
"cell_type": "code",
7265
"execution_count": null,
@@ -174,15 +167,7 @@
174167
"metadata": {},
175168
"outputs": [],
176169
"source": [
177-
"from ragas_annotator.backends.ragas_api_client import create_nano_id"
178-
]
179-
},
180-
{
181-
"cell_type": "code",
182-
"execution_count": null,
183-
"metadata": {},
184-
"outputs": [],
185-
"source": [
170+
"#| export\n",
186171
"async def create_dataset_columns(project_id, dataset_id, columns, create_dataset_column_func):\n",
187172
" tasks = []\n",
188173
" for column in columns:\n",

‎nbs/project/experiments.ipynb‎

Lines changed: 185 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,11 @@
3434
"from fastcore.utils import patch\n",
3535
"\n",
3636
"from ragas_annotator.project.core import Project\n",
37-
"from ragas_annotator.model.notion_model import NotionModel\n",
37+
"from ragas_annotator.model.pydantic_model import ExtendedPydanticBaseModel as BaseModel\n",
38+
"from ragas_annotator.utils import async_to_sync, create_nano_id\n",
39+
"from ragas_annotator.dataset import Dataset, BaseModelType\n",
3840
"from ragas_annotator.experiment import Experiment\n",
39-
"from ragas_annotator.dataset import Dataset"
41+
"import ragas_annotator.typing as rt"
4042
]
4143
},
4244
{
@@ -48,36 +50,125 @@
4850
"# | export\n",
4951
"@patch\n",
5052
"def create_experiment(\n",
51-
" self: Project, name: str, model: t.Type[NotionModel]\n",
53+
" self: Project, name: str, model: t.Type[BaseModel]\n",
5254
") -> Experiment:\n",
53-
" \"\"\"Create a new experiment view.\n",
55+
" \"\"\"Create a new experiment.\n",
5456
"\n",
5557
" Args:\n",
5658
" name: Name of the experiment\n",
5759
" model: Model class defining the experiment structure\n",
5860
"\n",
5961
" Returns:\n",
60-
" ExperimentView: View for managing experiment results\n",
62+
" Experiment: An experiment object for managing results\n",
6163
" \"\"\"\n",
62-
" if self.experiments_page_id == \"\":\n",
63-
" raise ValueError(\"Experiments page ID is not set\")\n",
64-
"\n",
65-
" # Collect all properties from model fields\n",
66-
" properties = {}\n",
67-
" for field_name, field in model._fields.items():\n",
68-
" properties.update(field._to_notion_property())\n",
69-
"\n",
70-
" # Create the database\n",
71-
" database_id = self._notion_backend.create_new_database(\n",
72-
" parent_page_id=self.experiments_page_id, title=name, properties=properties\n",
64+
" # Create the experiment\n",
65+
" sync_version = async_to_sync(self._ragas_api_client.create_experiment)\n",
66+
" experiment_info = sync_version(\n",
67+
" project_id=self.project_id,\n",
68+
" name=name,\n",
7369
" )\n",
7470
"\n",
71+
" # Create the columns for the experiment\n",
72+
" column_types = rt.ModelConverter.model_to_columns(model)\n",
73+
" sync_version = async_to_sync(create_experiment_columns)\n",
74+
" sync_version(\n",
75+
" project_id=self.project_id,\n",
76+
" experiment_id=experiment_info[\"id\"],\n",
77+
" columns=column_types,\n",
78+
" create_experiment_column_func=self._ragas_api_client.create_experiment_column,\n",
79+
" )\n",
80+
" \n",
81+
" # Return a new Experiment instance\n",
7582
" return Experiment(\n",
7683
" name=name,\n",
7784
" model=model,\n",
78-
" database_id=database_id,\n",
79-
" notion_backend=self._notion_backend,\n",
80-
" )"
85+
" project_id=self.project_id,\n",
86+
" experiment_id=experiment_info[\"id\"],\n",
87+
" ragas_api_client=self._ragas_api_client,\n",
88+
" )\n",
89+
"\n",
90+
"# Add this helper function similar to create_dataset_columns in core.ipynb\n",
91+
"async def create_experiment_columns(project_id, experiment_id, columns, create_experiment_column_func):\n",
92+
" tasks = []\n",
93+
" for column in columns:\n",
94+
" tasks.append(create_experiment_column_func(\n",
95+
" project_id=project_id,\n",
96+
" experiment_id=experiment_id,\n",
97+
" id=create_nano_id(),\n",
98+
" name=column[\"name\"],\n",
99+
" type=column[\"type\"],\n",
100+
" settings={\n",
101+
" \"max_length\": 255,\n",
102+
" \"is_required\": True,\n",
103+
" },\n",
104+
" ))\n",
105+
" return await asyncio.gather(*tasks)"
106+
]
107+
},
108+
{
109+
"cell_type": "code",
110+
"execution_count": null,
111+
"metadata": {},
112+
"outputs": [
113+
{
114+
"data": {
115+
"text/plain": [
116+
"Project(name='SuperMe')"
117+
]
118+
},
119+
"execution_count": null,
120+
"metadata": {},
121+
"output_type": "execute_result"
122+
}
123+
],
124+
"source": [
125+
"import os\n",
126+
"\n",
127+
"RAGAS_APP_TOKEN = \"apt.47bd-c55e4a45b27c-02f8-8446-1441f09b-651a8\"\n",
128+
"RAGAS_API_BASE_URL = \"https://api.dev.app.ragas.io\"\n",
129+
"\n",
130+
"os.environ[\"RAGAS_APP_TOKEN\"] = RAGAS_APP_TOKEN\n",
131+
"os.environ[\"RAGAS_API_BASE_URL\"] = RAGAS_API_BASE_URL\n",
132+
"\n",
133+
"PROJECT_ID = \"a6ccabe0-7b8d-4866-98af-f167a36b94ff\"\n",
134+
"p = Project(project_id=PROJECT_ID)\n",
135+
"p"
136+
]
137+
},
138+
{
139+
"cell_type": "code",
140+
"execution_count": null,
141+
"metadata": {},
142+
"outputs": [],
143+
"source": [
144+
"class TestModel(BaseModel):\n",
145+
" name: str\n",
146+
" description: str\n",
147+
" price: float\n"
148+
]
149+
},
150+
{
151+
"cell_type": "code",
152+
"execution_count": null,
153+
"metadata": {},
154+
"outputs": [
155+
{
156+
"data": {
157+
"text/plain": [
158+
"Experiment(name=just name, desc, price 2, model=TestModel)"
159+
]
160+
},
161+
"execution_count": null,
162+
"metadata": {},
163+
"output_type": "execute_result"
164+
}
165+
],
166+
"source": [
167+
"experiment_id = \"5d7752ab-17bf-46bc-a302-afe04ce1a763\"\n",
168+
"exp = p.create_experiment(name=\"just name, desc, price 2\", model=TestModel)\n",
169+
"#exp = p.create_dataset(name=\"just name and desc 2\", model=TestModel)\n",
170+
"\n",
171+
"exp"
81172
]
82173
},
83174
{
@@ -88,24 +179,84 @@
88179
"source": [
89180
"# | export\n",
90181
"@patch\n",
91-
"def get_experiment(self: Project, name: str, model: t.Type[NotionModel]) -> Experiment:\n",
92-
" \"\"\"Get an existing experiment by name.\"\"\"\n",
93-
" if self.experiments_page_id == \"\":\n",
94-
" raise ValueError(\"Experiments page ID is not set\")\n",
95-
"\n",
96-
" # Search for database with given name\n",
97-
" database_id = self._notion_backend.get_database_id(\n",
98-
" parent_page_id=self.experiments_page_id, name=name, return_multiple=False\n",
182+
"def get_experiment(self: Project, experiment_id: str, model: t.Type[BaseModel]) -> Experiment:\n",
183+
" \"\"\"Get an existing experiment by ID.\"\"\"\n",
184+
" # Get experiment info\n",
185+
" sync_version = async_to_sync(self._ragas_api_client.get_experiment)\n",
186+
" experiment_info = sync_version(\n",
187+
" project_id=self.project_id,\n",
188+
" experiment_id=experiment_id\n",
99189
" )\n",
100190
"\n",
101191
" return Experiment(\n",
102-
" name=name,\n",
192+
" name=experiment_info[\"name\"],\n",
103193
" model=model,\n",
104-
" database_id=database_id,\n",
105-
" notion_backend=self._notion_backend,\n",
194+
" project_id=self.project_id,\n",
195+
" experiment_id=experiment_id,\n",
196+
" ragas_api_client=self._ragas_api_client,\n",
106197
" )"
107198
]
108199
},
200+
{
201+
"cell_type": "code",
202+
"execution_count": null,
203+
"metadata": {},
204+
"outputs": [
205+
{
206+
"data": {
207+
"text/plain": [
208+
"'22bbb40c-1fc0-4a09-b26a-ccc93c8bd595'"
209+
]
210+
},
211+
"execution_count": null,
212+
"metadata": {},
213+
"output_type": "execute_result"
214+
}
215+
],
216+
"source": [
217+
"exp.dataset_id"
218+
]
219+
},
220+
{
221+
"cell_type": "code",
222+
"execution_count": null,
223+
"metadata": {},
224+
"outputs": [
225+
{
226+
"data": {
227+
"text/plain": [
228+
"Project(name='SuperMe')"
229+
]
230+
},
231+
"execution_count": null,
232+
"metadata": {},
233+
"output_type": "execute_result"
234+
}
235+
],
236+
"source": [
237+
"p"
238+
]
239+
},
240+
{
241+
"cell_type": "code",
242+
"execution_count": null,
243+
"metadata": {},
244+
"outputs": [
245+
{
246+
"data": {
247+
"text/plain": [
248+
"Experiment(name=just name, desc, price 2, model=TestModel)"
249+
]
250+
},
251+
"execution_count": null,
252+
"metadata": {},
253+
"output_type": "execute_result"
254+
}
255+
],
256+
"source": [
257+
"p.get_experiment(exp.dataset_id, TestModel)"
258+
]
259+
},
109260
{
110261
"cell_type": "code",
111262
"execution_count": null,
@@ -160,7 +311,7 @@
160311
"# | export\n",
161312
"@patch\n",
162313
"def experiment(\n",
163-
" self: Project, experiment_model: t.Type[NotionModel], name_prefix: str = \"\"\n",
314+
" self: Project, experiment_model, name_prefix: str = \"\"\n",
164315
"):\n",
165316
" \"\"\"Decorator for creating experiment functions without Langfuse integration.\n",
166317
"\n",
@@ -183,6 +334,8 @@
183334
" # if name is not provided, generate a memorable name\n",
184335
" if name is None:\n",
185336
" name = memorable_names.generate_unique_name()\n",
337+
" if name_prefix:\n",
338+
" name = f\"{name_prefix}-{name}\"\n",
186339
"\n",
187340
" # Create tasks for all items\n",
188341
" tasks = []\n",
@@ -219,7 +372,7 @@
219372
"# | export\n",
220373
"@patch\n",
221374
"def langfuse_experiment(\n",
222-
" self: Project, experiment_model: t.Type[NotionModel], name_prefix: str = \"\"\n",
375+
" self: Project, experiment_model, name_prefix: str = \"\"\n",
223376
"):\n",
224377
" \"\"\"Decorator for creating experiment functions with Langfuse integration.\n",
225378
"\n",

‎ragas_annotator/_modidx.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -558,7 +558,9 @@
558558
'ragas_annotator.project.core.Project.delete': ( 'project/core.html#project.delete',
559559
'ragas_annotator/project/core.py'),
560560
'ragas_annotator.project.core.Project.get_dataset': ( 'project/core.html#project.get_dataset',
561-
'ragas_annotator/project/core.py')},
561+
'ragas_annotator/project/core.py'),
562+
'ragas_annotator.project.core.create_dataset_columns': ( 'project/core.html#create_dataset_columns',
563+
'ragas_annotator/project/core.py')},
562564
'ragas_annotator.project.experiments': { 'ragas_annotator.project.experiments.ExperimentProtocol': ( 'project/experiments.html#experimentprotocol',
563565
'ragas_annotator/project/experiments.py'),
564566
'ragas_annotator.project.experiments.ExperimentProtocol.__call__': ( 'project/experiments.html#experimentprotocol.__call__',
@@ -572,7 +574,9 @@
572574
'ragas_annotator.project.experiments.Project.get_experiment': ( 'project/experiments.html#project.get_experiment',
573575
'ragas_annotator/project/experiments.py'),
574576
'ragas_annotator.project.experiments.Project.langfuse_experiment': ( 'project/experiments.html#project.langfuse_experiment',
575-
'ragas_annotator/project/experiments.py')},
577+
'ragas_annotator/project/experiments.py'),
578+
'ragas_annotator.project.experiments.create_experiment_columns': ( 'project/experiments.html#create_experiment_columns',
579+
'ragas_annotator/project/experiments.py')},
576580
'ragas_annotator.project.naming': { 'ragas_annotator.project.naming.MemorableNames': ( 'project/naming.html#memorablenames',
577581
'ragas_annotator/project/naming.py'),
578582
'ragas_annotator.project.naming.MemorableNames.__init__': ( 'project/naming.html#memorablenames.__init__',

‎ragas_annotator/experiment.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,11 @@ def __init__(
2121
name: str,
2222
model: t.Type[BaseModel],
2323
project_id: str,
24-
dataset_id: str,
24+
experiment_id: str,
2525
ragas_api_client: RagasApiClient,
2626
):
27-
super().__init__(name, model, project_id, dataset_id, ragas_api_client)
27+
self.experiment_id = experiment_id
28+
super().__init__(name, model, project_id, experiment_id, ragas_api_client)
2829

2930
def __str__(self):
3031
return f"Experiment(name={self.name}, model={self.model.__name__})"

0 commit comments

Comments
 (0)