diff --git a/nbs/backends/ragas_api_client.ipynb b/nbs/backends/ragas_api_client.ipynb
index 33e9f9f..0834e13 100644
--- a/nbs/backends/ragas_api_client.ipynb
+++ b/nbs/backends/ragas_api_client.ipynb
@@ -704,25 +704,7 @@
"outputs": [],
"source": [
"#| export\n",
- "from enum import Enum"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| export\n",
- "class ColumnType(str, Enum):\n",
- " NUMBER = \"number\"\n",
- " TEXT = \"text\"\n",
- " LONG_TEXT = \"longText\"\n",
- " SELECT = \"select\"\n",
- " DATE = \"date\"\n",
- " MULTI_SELECT = \"multiSelect\"\n",
- " CHECKBOX = \"checkbox\"\n",
- " CUSTOM = \"custom\""
+ "from ragas_experimental.typing import ColumnType"
]
},
{
diff --git a/nbs/dataset.ipynb b/nbs/dataset.ipynb
index 9ad3005..7eadb5d 100644
--- a/nbs/dataset.ipynb
+++ b/nbs/dataset.ipynb
@@ -149,7 +149,8 @@
"source": [
"# | hide\n",
"import ragas_experimental.typing as rt\n",
- "from ragas_experimental.backends.factory import RagasApiClientFactory"
+ "from ragas_experimental.backends.factory import RagasApiClientFactory\n",
+ "from ragas_experimental.metric.result import MetricResult"
]
},
{
@@ -162,27 +163,17 @@
"class TestModel(BaseModel):\n",
" id: int\n",
" name: str\n",
- " description: str"
+ " description: str\n",
+ " result: MetricResult"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "{'id': 'id', 'name': 'name', 'description': 'description'}"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
- "TestModel.__column_mapping__"
+ "TestModel.__column_mapping__ = {}"
]
},
{
@@ -193,7 +184,7 @@
{
"data": {
"text/plain": [
- "TestModel(id=0, name='test', description='test description')"
+ "TestModel(id=0, name='test', description='test description', result=0.5)"
]
},
"execution_count": null,
@@ -202,7 +193,7 @@
}
],
"source": [
- "test_model = TestModel(id=0, name=\"test\", description=\"test description\")\n",
+ "test_model = TestModel(id=0, name=\"test\", description=\"test description\", result=MetricResult(result=0.5, reason=\"test reason\"))\n",
"test_model"
]
},
@@ -214,7 +205,11 @@
{
"data": {
"text/plain": [
- "{'id': 'id', 'name': 'name', 'description': 'description'}"
+ "{'id': 'id',\n",
+ " 'name': 'name',\n",
+ " 'description': 'description',\n",
+ " 'result': 'result',\n",
+ " 'result_reason': 'result_reason'}"
]
},
"execution_count": null,
@@ -232,7 +227,8 @@
"metadata": {},
"outputs": [],
"source": [
- "import os"
+ "import os\n",
+ "from ragas_experimental import Project"
]
},
{
@@ -265,13 +261,35 @@
{
"data": {
"text/plain": [
- "{'id': '0fee5330-9f6e-44a9-a85c-e3b947b697de',\n",
- " 'name': 'TestModel',\n",
+ "Dataset(name=TestModel_with_long_text, model=TestModel, len=0)"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "p = Project(project_id=\"1ef0843b-231f-4a2c-b64d-d39bcee9d830\", ragas_app_client=ragas_api_client)\n",
+ "test_dataset = p.create_dataset(name=\"TestModel_with_long_text\", model=TestModel)\n",
+ "test_dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'id': 'a7bd8b1f-dc29-4ae0-b1e5-24f0dfb76f7a',\n",
+ " 'name': 'TestModel_with_long_text',\n",
" 'description': None,\n",
- " 'created_at': '2025-04-10T22:41:36.582714+00:00',\n",
- " 'updated_at': '2025-04-10T22:41:36.582714+00:00',\n",
+ " 'created_at': '2025-04-16T00:02:58.149112+00:00',\n",
+ " 'updated_at': '2025-04-16T00:02:58.149112+00:00',\n",
" 'version_counter': 0,\n",
- " 'project_id': 'bbe45632-3268-43a6-9694-b020b3f5226f'}"
+ " 'project_id': '1ef0843b-231f-4a2c-b64d-d39bcee9d830'}"
]
},
"execution_count": null,
@@ -281,13 +299,38 @@
],
"source": [
"# https://dev.app.ragas.io/dashboard/projects/0a7c4ecb-b313-4bb0-81c0-852c9634ce03/datasets/a4f0d169-ebce-4a2b-b758-0ff49c0c4312\n",
- "TEST_PROJECT_ID = \"bbe45632-3268-43a6-9694-b020b3f5226f\"\n",
- "TEST_DATASET_ID = \"0fee5330-9f6e-44a9-a85c-e3b947b697de\"\n",
+ "TEST_PROJECT_ID = p.project_id\n",
+ "TEST_DATASET_ID = test_dataset.dataset_id\n",
"test_project = await ragas_api_client.get_project(project_id=TEST_PROJECT_ID)\n",
"test_dataset = await ragas_api_client.get_dataset(project_id=TEST_PROJECT_ID, dataset_id=TEST_DATASET_ID)\n",
"test_dataset"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'items': [],\n",
+ " 'pagination': {'offset': 0,\n",
+ " 'limit': 50,\n",
+ " 'total': 0,\n",
+ " 'order_by': 'created_at',\n",
+ " 'sort_dir': 'asc'}}"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "await ragas_api_client.list_dataset_rows(project_id=TEST_PROJECT_ID, dataset_id=TEST_DATASET_ID)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -307,7 +350,11 @@
{
"data": {
"text/plain": [
- "{'id': 'fSh3ESKGNZLf', 'description': 'ftKwfDqnbinL', 'name': 'fVnsYEnvxARh'}"
+ "{'id': 'eCYjJtBbLSDL',\n",
+ " 'name': '1sfEb8iJ1TUg',\n",
+ " 'description': 'gYsqmPXHbOyy',\n",
+ " 'result': 'cAZanmY0CwWT',\n",
+ " 'result_reason': 'e7IgZXICkmhA'}"
]
},
"execution_count": null,
@@ -319,6 +366,16 @@
"dataset.model.__column_mapping__"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "import ragas_experimental.typing as rt"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -335,12 +392,12 @@
" column_id_map = self.model.__column_mapping__\n",
"\n",
" # create the rows\n",
- " row_dict = entry.model_dump()\n",
+ " row_dict_converted = rt.ModelConverter.instance_to_row(entry)\n",
" row_id = create_nano_id()\n",
" row_data = {}\n",
- " for key, value in row_dict.items():\n",
- " if key in column_id_map:\n",
- " row_data[column_id_map[key]] = value\n",
+ " for column in row_dict_converted[\"data\"]:\n",
+ " if column[\"column_id\"] in column_id_map:\n",
+ " row_data[column_id_map[column[\"column_id\"]]] = column[\"data\"]\n",
"\n",
" sync_func = async_to_sync(self._ragas_api_client.create_dataset_row)\n",
" response = sync_func(\n",
@@ -363,7 +420,7 @@
{
"data": {
"text/plain": [
- "3"
+ "1"
]
},
"execution_count": null,
@@ -418,7 +475,7 @@
{
"data": {
"text/plain": [
- "1"
+ "0"
]
},
"execution_count": null,
@@ -441,6 +498,29 @@
"test_eq(len(dataset), 0)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "12"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# now add some more entries\n",
+ "for i in range(10):\n",
+ " dataset.append(test_model)\n",
+ "len(dataset)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -534,8 +614,66 @@
{
"data": {
"text/plain": [
- "[{'id': 0, 'name': 'test', 'description': 'test description'},\n",
- " {'id': 0, 'name': 'test', 'description': 'test description'}]"
+ "[{'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0}]"
]
},
"execution_count": null,
@@ -597,6 +735,7 @@
" | id | \n",
" name | \n",
" description | \n",
+ " result | \n",
" \n",
" \n",
" \n",
@@ -605,21 +744,103 @@
" 0 | \n",
" test | \n",
" test description | \n",
+ " 0.5 | \n",
" \n",
" \n",
" | 1 | \n",
" 0 | \n",
" test | \n",
" test description | \n",
+ " 0.5 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ " 0.5 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ " 0.5 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ " 0.5 | \n",
+ "
\n",
+ " \n",
+ " | 5 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ " 0.5 | \n",
+ "
\n",
+ " \n",
+ " | 6 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ " 0.5 | \n",
+ "
\n",
+ " \n",
+ " | 7 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ " 0.5 | \n",
+ "
\n",
+ " \n",
+ " | 8 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ " 0.5 | \n",
+ "
\n",
+ " \n",
+ " | 9 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ " 0.5 | \n",
+ "
\n",
+ " \n",
+ " | 10 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ " 0.5 | \n",
+ "
\n",
+ " \n",
+ " | 11 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ " 0.5 | \n",
"
\n",
" \n",
"\n",
""
],
"text/plain": [
- " id name description\n",
- "0 0 test test description\n",
- "1 0 test test description"
+ " id name description result\n",
+ "0 0 test test description 0.5\n",
+ "1 0 test test description 0.5\n",
+ "2 0 test test description 0.5\n",
+ "3 0 test test description 0.5\n",
+ "4 0 test test description 0.5\n",
+ "5 0 test test description 0.5\n",
+ "6 0 test test description 0.5\n",
+ "7 0 test test description 0.5\n",
+ "8 0 test test description 0.5\n",
+ "9 0 test test description 0.5\n",
+ "10 0 test test description 0.5\n",
+ "11 0 test test description 0.5"
]
},
"execution_count": null,
@@ -661,12 +882,12 @@
" \n",
" # Get column mapping and prepare data\n",
" column_id_map = self.model.__column_mapping__\n",
- " row_dict = item.model_dump()\n",
+ " row_dict = rt.ModelConverter.instance_to_row(item)[\"data\"]\n",
" row_data = {}\n",
" \n",
- " for key, value in row_dict.items():\n",
- " if key in column_id_map:\n",
- " row_data[column_id_map[key]] = value\n",
+ " for column in row_dict:\n",
+ " if column[\"column_id\"] in column_id_map:\n",
+ " row_data[column_id_map[column[\"column_id\"]]] = column[\"data\"]\n",
" \n",
" # Update in backend\n",
" sync_func = async_to_sync(self._ragas_api_client.update_dataset_row)\n",
@@ -694,7 +915,7 @@
{
"data": {
"text/plain": [
- "TestModel(id=0, name='test', description='test description')"
+ "TestModel(id=0, name='updated name', description='test description', result=0.5)"
]
},
"execution_count": null,
@@ -737,8 +958,66 @@
{
"data": {
"text/plain": [
- "[{'id': 0, 'name': 'updated name', 'description': 'test description'},\n",
- " {'id': 0, 'name': 'test', 'description': 'test description'}]"
+ "[{'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'test',\n",
+ " 'id': 0},\n",
+ " {'description': 'test description',\n",
+ " 'result': 0.5,\n",
+ " 'result_reason': 'test reason',\n",
+ " 'name': 'updated name',\n",
+ " 'id': 0}]"
]
},
"execution_count": null,
@@ -808,7 +1087,7 @@
{
"data": {
"text/plain": [
- "'eWlNKu6F4jIX'"
+ "'eVpgxsmPGwa8'"
]
},
"execution_count": null,
@@ -828,7 +1107,7 @@
{
"data": {
"text/plain": [
- "TestModel(id=0, name='updated name', description='test description')"
+ "TestModel(id=0, name='updated name', description='test description', result=0.5)"
]
},
"execution_count": null,
diff --git a/nbs/metric/result.ipynb b/nbs/metric/result.ipynb
index ecbab85..68ccd6b 100644
--- a/nbs/metric/result.ipynb
+++ b/nbs/metric/result.ipynb
@@ -291,21 +291,53 @@
" return value\n",
" return MetricResult(result=value)\n",
"\n",
- "# Add Pydantic compatibility methods\n",
+ "@patch\n",
+ "def __json__(self: MetricResult):\n",
+ " \"\"\"Return data for JSON serialization.\n",
+ " \n",
+ " This method is used by json.dumps and other JSON serializers \n",
+ " to convert MetricResult to a JSON-compatible format.\n",
+ " \"\"\"\n",
+ " return {\n",
+ " \"result\": self._result,\n",
+ " \"reason\": self.reason,\n",
+ " }\n",
+ "\n",
"@patch(cls_method=True)\n",
"def __get_pydantic_core_schema__(\n",
" cls: MetricResult, \n",
" _source_type: t.Any, \n",
" _handler: GetCoreSchemaHandler\n",
") -> core_schema.CoreSchema:\n",
- " \"\"\"Generate a Pydantic core schema for MetricResult.\"\"\"\n",
- " return core_schema.with_info_plain_validator_function(cls.validate)\n",
- "\n",
- "\n",
- "@patch\n",
- "def model_dump(self: MetricResult):\n",
- " \"\"\"Support Pydantic's model_dump method.\"\"\"\n",
- " return self.to_dict()"
+ " \"\"\"Generate a Pydantic core schema for MetricResult.\n",
+ " \n",
+ " This custom schema handles different serialization behaviors:\n",
+ " - For model_dump(): Returns the original MetricResult instance\n",
+ " - For model_dump_json(): Converts to a JSON-compatible dict using __json__\n",
+ " \"\"\"\n",
+ " def serializer_function(instance, info):\n",
+ " \"\"\"Handle different serialization modes for MetricResult.\"\"\"\n",
+ " # For JSON serialization (model_dump_json), use __json__ method\n",
+ " if getattr(info, 'mode', None) == 'json':\n",
+ " return instance.__json__()\n",
+ " # For Python serialization (model_dump), return the instance itself\n",
+ " return instance\n",
+ " \n",
+ " return core_schema.union_schema([\n",
+ " # First schema: handles validation of MetricResult instances\n",
+ " core_schema.is_instance_schema(MetricResult),\n",
+ " \n",
+ " # Second schema: handles validation of other values and conversion to MetricResult\n",
+ " core_schema.chain_schema([\n",
+ " core_schema.any_schema(),\n",
+ " core_schema.no_info_plain_validator_function(\n",
+ " lambda value: MetricResult(result=value) if not isinstance(value, MetricResult) else value\n",
+ " ),\n",
+ " ]),\n",
+ " ], serialization=core_schema.plain_serializer_function_ser_schema(\n",
+ " serializer_function,\n",
+ " info_arg=True # Explicitly specify that we're using the info argument\n",
+ " ))"
]
},
{
@@ -328,9 +360,21 @@
"execution_count": null,
"id": "6ac6b955",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "TestModel(response='test', grade=1, faithfulness=1)"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "m = TestModel(response=\"test\", grade=MetricResult(result=1, reason=\"test\"), faithfulness=MetricResult(result=1, reason=\"test\"))"
+ "m = TestModel(response=\"test\", grade=MetricResult(result=1, reason=\"test\"), faithfulness=MetricResult(result=1, reason=\"test\"))\n",
+ "m"
]
},
{
@@ -342,7 +386,7 @@
{
"data": {
"text/plain": [
- "'test'"
+ "{'response': 'test', 'grade': 1, 'faithfulness': 1}"
]
},
"execution_count": null,
@@ -351,16 +395,29 @@
}
],
"source": [
- "m.grade.reason"
+ "m.model_dump()"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "9d32b10f",
+ "id": "0bc2a1ec",
"metadata": {},
- "outputs": [],
- "source": []
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'{\"response\":\"test\",\"grade\":{\"result\":1,\"reason\":\"test\"},\"faithfulness\":{\"result\":1,\"reason\":\"test\"}}'"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "m.model_dump_json()"
+ ]
}
],
"metadata": {
diff --git a/nbs/model/pydantic_mode.ipynb b/nbs/model/pydantic_mode.ipynb
index 5e644ba..59be463 100644
--- a/nbs/model/pydantic_mode.ipynb
+++ b/nbs/model/pydantic_mode.ipynb
@@ -70,6 +70,24 @@
" column_id = field_name\n",
" \n",
" cls.__column_mapping__[field_name] = column_id\n",
+ "\n",
+ " # check if the field is a MetricResult\n",
+ " if cls._is_metric_result_field(field_info.annotation):\n",
+ " # add additional mapping for the metric result\n",
+ " reason_field_name = f\"{field_name}_reason\"\n",
+ " reason_column_id = f\"{column_id}_reason\"\n",
+ " cls.__column_mapping__[reason_field_name] = reason_column_id\n",
+ "\n",
+ " @staticmethod\n",
+ " def _is_metric_result_field(annotation):\n",
+ " \"\"\"Check if a field annotation represents a MetricResult.\"\"\"\n",
+ " # Direct import of MetricResult\n",
+ " from ragas_experimental.metric.result import MetricResult\n",
+ " \n",
+ " # Check if annotation is or references MetricResult\n",
+ " return (annotation is MetricResult or \n",
+ " (hasattr(annotation, \"__origin__\") and annotation.__origin__ is MetricResult) or\n",
+ " (hasattr(annotation, \"__class__\") and annotation.__class__ is MetricResult))\n",
" \n",
" @classmethod\n",
" def get_column_id(cls, field_name: str) -> str:\n",
@@ -97,7 +115,8 @@
"metadata": {},
"outputs": [],
"source": [
- "import ragas_experimental.typing as rt"
+ "import ragas_experimental.typing as rt\n",
+ "from ragas_experimental.metric.result import MetricResult"
]
},
{
@@ -110,7 +129,17 @@
"class TestDataRow(ExtendedPydanticBaseModel):\n",
" id: t.Optional[int] = None\n",
" query: t.Annotated[str, rt.Text(id=\"search_query\")]\n",
- " persona: t.List[t.Literal[\"opt1\", \"opt2\", \"opt3\"]]"
+ " persona: t.List[t.Literal[\"opt1\", \"opt2\", \"opt3\"]]\n",
+ " result: MetricResult"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "TestDataRow.__column_mapping__ = {}"
]
},
{
@@ -121,7 +150,7 @@
{
"data": {
"text/plain": [
- "{}"
+ "TestDataRow(id=1, query='this is a test', persona=['opt1'], result=0.5)"
]
},
"execution_count": null,
@@ -130,16 +159,8 @@
}
],
"source": [
- "TestDataRow.__column_mapping__"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "t = TestDataRow(id=1, query=\"this is a test\", persona=[\"opt1\"])"
+ "test_data_row = TestDataRow(id=1, query=\"this is a test\", persona=[\"opt1\"], result=MetricResult(result=0.5, reason=\"test reason\"))\n",
+ "test_data_row"
]
},
{
@@ -150,7 +171,11 @@
{
"data": {
"text/plain": [
- "{'id': 'id', 'query': 'search_query', 'persona': 'persona'}"
+ "{'id': 'id',\n",
+ " 'query': 'search_query',\n",
+ " 'persona': 'persona',\n",
+ " 'result': 'result',\n",
+ " 'result_reason': 'result_reason'}"
]
},
"execution_count": null,
@@ -159,7 +184,7 @@
}
],
"source": [
- "t.__column_mapping__"
+ "test_data_row.__column_mapping__"
]
},
{
diff --git a/nbs/typing.ipynb b/nbs/typing.ipynb
index cbab5ff..3789ad8 100644
--- a/nbs/typing.ipynb
+++ b/nbs/typing.ipynb
@@ -53,7 +53,7 @@
"class ColumnType(str, Enum):\n",
" \"\"\"Column types supported by the Ragas API.\"\"\"\n",
" NUMBER = \"number\"\n",
- " TEXT = \"text\"\n",
+ " TEXT = \"longText\"\n",
" SELECT = \"select\"\n",
" MULTI_SELECT = \"multiSelect\"\n",
" CHECKBOX = \"checkbox\"\n",
@@ -404,141 +404,6 @@
" return [cls.instance_to_row(instance, model_class) for instance in instances]"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#| export\n",
- "# class ModelConverter:\n",
- "# \"\"\"Convert Pydantic models to Ragas API columns and rows.\"\"\"\n",
- " \n",
- "# @staticmethod\n",
- "# def infer_field_type(annotation, field_info=None):\n",
- "# \"\"\"Infer field type from Python type annotation.\"\"\"\n",
- "# # Check for Annotated with our custom metadata\n",
- "# origin = t.get_origin(annotation)\n",
- "# args = t.get_args(annotation)\n",
- " \n",
- "# # If this is an Annotated field with our metadata\n",
- "# if origin is t.Annotated and len(args) > 1:\n",
- "# for arg in args[1:]:\n",
- "# if isinstance(arg, FieldMeta):\n",
- "# return arg\n",
- " \n",
- "# # If no field metadata found, infer from the base type\n",
- "# return ModelConverter.infer_field_type(args[0], field_info)\n",
- " \n",
- "# # Handle Optional, List, etc.\n",
- "# if origin is t.Union:\n",
- "# if type(None) in args:\n",
- "# # This is Optional[T]\n",
- "# non_none_args = [arg for arg in args if arg is not type(None)]\n",
- "# if len(non_none_args) == 1:\n",
- "# # Get the field type of the non-None arg\n",
- "# field_meta = ModelConverter.infer_field_type(non_none_args[0], field_info)\n",
- "# field_meta.required = False\n",
- "# return field_meta\n",
- " \n",
- "# # Handle List and array types\n",
- "# # NOTE: here we are converting lists to strings, except for literal types\n",
- "# if origin is list or origin is t.List:\n",
- "# if len(args) > 0:\n",
- "# # Check if it's a list of literals\n",
- "# if t.get_origin(args[0]) is t.Literal:\n",
- "# literal_options = t.get_args(args[0])\n",
- "# return MultiSelect(options=list(literal_options))\n",
- "# # Otherwise just a regular list\n",
- "# return Text() # Default to Text for lists\n",
- " \n",
- "# # Handle Literal\n",
- "# if origin is t.Literal:\n",
- "# return Select(options=list(args))\n",
- " \n",
- "# # Basic type handling\n",
- "# if annotation is str:\n",
- "# return Text()\n",
- "# elif annotation is int or annotation is float:\n",
- "# return Number()\n",
- "# elif annotation is bool:\n",
- "# return Checkbox()\n",
- "# elif annotation is datetime or annotation is date:\n",
- "# return Date(include_time=annotation is datetime)\n",
- " \n",
- "# # Default to Text for complex or unknown types\n",
- "# return Text()\n",
- " \n",
- "# @classmethod\n",
- "# def model_to_columns(cls, model_class):\n",
- "# \"\"\"Convert a Pydantic model class to Ragas API column definitions.\"\"\"\n",
- "# columns = []\n",
- "# for field_name, field_info in model_class.model_fields.items():\n",
- "# # Get the field's type annotation\n",
- "# annotation = field_info.annotation\n",
- " \n",
- "# # Try to get field metadata\n",
- "# field_meta = cls.infer_field_type(annotation, field_info)\n",
- " \n",
- "# # Create column definition\n",
- "# column = {\n",
- "# \"id\": field_name,\n",
- "# \"name\": field_name,\n",
- "# \"type\": field_meta.type.value,\n",
- "# \"settings\": field_meta.settings.copy()\n",
- "# }\n",
- " \n",
- "# columns.append(column)\n",
- " \n",
- "# return columns\n",
- " \n",
- "# @classmethod\n",
- "# def instance_to_row(cls, instance, model_class=None):\n",
- "# \"\"\"Convert a Pydantic model instance to a Ragas API row.\"\"\"\n",
- "# if model_class is None:\n",
- "# model_class = instance.__class__\n",
- " \n",
- "# row_cells = []\n",
- "# model_data = instance.model_dump()\n",
- " \n",
- "# for field_name, field_info in model_class.model_fields.items():\n",
- "# if field_name in model_data:\n",
- "# value = model_data[field_name]\n",
- "# # Process value based on field type\n",
- "# annotation = field_info.annotation\n",
- "# field_meta = cls.infer_field_type(annotation, field_info)\n",
- " \n",
- "# # Special handling for various types\n",
- "# if field_meta.type == ColumnType.MULTI_SELECT and isinstance(value, list):\n",
- "# # Convert list to string format accepted by API\n",
- "# processed_value = value\n",
- "# elif field_meta.type == ColumnType.DATE and isinstance(value, (datetime, date)):\n",
- "# # Format date as string\n",
- "# processed_value = value.isoformat()\n",
- "# else:\n",
- "# processed_value = value\n",
- " \n",
- "# row_cells.append({\n",
- "# \"column_id\": field_name,\n",
- "# \"data\": processed_value\n",
- "# })\n",
- " \n",
- "# return {\n",
- "# \"data\": row_cells\n",
- "# }\n",
- " \n",
- "# @classmethod\n",
- "# def instances_to_rows(cls, instances, model_class=None):\n",
- "# \"\"\"Convert multiple Pydantic model instances to Ragas API rows.\"\"\"\n",
- "# if not instances:\n",
- "# return []\n",
- " \n",
- "# if model_class is None and instances:\n",
- "# model_class = instances[0].__class__\n",
- " \n",
- "# return [cls.instance_to_row(instance, model_class) for instance in instances]"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -607,24 +472,18 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Testing model_to_columns:\n"
- ]
- },
- {
- "ename": "ImportError",
- "evalue": "cannot import name 'Project' from partially initialized module 'ragas_experimental.project.core' (most likely due to a circular import) (/Users/jjmachan/workspace/eglabs/ragas_experimental/ragas_experimental/project/core.py)",
- "output_type": "error",
- "traceback": [
- "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
- "\u001b[31mImportError\u001b[39m Traceback (most recent call last)",
- "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[15]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# Test the model_to_columns method\u001b[39;00m\n\u001b[32m 2\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33mTesting model_to_columns:\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m columns = \u001b[43mModelConverter\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmodel_to_columns\u001b[49m\u001b[43m(\u001b[49m\u001b[43mTestModel\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m col \u001b[38;5;129;01min\u001b[39;00m columns:\n\u001b[32m 5\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m- \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcol[\u001b[33m'\u001b[39m\u001b[33mname\u001b[39m\u001b[33m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcol[\u001b[33m'\u001b[39m\u001b[33mtype\u001b[39m\u001b[33m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m): \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcol[\u001b[33m'\u001b[39m\u001b[33msettings\u001b[39m\u001b[33m'\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m)\n",
- "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[12]\u001b[39m\u001b[32m, line 90\u001b[39m, in \u001b[36mModelConverter.model_to_columns\u001b[39m\u001b[34m(cls, model_class)\u001b[39m\n\u001b[32m 87\u001b[39m \u001b[38;5;129m@classmethod\u001b[39m\n\u001b[32m 88\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mmodel_to_columns\u001b[39m(\u001b[38;5;28mcls\u001b[39m, model_class):\n\u001b[32m 89\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"Convert a Pydantic model class to Ragas API column definitions.\"\"\"\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m90\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mragas_experimental\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmetric\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mresult\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MetricResult\n\u001b[32m 92\u001b[39m columns = []\n\u001b[32m 93\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m field_name, field_info \u001b[38;5;129;01min\u001b[39;00m model_class.model_fields.items():\n\u001b[32m 94\u001b[39m \u001b[38;5;66;03m# Get the field's type annotation\u001b[39;00m\n",
- "\u001b[36mFile \u001b[39m\u001b[32m~/workspace/eglabs/ragas_experimental/ragas_experimental/__init__.py:8\u001b[39m\n\u001b[32m 5\u001b[39m __all__ = []\n\u001b[32m 7\u001b[39m \u001b[38;5;66;03m# %% ../nbs/init_module.ipynb 2\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m8\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mproject\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Project\n\u001b[32m 9\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mragas_experimental\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmodel\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mnotion_typing\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnmt\u001b[39;00m\n\u001b[32m 10\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmodel\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mnotion_model\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m NotionModel\n",
- "\u001b[36mFile \u001b[39m\u001b[32m~/workspace/eglabs/ragas_experimental/ragas_experimental/project/core.py:18\u001b[39m\n\u001b[32m 16\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mbackends\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mfactory\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m RagasApiClientFactory\n\u001b[32m 17\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mbackends\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mragas_api_client\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m RagasApiClient\n\u001b[32m---> \u001b[39m\u001b[32m18\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mragas_experimental\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mtyping\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mrt\u001b[39;00m\n\u001b[32m 19\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mutils\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m async_to_sync, create_nano_id\n\u001b[32m 20\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdataset\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Dataset\n",
- "\u001b[36mFile \u001b[39m\u001b[32m~/workspace/eglabs/ragas_experimental/ragas_experimental/typing.py:15\u001b[39m\n\u001b[32m 12\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mdatetime\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m datetime, date\n\u001b[32m 13\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01minspect\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m15\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmetric\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mresult\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MetricResult\n\u001b[32m 17\u001b[39m \u001b[38;5;66;03m# %% ../nbs/typing.ipynb 4\u001b[39;00m\n\u001b[32m 18\u001b[39m \u001b[38;5;28;01mclass\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mColumnType\u001b[39;00m(\u001b[38;5;28mstr\u001b[39m, Enum):\n",
- "\u001b[36mFile \u001b[39m\u001b[32m~/workspace/eglabs/ragas_experimental/ragas_experimental/metric/__init__.py:2\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mragas_experimental\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmetric\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mresult\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MetricResult\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mragas_experimental\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmetric\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mbase\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Metric\n\u001b[32m 3\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mragas_experimental\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmetric\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdiscrete\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m DiscreteMetric\n\u001b[32m 4\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mragas_experimental\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmetric\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mnumeric\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m NumericMetric\n",
- "\u001b[36mFile \u001b[39m\u001b[32m~/workspace/eglabs/ragas_experimental/ragas_experimental/metric/base.py:22\u001b[39m\n\u001b[32m 20\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m MetricResult\n\u001b[32m 21\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mllm\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m RagasLLM\n\u001b[32m---> \u001b[39m\u001b[32m22\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mproject\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mcore\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Project\n\u001b[32m 23\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmodel\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mnotion_model\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m NotionModel\n\u001b[32m 24\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mprompt\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdynamic_few_shot\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m DynamicFewShotPrompt\n",
- "\u001b[31mImportError\u001b[39m: cannot import name 'Project' from partially initialized module 'ragas_experimental.project.core' (most likely due to a circular import) (/Users/jjmachan/workspace/eglabs/ragas_experimental/ragas_experimental/project/core.py)"
+ "Testing model_to_columns:\n",
+ "- id (number): {}\n",
+ "- name (longText): {'max_length': 1000}\n",
+ "- is_active (checkbox): {}\n",
+ "- created_at (date): {'include_time': True}\n",
+ "- optional_text (longText): {'max_length': 1000}\n",
+ "- tags (longText): {'max_length': 1000}\n",
+ "- status (select): {'options': [{'name': 'pending'}, {'name': 'active'}, {'name': 'completed'}]}\n",
+ "- score (number): {}\n",
+ "- description (longText): {'max_length': 1000}\n",
+ "- category (select): {'options': [{'name': 'A'}, {'name': 'B'}, {'name': 'C'}]}\n",
+ "- features (longText): {'max_length': 1000}\n"
]
}
],
@@ -650,7 +509,7 @@
"- id: 1\n",
"- name: Test Item 1\n",
"- is_active: True\n",
- "- created_at: 2025-04-14T17:36:24.492518\n",
+ "- created_at: 2025-04-15T17:02:27.298241\n",
"- optional_text: None\n",
"- tags: ['tag1', 'tag2']\n",
"- status: active\n",
@@ -685,11 +544,11 @@
"\n",
"Testing type inference:\n",
"- : number (Required: True)\n",
- "- : text (Required: True)\n",
+ "- : longText (Required: True)\n",
"- : checkbox (Required: True)\n",
"- : date (Required: True)\n",
- "- typing.Optional[str]: text (Required: False)\n",
- "- typing.List[str]: text (Required: True)\n",
+ "- typing.Optional[str]: longText (Required: False)\n",
+ "- typing.List[str]: longText (Required: True)\n",
"- typing.Literal['a', 'b']: select (Required: True)\n",
" - Options: ['a', 'b']\n",
"- typing.List[typing.Literal['x', 'y']]: multiSelect (Required: True)\n",
@@ -744,7 +603,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "[{'id': 'id', 'name': 'id', 'type': 'number', 'settings': {}, 'editable': False}, {'id': 'name', 'name': 'name', 'type': 'text', 'settings': {'max_length': 1000}, 'editable': False}, {'id': 'created_at', 'name': 'created_at', 'type': 'date', 'settings': {'include_time': True}, 'editable': False}]\n"
+ "[{'id': 'id', 'name': 'id', 'type': 'number', 'settings': {}, 'editable': False}, {'id': 'name', 'name': 'name', 'type': 'longText', 'settings': {'max_length': 1000}, 'editable': False}, {'id': 'created_at', 'name': 'created_at', 'type': 'date', 'settings': {'include_time': True}, 'editable': False}]\n"
]
}
],
@@ -818,22 +677,22 @@
"text/plain": [
"[{'id': 'query',\n",
" 'name': 'query',\n",
- " 'type': 'text',\n",
+ " 'type': 'longText',\n",
" 'settings': {'max_length': 1000},\n",
" 'editable': False},\n",
" {'id': 'response',\n",
" 'name': 'response',\n",
- " 'type': 'text',\n",
+ " 'type': 'longText',\n",
" 'settings': {'max_length': 1000},\n",
" 'editable': False},\n",
" {'id': 'score',\n",
" 'name': 'score',\n",
- " 'type': 'text',\n",
+ " 'type': 'longText',\n",
" 'settings': {'max_length': 1000},\n",
" 'editable': True},\n",
" {'id': 'score_reason',\n",
" 'name': 'score_reason',\n",
- " 'type': 'text',\n",
+ " 'type': 'longText',\n",
" 'settings': {'max_length': 1000},\n",
" 'editable': True}]"
]
@@ -872,6 +731,29 @@
"ModelConverter.instance_to_row(m)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[{'data': [{'column_id': 'query', 'data': 'test'},\n",
+ " {'column_id': 'response', 'data': 'test'},\n",
+ " {'column_id': 'score', 'data': 1},\n",
+ " {'column_id': 'score_reason', 'data': 'test'}]}]"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "ModelConverter.instances_to_rows([m])"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/ragas_experimental/_modidx.py b/ragas_experimental/_modidx.py
index ad33045..e58fd07 100644
--- a/ragas_experimental/_modidx.py
+++ b/ragas_experimental/_modidx.py
@@ -95,8 +95,6 @@
'ragas_experimental/backends/notion_backend.py')},
'ragas_experimental.backends.ragas_api_client': { 'ragas_experimental.backends.ragas_api_client.Column': ( 'backends/ragas_api_client.html#column',
'ragas_experimental/backends/ragas_api_client.py'),
- 'ragas_experimental.backends.ragas_api_client.ColumnType': ( 'backends/ragas_api_client.html#columntype',
- 'ragas_experimental/backends/ragas_api_client.py'),
'ragas_experimental.backends.ragas_api_client.RagasApiClient': ( 'backends/ragas_api_client.html#ragasapiclient',
'ragas_experimental/backends/ragas_api_client.py'),
'ragas_experimental.backends.ragas_api_client.RagasApiClient.__init__': ( 'backends/ragas_api_client.html#ragasapiclient.__init__',
@@ -349,6 +347,8 @@
'ragas_experimental/metric/result.py'),
'ragas_experimental.metric.result.MetricResult.__iter__': ( 'metric/result.html#metricresult.__iter__',
'ragas_experimental/metric/result.py'),
+ 'ragas_experimental.metric.result.MetricResult.__json__': ( 'metric/result.html#metricresult.__json__',
+ 'ragas_experimental/metric/result.py'),
'ragas_experimental.metric.result.MetricResult.__le__': ( 'metric/result.html#metricresult.__le__',
'ragas_experimental/metric/result.py'),
'ragas_experimental.metric.result.MetricResult.__len__': ( 'metric/result.html#metricresult.__len__',
@@ -373,8 +373,6 @@
'ragas_experimental/metric/result.py'),
'ragas_experimental.metric.result.MetricResult.__truediv__': ( 'metric/result.html#metricresult.__truediv__',
'ragas_experimental/metric/result.py'),
- 'ragas_experimental.metric.result.MetricResult.model_dump': ( 'metric/result.html#metricresult.model_dump',
- 'ragas_experimental/metric/result.py'),
'ragas_experimental.metric.result.MetricResult.result': ( 'metric/result.html#metricresult.result',
'ragas_experimental/metric/result.py'),
'ragas_experimental.metric.result.MetricResult.to_dict': ( 'metric/result.html#metricresult.to_dict',
@@ -519,6 +517,8 @@
'ragas_experimental/model/pydantic_model.py'),
'ragas_experimental.model.pydantic_model.ExtendedPydanticBaseModel._initialize_column_mapping': ( 'model/pydantic_mode.html#extendedpydanticbasemodel._initialize_column_mapping',
'ragas_experimental/model/pydantic_model.py'),
+ 'ragas_experimental.model.pydantic_model.ExtendedPydanticBaseModel._is_metric_result_field': ( 'model/pydantic_mode.html#extendedpydanticbasemodel._is_metric_result_field',
+ 'ragas_experimental/model/pydantic_model.py'),
'ragas_experimental.model.pydantic_model.ExtendedPydanticBaseModel.get_column_id': ( 'model/pydantic_mode.html#extendedpydanticbasemodel.get_column_id',
'ragas_experimental/model/pydantic_model.py'),
'ragas_experimental.model.pydantic_model.ExtendedPydanticBaseModel.get_db_field_mapping': ( 'model/pydantic_mode.html#extendedpydanticbasemodel.get_db_field_mapping',
diff --git a/ragas_experimental/backends/ragas_api_client.py b/ragas_experimental/backends/ragas_api_client.py
index 2efe3bd..04ce591 100644
--- a/ragas_experimental/backends/ragas_api_client.py
+++ b/ragas_experimental/backends/ragas_api_client.py
@@ -3,7 +3,7 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/backends/ragas_api_client.ipynb.
# %% auto 0
-__all__ = ['DEFAULT_SETTINGS', 'RagasApiClient', 'ColumnType', 'create_nano_id', 'Column', 'RowCell', 'Row']
+__all__ = ['DEFAULT_SETTINGS', 'RagasApiClient', 'create_nano_id', 'Column', 'RowCell', 'Row']
# %% ../../nbs/backends/ragas_api_client.ipynb 4
import httpx
@@ -258,20 +258,9 @@ async def delete_experiment(self: RagasApiClient, project_id: str, experiment_id
# %% ../../nbs/backends/ragas_api_client.ipynb 25
-from enum import Enum
+from ..typing import ColumnType
# %% ../../nbs/backends/ragas_api_client.ipynb 26
-class ColumnType(str, Enum):
- NUMBER = "number"
- TEXT = "text"
- LONG_TEXT = "longText"
- SELECT = "select"
- DATE = "date"
- MULTI_SELECT = "multiSelect"
- CHECKBOX = "checkbox"
- CUSTOM = "custom"
-
-# %% ../../nbs/backends/ragas_api_client.ipynb 27
#---- Dataset Columns ----
@patch
async def list_dataset_columns(
@@ -342,7 +331,7 @@ async def delete_dataset_column(
f"projects/{project_id}/datasets/{dataset_id}/columns/{column_id}"
)
-# %% ../../nbs/backends/ragas_api_client.ipynb 35
+# %% ../../nbs/backends/ragas_api_client.ipynb 34
#---- Dataset Rows ----
@patch
async def list_dataset_rows(
@@ -404,11 +393,11 @@ async def delete_dataset_row(
)
-# %% ../../nbs/backends/ragas_api_client.ipynb 47
+# %% ../../nbs/backends/ragas_api_client.ipynb 46
import uuid
import string
-# %% ../../nbs/backends/ragas_api_client.ipynb 48
+# %% ../../nbs/backends/ragas_api_client.ipynb 47
def create_nano_id(size=12):
# Define characters to use (alphanumeric)
alphabet = string.ascii_letters + string.digits
@@ -425,11 +414,11 @@ def create_nano_id(size=12):
# Pad if necessary and return desired length
return result[:size]
-# %% ../../nbs/backends/ragas_api_client.ipynb 50
+# %% ../../nbs/backends/ragas_api_client.ipynb 49
import uuid
import string
-# %% ../../nbs/backends/ragas_api_client.ipynb 51
+# %% ../../nbs/backends/ragas_api_client.ipynb 50
def create_nano_id(size=12):
# Define characters to use (alphanumeric)
alphabet = string.ascii_letters + string.digits
@@ -446,7 +435,7 @@ def create_nano_id(size=12):
# Pad if necessary and return desired length
return result[:size]
-# %% ../../nbs/backends/ragas_api_client.ipynb 53
+# %% ../../nbs/backends/ragas_api_client.ipynb 52
# Default settings for columns
DEFAULT_SETTINGS = {
"is_required": False,
@@ -469,7 +458,7 @@ class Row(BaseModel):
id: str = Field(default_factory=create_nano_id)
data: t.List[RowCell] = Field(...)
-# %% ../../nbs/backends/ragas_api_client.ipynb 54
+# %% ../../nbs/backends/ragas_api_client.ipynb 53
#---- Resource With Data Helper Methods ----
@patch
async def _create_with_data(
@@ -596,7 +585,7 @@ async def create_dataset_with_data(
"dataset", project_id, name, description, columns, rows, batch_size
)
-# %% ../../nbs/backends/ragas_api_client.ipynb 60
+# %% ../../nbs/backends/ragas_api_client.ipynb 59
#---- Experiment Columns ----
@patch
async def list_experiment_columns(
@@ -727,7 +716,7 @@ async def delete_experiment_row(
f"projects/{project_id}/experiments/{experiment_id}/rows/{row_id}"
)
-# %% ../../nbs/backends/ragas_api_client.ipynb 63
+# %% ../../nbs/backends/ragas_api_client.ipynb 62
@patch
async def create_experiment_with_data(
self: RagasApiClient,
@@ -758,7 +747,7 @@ async def create_experiment_with_data(
"experiment", project_id, name, description, columns, rows, batch_size
)
-# %% ../../nbs/backends/ragas_api_client.ipynb 64
+# %% ../../nbs/backends/ragas_api_client.ipynb 63
#---- Utility Methods ----
@patch
def create_column(
diff --git a/ragas_experimental/dataset.py b/ragas_experimental/dataset.py
index 5a70c52..7f6df1c 100644
--- a/ragas_experimental/dataset.py
+++ b/ragas_experimental/dataset.py
@@ -102,7 +102,10 @@ def __len__(self) -> int:
def __iter__(self) -> t.Iterator[BaseModelType]:
return iter(self._entries)
-# %% ../nbs/dataset.ipynb 16
+# %% ../nbs/dataset.ipynb 18
+import ragas_experimental.typing as rt
+
+# %% ../nbs/dataset.ipynb 19
@patch
def append(self: Dataset, entry: BaseModelType) -> None:
"""Add a new entry to the dataset and sync to Notion."""
@@ -112,12 +115,12 @@ def append(self: Dataset, entry: BaseModelType) -> None:
column_id_map = self.model.__column_mapping__
# create the rows
- row_dict = entry.model_dump()
+ row_dict_converted = rt.ModelConverter.instance_to_row(entry)
row_id = create_nano_id()
row_data = {}
- for key, value in row_dict.items():
- if key in column_id_map:
- row_data[column_id_map[key]] = value
+ for column in row_dict_converted["data"]:
+ if column["column_id"] in column_id_map:
+ row_data[column_id_map[column["column_id"]]] = column["data"]
sync_func = async_to_sync(self._ragas_api_client.create_dataset_row)
response = sync_func(
@@ -131,7 +134,7 @@ def append(self: Dataset, entry: BaseModelType) -> None:
# Update entry with Notion data (like ID)
self._entries.append(entry)
-# %% ../nbs/dataset.ipynb 19
+# %% ../nbs/dataset.ipynb 22
@patch
def pop(self: Dataset, index: int = -1) -> BaseModelType:
"""Remove and return entry at index, sync deletion to Notion."""
@@ -148,7 +151,7 @@ def pop(self: Dataset, index: int = -1) -> BaseModelType:
# Remove from local cache
return self._entries.pop(index)
-# %% ../nbs/dataset.ipynb 22
+# %% ../nbs/dataset.ipynb 26
@patch
def load(self: Dataset) -> None:
"""Load all entries from the backend API."""
@@ -184,7 +187,7 @@ def load(self: Dataset) -> None:
self._entries.append(entry)
-# %% ../nbs/dataset.ipynb 24
+# %% ../nbs/dataset.ipynb 28
@patch
def load_as_dicts(self: Dataset) -> t.List[t.Dict]:
"""Load all entries as dictionaries."""
@@ -210,7 +213,7 @@ def load_as_dicts(self: Dataset) -> t.List[t.Dict]:
return result
-# %% ../nbs/dataset.ipynb 26
+# %% ../nbs/dataset.ipynb 30
@patch
def to_pandas(self: Dataset) -> "pd.DataFrame":
"""Convert dataset to pandas DataFrame."""
@@ -224,7 +227,7 @@ def to_pandas(self: Dataset) -> "pd.DataFrame":
data = [entry.model_dump() for entry in self._entries]
return pd.DataFrame(data)
-# %% ../nbs/dataset.ipynb 28
+# %% ../nbs/dataset.ipynb 32
@patch
def save(self: Dataset, item: BaseModelType) -> None:
"""Save changes to an item to the backend."""
@@ -248,12 +251,12 @@ def save(self: Dataset, item: BaseModelType) -> None:
# Get column mapping and prepare data
column_id_map = self.model.__column_mapping__
- row_dict = item.model_dump()
+ row_dict = rt.ModelConverter.instance_to_row(item)["data"]
row_data = {}
- for key, value in row_dict.items():
- if key in column_id_map:
- row_data[column_id_map[key]] = value
+ for column in row_dict:
+ if column["column_id"] in column_id_map:
+ row_data[column_id_map[column["column_id"]]] = column["data"]
# Update in backend
sync_func = async_to_sync(self._ragas_api_client.update_dataset_row)
@@ -272,7 +275,7 @@ def save(self: Dataset, item: BaseModelType) -> None:
self._entries[i] = item
break
-# %% ../nbs/dataset.ipynb 32
+# %% ../nbs/dataset.ipynb 36
@patch
def get(self: Dataset, field_value: str, field_name: str = "_row_id") -> t.Optional[BaseModelType]:
"""Get an entry by field value.
diff --git a/ragas_experimental/metric/result.py b/ragas_experimental/metric/result.py
index 5a6dc22..10887d9 100644
--- a/ragas_experimental/metric/result.py
+++ b/ragas_experimental/metric/result.py
@@ -188,18 +188,50 @@ def validate(cls: MetricResult, value: t.Any, info: ValidationInfo):
return value
return MetricResult(result=value)
-# Add Pydantic compatibility methods
+@patch
+def __json__(self: MetricResult):
+ """Return data for JSON serialization.
+
+ This method is used by json.dumps and other JSON serializers
+ to convert MetricResult to a JSON-compatible format.
+ """
+ return {
+ "result": self._result,
+ "reason": self.reason,
+ }
+
@patch(cls_method=True)
def __get_pydantic_core_schema__(
cls: MetricResult,
_source_type: t.Any,
_handler: GetCoreSchemaHandler
) -> core_schema.CoreSchema:
- """Generate a Pydantic core schema for MetricResult."""
- return core_schema.with_info_plain_validator_function(cls.validate)
-
-
-@patch
-def model_dump(self: MetricResult):
- """Support Pydantic's model_dump method."""
- return self.to_dict()
+ """Generate a Pydantic core schema for MetricResult.
+
+ This custom schema handles different serialization behaviors:
+ - For model_dump(): Returns the original MetricResult instance
+ - For model_dump_json(): Converts to a JSON-compatible dict using __json__
+ """
+ def serializer_function(instance, info):
+ """Handle different serialization modes for MetricResult."""
+ # For JSON serialization (model_dump_json), use __json__ method
+ if getattr(info, 'mode', None) == 'json':
+ return instance.__json__()
+ # For Python serialization (model_dump), return the instance itself
+ return instance
+
+ return core_schema.union_schema([
+ # First schema: handles validation of MetricResult instances
+ core_schema.is_instance_schema(MetricResult),
+
+ # Second schema: handles validation of other values and conversion to MetricResult
+ core_schema.chain_schema([
+ core_schema.any_schema(),
+ core_schema.no_info_plain_validator_function(
+ lambda value: MetricResult(result=value) if not isinstance(value, MetricResult) else value
+ ),
+ ]),
+ ], serialization=core_schema.plain_serializer_function_ser_schema(
+ serializer_function,
+ info_arg=True # Explicitly specify that we're using the info argument
+ ))
diff --git a/ragas_experimental/model/pydantic_model.py b/ragas_experimental/model/pydantic_model.py
index c3b3eca..4fc9875 100644
--- a/ragas_experimental/model/pydantic_model.py
+++ b/ragas_experimental/model/pydantic_model.py
@@ -44,6 +44,24 @@ def _initialize_column_mapping(cls):
column_id = field_name
cls.__column_mapping__[field_name] = column_id
+
+ # check if the field is a MetricResult
+ if cls._is_metric_result_field(field_info.annotation):
+ # add additional mapping for the metric result
+ reason_field_name = f"{field_name}_reason"
+ reason_column_id = f"{column_id}_reason"
+ cls.__column_mapping__[reason_field_name] = reason_column_id
+
+ @staticmethod
+ def _is_metric_result_field(annotation):
+ """Check if a field annotation represents a MetricResult."""
+ # Direct import of MetricResult
+ from ragas_experimental.metric.result import MetricResult
+
+ # Check if annotation is or references MetricResult
+ return (annotation is MetricResult or
+ (hasattr(annotation, "__origin__") and annotation.__origin__ is MetricResult) or
+ (hasattr(annotation, "__class__") and annotation.__class__ is MetricResult))
@classmethod
def get_column_id(cls, field_name: str) -> str:
diff --git a/ragas_experimental/typing.py b/ragas_experimental/typing.py
index f537ffe..fb39874 100644
--- a/ragas_experimental/typing.py
+++ b/ragas_experimental/typing.py
@@ -18,7 +18,7 @@
class ColumnType(str, Enum):
"""Column types supported by the Ragas API."""
NUMBER = "number"
- TEXT = "text"
+ TEXT = "longText"
SELECT = "select"
MULTI_SELECT = "multiSelect"
CHECKBOX = "checkbox"
@@ -301,131 +301,3 @@ def instances_to_rows(cls, instances, model_class=None):
model_class = instances[0].__class__
return [cls.instance_to_row(instance, model_class) for instance in instances]
-
-# %% ../nbs/typing.ipynb 15
-# class ModelConverter:
-# """Convert Pydantic models to Ragas API columns and rows."""
-
-# @staticmethod
-# def infer_field_type(annotation, field_info=None):
-# """Infer field type from Python type annotation."""
-# # Check for Annotated with our custom metadata
-# origin = t.get_origin(annotation)
-# args = t.get_args(annotation)
-
-# # If this is an Annotated field with our metadata
-# if origin is t.Annotated and len(args) > 1:
-# for arg in args[1:]:
-# if isinstance(arg, FieldMeta):
-# return arg
-
-# # If no field metadata found, infer from the base type
-# return ModelConverter.infer_field_type(args[0], field_info)
-
-# # Handle Optional, List, etc.
-# if origin is t.Union:
-# if type(None) in args:
-# # This is Optional[T]
-# non_none_args = [arg for arg in args if arg is not type(None)]
-# if len(non_none_args) == 1:
-# # Get the field type of the non-None arg
-# field_meta = ModelConverter.infer_field_type(non_none_args[0], field_info)
-# field_meta.required = False
-# return field_meta
-
-# # Handle List and array types
-# # NOTE: here we are converting lists to strings, except for literal types
-# if origin is list or origin is t.List:
-# if len(args) > 0:
-# # Check if it's a list of literals
-# if t.get_origin(args[0]) is t.Literal:
-# literal_options = t.get_args(args[0])
-# return MultiSelect(options=list(literal_options))
-# # Otherwise just a regular list
-# return Text() # Default to Text for lists
-
-# # Handle Literal
-# if origin is t.Literal:
-# return Select(options=list(args))
-
-# # Basic type handling
-# if annotation is str:
-# return Text()
-# elif annotation is int or annotation is float:
-# return Number()
-# elif annotation is bool:
-# return Checkbox()
-# elif annotation is datetime or annotation is date:
-# return Date(include_time=annotation is datetime)
-
-# # Default to Text for complex or unknown types
-# return Text()
-
-# @classmethod
-# def model_to_columns(cls, model_class):
-# """Convert a Pydantic model class to Ragas API column definitions."""
-# columns = []
-# for field_name, field_info in model_class.model_fields.items():
-# # Get the field's type annotation
-# annotation = field_info.annotation
-
-# # Try to get field metadata
-# field_meta = cls.infer_field_type(annotation, field_info)
-
-# # Create column definition
-# column = {
-# "id": field_name,
-# "name": field_name,
-# "type": field_meta.type.value,
-# "settings": field_meta.settings.copy()
-# }
-
-# columns.append(column)
-
-# return columns
-
-# @classmethod
-# def instance_to_row(cls, instance, model_class=None):
-# """Convert a Pydantic model instance to a Ragas API row."""
-# if model_class is None:
-# model_class = instance.__class__
-
-# row_cells = []
-# model_data = instance.model_dump()
-
-# for field_name, field_info in model_class.model_fields.items():
-# if field_name in model_data:
-# value = model_data[field_name]
-# # Process value based on field type
-# annotation = field_info.annotation
-# field_meta = cls.infer_field_type(annotation, field_info)
-
-# # Special handling for various types
-# if field_meta.type == ColumnType.MULTI_SELECT and isinstance(value, list):
-# # Convert list to string format accepted by API
-# processed_value = value
-# elif field_meta.type == ColumnType.DATE and isinstance(value, (datetime, date)):
-# # Format date as string
-# processed_value = value.isoformat()
-# else:
-# processed_value = value
-
-# row_cells.append({
-# "column_id": field_name,
-# "data": processed_value
-# })
-
-# return {
-# "data": row_cells
-# }
-
-# @classmethod
-# def instances_to_rows(cls, instances, model_class=None):
-# """Convert multiple Pydantic model instances to Ragas API rows."""
-# if not instances:
-# return []
-
-# if model_class is None and instances:
-# model_class = instances[0].__class__
-
-# return [cls.instance_to_row(instance, model_class) for instance in instances]