diff --git a/.gitignore b/.gitignore
index bec600f..0f2c2f2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -172,3 +172,4 @@ cython_debug/
.python-version
uv.lock
_proc
+experiments
diff --git a/ragas_annotator/backends/__init__.py b/nbs/.notest
similarity index 100%
rename from ragas_annotator/backends/__init__.py
rename to nbs/.notest
diff --git a/nbs/backends/factory.ipynb b/nbs/backends/factory.ipynb
index c9d5595..388bf14 100644
--- a/nbs/backends/factory.ipynb
+++ b/nbs/backends/factory.ipynb
@@ -24,13 +24,10 @@
"metadata": {},
"outputs": [],
"source": [
- "# | export\n",
+ "#| export\n",
"import typing as t\n",
- "import os\n",
"\n",
- "from notion_client import Client as NotionClient\n",
- "from ragas_annotator.backends.mock_notion import MockNotionClient\n",
- "from ragas_annotator.backends.notion_backend import NotionBackend"
+ "from ragas_experimental.backends.ragas_api_client import RagasApiClient"
]
},
{
@@ -39,204 +36,37 @@
"metadata": {},
"outputs": [],
"source": [
- "# | export\n",
- "class NotionClientFactory:\n",
- " \"\"\"Factory for creating Notion client instances.\"\"\"\n",
+ "#| export\n",
+ "class RagasApiClientFactory:\n",
+ " \"\"\"Factory for creating Ragas API client instances.\"\"\"\n",
"\n",
" @staticmethod\n",
" def create(\n",
- " use_mock: bool = False,\n",
- " api_key: t.Optional[str] = None,\n",
- " initialize_project: bool = False,\n",
- " root_page_id: t.Optional[str] = None,\n",
- " ) -> t.Union[NotionClient, MockNotionClient]:\n",
- " \"\"\"Create a Notion client.\n",
+ " app_token: t.Optional[str] = None,\n",
+ " base_url: t.Optional[str] = None,\n",
+ " ) -> RagasApiClient:\n",
+ " \"\"\"Create a Ragas API client.\n",
"\n",
" Args:\n",
- " use_mock: If True, create a mock client\n",
- " api_key: Notion API key (only used for real client)\n",
- " initialize_project: If True and using mock, initialize project structure\n",
- " root_page_id: Required if initialize_project is True\n",
+ " api_key: The API key for the Ragas API\n",
+ " base_url: The base URL for the Ragas API\n",
"\n",
" Returns:\n",
- " Union[NotionClient, MockNotionClient]: A real or mock client\n",
+ " RagasApiClient: A Ragas API client instance\n",
" \"\"\"\n",
- " if use_mock:\n",
- " client = MockNotionClient()\n",
+ " if app_token is None:\n",
+ " app_token = os.getenv(\"RAGAS_APP_TOKEN\")\n",
"\n",
- " # Optionally initialize project structure\n",
- " if initialize_project and root_page_id:\n",
- " # Create root page if it doesn't exist in the mock client\n",
- " if root_page_id not in client._pages:\n",
- " # Create root page\n",
- " root_page = {\n",
- " \"id\": root_page_id,\n",
- " \"object\": \"page\",\n",
- " \"created_time\": client._get_timestamp(),\n",
- " \"last_edited_time\": client._get_timestamp(),\n",
- " \"archived\": False,\n",
- " \"properties\": {\n",
- " \"title\": {\n",
- " \"type\": \"title\",\n",
- " \"title\": [\n",
- " {\n",
- " \"plain_text\": \"Root Page\",\n",
- " \"type\": \"text\",\n",
- " \"text\": {\"content\": \"Root Page\"},\n",
- " }\n",
- " ],\n",
- " }\n",
- " },\n",
- " }\n",
- " client.add_page(root_page)\n",
+ " if app_token is None:\n",
+ " raise ValueError(\"RAGAS_API_KEY environment variable is not set\")\n",
"\n",
- " # Create required sub-pages\n",
- " for page_name in [\"Datasets\", \"Experiments\", \"Comparisons\"]:\n",
- " # Create page ID\n",
- " page_id = client._create_id()\n",
+ " if base_url is None:\n",
+ " base_url = os.getenv(\"RAGAS_API_BASE_URL\")\n",
"\n",
- " # Create page\n",
- " page = {\n",
- " \"id\": page_id,\n",
- " \"object\": \"page\",\n",
- " \"created_time\": client._get_timestamp(),\n",
- " \"last_edited_time\": client._get_timestamp(),\n",
- " \"archived\": False,\n",
- " \"properties\": {\n",
- " \"title\": {\n",
- " \"type\": \"title\",\n",
- " \"title\": [\n",
- " {\n",
- " \"plain_text\": page_name,\n",
- " \"type\": \"text\",\n",
- " \"text\": {\"content\": page_name},\n",
- " }\n",
- " ],\n",
- " }\n",
- " },\n",
- " \"parent\": {\"type\": \"page_id\", \"page_id\": root_page_id},\n",
- " }\n",
- " client.add_page(page)\n",
+ " if base_url is None:\n",
+ " base_url = \"https://api.dev.app.ragas.io\"\n",
"\n",
- " # Add child block to root\n",
- " child_block = {\n",
- " \"id\": client._create_id(),\n",
- " \"object\": \"block\",\n",
- " \"type\": \"child_page\",\n",
- " \"created_time\": client._get_timestamp(),\n",
- " \"last_edited_time\": client._get_timestamp(),\n",
- " \"child_page\": {\"title\": page_name},\n",
- " }\n",
- "\n",
- " client.add_children(root_page_id, [child_block])\n",
- "\n",
- " return client\n",
- " else:\n",
- " # For real client, use provided API key or environment variable\n",
- " if api_key is None:\n",
- " api_key = os.getenv(\"NOTION_API_KEY\")\n",
- "\n",
- " if api_key is None:\n",
- " raise ValueError(\n",
- " \"api_key must be provided or set as NOTION_API_KEY environment variable\"\n",
- " )\n",
- "\n",
- " return NotionClient(auth=api_key)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "MockNotionClient(num_pages=0, num_databases=0, num_blocks=0)"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "# create the mock notion client\n",
- "mock_notion_client = NotionClientFactory.create(use_mock=True)\n",
- "mock_notion_client"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "the `initialize_project` adds the project pages too for you."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "MockNotionClient(num_pages=4, num_databases=0, num_blocks=0)"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "mock_notion_client = NotionClientFactory.create(\n",
- " use_mock=True, initialize_project=True, root_page_id=\"your_root_page_id\"\n",
- ")\n",
- "mock_notion_client"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "class NotionBackendFactory:\n",
- " \"\"\"Factory for creating NotionBackend instances.\"\"\"\n",
- "\n",
- " @staticmethod\n",
- " def create(\n",
- " root_page_id: str,\n",
- " use_mock: bool = False,\n",
- " api_key: t.Optional[str] = None,\n",
- " initialize_project: bool = False,\n",
- " notion_client: t.Optional[t.Union[NotionClient, MockNotionClient]] = None,\n",
- " ) -> NotionBackend:\n",
- " \"\"\"Create a NotionBackend instance.\n",
- "\n",
- " Args:\n",
- " root_page_id: The ID of the root page\n",
- " use_mock: If True, create a backend with a mock client\n",
- " api_key: Notion API key (only used for real client)\n",
- " initialize_project: If True and using mock, initialize project structure\n",
- " notion_client: Optional pre-configured Notion client\n",
- "\n",
- " Returns:\n",
- " NotionBackend: A backend instance with either real or mock client\n",
- " \"\"\"\n",
- " # Use provided client or create one\n",
- " if notion_client is None:\n",
- " notion_client = NotionClientFactory.create(\n",
- " use_mock=use_mock,\n",
- " api_key=api_key,\n",
- " initialize_project=initialize_project,\n",
- " root_page_id=root_page_id,\n",
- " )\n",
- "\n",
- " # Create and return the backend\n",
- " return NotionBackend(root_page_id=root_page_id, notion_client=notion_client)"
+ " return RagasApiClient(app_token=app_token, base_url=base_url)\n"
]
}
],
diff --git a/nbs/backends/mock_notion_client.ipynb b/nbs/backends/mock_notion_client.ipynb
deleted file mode 100644
index 1c26c2f..0000000
--- a/nbs/backends/mock_notion_client.ipynb
+++ /dev/null
@@ -1,328 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# `MockNotionClient`\n",
- "\n",
- "> Helps with testing `ragas_annotator` better."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | default_exp backends.mock_notion"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "import typing as t\n",
- "import uuid\n",
- "from copy import deepcopy\n",
- "from datetime import datetime\n",
- "\n",
- "from ragas_annotator.exceptions import NotFoundError"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "class MockPagesAPI:\n",
- " \"\"\"Mock implementation of notion_client.Client.pages\"\"\"\n",
- "\n",
- " def __init__(self, client):\n",
- " self.client = client\n",
- "\n",
- " def create(self, parent, properties, **kwargs):\n",
- " \"\"\"Create a new page.\"\"\"\n",
- " page_id = self.client._create_id()\n",
- "\n",
- " # Create the page object\n",
- " page = {\n",
- " \"id\": page_id,\n",
- " \"object\": \"page\",\n",
- " \"created_time\": self.client._get_timestamp(),\n",
- " \"last_edited_time\": self.client._get_timestamp(),\n",
- " \"archived\": False,\n",
- " \"properties\": deepcopy(properties),\n",
- " \"parent\": deepcopy(parent),\n",
- " }\n",
- "\n",
- " # Add page to storage\n",
- " self.client._pages[page_id] = page\n",
- "\n",
- " # Add child reference to parent\n",
- " parent_type = parent.get(\"type\")\n",
- " parent_id = parent.get(f\"{parent_type}_id\")\n",
- "\n",
- " if parent_id:\n",
- " child_block = {\n",
- " \"id\": self.client._create_id(),\n",
- " \"object\": \"block\",\n",
- " \"type\": \"child_page\",\n",
- " \"created_time\": self.client._get_timestamp(),\n",
- " \"last_edited_time\": self.client._get_timestamp(),\n",
- " \"child_page\": {\"title\": self._extract_title(properties)},\n",
- " }\n",
- "\n",
- " if parent_id not in self.client._children:\n",
- " self.client._children[parent_id] = []\n",
- "\n",
- " self.client._children[parent_id].append(child_block)\n",
- "\n",
- " return deepcopy(page)\n",
- "\n",
- " def retrieve(self, page_id):\n",
- " \"\"\"Retrieve a page by ID.\"\"\"\n",
- " if page_id not in self.client._pages:\n",
- " raise NotFoundError(f\"Page {page_id} not found\")\n",
- "\n",
- " return deepcopy(self.client._pages[page_id])\n",
- "\n",
- " def update(self, page_id, properties=None, archived=None, **kwargs):\n",
- " \"\"\"Update a page.\"\"\"\n",
- " if page_id not in self.client._pages:\n",
- " raise NotFoundError(f\"Page {page_id} not found\")\n",
- "\n",
- " page = self.client._pages[page_id]\n",
- "\n",
- " if properties:\n",
- " # Update properties\n",
- " for key, value in properties.items():\n",
- " page[\"properties\"][key] = deepcopy(value)\n",
- "\n",
- " if archived is not None:\n",
- " page[\"archived\"] = archived\n",
- "\n",
- " page[\"last_edited_time\"] = self.client._get_timestamp()\n",
- "\n",
- " return deepcopy(page)\n",
- "\n",
- " def _extract_title(self, properties):\n",
- " \"\"\"Extract page title from properties.\"\"\"\n",
- " for prop in properties.values():\n",
- " if prop.get(\"type\") == \"title\" and prop.get(\"title\"):\n",
- " for text_obj in prop[\"title\"]:\n",
- " if text_obj.get(\"type\") == \"text\" and \"content\" in text_obj.get(\n",
- " \"text\", {}\n",
- " ):\n",
- " return text_obj[\"text\"][\"content\"]\n",
- " return \"Untitled\""
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "class MockDatabasesAPI:\n",
- " \"\"\"Mock implementation of notion_client.Client.databases\"\"\"\n",
- "\n",
- " def __init__(self, client):\n",
- " self.client = client\n",
- "\n",
- " def create(self, parent, title, properties, **kwargs):\n",
- " \"\"\"Create a new database.\"\"\"\n",
- " database_id = self.client._create_id()\n",
- "\n",
- " # Create database object\n",
- " database = {\n",
- " \"id\": database_id,\n",
- " \"object\": \"database\",\n",
- " \"created_time\": self.client._get_timestamp(),\n",
- " \"last_edited_time\": self.client._get_timestamp(),\n",
- " \"title\": deepcopy(title),\n",
- " \"properties\": deepcopy(properties),\n",
- " \"parent\": deepcopy(parent),\n",
- " }\n",
- "\n",
- " # Add database to storage\n",
- " self.client._databases[database_id] = database\n",
- "\n",
- " # Add child reference to parent\n",
- " parent_type = parent.get(\"type\")\n",
- " parent_id = parent.get(f\"{parent_type}_id\")\n",
- "\n",
- " if parent_id:\n",
- " child_block = {\n",
- " \"id\": self.client._create_id(),\n",
- " \"object\": \"block\",\n",
- " \"type\": \"child_database\",\n",
- " \"created_time\": self.client._get_timestamp(),\n",
- " \"last_edited_time\": self.client._get_timestamp(),\n",
- " \"child_database\": {\"title\": self._extract_title(title)},\n",
- " }\n",
- "\n",
- " if parent_id not in self.client._children:\n",
- " self.client._children[parent_id] = []\n",
- "\n",
- " self.client._children[parent_id].append(child_block)\n",
- "\n",
- " return deepcopy(database)\n",
- "\n",
- " def retrieve(self, database_id):\n",
- " \"\"\"Retrieve a database by ID.\"\"\"\n",
- " if database_id not in self.client._databases:\n",
- " raise NotFoundError(f\"Database {database_id} not found\")\n",
- "\n",
- " return deepcopy(self.client._databases[database_id])\n",
- "\n",
- " def query(\n",
- " self,\n",
- " database_id,\n",
- " filter=None,\n",
- " sorts=None,\n",
- " start_cursor=None,\n",
- " page_size=100,\n",
- " **kwargs,\n",
- " ):\n",
- " \"\"\"Query a database.\"\"\"\n",
- " if database_id not in self.client._databases:\n",
- " raise NotFoundError(f\"Database {database_id} not found\")\n",
- "\n",
- " # Get all pages in the database\n",
- " results = []\n",
- " for page_id, page in self.client._pages.items():\n",
- " parent = page.get(\"parent\", {})\n",
- " if (\n",
- " parent.get(\"type\") == \"database_id\"\n",
- " and parent.get(\"database_id\") == database_id\n",
- " ):\n",
- " results.append(deepcopy(page))\n",
- "\n",
- " # TODO: Implement filtering, sorting, and pagination if needed\n",
- "\n",
- " return {\"results\": results, \"has_more\": False, \"next_cursor\": None}\n",
- "\n",
- " def _extract_title(self, title):\n",
- " \"\"\"Extract database title from title array.\"\"\"\n",
- " for text_obj in title:\n",
- " if text_obj.get(\"type\") == \"text\" and \"content\" in text_obj.get(\"text\", {}):\n",
- " return text_obj[\"text\"][\"content\"]\n",
- " return \"Untitled\""
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "class MockBlocksAPI:\n",
- " \"\"\"Mock implementation of notion_client.Client.blocks\"\"\"\n",
- "\n",
- " def __init__(self, client):\n",
- " self.client = client\n",
- " self.children = MockBlockChildrenAPI(client)\n",
- "\n",
- " def retrieve(self, block_id):\n",
- " \"\"\"Retrieve a block by ID.\"\"\"\n",
- " if block_id not in self.client._blocks:\n",
- " raise NotFoundError(f\"Block {block_id} not found\")\n",
- "\n",
- " return deepcopy(self.client._blocks[block_id])\n",
- "\n",
- "\n",
- "class MockBlockChildrenAPI:\n",
- " \"\"\"Mock implementation of notion_client.Client.blocks.children\"\"\"\n",
- "\n",
- " def __init__(self, client):\n",
- " self.client = client\n",
- "\n",
- " def list(self, block_id, start_cursor=None, page_size=100):\n",
- " \"\"\"List children of a block.\"\"\"\n",
- " children = self.client._children.get(block_id, [])\n",
- "\n",
- " # TODO: Implement pagination if needed\n",
- "\n",
- " return {\"results\": deepcopy(children), \"has_more\": False, \"next_cursor\": None}"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "class MockNotionClient:\n",
- " \"\"\"Mock implementation of notion_client.Client for testing.\"\"\"\n",
- "\n",
- " def __init__(self, auth=None):\n",
- " \"\"\"Initialize the mock client with in-memory storage.\n",
- "\n",
- " Args:\n",
- " auth: Ignored in mock implementation\n",
- " \"\"\"\n",
- " # In-memory storage\n",
- " self._pages = {} # page_id -> page object\n",
- " self._databases = {} # database_id -> database object\n",
- " self._blocks = {} # block_id -> block object\n",
- " self._children = {} # parent_id -> list of child blocks\n",
- "\n",
- " # Create API namespaces to match real client\n",
- " self.pages = MockPagesAPI(self)\n",
- " self.databases = MockDatabasesAPI(self)\n",
- " self.blocks = MockBlocksAPI(self)\n",
- "\n",
- " def _get_timestamp(self):\n",
- " \"\"\"Generate a timestamp in Notion API format.\"\"\"\n",
- " return datetime.utcnow().isoformat() + \"Z\"\n",
- "\n",
- " def _create_id(self):\n",
- " \"\"\"Generate a random ID in Notion format.\"\"\"\n",
- " return str(uuid.uuid4()).replace(\"-\", \"\")\n",
- "\n",
- " def add_page(self, page_data):\n",
- " \"\"\"Add a page to the mock storage.\"\"\"\n",
- " self._pages[page_data[\"id\"]] = deepcopy(page_data)\n",
- "\n",
- " def add_database(self, database_data):\n",
- " \"\"\"Add a database to the mock storage.\"\"\"\n",
- " self._databases[database_data[\"id\"]] = deepcopy(database_data)\n",
- "\n",
- " def add_block(self, block_data):\n",
- " \"\"\"Add a block to the mock storage.\"\"\"\n",
- " self._blocks[block_data[\"id\"]] = deepcopy(block_data)\n",
- "\n",
- " def add_children(self, parent_id, children):\n",
- " \"\"\"Add children to a parent.\"\"\"\n",
- " if parent_id not in self._children:\n",
- " self._children[parent_id] = []\n",
- " self._children[parent_id].extend(deepcopy(children))\n",
- "\n",
- " def __str__(self):\n",
- " return \"MockNotionClient(num_pages={}, num_databases={}, num_blocks={})\".format(\n",
- " len(self._pages), len(self._databases), len(self._blocks)\n",
- " )\n",
- "\n",
- " __repr__ = __str__"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "python3",
- "language": "python",
- "name": "python3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/nbs/backends/notion.ipynb b/nbs/backends/notion.ipynb
deleted file mode 100644
index b6da42b..0000000
--- a/nbs/backends/notion.ipynb
+++ /dev/null
@@ -1,637 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# `NotionBackend`\n",
- "\n",
- "> `Project` uses this backend to interact with the Notion API."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | default_exp backends.notion_backend"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | hide\n",
- "from nbdev.showdoc import *"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "import typing as t\n",
- "import os\n",
- "from datetime import datetime\n",
- "import uuid\n",
- "\n",
- "from notion_client import Client as NotionClient\n",
- "from fastcore.utils import patch_to, patch\n",
- "\n",
- "from ragas_annotator.exceptions import DuplicateError, NotFoundError"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "class NotionBackend:\n",
- " \"\"\"A backend for interacting with the Notion API\"\"\"\n",
- "\n",
- " def __init__(\n",
- " self, root_page_id: str, notion_client: t.Optional[NotionClient] = None\n",
- " ):\n",
- " self.root_page_id = root_page_id\n",
- " if notion_client is None:\n",
- " self.client = NotionClient(auth=os.getenv(\"NOTION_API_KEY\"))\n",
- " else:\n",
- " self.client = notion_client\n",
- "\n",
- " def __repr__(self):\n",
- " return f\"NotionBackend(root_page_id={self.root_page_id})\"\n",
- "\n",
- " def validate_project_structure(self, root_page_id):\n",
- " \"\"\"\n",
- " Validate the project structure by checking if the root page exists and has the correct sub-pages.\n",
- " Structure is as follows:\n",
- " - Root Page\n",
- " - Datasets\n",
- " - Experiments\n",
- " - Comparisons\n",
- " \"\"\"\n",
- " # Check if root page exists\n",
- " if not self.page_exists(root_page_id):\n",
- " return False\n",
- "\n",
- " # Search for required sub-pages under root\n",
- " required_pages = {\"Datasets\", \"Experiments\", \"Comparisons\"}\n",
- " found_pages = set()\n",
- "\n",
- " # Search for child pages\n",
- " children = self.client.blocks.children.list(root_page_id)\n",
- " for block in children[\"results\"]:\n",
- " if block[\"type\"] == \"child_page\":\n",
- " found_pages.add(block[\"child_page\"][\"title\"])\n",
- "\n",
- " # Verify all required pages exist\n",
- " return required_pages.issubset(found_pages)\n",
- "\n",
- " def create_new_page(self, parent_page_id, page_name) -> str:\n",
- " \"\"\"\n",
- " Create a new page inside the given parent page and return the page id.\n",
- "\n",
- " Args:\n",
- " parent_page_id (str): The ID of the parent page\n",
- " page_name (str): The title for the new page\n",
- "\n",
- " Returns:\n",
- " str: The ID of the newly created page\n",
- "\n",
- " Raises:\n",
- " ValueError: If the parent page does not exist\n",
- " \"\"\"\n",
- " # First check if parent page exists\n",
- " if not self.page_exists(parent_page_id):\n",
- " raise ValueError(f\"Parent page {parent_page_id} does not exist\")\n",
- "\n",
- " # Create a new child page\n",
- " response = self.client.pages.create(\n",
- " parent={\"type\": \"page_id\", \"page_id\": parent_page_id},\n",
- " properties={\"title\": [{\"type\": \"text\", \"text\": {\"content\": page_name}}]},\n",
- " )\n",
- "\n",
- " # Return the ID of the newly created page\n",
- " return response[\"id\"]\n",
- "\n",
- " def page_exists(self, page_id):\n",
- " \"\"\"Check if a page exists by attempting to retrieve it.\"\"\"\n",
- " try:\n",
- " self.client.pages.retrieve(page_id)\n",
- " return True\n",
- " except:\n",
- " return False\n",
- "\n",
- " def create_new_database(\n",
- " self, parent_page_id: str, title: str, properties: dict\n",
- " ) -> str:\n",
- " \"\"\"Create a new database inside the given parent page.\n",
- "\n",
- " Args:\n",
- " parent_page_id (str): The ID of the parent page\n",
- " title (str): The title for the new database\n",
- " properties (dict): The database properties definition\n",
- "\n",
- " Returns:\n",
- " str: The ID of the newly created database\n",
- " \"\"\"\n",
- " response = self.client.databases.create(\n",
- " parent={\"type\": \"page_id\", \"page_id\": parent_page_id},\n",
- " title=[{\"type\": \"text\", \"text\": {\"content\": title}}],\n",
- " properties=properties,\n",
- " )\n",
- " return response[\"id\"]"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Lets create a new project in Notion with the `NotionBackend` class. In order to get the root_page_id you can click on the \"share\" button on the page you want to use as the root page. This will give a link like this:\n",
- "\n",
- "```\n",
- "https://www.notion.so/ragas/nbdev_notion_annotator-1b05d9bf94ff8092b52ae8d676e6abf2?pvs=4\n",
- "```\n",
- "\n",
- "The page id is the uuid after the last dash (here it is `1b05d9bf94ff8092b52ae8d676e6abf2`)."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "NotionBackend(root_page_id=your_root_page_id)"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "root_page_id = \"your_root_page_id\"\n",
- "\n",
- "notion_backend = NotionBackend(root_page_id)\n",
- "notion_backend"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | hide\n",
- "\n",
- "# uncomment if you want to test this on the notion page \"nbdev_notion_annotator\"\n",
- "# root_page_id = \"1b05d9bf94ff8092b52ae8d676e6abf2\"\n",
- "# notion_backend = NotionBackend(root_page_id)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | hide\n",
- "from unittest.mock import MagicMock"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | hide\n",
- "\n",
- "# patch\n",
- "mocked_notion_client = MagicMock()\n",
- "mocked_notion_client.blocks.children.list.return_value = {\n",
- " \"results\": [\n",
- " # {\"type\": \"child_page\", \"child_page\": {\"title\": \"Datasets\"}},\n",
- " # {\"type\": \"child_page\", \"child_page\": {\"title\": \"Experiments\"}},\n",
- " # {\"type\": \"child_page\", \"child_page\": {\"title\": \"Comparisons\"}}\n",
- " ]\n",
- "}\n",
- "notion_backend = NotionBackend(root_page_id, mocked_notion_client)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "False"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "notion_backend.validate_project_structure(root_page_id)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | hide\n",
- "# | test\n",
- "\n",
- "# mock page creation\n",
- "mocked_notion_client.pages.create.return_value = {\"id\": \"1234567890\"}"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# create the structure inside the project root page\n",
- "pages = [\"Datasets\", \"Experiments\", \"Comparisons\"]\n",
- "for page in pages:\n",
- " notion_backend.create_new_page(root_page_id, page)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now you should have the following structure for your project:\n",
- "\n",
- "```\n",
- " (Root Project Page)\n",
- " ├── Datasets\n",
- " ├── Experiments\n",
- " └─ Comparisons\n",
- "```\n",
- "\n",
- "and `notion_backend.validate_project_structure(root_page_id)` should return `True`"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | hide\n",
- "# | test\n",
- "\n",
- "# now that we have the structure, lets return the correct structure\n",
- "mocked_notion_client.blocks.children.list.return_value = {\n",
- " \"results\": [\n",
- " {\"type\": \"child_page\", \"child_page\": {\"title\": \"Datasets\"}},\n",
- " {\"type\": \"child_page\", \"child_page\": {\"title\": \"Experiments\"}},\n",
- " {\"type\": \"child_page\", \"child_page\": {\"title\": \"Comparisons\"}},\n",
- " ]\n",
- "}"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "True"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "notion_backend.validate_project_structure(root_page_id)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "@t.overload\n",
- "def get_page_id(\n",
- " self, parent_id: str, page_name: str, return_multiple: t.Literal[False] = False\n",
- ") -> str: ...\n",
- "@t.overload\n",
- "def get_page_id(\n",
- " self, parent_id: str, page_name: str, return_multiple: t.Literal[True]\n",
- ") -> t.List[str]: ...\n",
- "@patch_to(NotionBackend)\n",
- "def get_page_id(\n",
- " self, parent_id: str, page_name: str, return_multiple: bool = False\n",
- ") -> t.Union[str, t.List[str]]:\n",
- " \"\"\"Get page ID(s) by name under a parent page.\n",
- "\n",
- " Args:\n",
- " parent_id (str): The ID of the parent page to search under\n",
- " page_name (str): The title of the page to find\n",
- " return_multiple (bool): If True, returns all matching page IDs\n",
- "\n",
- " Returns:\n",
- " Union[str, List[str]]: Single page ID or list of page IDs\n",
- "\n",
- " Raises:\n",
- " DuplicateError: If return_multiple is False and multiple pages found\n",
- " ValueError: If no pages found\n",
- " \"\"\"\n",
- " matching_pages = []\n",
- " next_cursor = None\n",
- "\n",
- " while True:\n",
- " # Get page of results, using cursor if we have one\n",
- " response = self.client.blocks.children.list(parent_id, start_cursor=next_cursor)\n",
- "\n",
- " # Check each block in current page\n",
- " for block in response[\"results\"]:\n",
- " if (\n",
- " block[\"type\"] == \"child_page\"\n",
- " and block[\"child_page\"][\"title\"] == page_name\n",
- " ):\n",
- " matching_pages.append(block[\"id\"])\n",
- "\n",
- " # Check if there are more results\n",
- " if not response.get(\"has_more\", False):\n",
- " break\n",
- "\n",
- " next_cursor = response.get(\"next_cursor\")\n",
- "\n",
- " if not matching_pages:\n",
- " raise NotFoundError(f\"No page found with name '{page_name}'\")\n",
- "\n",
- " if return_multiple:\n",
- " return matching_pages\n",
- " else:\n",
- " if len(matching_pages) > 1:\n",
- " raise DuplicateError(f\"Multiple pages found with name '{page_name}'\")\n",
- " return matching_pages[0]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "@t.overload\n",
- "def get_database_id(\n",
- " self, parent_page_id: str, name: str, return_multiple: t.Literal[False] = False\n",
- ") -> str: ...\n",
- "\n",
- "\n",
- "@t.overload\n",
- "def get_database_id(\n",
- " self, parent_page_id: str, name: str, return_multiple: t.Literal[True]\n",
- ") -> t.List[str]: ...\n",
- "\n",
- "\n",
- "@patch_to(NotionBackend)\n",
- "def get_database_id(\n",
- " self, parent_page_id: str, name: str, return_multiple: bool = False\n",
- ") -> t.Union[str, t.List[str]]:\n",
- " \"\"\"Get the database ID(s) by name under a parent page.\n",
- "\n",
- " Args:\n",
- " parent_page_id (str): The ID of the parent page to search under\n",
- " name (str): The name of the database to find\n",
- " return_multiple (bool): If True, returns all matching database IDs\n",
- "\n",
- " Returns:\n",
- " Union[str, List[str]]: Single database ID or list of database IDs\n",
- "\n",
- " Raises:\n",
- " NotFoundError: If no database found with given name\n",
- " DuplicateError: If return_multiple is False and multiple databases found\n",
- " \"\"\"\n",
- " matching_databases = []\n",
- " next_cursor = None\n",
- "\n",
- " while True:\n",
- " response = self.client.blocks.children.list(\n",
- " parent_page_id, start_cursor=next_cursor\n",
- " )\n",
- "\n",
- " for block in response[\"results\"]:\n",
- " if block[\"type\"] == \"child_database\":\n",
- " database = self.client.databases.retrieve(database_id=block[\"id\"])\n",
- " if database[\"title\"][0][\"plain_text\"].lower() == name.lower():\n",
- " matching_databases.append(block[\"id\"])\n",
- "\n",
- " if not response.get(\"has_more\", False):\n",
- " break\n",
- "\n",
- " next_cursor = response.get(\"next_cursor\")\n",
- "\n",
- " if not matching_databases:\n",
- " raise NotFoundError(f\"No database found with name '{name}'\")\n",
- "\n",
- " if return_multiple:\n",
- " return matching_databases\n",
- " else:\n",
- " if len(matching_databases) > 1:\n",
- " raise DuplicateError(f\"Multiple databases found with name '{name}'\")\n",
- " return matching_databases[0]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "@patch\n",
- "def create_page_in_database(\n",
- " self: NotionBackend,\n",
- " database_id: str,\n",
- " properties: dict,\n",
- " parent: t.Optional[dict] = None,\n",
- ") -> dict:\n",
- " \"\"\"Create a new page in a database.\n",
- "\n",
- " Args:\n",
- " database_id: The ID of the database to create the page in\n",
- " properties: The page properties\n",
- " parent: Optional parent object (defaults to database parent)\n",
- "\n",
- " Returns:\n",
- " dict: The created page object\n",
- " \"\"\"\n",
- " if parent is None:\n",
- " parent = {\"type\": \"database_id\", \"database_id\": database_id}\n",
- "\n",
- " # Remove any unique_id properties as they cannot be updated directly\n",
- " filtered_properties = {\n",
- " k: v\n",
- " for k, v in properties.items()\n",
- " if not (isinstance(v, dict) and v.get(\"type\") == \"unique_id\")\n",
- " }\n",
- "\n",
- " response = self.client.pages.create(parent=parent, properties=filtered_properties)\n",
- "\n",
- " return response"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "@patch\n",
- "def get_database(self: NotionBackend, database_id: str) -> dict:\n",
- " \"\"\"Get a database by ID.\n",
- "\n",
- " Args:\n",
- " database_id: The ID of the database to retrieve\n",
- "\n",
- " Returns:\n",
- " dict: The database object\n",
- " \"\"\"\n",
- " return self.client.databases.retrieve(database_id=database_id)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "@patch\n",
- "def query_database(\n",
- " self: NotionBackend,\n",
- " database_id: str,\n",
- " filter: t.Optional[dict] = None,\n",
- " sorts: t.Optional[t.List[dict]] = None,\n",
- " archived: bool = False,\n",
- ") -> dict:\n",
- " \"\"\"Query a database with optional filtering and sorting.\n",
- "\n",
- " Args:\n",
- " database_id: The ID of the database to query\n",
- " filter: Optional filter conditions\n",
- " sorts: Optional sort conditions\n",
- " archived: If True, include archived pages. If False, only return non-archived pages\n",
- "\n",
- " Returns:\n",
- " dict: Query response containing all results\n",
- " \"\"\"\n",
- " query_params = {\n",
- " \"database_id\": database_id,\n",
- " \"page_size\": 100, # Maximum allowed by Notion API\n",
- " }\n",
- "\n",
- " if filter:\n",
- " query_params[\"filter\"] = filter\n",
- " if sorts:\n",
- " query_params[\"sorts\"] = sorts\n",
- "\n",
- " # Initialize results\n",
- " all_results = []\n",
- " has_more = True\n",
- " next_cursor = None\n",
- "\n",
- " # Fetch all pages\n",
- " while has_more:\n",
- " if next_cursor:\n",
- " query_params[\"start_cursor\"] = next_cursor\n",
- "\n",
- " response = self.client.databases.query(**query_params)\n",
- "\n",
- " # Filter results based on archived status\n",
- " filtered_results = [\n",
- " page\n",
- " for page in response[\"results\"]\n",
- " if page.get(\"archived\", False) == archived\n",
- " ]\n",
- " all_results.extend(filtered_results)\n",
- "\n",
- " has_more = response.get(\"has_more\", False)\n",
- " next_cursor = response.get(\"next_cursor\")\n",
- "\n",
- " # Return combined results\n",
- " return {\"results\": all_results, \"has_more\": False, \"next_cursor\": None}"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# | export\n",
- "@patch\n",
- "def update_page(\n",
- " self: NotionBackend,\n",
- " page_id: str,\n",
- " properties: t.Optional[t.Dict[str, t.Any]] = None,\n",
- " archived: bool = False,\n",
- ") -> dict:\n",
- " \"\"\"Update a page's properties and/or archive status.\n",
- "\n",
- " Args:\n",
- " page_id: The ID of the page to update\n",
- " properties: Optional properties to update\n",
- " archived: Whether to archive the page\n",
- "\n",
- " Returns:\n",
- " dict: The updated page object\n",
- " \"\"\"\n",
- " update_params = {\"page_id\": page_id}\n",
- "\n",
- " if properties:\n",
- " # Remove any unique_id properties as they cannot be updated directly\n",
- " filtered_properties = {\n",
- " k: v\n",
- " for k, v in properties.items()\n",
- " if not (isinstance(v, dict) and v.get(\"type\") == \"unique_id\")\n",
- " }\n",
- " update_params[\"properties\"] = filtered_properties\n",
- "\n",
- " if archived:\n",
- " update_params[\"archived\"] = True # type: ignore\n",
- "\n",
- " return self.client.pages.update(**update_params)"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "python3",
- "language": "python",
- "name": "python3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/nbs/backends/ragas_api_client.ipynb b/nbs/backends/ragas_api_client.ipynb
index 0157579..33e9f9f 100644
--- a/nbs/backends/ragas_api_client.ipynb
+++ b/nbs/backends/ragas_api_client.ipynb
@@ -9,6 +9,13 @@
"> Python client to api.ragas.io"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -37,12 +44,8 @@
"#| export\n",
"import httpx\n",
"import asyncio\n",
- "import functools\n",
"import typing as t\n",
- "import inspect\n",
"from pydantic import BaseModel, Field\n",
- "from enum import StrEnum\n",
- "import uuid\n",
"from fastcore.utils import patch"
]
},
@@ -701,7 +704,17 @@
"outputs": [],
"source": [
"#| export\n",
- "class ColumnType(StrEnum):\n",
+ "from enum import Enum"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "class ColumnType(str, Enum):\n",
" NUMBER = \"number\"\n",
" TEXT = \"text\"\n",
" LONG_TEXT = \"longText\"\n",
@@ -1374,6 +1387,63 @@
"nano_id"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "import uuid\n",
+ "import string"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def create_nano_id(size=12):\n",
+ " # Define characters to use (alphanumeric)\n",
+ " alphabet = string.ascii_letters + string.digits\n",
+ " \n",
+ " # Generate UUID and convert to int\n",
+ " uuid_int = uuid.uuid4().int\n",
+ " \n",
+ " # Convert to base62\n",
+ " result = \"\"\n",
+ " while uuid_int:\n",
+ " uuid_int, remainder = divmod(uuid_int, len(alphabet))\n",
+ " result = alphabet[remainder] + result\n",
+ " \n",
+ " # Pad if necessary and return desired length\n",
+ " return result[:size]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'Anvz5k9geU7T'"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Usage\n",
+ "nano_id = create_nano_id() # e.g., \"8dK9cNw3mP5x\"\n",
+ "nano_id"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/nbs/dataset.ipynb b/nbs/dataset.ipynb
index 545c331..9ad3005 100644
--- a/nbs/dataset.ipynb
+++ b/nbs/dataset.ipynb
@@ -41,8 +41,9 @@
"\n",
"from fastcore.utils import patch\n",
"\n",
- "from ragas_annotator.model.notion_model import NotionModel\n",
- "from ragas_annotator.backends.notion_backend import NotionBackend"
+ "from ragas_experimental.model.pydantic_model import ExtendedPydanticBaseModel as BaseModel\n",
+ "from ragas_experimental.utils import create_nano_id, async_to_sync\n",
+ "from ragas_experimental.backends.ragas_api_client import RagasApiClient"
]
},
{
@@ -52,71 +53,91 @@
"outputs": [],
"source": [
"# | export\n",
- "NotionModelType = t.TypeVar(\"NotionModelType\", bound=NotionModel)\n",
+ "BaseModelType = t.TypeVar(\"BaseModelType\", bound=BaseModel)\n",
"\n",
- "\n",
- "class Dataset(t.Generic[NotionModelType]):\n",
- " \"\"\"A list-like interface for managing NotionModel instances in a Notion database.\"\"\"\n",
+ "class Dataset(t.Generic[BaseModelType]):\n",
+ " \"\"\"A list-like interface for managing dataset entries with backend synchronization.\n",
+ " \n",
+ " This class behaves like a Python list while synchronizing operations with the\n",
+ " Ragas backend API.\n",
+ " \"\"\"\n",
"\n",
" def __init__(\n",
" self,\n",
" name: str,\n",
- " model: t.Type[NotionModel],\n",
- " database_id: str,\n",
- " notion_backend: NotionBackend,\n",
+ " model: t.Type[BaseModel],\n",
+ " project_id: str,\n",
+ " dataset_id: str,\n",
+ " ragas_api_client: RagasApiClient,\n",
" ):\n",
" self.name = name\n",
" self.model = model\n",
- " self.database_id = database_id\n",
- " self._notion_backend = notion_backend\n",
- " self._entries: t.List[NotionModelType] = []\n",
+ " self.project_id = project_id\n",
+ " self.dataset_id = dataset_id\n",
+ " self._ragas_api_client = ragas_api_client\n",
+ " self._entries: t.List[BaseModelType] = []\n",
+ "\n",
+ " # Initialize column mapping if it doesn't exist yet\n",
+ " if not hasattr(self.model, \"__column_mapping__\"):\n",
+ " self.model.__column_mapping__ = {}\n",
+ " \n",
+ " # Get column mappings from API and update the model's mapping\n",
+ " column_id_map = self._get_column_id_map(dataset_id=dataset_id)\n",
+ " \n",
+ " # Update the model's column mapping with the values from the API\n",
+ " for field_name, column_id in column_id_map.items():\n",
+ " self.model.__column_mapping__[field_name] = column_id\n",
+ "\n",
+ " def _get_column_id_map(self: \"Dataset\", dataset_id: str) -> dict:\n",
+ " \"\"\"Get a map of column name to column id\"\"\"\n",
+ " sync_func = async_to_sync(self._ragas_api_client.list_dataset_columns)\n",
+ " columns = sync_func(project_id=self.project_id, dataset_id=dataset_id)\n",
+ " column_id_map = {column[\"name\"]: column[\"id\"] for column in columns[\"items\"]}\n",
+ "\n",
+ " # add the column id map to the model, selectively overwriting existing column mapping\n",
+ " for field in self.model.__column_mapping__.keys():\n",
+ " if field in column_id_map:\n",
+ " self.model.__column_mapping__[field] = column_id_map[field]\n",
+ " return column_id_map\n",
"\n",
" def __getitem__(\n",
" self, key: t.Union[int, slice]\n",
- " ) -> t.Union[NotionModelType, \"Dataset[NotionModelType]\"]:\n",
+ " ) -> t.Union[BaseModelType, \"Dataset[BaseModelType]\"]:\n",
" \"\"\"Get an entry by index or slice.\"\"\"\n",
" if isinstance(key, slice):\n",
" new_dataset = type(self)(\n",
" name=self.name,\n",
" model=self.model,\n",
- " database_id=self.database_id,\n",
- " notion_backend=self._notion_backend,\n",
+ " project_id=self.project_id,\n",
+ " dataset_id=self.dataset_id,\n",
+ " ragas_api_client=self._ragas_api_client,\n",
" )\n",
" new_dataset._entries = self._entries[key]\n",
" return new_dataset\n",
" else:\n",
" return self._entries[key]\n",
"\n",
- " def __setitem__(self, index: int, entry: NotionModelType) -> None:\n",
- " \"\"\"Update an entry at the given index and sync to Notion.\"\"\"\n",
+ " def __setitem__(self, index: int, entry: BaseModelType) -> None:\n",
+ " \"\"\"Update an entry at the given index and sync to backend.\"\"\"\n",
" if not isinstance(entry, self.model):\n",
" raise TypeError(f\"Entry must be an instance of {self.model.__name__}\")\n",
"\n",
- " # Get existing entry to get Notion page ID\n",
+ " # Get existing entry to get its ID\n",
" existing = self._entries[index]\n",
- " if not hasattr(existing, \"_page_id\"):\n",
- " raise ValueError(\"Existing entry has no page_id\")\n",
- "\n",
- " # Update in Notion\n",
- " assert (\n",
- " existing._page_id is not None\n",
- " ) # mypy fails to infer that we check for it above\n",
- " response = self._notion_backend.update_page(\n",
- " page_id=existing._page_id, properties=entry.to_notion()[\"properties\"]\n",
- " )\n",
- "\n",
- " # Update local cache with response data\n",
- " self._entries[index] = self.model.from_notion(response)\n",
+ " \n",
+ " # Update in backend\n",
+ " self.save(entry)\n",
+ " \n",
+ " # Update local cache\n",
+ " self._entries[index] = entry\n",
"\n",
" def __repr__(self) -> str:\n",
- " return (\n",
- " f\"Dataset(name={self.name}, model={self.model.__name__}, len={len(self)})\"\n",
- " )\n",
+ " return f\"Dataset(name={self.name}, model={self.model.__name__}, len={len(self)})\"\n",
"\n",
" def __len__(self) -> int:\n",
" return len(self._entries)\n",
"\n",
- " def __iter__(self) -> t.Iterator[NotionModelType]:\n",
+ " def __iter__(self) -> t.Iterator[BaseModelType]:\n",
" return iter(self._entries)"
]
},
@@ -127,10 +148,21 @@
"outputs": [],
"source": [
"# | hide\n",
- "import ragas_annotator.model.notion_typing as nmt\n",
- "from ragas_annotator.backends.mock_notion import MockNotionClient\n",
- "from ragas_annotator.backends.factory import NotionClientFactory\n",
- "from ragas_annotator.backends.notion_backend import NotionBackend"
+ "import ragas_experimental.typing as rt\n",
+ "from ragas_experimental.backends.factory import RagasApiClientFactory"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# test model\n",
+ "class TestModel(BaseModel):\n",
+ " id: int\n",
+ " name: str\n",
+ " description: str"
]
},
{
@@ -141,7 +173,7 @@
{
"data": {
"text/plain": [
- "TestModel(name='test' description='test description')"
+ "{'id': 'id', 'name': 'name', 'description': 'description'}"
]
},
"execution_count": null,
@@ -150,45 +182,110 @@
}
],
"source": [
- "# test model\n",
- "class TestModel(NotionModel):\n",
- " id: int = nmt.ID()\n",
- " name: str = nmt.Title()\n",
- " description: str = nmt.Text()\n",
- "\n",
- "\n",
- "test_model = TestModel(name=\"test\", description=\"test description\")\n",
+ "TestModel.__column_mapping__"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "TestModel(id=0, name='test', description='test description')"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "test_model = TestModel(id=0, name=\"test\", description=\"test description\")\n",
"test_model"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'id': 'id', 'name': 'name', 'description': 'description'}"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "test_model.__column_mapping__"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
- "# | hide\n",
- "# Set up a test environment with mock Notion client and a test database.\n",
- "# root page id\n",
- "root_page_id = \"test-root-id\"\n",
- "# Create a mock client\n",
- "mock_client = NotionClientFactory.create(\n",
- " use_mock=True, initialize_project=True, root_page_id=root_page_id\n",
- ")\n",
- "\n",
- "# Create NotionBackend with mock client\n",
- "backend = NotionBackend(root_page_id=root_page_id, notion_client=mock_client)\n",
- "\n",
- "# get the page id of the datasets page\n",
- "dataset_page_id = backend.get_page_id(parent_id=root_page_id, page_name=\"Datasets\")\n",
+ "import os"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "RAGAS_APP_TOKEN = \"apt.47bd-c55e4a45b27c-02f8-8446-1441f09b-651a8\"\n",
+ "RAGAS_API_BASE_URL = \"https://api.dev.app.ragas.io\"\n",
"\n",
- "# create a new database in the datasets page\n",
- "properties = {}\n",
- "for _, field in TestModel._fields.items():\n",
- " properties.update(field._to_notion_property())\n",
- "datasets_id = backend.create_new_database(\n",
- " parent_page_id=dataset_page_id, title=\"TestModel\", properties=properties\n",
- ")"
+ "os.environ[\"RAGAS_APP_TOKEN\"] = RAGAS_APP_TOKEN\n",
+ "os.environ[\"RAGAS_API_BASE_URL\"] = RAGAS_API_BASE_URL"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "ragas_api_client = RagasApiClientFactory.create()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'id': '0fee5330-9f6e-44a9-a85c-e3b947b697de',\n",
+ " 'name': 'TestModel',\n",
+ " 'description': None,\n",
+ " 'created_at': '2025-04-10T22:41:36.582714+00:00',\n",
+ " 'updated_at': '2025-04-10T22:41:36.582714+00:00',\n",
+ " 'version_counter': 0,\n",
+ " 'project_id': 'bbe45632-3268-43a6-9694-b020b3f5226f'}"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# https://dev.app.ragas.io/dashboard/projects/0a7c4ecb-b313-4bb0-81c0-852c9634ce03/datasets/a4f0d169-ebce-4a2b-b758-0ff49c0c4312\n",
+ "TEST_PROJECT_ID = \"bbe45632-3268-43a6-9694-b020b3f5226f\"\n",
+ "TEST_DATASET_ID = \"0fee5330-9f6e-44a9-a85c-e3b947b697de\"\n",
+ "test_project = await ragas_api_client.get_project(project_id=TEST_PROJECT_ID)\n",
+ "test_dataset = await ragas_api_client.get_dataset(project_id=TEST_PROJECT_ID, dataset_id=TEST_DATASET_ID)\n",
+ "test_dataset"
]
},
{
@@ -198,10 +295,30 @@
"outputs": [],
"source": [
"dataset = Dataset(\n",
- " name=\"TestModel\", model=TestModel, database_id=datasets_id, notion_backend=backend\n",
+ " name=\"TestModel\", model=TestModel, project_id=TEST_PROJECT_ID, dataset_id=TEST_DATASET_ID, ragas_api_client=ragas_api_client\n",
")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'id': 'fSh3ESKGNZLf', 'description': 'ftKwfDqnbinL', 'name': 'fVnsYEnvxARh'}"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "dataset.model.__column_mapping__"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -210,19 +327,32 @@
"source": [
"# | export\n",
"@patch\n",
- "def append(self: Dataset, entry: NotionModelType) -> None:\n",
+ "def append(self: Dataset, entry: BaseModelType) -> None:\n",
" \"\"\"Add a new entry to the dataset and sync to Notion.\"\"\"\n",
- " # if not isinstance(entry, self.model):\n",
- " # raise TypeError(f\"Entry must be an instance of {self.model.__name__}\")\n",
- "\n",
- " # Create in Notion and get response\n",
- " response = self._notion_backend.create_page_in_database(\n",
- " database_id=self.database_id, properties=entry.to_notion()[\"properties\"]\n",
+ " # Create row inside the table\n",
+ "\n",
+ " # first get the columns for the dataset\n",
+ " column_id_map = self.model.__column_mapping__\n",
+ "\n",
+ " # create the rows\n",
+ " row_dict = entry.model_dump()\n",
+ " row_id = create_nano_id()\n",
+ " row_data = {}\n",
+ " for key, value in row_dict.items():\n",
+ " if key in column_id_map:\n",
+ " row_data[column_id_map[key]] = value\n",
+ "\n",
+ " sync_func = async_to_sync(self._ragas_api_client.create_dataset_row)\n",
+ " response = sync_func(\n",
+ " project_id=self.project_id,\n",
+ " dataset_id=self.dataset_id,\n",
+ " id=row_id,\n",
+ " data=row_data,\n",
" )\n",
- "\n",
+ " # add the row id to the entry\n",
+ " entry._row_id = response[\"id\"]\n",
" # Update entry with Notion data (like ID)\n",
- " updated_entry = self.model.from_notion(response)\n",
- " self._entries.append(updated_entry)"
+ " self._entries.append(entry)"
]
},
{
@@ -233,7 +363,7 @@
{
"data": {
"text/plain": [
- "1"
+ "3"
]
},
"execution_count": null,
@@ -264,15 +394,17 @@
"source": [
"# | export\n",
"@patch\n",
- "def pop(self: Dataset, index: int = -1) -> NotionModelType:\n",
+ "def pop(self: Dataset, index: int = -1) -> BaseModelType:\n",
" \"\"\"Remove and return entry at index, sync deletion to Notion.\"\"\"\n",
" entry = self._entries[index]\n",
- " if not hasattr(entry, \"_page_id\"):\n",
- " raise ValueError(\"Entry has no page_id\")\n",
+ " # get the row id\n",
+ " row_id = entry._row_id\n",
+ " if row_id is None:\n",
+ " raise ValueError(\"Entry has no row id. This likely means it was not added or synced to the dataset.\")\n",
"\n",
- " # Archive in Notion (soft delete)\n",
- " assert entry._page_id is not None # mypy fails to infer that we check for it above\n",
- " self._notion_backend.update_page(page_id=entry._page_id, archived=True)\n",
+ " # soft delete the row\n",
+ " sync_func = async_to_sync(self._ragas_api_client.delete_dataset_row)\n",
+ " sync_func(project_id=self.project_id, dataset_id=self.dataset_id, row_id=row_id)\n",
"\n",
" # Remove from local cache\n",
" return self._entries.pop(index)"
@@ -286,7 +418,7 @@
{
"data": {
"text/plain": [
- "0"
+ "1"
]
},
"execution_count": null,
@@ -318,18 +450,37 @@
"# | export\n",
"@patch\n",
"def load(self: Dataset) -> None:\n",
- " \"\"\"Load all entries from the Notion database.\"\"\"\n",
- " # Query the database\n",
- " response = self._notion_backend.query_database(\n",
- " database_id=self.database_id, archived=False\n",
+ " \"\"\"Load all entries from the backend API.\"\"\"\n",
+ " # Get all rows\n",
+ " sync_func = async_to_sync(self._ragas_api_client.list_dataset_rows)\n",
+ " response = sync_func(\n",
+ " project_id=self.project_id,\n",
+ " dataset_id=self.dataset_id\n",
" )\n",
- "\n",
+ " \n",
+ " # Get column mapping (ID -> name)\n",
+ " column_map = {v: k for k, v in self.model.__column_mapping__.items()}\n",
+ " \n",
" # Clear existing entries\n",
" self._entries.clear()\n",
- "\n",
- " # Convert results to model instances\n",
- " for page in response.get(\"results\", []):\n",
- " entry = self.model.from_notion(page)\n",
+ " \n",
+ " # Process rows\n",
+ " for row in response.get(\"items\", []):\n",
+ " model_data = {}\n",
+ " row_id = row.get(\"id\")\n",
+ " \n",
+ " # Convert from API data format to model fields\n",
+ " for col_id, value in row.get(\"data\", {}).items():\n",
+ " if col_id in column_map:\n",
+ " field_name = column_map[col_id]\n",
+ " model_data[field_name] = value\n",
+ " \n",
+ " # Create model instance\n",
+ " entry = self.model(**model_data)\n",
+ " \n",
+ " # Store row ID for future operations\n",
+ " entry._row_id = row_id\n",
+ " \n",
" self._entries.append(entry)"
]
},
@@ -342,6 +493,39 @@
"dataset.load()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# | export\n",
+ "@patch\n",
+ "def load_as_dicts(self: Dataset) -> t.List[t.Dict]:\n",
+ " \"\"\"Load all entries as dictionaries.\"\"\"\n",
+ " # Get all rows\n",
+ " sync_func = async_to_sync(self._ragas_api_client.list_dataset_rows)\n",
+ " response = sync_func(\n",
+ " project_id=self.project_id,\n",
+ " dataset_id=self.dataset_id\n",
+ " )\n",
+ " \n",
+ " # Get column mapping (ID -> name)\n",
+ " column_map = {v: k for k, v in self.model.__column_mapping__.items()}\n",
+ " \n",
+ " # Convert to dicts with field names\n",
+ " result = []\n",
+ " for row in response.get(\"items\", []):\n",
+ " item_dict = {}\n",
+ " for col_id, value in row.get(\"data\", {}).items():\n",
+ " if col_id in column_map:\n",
+ " field_name = column_map[col_id]\n",
+ " item_dict[field_name] = value\n",
+ " result.append(item_dict)\n",
+ " \n",
+ " return result"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -350,7 +534,8 @@
{
"data": {
"text/plain": [
- "3"
+ "[{'id': 0, 'name': 'test', 'description': 'test description'},\n",
+ " {'id': 0, 'name': 'test', 'description': 'test description'}]"
]
},
"execution_count": null,
@@ -359,9 +544,28 @@
}
],
"source": [
- "for i in range(3):\n",
- " dataset.append(test_model)\n",
- "len(dataset)"
+ "dataset.load_as_dicts()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# | export\n",
+ "@patch\n",
+ "def to_pandas(self: Dataset) -> \"pd.DataFrame\":\n",
+ " \"\"\"Convert dataset to pandas DataFrame.\"\"\"\n",
+ " import pandas as pd\n",
+ " \n",
+ " # Make sure we have data\n",
+ " if not self._entries:\n",
+ " self.load()\n",
+ " \n",
+ " # Convert entries to dictionaries\n",
+ " data = [entry.model_dump() for entry in self._entries]\n",
+ " return pd.DataFrame(data)"
]
},
{
@@ -371,8 +575,51 @@
"outputs": [
{
"data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " id | \n",
+ " name | \n",
+ " description | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 0 | \n",
+ " test | \n",
+ " test description | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "