Merge pull request #2 from explodinggradients/feat/vanilla-exp

added experiment without langfuse

Author: jjmachan

README.md

@@ -22,10 +22,6 @@ $ pip install ragas_annotator
 
 ## Getting Started
 
-Fill me in please! Don’t forget code examples:
-
-``` python
-1 + 1
-```
-
-    2
+First lets init a
+[`Project`](https://explodinggradients.github.io/ragas_annotator/project/core.html#project)
+from notion.

nbs/index.ipynb

@@ -57,35 +57,8 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Fill me in please! Don't forget code examples:"
+    "First lets init a `Project` from notion."
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "2"
-      ]
-     },
-     "execution_count": null,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "1 + 1"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

nbs/project/core.ipynb

@@ -42,6 +42,7 @@
     "from fastcore.utils import patch\n",
     "\n",
     "from ragas_annotator.backends.notion_backend import NotionBackend\n",
+    "from ragas_annotator.backends.factory import NotionBackendFactory\n",
     "from ragas_annotator.model.notion_model import NotionModel\n",
     "import ragas_annotator.model.notion_typing as nmt\n",
     "from ragas_annotator.dataset import Dataset\n",
@@ -81,10 +82,17 @@
     "            if notion_root_page_id is None:\n",
     "                raise ValueError(\"NOTION_ROOT_PAGE_ID is not set\")\n",
     "\n",
-    "            self._notion_backend = NotionBackend(\n",
-    "                notion_client=NotionClient(auth=notion_api_key),\n",
-    "                root_page_id=notion_root_page_id,\n",
-    "            )\n",
+    "            if notion_api_key == \"TEST\":\n",
+    "                self._notion_backend = NotionBackendFactory.create(\n",
+    "                    root_page_id=notion_root_page_id,\n",
+    "                    use_mock=True,\n",
+    "                    initialize_project=True,\n",
+    "                )\n",
+    "            else:\n",
+    "                self._notion_backend = NotionBackend(\n",
+    "                    notion_client=NotionClient(auth=notion_api_key),\n",
+    "                    root_page_id=notion_root_page_id,\n",
+    "                )\n",
     "        else:\n",
     "            self._notion_backend = notion_backend\n",
     "\n",

nbs/project/experiments.ipynb

@@ -159,12 +159,13 @@
    "source": [
     "# | export\n",
     "@patch\n",
-    "def langfuse_experiment(\n",
+    "def experiment(\n",
     "    self: Project, experiment_model: t.Type[NotionModel], name_prefix: str = \"\"\n",
     "):\n",
-    "    \"\"\"Decorator for creating experiment functions.\n",
+    "    \"\"\"Decorator for creating experiment functions without Langfuse integration.\n",
     "\n",
     "    Args:\n",
+    "        experiment_model: The NotionModel type to use for experiment results\n",
     "        name_prefix: Optional prefix for experiment names\n",
     "\n",
     "    Returns:\n",
@@ -174,11 +175,8 @@
     "    def decorator(func: t.Callable) -> ExperimentProtocol:\n",
     "        @wraps(func)\n",
     "        async def wrapped_experiment(*args, **kwargs):\n",
-    "            # wrap the function with langfuse observation so that it can be traced\n",
-    "            # and spans inside the function can be retrieved with sync_trace()\n",
-    "            observed_func = observe(name=f\"{name_prefix}-{func.__name__}\")(func)\n",
-    "\n",
-    "            return await observed_func(*args, **kwargs)\n",
+    "            # Simply call the function without Langfuse observation\n",
+    "            return await func(*args, **kwargs)\n",
     "\n",
     "        # Add run method to the wrapped function\n",
     "        async def run_async(dataset: Dataset, name: t.Optional[str] = None):\n",
@@ -196,7 +194,8 @@
     "            for future in tqdm(asyncio.as_completed(tasks), total=len(tasks)):\n",
     "                result = await future\n",
     "                # Add each result to experiment view as it completes\n",
-    "                results.append(result) if result is not None else None\n",
+    "                if result is not None:\n",
+    "                    results.append(result)\n",
     "\n",
     "            # upload results to experiment view\n",
     "            experiment_view = self.create_experiment(name=name, model=experiment_model)\n",
@@ -208,6 +207,57 @@
     "        wrapped_experiment.__setattr__(\"run_async\", run_async)\n",
     "        return t.cast(ExperimentProtocol, wrapped_experiment)\n",
     "\n",
+    "    return decorator\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# | export\n",
+    "@patch\n",
+    "def langfuse_experiment(\n",
+    "    self: Project, experiment_model: t.Type[NotionModel], name_prefix: str = \"\"\n",
+    "):\n",
+    "    \"\"\"Decorator for creating experiment functions with Langfuse integration.\n",
+    "\n",
+    "    Args:\n",
+    "        experiment_model: The NotionModel type to use for experiment results\n",
+    "        name_prefix: Optional prefix for experiment names\n",
+    "\n",
+    "    Returns:\n",
+    "        Decorator function that wraps experiment functions with Langfuse observation\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def decorator(func: t.Callable) -> ExperimentProtocol:\n",
+    "        # First, create a base experiment wrapper\n",
+    "        base_experiment = self.experiment(experiment_model, name_prefix)(func)\n",
+    "        \n",
+    "        # Override the wrapped function to add Langfuse observation\n",
+    "        @wraps(func)\n",
+    "        async def wrapped_with_langfuse(*args, **kwargs):\n",
+    "            # wrap the function with langfuse observation\n",
+    "            observed_func = observe(name=f\"{name_prefix}-{func.__name__}\")(func)\n",
+    "            return await observed_func(*args, **kwargs)\n",
+    "        \n",
+    "        # Replace the async function to use Langfuse\n",
+    "        original_run_async = base_experiment.run_async\n",
+    "        \n",
+    "        # Use the original run_async but with the Langfuse-wrapped function\n",
+    "        async def run_async_with_langfuse(dataset: Dataset, name: t.Optional[str] = None):\n",
+    "            # Override the internal wrapped_experiment with our Langfuse version\n",
+    "            base_experiment.__wrapped__ = wrapped_with_langfuse\n",
+    "            \n",
+    "            # Call the original run_async which will now use our Langfuse-wrapped function\n",
+    "            return await original_run_async(dataset, name)\n",
+    "        \n",
+    "        # Replace the run_async method\n",
+    "        base_experiment.__setattr__(\"run_async\", run_async_with_langfuse)\n",
+    "        \n",
+    "        return t.cast(ExperimentProtocol, base_experiment)\n",
+    "\n",
     "    return decorator"
    ]
   },

ragas_annotator/_modidx.py

@@ -376,6 +376,8 @@
                                                                                                                            'ragas_annotator/project/experiments.py'),
                                                      'ragas_annotator.project.experiments.Project.create_experiment': ( 'project/experiments.html#project.create_experiment',
                                                                                                                         'ragas_annotator/project/experiments.py'),
+                                                     'ragas_annotator.project.experiments.Project.experiment': ( 'project/experiments.html#project.experiment',
+                                                                                                                 'ragas_annotator/project/experiments.py'),
                                                      'ragas_annotator.project.experiments.Project.get_experiment': ( 'project/experiments.html#project.get_experiment',
                                                                                                                      'ragas_annotator/project/experiments.py'),
                                                      'ragas_annotator.project.experiments.Project.langfuse_experiment': ( 'project/experiments.html#project.langfuse_experiment',

ragas_annotator/project/core.py

@@ -13,6 +13,7 @@
 from fastcore.utils import patch
 
 from ..backends.notion_backend import NotionBackend
+from ..backends.factory import NotionBackendFactory
 from ..model.notion_model import NotionModel
 import ragas_annotator.model.notion_typing as nmt
 from ..dataset import Dataset
@@ -45,10 +46,17 @@ def __init__(
             if notion_root_page_id is None:
                 raise ValueError("NOTION_ROOT_PAGE_ID is not set")
 
-            self._notion_backend = NotionBackend(
-                notion_client=NotionClient(auth=notion_api_key),
-                root_page_id=notion_root_page_id,
-            )
+            if notion_api_key == "TEST":
+                self._notion_backend = NotionBackendFactory.create(
+                    root_page_id=notion_root_page_id,
+                    use_mock=True,
+                    initialize_project=True,
+                )
+            else:
+                self._notion_backend = NotionBackend(
+                    notion_client=NotionClient(auth=notion_api_key),
+                    root_page_id=notion_root_page_id,
+                )
         else:
             self._notion_backend = notion_backend
 

ragas_annotator/project/experiments.py

@@ -90,12 +90,13 @@ async def run_async(self, name: str, dataset: Dataset): ...
 
 # %% ../../nbs/project/experiments.ipynb 9
 @patch
-def langfuse_experiment(
+def experiment(
     self: Project, experiment_model: t.Type[NotionModel], name_prefix: str = ""
 ):
-    """Decorator for creating experiment functions.
+    """Decorator for creating experiment functions without Langfuse integration.
 
     Args:
+        experiment_model: The NotionModel type to use for experiment results
         name_prefix: Optional prefix for experiment names
 
     Returns:
@@ -105,11 +106,8 @@ def langfuse_experiment(
     def decorator(func: t.Callable) -> ExperimentProtocol:
         @wraps(func)
         async def wrapped_experiment(*args, **kwargs):
-            # wrap the function with langfuse observation so that it can be traced
-            # and spans inside the function can be retrieved with sync_trace()
-            observed_func = observe(name=f"{name_prefix}-{func.__name__}")(func)
-
-            return await observed_func(*args, **kwargs)
+            # Simply call the function without Langfuse observation
+            return await func(*args, **kwargs)
 
         # Add run method to the wrapped function
         async def run_async(dataset: Dataset, name: t.Optional[str] = None):
@@ -127,7 +125,8 @@ async def run_async(dataset: Dataset, name: t.Optional[str] = None):
             for future in tqdm(asyncio.as_completed(tasks), total=len(tasks)):
                 result = await future
                 # Add each result to experiment view as it completes
-                results.append(result) if result is not None else None
+                if result is not None:
+                    results.append(result)
 
             # upload results to experiment view
             experiment_view = self.create_experiment(name=name, model=experiment_model)
@@ -140,3 +139,48 @@ async def run_async(dataset: Dataset, name: t.Optional[str] = None):
         return t.cast(ExperimentProtocol, wrapped_experiment)
 
     return decorator
+
+
+# %% ../../nbs/project/experiments.ipynb 10
+@patch
+def langfuse_experiment(
+    self: Project, experiment_model: t.Type[NotionModel], name_prefix: str = ""
+):
+    """Decorator for creating experiment functions with Langfuse integration.
+
+    Args:
+        experiment_model: The NotionModel type to use for experiment results
+        name_prefix: Optional prefix for experiment names
+
+    Returns:
+        Decorator function that wraps experiment functions with Langfuse observation
+    """
+
+    def decorator(func: t.Callable) -> ExperimentProtocol:
+        # First, create a base experiment wrapper
+        base_experiment = self.experiment(experiment_model, name_prefix)(func)
+        
+        # Override the wrapped function to add Langfuse observation
+        @wraps(func)
+        async def wrapped_with_langfuse(*args, **kwargs):
+            # wrap the function with langfuse observation
+            observed_func = observe(name=f"{name_prefix}-{func.__name__}")(func)
+            return await observed_func(*args, **kwargs)
+        
+        # Replace the async function to use Langfuse
+        original_run_async = base_experiment.run_async
+        
+        # Use the original run_async but with the Langfuse-wrapped function
+        async def run_async_with_langfuse(dataset: Dataset, name: t.Optional[str] = None):
+            # Override the internal wrapped_experiment with our Langfuse version
+            base_experiment.__wrapped__ = wrapped_with_langfuse
+            
+            # Call the original run_async which will now use our Langfuse-wrapped function
+            return await original_run_async(dataset, name)
+        
+        # Replace the run_async method
+        base_experiment.__setattr__("run_async", run_async_with_langfuse)
+        
+        return t.cast(ExperimentProtocol, base_experiment)
+
+    return decorator