changes for few shot learning

Author: shahules786

nbs/metric/base.ipynb

@@ -40,21 +40,35 @@
     "from abc import ABC, abstractmethod\n",
     "import asyncio\n",
     "from dataclasses import dataclass, field\n",
+    "import datasets\n",
     "from pydantic import BaseModel\n",
     "import typing as t\n",
+    "import json\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "from ragas_annotator.prompt.base import Prompt\n",
+    "from ragas_annotator.embedding.base import RagasEmbedding\n",
     "from ragas_annotator.metric import MetricResult\n",
     "from ragas_annotator.llm import RagasLLM\n",
+    "from ragas_annotator.project.core import Project\n",
+    "from ragas_annotator.model.notion_model import NotionModel\n",
+    "from ragas_annotator.prompt.dynamic_few_shot import DynamicFewShotPrompt\n",
+    "\n",
     "\n",
     "@dataclass\n",
     "class Metric(ABC):\n",
     "    \"\"\"Base class for all metrics in the LLM evaluation library.\"\"\"\n",
     "    name: str\n",
-    "    prompt: str\n",
+    "    prompt: str | Prompt\n",
     "    llm: RagasLLM\n",
     "    _response_models: t.Dict[bool, t.Type[BaseModel]] = field(\n",
     "        default_factory=dict, init=False, repr=False\n",
     "    )\n",
     "    \n",
+    "    def __post_init__(self):\n",
+    "        if isinstance(self.prompt,str):\n",
+    "            self.prompt = Prompt(self.prompt)\n",
+    "    \n",
     "    @abstractmethod\n",
     "    def _get_response_model(self, with_reasoning: bool) -> t.Type[BaseModel]:\n",
     "        \"\"\"Get the appropriate response model.\"\"\"\n",
@@ -67,22 +81,32 @@
     "    \n",
     "    def score(self, reasoning: bool = True, n: int = 1, **kwargs) -> t.Any:\n",
     "        responses = []\n",
+    "        traces = {}\n",
+    "        traces[\"input\"] = kwargs\n",
     "        prompt_input = self.prompt.format(**kwargs)\n",
     "        for _ in range(n):\n",
     "            response = self.llm.generate(prompt_input, response_model = self._get_response_model(reasoning)) \n",
+    "            traces['output'] = response.model_dump()\n",
     "            response = MetricResult(**response.model_dump())\n",
     "            responses.append(response)\n",
-    "        return self._ensemble(responses)\n",
+    "        results = self._ensemble(responses)\n",
+    "        results.traces = traces\n",
+    "        return results\n",
     "\n",
     "\n",
     "    async def ascore(self, reasoning: bool = True, n: int = 1, **kwargs) -> MetricResult:\n",
     "        responses = []  # Added missing initialization\n",
+    "        traces = {}\n",
+    "        traces[\"input\"] = kwargs\n",
     "        prompt_input = self.prompt.format(**kwargs)\n",
     "        for _ in range(n):\n",
     "            response = await self.llm.agenerate(prompt_input, response_model = self._get_response_model(reasoning))\n",
+    "            traces['output'] = response.model_dump()\n",
     "            response = MetricResult(**response.model_dump())  # Fixed missing parentheses\n",
     "            responses.append(response)\n",
-    "        return self._ensemble(responses)\n",
+    "        results = self._ensemble(responses)\n",
+    "        results.traces = traces\n",
+    "        return results\n",
     "        \n",
     "    def batch_score(self, inputs: t.List[t.Dict[str, t.Any]], reasoning: bool = True, n: int = 1) -> t.List[t.Any]:\n",
     "        return [self.score(reasoning, n, **input_dict) for input_dict in inputs]\n",
@@ -95,9 +119,33 @@
     "            \n",
     "        # Run all tasks concurrently and return results\n",
     "        return await asyncio.gather(*async_tasks)\n",
-    "            \n",
+    "    \n",
+    "    def train(self,project:Project, experiment_names: t.List[str], model:NotionModel, embedding_model: RagasEmbedding,method: t.Dict[str, t.Any]):\n",
+    "        \n",
+    "        assert isinstance(self.prompt, Prompt)\n",
+    "        self.prompt = DynamicFewShotPrompt.from_prompt(self.prompt,embedding_model)\n",
+    "        datasets = []\n",
+    "        for experiment_name in experiment_names:\n",
+    "            experiment_data = project.get_experiment(experiment_name,model)\n",
+    "            experiment_data.load()\n",
+    "            datasets.append(experiment_data)\n",
+    "        \n",
+    "        total_items = sum([len(dataset) for dataset in datasets])\n",
+    "        with tqdm(total=total_items, desc=\"Processing examples\") as pbar:\n",
+    "            for dataset in datasets:\n",
+    "                for row in dataset:\n",
+    "                    if hasattr(row, f'{self.name}_traces'):\n",
+    "                        traces = json.loads(getattr(row, f'{self.name}_traces'))\n",
+    "                        if traces:\n",
+    "                            self.prompt.add_example(traces['input'],traces['output'])\n",
+    "                    pbar.update(1)\n",
+    "        \n",
+    "                \n",
+    "                \n",
+    "        \n",
+    "        \n",
     "        \n",
-    "        "
+    "                "
    ]
   },
   {

nbs/metric/decorator.ipynb

@@ -66,15 +66,6 @@
     "            \n",
     "            @dataclass\n",
     "            class CustomMetric(metric_class):\n",
-    "                def _extract_result(self, result, reasoning: bool):\n",
-    "                    \"\"\"Extract score and reason from the result.\"\"\"\n",
-    "                    if isinstance(result, tuple) and len(result) == 2:\n",
-    "                        score, reason = result\n",
-    "                    else:\n",
-    "                        score, reason = result, None\n",
-    "                    \n",
-    "                    # Use \"result\" instead of \"score\" for the new MetricResult implementation\n",
-    "                    return MetricResult(result=score, reason=reason if reasoning else None)\n",
     "                \n",
     "                def _run_sync_in_async(self, func, *args, **kwargs):\n",
     "                    \"\"\"Run a synchronous function in an async context.\"\"\"\n",
@@ -101,7 +92,7 @@
     "                            # Sync function implementation\n",
     "                            result = func(self.llm, self.prompt, **kwargs)\n",
     "                        \n",
-    "                        return self._extract_result(result, reasoning)\n",
+    "                        return result\n",
     "                    except Exception as e:\n",
     "                        # Handle errors gracefully\n",
     "                        error_msg = f\"Error executing metric {self.name}: {str(e)}\"\n",
@@ -120,7 +111,7 @@
     "                    else:\n",
     "                        # For sync functions, run normally\n",
     "                        result = self._run_sync_in_async(func, self.llm, self.prompt, **kwargs)\n",
-    "                        return self._extract_result(result, reasoning)\n",
+    "                        return result\n",
     "            \n",
     "            # Create the metric instance with all parameters\n",
     "            metric_instance = CustomMetric(\n",
@@ -168,7 +159,7 @@
     "#| eval: false\n",
     "\n",
     "\n",
-    "from ragas_annotator.metric import DiscreteMetric\n",
+    "from ragas_annotator.metric import DiscreteMetric, MetricResult\n",
     "from pydantic import BaseModel\n",
     "\n",
     "from ragas_annotator.llm import ragas_llm\n",
@@ -193,7 +184,7 @@
     "            score = 'low'\n",
     "        else:\n",
     "            score = 'high'\n",
-    "        return score,\"reason\"\n",
+    "        return MetricResult(result=score, reason=response.reason)\n",
     "\n",
     "result = my_metric.score(response='my response') # result\n",
     "print(result)\n",

nbs/metric/discrete.ipynb

@@ -21,16 +21,7 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/opt/homebrew/Caskroom/miniforge/base/envs/random/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "#| export\n",
     "import typing as t\n",
@@ -99,8 +90,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "low\n",
-      "The response does not provide any specific information or context that can help evaluate its helpfulness.\n"
+      "med\n",
+      "The response 'this is my response' is somewhat helpful because it acknowledges receipt or presence of a response but lacks specific information or relevance that would make it more beneficial or informative.\n"
      ]
     }
    ],
@@ -143,13 +134,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "high\n",
-      "reason\n"
+      "None\n",
+      "Error executing metric new_metric: MetricResult.__init__() got an unexpected keyword argument 'traces'\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/var/folders/ww/sk5dkfhn673234cmy5w7008r0000gn/T/ipykernel_31199/2702091914.py:15: PydanticDeprecatedSince20: The `dict` method is deprecated; use `model_dump` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/\n",
+      "  traces['output'] = response.dict()\n"
      ]
     }
    ],
    "source": [
     "#| eval: false\n",
+    "from ragas_annotator.metric.result import MetricResult\n",
+    "\n",
     "@discrete_metric(llm=llm,\n",
     "    prompt=\"Evaluate if given answer is helpful\\n\\n{response}\",\n",
     "    name='new_metric',values=[\"low\",\"med\",\"high\"])\n",
@@ -158,14 +159,17 @@
     "        class response_model(BaseModel):\n",
     "             output: t.List[bool]\n",
     "             reason: str\n",
-    "        \n",
+    "        traces = {}\n",
+    "        traces['input'] = kwargs\n",
     "        response = llm.generate(prompt.format(**kwargs),response_model=response_model)\n",
+    "        traces['output'] = response.dict()\n",
     "        total = sum(response.output)\n",
     "        if total < 1:\n",
     "            score = 'low'\n",
     "        else:\n",
     "            score = 'high'\n",
-    "        return score,\"reason\"\n",
+    "            \n",
+    "        return MetricResult(result=score,reason=response.reason,traces=traces)\n",
     "\n",
     "result = my_metric.score(response='my response') # result\n",
     "print(result)\n",

nbs/metric/numeric.ipynb

@@ -147,6 +147,7 @@
    "source": [
     "\n",
     "#| eval: false\n",
+    "from ragas_annotator.metric import MetricResult\n",
     "\n",
     "@numeric_metric(llm=llm,\n",
     "    prompt=\"Evaluate if given answer is helpful\\n\\n{response}\",\n",
@@ -157,13 +158,16 @@
     "             output: int\n",
     "             reason: str\n",
     "        \n",
+    "        traces = {}\n",
+    "        traces['input'] = kwargs\n",
     "        response = llm.generate(prompt.format(**kwargs),response_model=response_model)\n",
+    "        traces['output'] = response.dict()\n",
     "        total = response.output\n",
     "        if total < 1:\n",
     "            score = 0\n",
     "        else:\n",
     "            score = 10\n",
-    "        return score,\"reason\"\n",
+    "        return MetricResult(result=score,reason=response.reason,traces=traces)\n",
     "\n",
     "result = my_metric.score(response='my response') # result\n",
     "result # 10\n",

nbs/metric/ranking.ipynb

@@ -185,6 +185,7 @@
    "source": [
     "#| eval: false\n",
     "\n",
+    "from ragas_annotator.metric import MetricResult\n",
     "\n",
     "@ranking_metric(\n",
     "    llm=llm,  # Your language model instance\n",
@@ -197,7 +198,7 @@
     "    # For example, process the prompt (formatted with candidates) and produce a ranking.\n",
     "    ranking = [1, 0, 2]  # Dummy ranking: second candidate is best, then first, then third.\n",
     "    reason = \"Ranked based on response clarity and detail.\"\n",
-    "    return ranking, reason\n",
+    "    return MetricResult(result=ranking, reason=reason)\n",
     "\n",
     "# Using the decorator-based ranking metric:\n",
     "result = my_ranking_metric.score(candidates=[\n",

nbs/metric/result.ipynb

@@ -46,9 +46,14 @@
     "    - RankingMetrics (list results)\n",
     "    \"\"\"\n",
     "    \n",
-    "    def __init__(self, result: t.Any, reason: t.Optional[str] = None):\n",
+    "    def __init__(self, result: t.Any, reason: t.Optional[str] = None, traces: t.Optional[t.Dict[str, t.Any]] = None):\n",
+    "        if traces is not None:\n",
+    "            invalid_keys = [key for key in traces.keys() if key not in {\"input\", \"output\"}]\n",
+    "            if invalid_keys:\n",
+    "                raise ValueError(f\"Invalid keys in traces: {invalid_keys}. Allowed keys are 'input' and 'output'.\")\n",
     "        self._result = result\n",
     "        self.reason = reason\n",
+    "        self.traces = traces\n",
     "    \n",
     "    def __repr__(self):\n",
     "        return repr(self._result)\n",

nbs/prompt/dynamic_few_shot.ipynb

@@ -194,10 +194,11 @@
     "        TypeError\n",
     "            If inputs or output is not a dictionary\n",
     "        \"\"\"\n",
-    "        \n",
-    "        self.examples.append((inputs, output))\n",
+    "        if (inputs, output) not in self.examples:\n",
+    "            self.examples.append((inputs, output))\n",
+    "            \n",
     "        # Add to example store\n",
-    "        if self.example_store:\n",
+    "        if isinstance(self.example_store, ExampleStore) and (inputs, output) not in self.example_store._examples:\n",
     "            self.example_store.add_example(inputs, output)\n",
     "    \n",
     "    @classmethod\n",
@@ -230,7 +231,24 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Evaluate if given answer Regularly updating your software reduces the risk of vulnerabilities. is same as expected answer Keeping software up to date helps patch known security flaws and prevents exploits.\n",
+      "\n",
+      "Examples:\n",
+      "\n",
+      "Example 1:\n",
+      "Input:\n",
+      "response: Using two-factor authentication greatly enhances account security.\n",
+      "expected_answer: Two-factor authentication adds a layer of protection by requiring a second form of identity verification.\n",
+      "Output:\n",
+      "score: fail\n"
+     ]
+    }
+   ],
    "source": [
     "#| eval: false\n",
     "from ragas_annotator.prompt import Prompt\n",
@@ -255,22 +273,46 @@
     "prompt = DynamicFewShotPrompt.from_prompt(\n",
     "    prompt,\n",
     "    embedding_model=embedding,\n",
-    "    num_examples=3\n",
+    "    num_examples=1\n",
+    ")\n",
+    "\n",
+    "prompt.add_example(\n",
+    "    {\n",
+    "        \"response\": \"Bananas are high in potassium and great for quick energy.\",\n",
+    "        \"expected_answer\": \"Bananas provide potassium and are a good source of fast-digesting carbohydrates.\"\n",
+    "    },\n",
+    "    {\"score\": \"pass\"}\n",
     ")\n",
     "\n",
+    "prompt.add_example(\n",
+    "    {\n",
+    "        \"response\": \"Using two-factor authentication greatly enhances account security.\",\n",
+    "        \"expected_answer\": \"Two-factor authentication adds a layer of protection by requiring a second form of identity verification.\"\n",
+    "    },\n",
+    "    {\"score\": \"fail\"}\n",
+    ")\n",
+    "\n",
+    "\n",
     "prompt.example_store.get_examples(\n",
-    "    {\"response\": \"You can get a full refund if you miss your flight.\", \"expected_answer\": \"Refunds depend on ticket type; only refundable tickets qualify for full refunds.\"})"
+    "{\n",
+    "        \"response\": \"Regularly updating your software reduces the risk of vulnerabilities.\",\n",
+    "        \"expected_answer\": \"Keeping software up to date helps patch known security flaws and prevents exploits.\"\n",
+    "    })\n",
+    "\n",
+    "print(prompt.format(**{\n",
+    "        \"response\": \"Regularly updating your software reduces the risk of vulnerabilities.\",\n",
+    "        \"expected_answer\": \"Keeping software up to date helps patch known security flaws and prevents exploits.\"\n",
+    "    }))"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
- "metadata": {},
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
  "nbformat": 4,
  "nbformat_minor": 2
 }