feat: deeper helicon integration (#1196)

based on #1182

---------

Co-authored-by: Justin <[email protected]>

Author: jjmachan

docs/howtos/integrations/helicone.ipynb

@@ -53,15 +53,22 @@
     "from datasets import Dataset\n",
     "from ragas import evaluate\n",
     "from ragas.metrics import faithfulness, answer_relevancy, context_precision\n",
+    "from ragas.integrations.helicone import helicone_config  # import helicone_config\n",
+    "\n",
     "\n",
     "# Set up Helicone\n",
-    "HELICONE_API_KEY = \"your_helicone_api_key_here\"  # Replace with your actual Helicone API key\n",
-    "os.environ[\"OPENAI_API_BASE\"] = f\"https://oai.helicone.ai/{HELICONE_API_KEY}/v1\"\n",
-    "os.environ[\"OPENAI_API_KEY\"] = \"your_openai_api_key_here\"  # Replace with your actual OpenAI API key\n",
+    "helicone_config.api_key = (\n",
+    "    \"your_helicone_api_key_here\"  # Replace with your actual Helicone API key\n",
+    ")\n",
+    "os.environ[\n",
+    "    \"OPENAI_API_KEY\"\n",
+    "] = \"your_openai_api_key_here\"  # Replace with your actual OpenAI API key\n",
     "\n",
     "# Verify Helicone API key is set\n",
     "if HELICONE_API_KEY == \"your_helicone_api_key_here\":\n",
-    "    raise ValueError(\"Please replace 'your_helicone_api_key_here' with your actual Helicone API key.\")"
+    "    raise ValueError(\n",
+    "        \"Please replace 'your_helicone_api_key_here' with your actual Helicone API key.\"\n",
+    "    )"
    ]
   },
   {
@@ -80,13 +87,23 @@
    "outputs": [],
    "source": [
     "data_samples = {\n",
-    "    'question': ['When was the first Super Bowl?', 'Who has won the most Super Bowls?'],\n",
-    "    'answer': ['The first Super Bowl was held on January 15, 1967.', 'The New England Patriots have won the most Super Bowls, with six championships.'],\n",
-    "    'contexts': [\n",
-    "        ['The First AFL–NFL World Championship Game, later known as Super Bowl I, was played on January 15, 1967, at the Los Angeles Memorial Coliseum in Los Angeles, California.'],\n",
-    "        ['As of 2021, the New England Patriots have won the most Super Bowls with six championships, all under the leadership of quarterback Tom Brady and head coach Bill Belichick.']\n",
+    "    \"question\": [\"When was the first Super Bowl?\", \"Who has won the most Super Bowls?\"],\n",
+    "    \"answer\": [\n",
+    "        \"The first Super Bowl was held on January 15, 1967.\",\n",
+    "        \"The New England Patriots have won the most Super Bowls, with six championships.\",\n",
+    "    ],\n",
+    "    \"contexts\": [\n",
+    "        [\n",
+    "            \"The First AFL–NFL World Championship Game, later known as Super Bowl I, was played on January 15, 1967, at the Los Angeles Memorial Coliseum in Los Angeles, California.\"\n",
+    "        ],\n",
+    "        [\n",
+    "            \"As of 2021, the New England Patriots have won the most Super Bowls with six championships, all under the leadership of quarterback Tom Brady and head coach Bill Belichick.\"\n",
+    "        ],\n",
+    "    ],\n",
+    "    \"ground_truth\": [\n",
+    "        \"The first Super Bowl was held on January 15, 1967.\",\n",
+    "        \"The New England Patriots have won the most Super Bowls, with six championships as of 2021.\",\n",
     "    ],\n",
-    "    'ground_truth': ['The first Super Bowl was held on January 15, 1967.', 'The New England Patriots have won the most Super Bowls, with six championships as of 2021.']\n",
     "}\n",
     "\n",
     "dataset = Dataset.from_dict(data_samples)\n",

src/ragas/evaluation.py

@@ -19,6 +19,7 @@
 )
 from ragas.exceptions import ExceptionInRunner
 from ragas.executor import Executor
+from ragas.integrations.helicone import helicone_config
 from ragas.llms import llm_factory
 from ragas.llms.base import BaseRagasLLM, LangchainLLMWrapper
 from ragas.metrics._answer_correctness import AnswerCorrectness
@@ -136,6 +137,12 @@ def evaluate(
     column_map = column_map or {}
     callbacks = callbacks or []
 
+    if helicone_config.is_enabled:
+        import uuid
+
+        helicone_config.session_name = "ragas-evaluation"
+        helicone_config.session_id = str(uuid.uuid4())
+
     if dataset is None:
         raise ValueError("Provide dataset!")
 

src/ragas/integrations/helicone.py

@@ -0,0 +1,101 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
+
+
+@dataclass
+class CacheConfig:
+    ttl: int = 60 * 60 * 24 * 30  # 30 days
+    maxsize: int = 1000
+
+
+@dataclass
+class HeliconeSingleton:
+    api_key: Optional[str] = None
+    base_url: Optional[str] = "https://oai.helicone.ai"
+    cache_config: Optional[CacheConfig] = None
+    _instance: Optional["HeliconeSingleton"] = None
+
+    # New fields for configurable headers
+    target_url: Optional[str] = None
+    openai_api_base: Optional[str] = None
+    request_id: Optional[str] = None
+    model_override: Optional[str] = None
+    prompt_id: Optional[str] = None
+    user_id: Optional[str] = None
+    fallbacks: Optional[str] = None
+    rate_limit_policy: Optional[str] = None
+    session_id: Optional[str] = None
+    session_path: Optional[str] = None
+    session_name: Optional[str] = None
+    posthog_key: Optional[str] = None
+    posthog_host: Optional[str] = None
+    omit_response: Optional[bool] = None
+    omit_request: Optional[bool] = None
+    cache_enabled: Optional[bool] = None
+    retry_enabled: Optional[bool] = None
+    moderations_enabled: Optional[bool] = None
+    llm_security_enabled: Optional[bool] = None
+    stream_force_format: Optional[bool] = None
+    custom_properties: Dict[str, str] = field(default_factory=dict)
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def default_headers(self) -> Dict[str, Any]:
+        headers = {"Helicone-Auth": f"Bearer {self.api_key}"}
+
+        if self.target_url:
+            headers["Helicone-Target-URL"] = self.target_url
+        if self.openai_api_base:
+            headers["Helicone-OpenAI-Api-Base"] = self.openai_api_base
+        if self.request_id:
+            headers["Helicone-Request-Id"] = self.request_id
+        if self.model_override:
+            headers["Helicone-Model-Override"] = self.model_override
+        if self.prompt_id:
+            headers["Helicone-Prompt-Id"] = self.prompt_id
+        if self.user_id:
+            headers["Helicone-User-Id"] = self.user_id
+        if self.fallbacks:
+            headers["Helicone-Fallbacks"] = self.fallbacks
+        if self.rate_limit_policy:
+            headers["Helicone-RateLimit-Policy"] = self.rate_limit_policy
+        if self.session_id:
+            headers["Helicone-Session-Id"] = self.session_id
+        if self.session_path:
+            headers["Helicone-Session-Path"] = self.session_path
+        if self.session_name:
+            headers["Helicone-Session-Name"] = self.session_name
+        if self.posthog_key:
+            headers["Helicone-Posthog-Key"] = self.posthog_key
+        if self.posthog_host:
+            headers["Helicone-Posthog-Host"] = self.posthog_host
+
+        # Boolean headers
+        for header, value in {
+            "Helicone-Omit-Response": self.omit_response,
+            "Helicone-Omit-Request": self.omit_request,
+            "Helicone-Cache-Enabled": (self.cache_enabled and "true")
+            or (self.cache_config.maxsize or self.cache_config.ttl and "true"),  # type: ignore
+            "Helicone-Retry-Enabled": self.retry_enabled,
+            "Helicone-Moderations-Enabled": self.moderations_enabled,
+            "Helicone-LLM-Security-Enabled": self.llm_security_enabled,
+            "Helicone-Stream-Force-Format": self.stream_force_format,
+        }.items():
+            if value is not None:
+                headers[header] = str(value).lower()
+
+        # Custom properties
+        for key, value in self.custom_properties.items():
+            headers[f"Helicone-Property-{key}"] = value
+
+        return headers
+
+    @property
+    def is_enabled(self):
+        return self.api_key is not None
+
+
+helicone_config = HeliconeSingleton()

src/ragas/llms/base.py

@@ -15,6 +15,7 @@
 from langchain_openai.llms import AzureOpenAI, OpenAI
 from langchain_openai.llms.base import BaseOpenAI
 
+from ragas.integrations.helicone import helicone_config
 from ragas.run_config import RunConfig, add_async_retry, add_retry
 
 if t.TYPE_CHECKING:
@@ -289,10 +290,21 @@ async def agenerate_text(
 
 
 def llm_factory(
-    model: str = "gpt-4o-mini", run_config: t.Optional[RunConfig] = None
+    model: str = "gpt-4o-mini",
+    run_config: t.Optional[RunConfig] = None,
+    default_headers: t.Optional[t.Dict[str, str]] = None,
+    base_url: t.Optional[str] = None,
 ) -> BaseRagasLLM:
     timeout = None
     if run_config is not None:
         timeout = run_config.timeout
-    openai_model = ChatOpenAI(model=model, timeout=timeout)
+
+    # if helicone is enabled, use the helicone
+    if helicone_config.is_enabled:
+        default_headers = helicone_config.default_headers()
+        base_url = helicone_config.base_url
+
+    openai_model = ChatOpenAI(
+        model=model, timeout=timeout, default_headers=default_headers, base_url=base_url
+    )
     return LangchainLLMWrapper(openai_model, run_config)