feat: added track_completed decorator to track completed functions (#1171)

adding analytics to track completed functions.This is to measure the
errors that are happening with evals and testset generation

Author: jjmachan

src/ragas/_analytics.py

@@ -111,3 +111,24 @@ def track(event_properties: BaseEvent):
         return
 
     requests.post(USAGE_TRACKING_URL, json=payload, timeout=USAGE_REQUESTS_TIMEOUT_SEC)
+
+
+class IsCompleteEvent(BaseEvent):
+    is_completed: bool = True  # True if the event was completed, False otherwise
+
+
+@silent
+def track_was_completed(func: t.Callable[P, T]) -> t.Callable[P, T]:  # pragma: no cover
+    """
+    Track if the function was completed. This helps us understand failure cases and improve the user experience. Disable tracking by setting the environment variable RAGAS_DO_NOT_TRACK to True as usual.
+    """
+
+    @wraps(func)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> t.Any:
+        track(IsCompleteEvent(event_type=func.__name__, is_completed=False))
+        result = func(*args, **kwargs)
+        track(IsCompleteEvent(event_type=func.__name__, is_completed=True))
+
+        return result
+
+    return wrapper

src/ragas/evaluation.py

@@ -9,7 +9,7 @@
 from langchain_core.embeddings import Embeddings as LangchainEmbeddings
 from langchain_core.language_models import BaseLanguageModel as LangchainLLM
 
-from ragas._analytics import EvaluationEvent, track
+from ragas._analytics import EvaluationEvent, track, track_was_completed
 from ragas.callbacks import new_group
 from ragas.cost import TokenUsage
 from ragas.embeddings.base import (
@@ -46,6 +46,7 @@
     from ragas.cost import CostCallbackHandler, TokenUsageParser
 
 
+@track_was_completed
 def evaluate(
     dataset: Dataset,
     metrics: list[Metric] | None = None,

src/ragas/testset/generator.py

@@ -10,7 +10,7 @@
 from langchain_openai.chat_models import ChatOpenAI
 from langchain_openai.embeddings import OpenAIEmbeddings
 
-from ragas._analytics import TestsetGenerationEvent, track
+from ragas._analytics import TestsetGenerationEvent, track, track_was_completed
 from ragas.embeddings.base import (
     BaseRagasEmbeddings,
     LangchainEmbeddingsWrapper,
@@ -231,6 +231,7 @@ def init_evolution(self, evolution: Evolution) -> None:
                 if evolution.evolution_filter is None:
                     evolution.evolution_filter = EvolutionFilter(llm=self.critic_llm)
 
+    @track_was_completed
     def generate(
         self,
         test_size: int,

tests/unit/test_analytics.py

@@ -1,4 +1,6 @@
 from __future__ import annotations
+import typing as t
+import pytest
 
 
 def test_debug_tracking_flag():
@@ -125,3 +127,32 @@ def test_testset_generation_tracking(monkeypatch):
         monkeypatch.setattr(analyticsmodule, "do_not_track", lambda: False)
         monkeypatch.setattr(analyticsmodule, "_usage_event_debugging", lambda: False)
         track(testset_event_payload)
+
+
+def test_was_completed(monkeypatch):
+    from ragas._analytics import track_was_completed, IsCompleteEvent
+
+    event_properties_list: t.List[IsCompleteEvent] = []
+
+    def echo_track(event_properties):
+        event_properties_list.append(event_properties)
+
+    monkeypatch.setattr("ragas._analytics.track", echo_track)
+
+    @track_was_completed
+    def test(raise_error=True):
+        if raise_error:
+            raise ValueError("test")
+        else:
+            pass
+
+    with pytest.raises(ValueError):
+        test(raise_error=True)
+
+    assert event_properties_list[-1].event_type == "test"
+    assert event_properties_list[-1].is_completed is False
+
+    test(raise_error=False)
+
+    assert event_properties_list[-1].event_type == "test"
+    assert event_properties_list[-1].is_completed is True