feat: added tracking of minimal stuff in testset generation (#509)

Author: jjmachan

src/ragas/_analytics.py

@@ -89,6 +89,11 @@ class EvaluationEvent(BaseEvent):
     num_rows: int
 
 
+class TesetGenerationEvent(BaseEvent):
+    evolutions: t.Dict[str, float]
+    num_rows: int
+
+
 @silent
 def track(event_properties: BaseEvent):
     if do_not_track():

src/ragas/testset/generator.py

@@ -9,6 +9,7 @@
 from langchain_openai.embeddings import OpenAIEmbeddings
 from llama_index.readers.schema import Document as LlamaindexDocument
 
+from ragas._analytics import TesetGenerationEvent, track
 from ragas.embeddings import BaseRagasEmbeddings
 from ragas.executor import Executor
 from ragas.llms import BaseRagasLLM, LangchainLLMWrapper
@@ -143,4 +144,15 @@ def generate(
             test_data_rows = exec.results()
         except ValueError as e:
             raise e
-        return TestDataset(test_data=test_data_rows)
+        test_dataset = TestDataset(test_data=test_data_rows)
+        track(
+            TesetGenerationEvent(
+                event_type="testset_generation",
+                evolutions={
+                    k.__class__.__name__.lower(): v for k, v in distributions.items()
+                },
+                num_rows=len(test_dataset.test_data),
+            )
+        )
+
+        return test_dataset

tests/unit/test_analytics.py

@@ -87,3 +87,31 @@ def test_load_userid_from_json_file(tmp_path, monkeypatch):
     get_userid.cache_clear()
 
     assert get_userid() == "test-userid"
+
+
+def test_testset_generation_tracking(monkeypatch):
+
+    import ragas._analytics as analyticsmodule
+    from ragas._analytics import TesetGenerationEvent, track
+    from ragas.testset.evolutions import multi_context, reasoning, simple
+
+    distributions = {simple: 0.5, multi_context: 0.3, reasoning: 0.2}
+
+    testset_event_payload = TesetGenerationEvent(
+        event_type="testset_generation",
+        evolutions={k.__class__.__name__.lower(): v for k, v in distributions.items()},
+        num_rows=10,
+    )
+
+    assert dict(testset_event_payload)["evolutions"] == {
+        "simpleevolution": 0.5,
+        "multicontextevolution": 0.3,
+        "reasoningevolution": 0.2,
+    }
+
+    # just in the case you actually want to check if tracking is working in the
+    # dashboard
+    if False:
+        monkeypatch.setattr(analyticsmodule, "do_not_track", lambda: False)
+        monkeypatch.setattr(analyticsmodule, "_usage_event_debugging", lambda: False)
+        track(testset_event_payload)