docs: update ab testing cookbook

Author: joschkabraun

cookbook/ab_testing.py

@@ -5,8 +5,8 @@
 
 from openai import OpenAI
 
-from parea import Parea, get_current_trace_id, trace, trace_insert
-from parea.schemas import FeedbackRequest
+from parea import Parea, get_current_trace_id, parea_logger, trace, trace_insert
+from parea.schemas import EvaluationResult, UpdateLog
 
 client = OpenAI()
 # instantiate Parea client
@@ -15,16 +15,27 @@
 p.wrap_openai_client(client)
 
 
-@trace
-def generate_email(user: str) -> Tuple[str, str]:
-    """Randomly chooses a prompt to perform an A/B test for generating email. Returns the email and the trace ID.
-    The latter is used to tie-back the collected feedback from the user."""
+ab_test_name = "long-vs-short-emails"
+
+
+@trace  # decorator to trace functions with Parea
+def generate_email(user: str) -> Tuple[str, str, str]:
+    # randomly choose to generate a long or short email
     if random.random() < 0.5:
-        trace_insert({"metadata": {"ab_test_0": "variant_0"}})
+        variant = "variant_0"
         prompt = f"Generate a long email for {user}"
     else:
-        trace_insert({"metadata": {"ab_test_0": "variant_1"}})
+        variant = "variant_1"
         prompt = f"Generate a short email for {user}"
+    # tag the requests with the A/B test name & chosen variant
+    trace_insert(
+        {
+            "metadata": {
+                "ab_test_name": ab_test_name,
+                f"ab_test_{ab_test_name}": variant,
+            }
+        }
+    )
 
     email = (
         client.chat.completions.create(
@@ -39,22 +50,37 @@ def generate_email(user: str) -> Tuple[str, str]:
         .choices[0]
         .message.content
     )
+    # need to return in addition to the email, the trace_id and the chosen variant
+    return email, get_current_trace_id(), variant
 
-    return email, get_current_trace_id()
 
+def capture_feedback(feedback: float, trace_id: str, ab_test_variant: str, user_corrected_email: str = None) -> None:
+    field_name_to_value_map = {
+        "scores": [EvaluationResult(name=f"ab_test_{ab_test_variant}", score=feedback, reason="any additional user feedback on why it's good/bad")],
+    }
+    if user_corrected_email:
+        field_name_to_value_map["target"] = user_corrected_email
 
-def main():
-    # generate email and get trace ID
-    email, trace_id = generate_email("Max Mustermann")
-
-    # log user feedback on email using trace ID
-    p.record_feedback(
-        FeedbackRequest(
+    parea_logger.update_log(
+        UpdateLog(
             trace_id=trace_id,
-            score=1.0,
+            field_name_to_value_map=field_name_to_value_map,
         )
     )
 
 
+def main():
+    # generate email and get trace ID
+    email, trace_id, ab_test_variant = generate_email("Max Mustermann")
+
+    # create a biased feedback for shorter emals
+    if ab_test_variant == "variant_1":
+        user_feedback = 0.0 if random.random() < 0.7 else 1.0
+    else:
+        user_feedback = 0.0 if random.random() < 0.3 else 1.0
+
+    capture_feedback(user_feedback, trace_id, ab_test_variant, "Hi Max")
+
+
 if __name__ == "__main__":
     main()

cookbook/evals_and_experiments/run_experiment_agreement_among_evals.py

@@ -1,11 +1,12 @@
-import random
 from typing import List
+
 import os
+import random
 
 from dotenv import load_dotenv
 
 from parea import Parea, trace
-from parea.schemas import EvaluatedLog, Log, EvaluationResult
+from parea.schemas import EvaluatedLog, EvaluationResult, Log
 
 load_dotenv()
 
@@ -14,10 +15,8 @@
 
 def random_eval_factory(trial: int):
     def random_eval(log: Log) -> EvaluationResult:
-        return EvaluationResult(
-            score=1 if random.random() < 0.5 else 0,
-            name=f'random_eval_{trial}'
-        )
+        return EvaluationResult(score=1 if random.random() < 0.5 else 0, name=f"random_eval_{trial}")
+
     return random_eval
 
 
@@ -58,9 +57,4 @@ def percent_evals_agree(logs: List[EvaluatedLog]) -> float:
 
 # You can optionally run the experiment manually by calling `.run()`
 if __name__ == "__main__":
-    p.experiment(
-        name="Greeting",
-        data=data,
-        func=starts_with_f,
-        dataset_level_evals=[percent_evals_agree]
-    ).run()
+    p.experiment(name="Greeting", data=data, func=starts_with_f, dataset_level_evals=[percent_evals_agree]).run()

parea/helpers.py

@@ -4,7 +4,7 @@
 import random
 import uuid
 from copy import deepcopy
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import pytz
 from attr import asdict, fields_dict
@@ -79,6 +79,7 @@ def serialize_values(metadata: Dict[str, Any]) -> Dict[str, str]:
 
 def timezone_aware_now() -> datetime:
     return datetime.now(pytz.utc)
+    # return datetime.now(pytz.utc) - timedelta(days=6)
 
 
 def structure_trace_log_from_api(d: dict) -> TraceLogTree: