wip

Author: jalexanderII

parea/cookbook/simple_experiment_with_openai.py

@@ -6,7 +6,7 @@
 from openai import OpenAI
 
 from parea import Parea, trace
-from parea.schemas import Log
+from parea.schemas import Log, EvaluationResult
 
 load_dotenv()
 
@@ -15,12 +15,16 @@
 p.wrap_openai_client(client)
 
 
-def eval_func(log: Log) -> float:
+def eval_func(log: Log) -> EvaluationResult:
     from random import random
     from time import sleep
 
     sleep(random() * 10)
-    return random()
+    return EvaluationResult(
+        name="eval_func",
+        score=random(),
+        reason="Random score",
+    )
 
 
 # @trace(eval_funcs=[eval_func])
@@ -43,9 +47,9 @@ def func(topic: str) -> dict[str, str | None]:
 
 
 if __name__ == "__main__":
-    # print(func("Python"))
-    p.experiment(
-        name="hello-world-example-ch",
-        data=[{"topic": "Python"}, {"topic": "javascript"}],
-        func=func,
-    ).run()
+    print(func("Python"))
+    # p.experiment(
+    #     name="hello-world-example-ch",
+    #     data=[{"topic": "Fish"}, {"topic": "Python"}],
+    #     func=func,
+    # ).run()

parea/cookbook/tracing_without_deployed_prompt.py

@@ -13,18 +13,15 @@
 p = Parea(api_key=os.getenv("PAREA_API_KEY"))
 
 
-@trace  # <--- If you want to log the inputs to the LLM call you can optionally add a trace decorator here
 def call_llm(
     data: List[dict],
     model: str = "gpt-3.5-turbo-1106",
-    provider: str = "openai",
     temperature: float = 0.0,
 ) -> CompletionResponse:
     return p.completion(
         data=Completion(
             llm_configuration=LLMInputs(
                 model=model,
-                provider=provider,
                 model_params=ModelParams(temp=temperature),
                 messages=[Message(**d) for d in data],
             )
@@ -106,8 +103,7 @@ def refiner2(query: str, additional_description: str, current_arg: str, criticis
                 "content": "Please generate a new argument that incorporates the feedback from the user.",
             },
         ],
-        model="claude-2",
-        provider="anthropic",
+        model="claude-3-haiku-20240307",
     )
 
 
@@ -128,7 +124,6 @@ def json_call():
         data=Completion(
             llm_configuration=LLMInputs(
                 model="gpt-3.5-turbo-1106",
-                provider="openai",
                 model_params=ModelParams(temp=0.0, response_format={"type": "json_object"}),
                 messages=[Message(**d) for d in json_messages],
             )
@@ -147,12 +142,12 @@ def json_call():
         "Whether wine is good for you.",
         additional_description="Provide a concise, few sentence argument on why wine is good for you.",
     )
-    print(result2)
+    print(trace_id2, result2)
     p.record_feedback(
         FeedbackRequest(
             trace_id=trace_id2,
-            score=0.0,  # 0.0 (bad) to 1.0 (good)
-            target="Moonshine is wonderful.",
+            score=0.7,  # 0.0 (bad) to 1.0 (good)
+            target="Wine is wonderful.",
         )
     )
 
@@ -164,7 +159,7 @@ def json_call():
     p.record_feedback(
         FeedbackRequest(
             trace_id=result3.inference_id,
-            score=0.7,  # 0.0 (bad) to 1.0 (good)
+            score=0.5,  # 0.0 (bad) to 1.0 (good)
             target="Moonshine is wonderful. End of story.",
         )
     )

parea/schemas/models.py

@@ -77,7 +77,6 @@ class UseDeployedPromptResponse:
 class FeedbackRequest:
     score: float = field(validator=[validators.ge(0), validators.le(1)])
     trace_id: Optional[str] = None
-    inference_id: Optional[str] = None
     name: Optional[str] = None
     target: Optional[str] = None