struc outputs support

Author: jalexanderII

cookbook/openai/tracing_with_openai_with_structured_output.py

@@ -28,11 +28,80 @@ class CalendarEvent(BaseModel):
 )
 
 
-if __name__ == "__main__":
-    event = completion.choices[0].message.parsed
-    print(type(event))
-    print(event)
+def main():
+    response = client.chat.completions.create(
+        model="gpt-4o-2024-08-06",
+        messages=[
+            {"role": "system", "content": "You are a helpful math tutor. Guide the user through the solution step by step."},
+            {"role": "user", "content": "how can I solve 8x + 7 = -23"},
+        ],
+        response_format={
+            "type": "json_schema",
+            "json_schema": {
+                "name": "math_response",
+                "schema": {
+                    "type": "object",
+                    "properties": {
+                        "steps": {
+                            "type": "array",
+                            "items": {
+                                "type": "object",
+                                "properties": {"explanation": {"type": "string"}, "output": {"type": "string"}},
+                                "required": ["explanation", "output"],
+                                "additionalProperties": False,
+                            },
+                        },
+                        "final_answer": {"type": "string"},
+                    },
+                    "required": ["steps", "final_answer"],
+                    "additionalProperties": False,
+                },
+                "strict": True,
+            },
+        },
+    )
+
+    print(response.choices[0].message.content)
+
+
+def main2():
+    tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_delivery_date",
+                "description": "Get the delivery date for a customer's order. Call this whenever you need to know the delivery date, for example when a customer asks 'Where is my package'",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "order_id": {
+                            "type": "string",
+                            "description": "The customer's order ID.",
+                        },
+                    },
+                    "required": ["order_id"],
+                    "additionalProperties": False,
+                },
+            },
+            "strict": True,
+        }
+    ]
 
+    messages = [
+        {"role": "system", "content": "You are a helpful customer support assistant. Use the supplied tools to assist the user."},
+        {"role": "user", "content": "Hi, can you tell me the delivery date for my order with id 5?"},
+    ]
 
-TraceLog(configuration=LLMInputs(model='gpt-4o-2024-08-06', provider='openai', model_params=ModelParams(temp=1.0, top_p=1.0, frequency_penalty=0.0, presence_penalty=0.0, max_length=None, response_format="<class '__main__.CalendarEvent'>", safe_prompt=None), messages=[{'role': 'system', 'content': 'Extract the event information.'}, {'role': 'user', 'content': 'Alice and Bob are going to a science fair on Friday.'}], history=None, functions=[], function_call=None), inputs=None, output='{"name":"Science Fair","date":"Friday","participants":["Alice","Bob"]}', target=None, latency=1.771622, time_to_first_token=None, input_tokens=32, output_tokens=17, total_tokens=49, cost=0.000415, scores=[], trace_id='e894b955-c844-49f5-8480-71b0841f10b5', parent_trace_id='e894b955-c844-49f5-8480-71b0841f10b5', root_trace_id='e894b955-c844-49f5-8480-71b0841f10b5', start_timestamp='2024-08-06T20:26:42.365024+00:00', organization_id=None, project_uuid=None, error=None, status='success', deployment_id=None, cache_hit=False, output_for_eval_metrics=None, evaluation_metric_names=[], apply_eval_frac=1.0, feedback_score=None, trace_name='llm-openai', children=['600b25b7-417a-4409-8f96-afe8dd8fe8cf'], end_timestamp='2024-08-06T20:26:44.136646+00:00', end_user_identifier=None, session_id=None, metadata=None, tags=None, experiment_uuid=None, images=[], comments=None, annotations=None, depth=0, execution_order=0)
-D {'configuration': {'model': 'gpt-4o-2024-08-06', 'provider': 'openai', 'model_params': {'temp': 1.0, 'top_p': 1.0, 'frequency_penalty': 0.0, 'presence_penalty': 0.0, 'max_length': None, 'response_format': "<class '__main__.CalendarEvent'>", 'safe_prompt': None}, 'messages': [{'role': 'system', 'content': 'Extract the event information.'}, {'role': 'user', 'content': 'Alice and Bob are going to a science fair on Friday.'}], 'history': None, 'functions': [], 'function_call': None}, 'inputs': None, 'output': '{"name":"Science Fair","date":"Friday","participants":["Alice","Bob"]}', 'target': None, 'latency': 1.771622, 'time_to_first_token': None, 'input_tokens': 32, 'output_tokens': 17, 'total_tokens': 49, 'cost': 0.000415, 'scores': [], 'trace_id': 'e894b955-c844-49f5-8480-71b0841f10b5', 'parent_trace_id': 'e894b955-c844-49f5-8480-71b0841f10b5', 'root_trace_id': 'e894b955-c844-49f5-8480-71b0841f10b5', 'start_timestamp': '2024-08-06T20:26:42.365024+00:00', 'organization_id': None, 'project_uuid': '1c4dfe49-bf84-11ee-92b3-3a9b36099f82', 'error': None, 'status': 'success', 'deployment_id': None, 'cache_hit': False, 'output_for_eval_metrics': None, 'evaluation_metric_names': [], 'apply_eval_frac': 1.0, 'feedback_score': None, 'trace_name': 'llm-openai', 'children': ['600b25b7-417a-4409-8f96-afe8dd8fe8cf'], 'end_timestamp': '2024-08-06T20:26:44.136646+00:00', 'end_user_identifier': None, 'session_id': None, 'metadata': None, 'tags': None, 'experiment_uuid': None, 'images': [], 'comments': None, 'annotations': None, 'depth': 0, 'execution_order': 0}
+    response = client.chat.completions.create(
+        model="gpt-4o-2024-08-06",
+        messages=messages,
+        tools=tools,
+    )
+    print(response.choices[0].message.tool_calls)
+
+
+if __name__ == "__main__":
+    # event = completion.choices[0].message.parsed
+    # print(event)
+    # main()
+    main2()

parea/parea_logger.py

@@ -57,11 +57,8 @@ def update_log(self, data: UpdateLog) -> None:
         )
 
     def record_log(self, data: TraceLog) -> None:
-        print(data)
         data = serialize_metadata_values(data)
         data.project_uuid = self._get_project_uuid()
-        d = asdict(data)
-        print("D", d)
         self._client.request(
             "POST",
             LOG_ENDPOINT,

parea/utils/trace_utils.py

@@ -1,5 +1,3 @@
-from typing import Any, AsyncGenerator, AsyncIterator, Callable, Dict, Generator, Iterator, List, Optional, Tuple
-
 import contextvars
 import inspect
 import json
@@ -11,6 +9,7 @@
 from datetime import datetime
 from functools import wraps
 from random import random
+from typing import Any, AsyncGenerator, AsyncIterator, Callable, Dict, Generator, Iterator, List, Optional, Tuple
 
 from parea.constants import PAREA_OS_ENV_EXPERIMENT_UUID, TURN_OFF_PAREA_EVAL_LOGGING
 from parea.helpers import gen_trace_id, is_logging_disabled, timezone_aware_now
@@ -49,7 +48,9 @@ def log_in_thread(target_func: Callable, data: Dict[str, Any]):
     logging_thread.start()
 
 
-def merge(old, new):
+def merge(old, new, key=None):
+    if key == "error" and old:
+        return json_dumps([old, new])
     if isinstance(old, dict) and isinstance(new, dict):
         return dict(ChainMap(new, old))
     if isinstance(old, list) and isinstance(new, list):
@@ -112,7 +113,7 @@ def trace_insert(data: Dict[str, Any], trace_id: Optional[str] = None):
             return
         for key, new_value in data.items():
             existing_value = current_trace_data.__getattribute__(key)
-            current_trace_data.__setattr__(key, merge(existing_value, new_value) if existing_value else new_value)
+            current_trace_data.__setattr__(key, merge(existing_value, new_value, key) if existing_value else new_value)
     except Exception as e:
         logger.debug(f"Error occurred inserting data into trace log, {e}", exc_info=e)
 

parea/utils/universal_encoder.py

@@ -80,6 +80,18 @@ def handle_dspy_response(self, obj) -> Any:
         else:
             return None
 
+    def handle_openai_not_given(self, obj) -> Any:
+        try:
+            from openai import NotGiven
+        except ImportError:
+            return None
+
+        from openai import NotGiven
+
+        if isinstance(obj, NotGiven):
+            return {"not_given": None}
+        return None
+
     def default(self, obj: Any):
         if isinstance(obj, str):
             return obj
@@ -115,6 +127,8 @@ def default(self, obj: Any):
             return obj.to_dict(orient="records")
         elif dspy_response := self.handle_dspy_response(obj):
             return dspy_response
+        elif is_openai_not_given := self.handle_openai_not_given(obj):
+            return is_openai_not_given["not_given"]
         elif callable(obj):
             try:
                 return f"<callable {obj.__name__}>"

parea/wrapper/utils.py

@@ -5,7 +5,7 @@
 from typing import Callable, Dict, List, Optional, Union
 
 import tiktoken
-from openai import __version__ as openai_version
+from openai import __version__ as openai_version, NotGiven
 from parea.constants import ALL_NON_AZURE_MODELS_INFO, AZURE_MODEL_INFO, TURN_OFF_PAREA_EVAL_LOGGING
 from parea.parea_logger import parea_logger
 from parea.schemas.log import LLMInputs, Message, ModelParams, Role
@@ -219,9 +219,12 @@ def clean_json_string(s):
 
 def _resolve_functions(kwargs):
     if "functions" in kwargs:
-        return kwargs.get("functions", [])
+        f = kwargs.get("functions", [])
+        return None if isinstance(f, NotGiven) else f
     elif "tools" in kwargs:
         tools = kwargs["tools"]
+        if isinstance(tools, NotGiven):
+            return None
         if isinstance(tools, list):
             return [d.get("function", {}) for d in tools]
 
@@ -233,21 +236,26 @@ def _resolve_functions(kwargs):
 def _kwargs_to_llm_configuration(kwargs, model=None) -> LLMInputs:
     functions = _resolve_functions(kwargs)
     function_call_default = "auto" if functions else None
+    function_call = kwargs.get("function_call", function_call_default) or kwargs.get("tool_choice", function_call_default)
     response_format = kwargs.get("response_format", None)
-    response_format = str(response_format) if isinstance(response_format, ModelMetaclass) else response_format
-
+    response_format = {"type": "json_schema", "json_schema": str(response_format)} if isinstance(response_format, ModelMetaclass) else response_format
+    temp = kwargs.get("temperature", 1.0)
+    max_length = kwargs.get("max_tokens", None)
+    top_p = kwargs.get("top_p", 1.0)
+    frequency_penalty = kwargs.get("frequency_penalty", 0.0)
+    presence_penalty = kwargs.get("presence_penalty", 0.0)
     return LLMInputs(
         model=model or kwargs.get("model", None),
         provider="openai",
         messages=_convert_oai_messages(kwargs.get("messages", None)),
         functions=functions,
-        function_call=kwargs.get("function_call", function_call_default) or kwargs.get("tool_choice", function_call_default),
+        function_call=None if isinstance(function_call, NotGiven) else function_call,
         model_params=ModelParams(
-            temp=kwargs.get("temperature", 1.0),
-            max_length=kwargs.get("max_tokens", None),
-            top_p=kwargs.get("top_p", 1.0),
-            frequency_penalty=kwargs.get("frequency_penalty", 0.0),
-            presence_penalty=kwargs.get("presence_penalty", 0.0),
+            temp=None if isinstance(temp, NotGiven) else temp,
+            max_length=None if isinstance(max_length, NotGiven) else max_length,
+            top_p=None if isinstance(top_p, NotGiven) else top_p,
+            frequency_penalty=None if isinstance(frequency_penalty, NotGiven) else frequency_penalty,
+            presence_penalty=None if isinstance(presence_penalty, NotGiven) else presence_penalty,
             response_format=response_format,
         ),
     )
@@ -304,7 +312,11 @@ def _compute_cost(prompt_tokens: int, completion_tokens: int, model: str) -> flo
 
 def _process_response(response, model_inputs, trace_id):
     response_message = response.choices[0].message
-    if response_message.content:
+    if response_message.finish_reason == "content_filter":
+        trace_insert({"error": "Error: The content was filtered due to policy violations."}, trace_id)
+    if hasattr(response_message, "refusal"):
+        completion = response_message.refusal
+    elif response_message.content:
         completion = response_message.content
     else:
         completion = _format_function_call(response_message)

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 [tool.poetry]
 name = "parea-ai"
 packages = [{ include = "parea" }]
-version = "0.2.192"
+version = "0.2.193"
 description = "Parea python sdk"
 readme = "README.md"
 authors = ["joel-parea-ai <[email protected]>"]