Improve handling of literals and optionals (#8143)

Author: okhat

dspy/adapters/json_adapter.py

@@ -126,6 +126,10 @@ def parse(self, signature: Type[Signature], completion: str) -> dict[str, Any]:
         if match:  
             completion = match.group(0)
         fields = json_repair.loads(completion)
+
+        if not isinstance(fields, dict):
+            raise ValueError(f"Expected a JSON object but parsed a {type(fields)}")
+
         fields = {k: v for k, v in fields.items() if k in signature.output_fields}
 
         # Attempt to cast each value to type signature.output_fields[k].annotation.

dspy/adapters/utils.py

@@ -138,6 +138,25 @@ def parse_value(value, annotation):
     if isinstance(annotation, enum.EnumMeta):
         return find_enum_member(annotation, value)
 
+    origin = get_origin(annotation)
+
+    if origin is Literal:
+        allowed = get_args(annotation)
+        if value in allowed:
+            return value
+
+        if isinstance(value, str):
+            v = value.strip()
+            if v.startswith(("Literal[", "str[")) and v.endswith("]"):
+                v = v[v.find("[") + 1 : -1]
+            if len(v) > 1 and v[0] == v[-1] and v[0] in "\"'":
+                v = v[1:-1]
+
+            if v in allowed:
+                return v
+    
+        raise ValueError(f"{value!r} is not one of {allowed!r}")
+
     if not isinstance(value, str):
         return TypeAdapter(annotation).validate_python(value)
 
@@ -147,15 +166,13 @@ def parse_value(value, annotation):
             candidate = ast.literal_eval(value)
         except (ValueError, SyntaxError):
             candidate = value
-    
+
     try:
         return TypeAdapter(annotation).validate_python(candidate)
-    except pydantic.ValidationError as e:
-        # if the annotation is Optional[str], return just the string value
-        if annotation.__origin__ is Union and type(None) in get_args(annotation):
-            if len(get_args(annotation)) == 2 and str in get_args(annotation):
-                return str(candidate)
-        raise e
+    except pydantic.ValidationError:
+        if origin is Union and type(None) in get_args(annotation) and str in get_args(annotation):
+            return str(candidate)
+        raise
 
 def get_annotation_name(annotation):
     origin = get_origin(annotation)