Skip to content

fix(litellm): enhance structured output handling #1021

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+24 −12
Open

fix(litellm): enhance structured output handling #1021

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 17 additions & 12 deletions src/strands/models/litellm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,20 +238,25 @@ async def _structured_output_using_response_schema(

if len(response.choices) > 1:
raise ValueError("Multiple choices found in the response.")
if not response.choices or response.choices[0].finish_reason != "tool_calls":
raise ValueError("No tool_calls found in response")
if not response.choices:
raise ValueError("No choices found in response")

choice = response.choices[0]
try:
# Parse the message content as JSON
tool_call_data = json.loads(choice.message.content)
# Instantiate the output model with the parsed data
return output_model(**tool_call_data)
except ContextWindowExceededError as e:
logger.warning("litellm client raised context window overflow in structured_output")
raise ContextWindowOverflowException(e) from e
except (json.JSONDecodeError, TypeError, ValueError) as e:
raise ValueError(f"Failed to parse or load content into model: {e}") from e
if choice.finish_reason == "tool_calls" and getattr(choice.message, "tool_calls", None):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we need this logic, or at least don't need it now.

In the previous state we looked at just choice.message.content even when tool_calls was the finished reason. So the proposal you have here is to switch to look at the function arguments.

I'd rather make one small change at a time, which I believe is simply removing the check as you did in

response.choices[0].finish_reason != "tool_calls"

but I don't think we need to introduce the logic you have here and can instead just rely on the existing as we had before unless there is a compelling reason to

        try:
            # Parse the message content as JSON
            tool_call_data = json.loads(choice.message.content)
            # Instantiate the output model with the parsed data
            return output_model(**tool_call_data)
        except ContextWindowExceededError as e:
            logger.warning("litellm client raised context window overflow in structured_output")
            raise ContextWindowOverflowException(e) from e
        except (json.JSONDecodeError, TypeError, ValueError) as e:
            raise ValueError(f"Failed to parse or load content into model: {e}") from e

try:
tool_call = choice.message.tool_calls[0]
tool_call_data = json.loads(tool_call.function.arguments)
return output_model(**tool_call_data)
except ContextWindowExceededError as e:
logger.warning("litellm client raised context window overflow in structured_output")
raise ContextWindowOverflowException(e) from e
except (json.JSONDecodeError, TypeError, ValueError) as e:
raise ValueError(f"Failed to parse or load content into model: {e}") from e
else:
try:
return output_model.parse_raw(choice.message.content)
except Exception as e:
raise ValueError("No tool_calls found in response and could not parse plain text as model") from e

async def _structured_output_using_tool(
self, output_model: Type[T], prompt: Messages, system_prompt: Optional[str] = None
Expand Down
7 changes: 7 additions & 0 deletions tests/strands/models/test_litellm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,13 @@ async def test_structured_output(litellm_acompletion, model, test_output_model_c
mock_choice = unittest.mock.Mock()
mock_choice.finish_reason = "tool_calls"
mock_choice.message.content = '{"name": "John", "age": 30}'
# PATCH START: mock tool_calls as list with .function.arguments
tool_call_mock = unittest.mock.Mock()
tool_call_function_mock = unittest.mock.Mock()
tool_call_function_mock.arguments = '{"name": "John", "age": 30}'
tool_call_mock.function = tool_call_function_mock
mock_choice.message.tool_calls = [tool_call_mock]
# PATCH END
mock_response = unittest.mock.Mock()
mock_response.choices = [mock_choice]

Expand Down