Role issue when combining multiple tools and memory

[chazlupei]

I have a simple example that combines the basic memory and tools demo. The agent is able to answer one set of tool calls (can use one or more tool successfully), but then will immediately throw an error when it gets the next query from the user (whether or not that query would call for tools).

Graph

Example Convo with Error

You: what is 3+3?
  --> Thinking
tool: add
  --> Thinking
Bot:  The sum of 3 and 3 is 6.
You: and if we add 3 more?
  --> Thinking
Traceback (most recent call last):
  File "filepath/Projects/heyjaime/shareable", line 155, in <module>
    output_message = graph.invoke({"messages": [input_message]}, config) 
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/langgraph/pregel/__init__.py", line 1608, in invoke
    for chunk in self.stream(
                 ^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/langgraph/pregel/__init__.py", line 1336, in stream
    for _ in runner.tick(
             ^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/langgraph/pregel/runner.py", line 58, in tick
    run_with_retry(t, retry_policy)
  File "filepath/lib/python3.12/site-packages/langgraph/pregel/retry.py", line 29, in run_with_retry
    task.proc.invoke(task.input, config)
  File "filepath/lib/python3.12/site-packages/langgraph/utils/runnable.py", line 410, in invoke
    input = context.run(step.invoke, input, config, **kwargs)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/langgraph/utils/runnable.py", line 184, in invoke
    ret = context.run(self.func, input, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "filepath/Projects/heyjaime/shareable", line 84, in call_model
    response = model.invoke(messages)
               ^^^^^^^^^^^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/langchain_core/runnables/base.py", line 5354, in invoke
    return self.bound.invoke(
           ^^^^^^^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py", line 286, in invoke
    self.generate_prompt(
  File "filepath/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py", line 786, in generate_prompt
    return self.generate(prompt_messages, stop=stop, callbacks=callbacks, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py", line 643, in generate
    raise e
  File "filepath/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py", line 633, in generate
    self._generate_with_cache(
  File "filepath/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py", line 851, in _generate_with_cache
    result = self._generate(
             ^^^^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/langchain_openai/chat_models/base.py", line 689, in _generate
    response = self.client.create(**payload)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/openai/_utils/_utils.py", line 275, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/openai/resources/chat/completions.py", line 829, in create
    return self._post(
           ^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/openai/_base_client.py", line 1277, in post
    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/openai/_base_client.py", line 954, in request
    return self._request(
           ^^^^^^^^^^^^^^
  File "filepath/lib/python3.12/site-packages/openai/_base_client.py", line 1058, in _request
    raise self._make_status_error_from_response(err.response) from None
openai.BadRequestError: Error code: 400 - {'error': {'message': "Invalid parameter: messages with role 'tool' must be a response to a preceeding message with 'tool_calls'.", 'type': 'invalid_request_error', 'param': 'messages.[1].role', 'code': None}}

Code - requires an openai api

import os
from langchain_openai import ChatOpenAI
from langchain_core.messages import SystemMessage, HumanMessage, RemoveMessage, AIMessage
from langchain_core.tools import tool
from langgraph.prebuilt import ToolNode
from langgraph.graph import MessagesState, StateGraph, START, END
from langgraph.checkpoint.memory import MemorySaver
from IPython.display import Image, display

#use the built-in message state from langgraph
class State(MessagesState):
    summary: str

@tool
def add(first_int: int, second_int: int) -> int:
    """Return sum of two integers.
    Args:
        first_int (Optional[int]): first number
        second_int (Optional[int]): second number
    """
    print("tool: add")
    return first_int + second_int

@tool
def subtract(first_int: int, second_int: int) -> int:
    """Return difference of two integers.
    Args:
        first_int (Optional[int]): first number
        second_int (Optional[int]): second number
    """
    print("tool: subtract")
    return first_int - second_int

@tool
def multiply(first_int: int, second_int: int) -> int:
    """Return product of two integers.
    Args:
        first_int (Optional[int]): first number
        second_int (Optional[int]): second number
    """

    return first_int * second_int

@tool
def divide(first_int: int, second_int: int) -> int:
    """Return quotient of two integers.
    Args:
        first_int (Optional[int]): first number
        second_int (Optional[int]): second number
    """
    print("tool: divide")
    return first_int / second_int


tools = [add, multiply, subtract, divide]
base_llm = ChatOpenAI(model="gpt-4o",temperature=0)
model = base_llm.bind_tools(tools, parallel_tool_calls=True)

# Define the logic to call the model
def call_model(state: State):
    
    #default system instructions
    system_message = f"""
    You help users with simple integer math problems. Please use your tools.
    """
    
    # Get summary if it exists
    summary = state.get("summary", "")

    # If there is summary, then we add it
    if summary:
        
        # Add summary to system message
        system_message = system_message + f"\n\nSummary of conversation earlier: {summary}"

        # Append summary to any newer messages
        
    messages = [SystemMessage(content=system_message)] + state["messages"]
    print("  --> Thinking")

    response = model.invoke(messages)
    
    #print(response)
    return {"messages": response}

#prompt for summarizing the conversation
def summarize_conversation(state: State):
    
    # First, we get any existing summary
    summary = state.get("summary", "")

    # Create our summarization prompt 
    if summary:
        
        # A summary already exists
        summary_message = (
            f"This is summary of the conversation to date: {summary}\n\n"
            "Extend the summary by taking into account the new messages above:"
        )
        
    else:
        summary_message = "Create a summary of the conversation above:"

    # Add prompt to our history
    messages = state["messages"] + [HumanMessage(content=summary_message)]
    response = model.invoke(messages)
    
    # Delete all but the 2 most recent messages
    delete_messages = [RemoveMessage(id=m.id) for m in state["messages"][:-2]]
    return {"summary": response.content, "messages": delete_messages}

def tools_condition_check(state: State):
    messages = state.get('messages', [])
    
    # Check if the list is not empty and access the last message
    if messages:
        last_message = messages[-1]
        # Check for 'tool_calls' in 'additional_kwargs' of the last message
        if "tool_calls" in last_message.additional_kwargs and last_message.additional_kwargs["tool_calls"]:
            return "tools"
        
    return "summarize_conversation"

#Define the grapth
workflow = StateGraph(State)
workflow.add_node("conversation", call_model)
workflow.add_node("tools", ToolNode(tools))
workflow.add_node("summarize_conversation",summarize_conversation,)


workflow.add_edge(START, "conversation")
workflow.add_conditional_edges("conversation", tools_condition_check, {"tools": "tools", "summarize_conversation": "summarize_conversation"})
workflow.add_edge("tools", "conversation")
workflow.add_edge("summarize_conversation", END)

# Compile
memory = MemorySaver()
graph = workflow.compile(checkpointer=memory)

#plots the lang graph
react_graph = workflow.compile()
display(Image(react_graph.get_graph(xray=True).draw_mermaid_png()))

# Create a thread
config = {"configurable": {"thread_id": "1"}}

while True:
    input_message = input("You: ")
    if input_message.lower() == "quit":
        break
    
    output_message = graph.invoke({"messages": [input_message]}, config) 
    
    print("Bot: ", output_message['messages'][-1].content)    

[chazlupei]

resolved:

def call_model(state: State):
system_message = "You help users with simple integer math problems. Please use your tools."
summary = state.get("summary", "")
if summary:
system_message += f"\n\nSummary of conversation earlier: {summary}"

messages = state.get("messages", [])

# Convert string messages to HumanMessage objects
messages = [HumanMessage(content=m) if isinstance(m, str) else m for m in messages]

full_messages = [SystemMessage(content=system_message)] + messages
print("  --> Thinking")
response = model.invoke(full_messages)
return {"messages": full_messages + [response]}