Youtube tool and gpt-4o-mini

[dzianisv]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

import os
import sys
import logging
import httpx

from langchain_openai import ChatOpenAI
from langchain.memory import ConversationBufferWindowMemory
from langchain.chains import LLMChain
from contextlib import contextmanager

from langchain.prompts import (
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
    MessagesPlaceholder,
    SystemMessagePromptTemplate,
)
from langchain import hub
from langchain.agents import create_structured_chat_agent, AgentExecutor, load_tools
from langchain_community.tools import YouTubeSearchTool
from langchain import hub

logger = logging.getLogger(__name__)


def check_package(package_name):
    try:
        __import__(package_name)
        print(f"The package '{package_name}' is installed.")
        return True
    except ImportError:
        print(f"The package '{package_name}' is not installed.")
        return False


class LLM(object):
    def __init__(self, memory_window=8):
        self.memory = ConversationBufferWindowMemory(
            k=memory_window, memory_key="chat_history", return_messages=True
        )
        
        proxy_url = os.environ.get("OPENAI_PROXY")
        self.llm = ChatOpenAI(
            openai_api_base=os.environ.get("OPENAI_API_BASE"), 
            openai_api_key=os.environ.get("OPENAI_API_KEY"),
            openai_proxy=proxy_url,
            temperature=0.7, 
            model=os.environ.get("OPENAI_MODEL", "gpt-4o-mini"),
        )

        # self.chain = self._init_chain()
        self.chain = self._init_agent()

    def _init_agent(self):
        # https://python.langchain.com/docs/integrations/tools/openweathermap
        skils = []
        if check_package("pyowm"):
            skils.append("openweathermap-api")
        if check_package("arxiv"):
            skils.append("arxiv")
        if check_package("duckduckgo-search"): # requires curl-impersonate that is not available for armv7
            skils.append("ddg-search")
        # if check_package("wikipedia"):
        #     skils.append("wikipedia")

        tools = load_tools(skils)
        tools.append(YouTubeSearchTool())
       
        # Get the prompt to use - you can modify this!
        prompt = hub.pull("hwchase17/structured-chat-agent")    
        print(f"Prompt: {prompt}")

        agent = create_structured_chat_agent(
            tools=tools,
            llm=self.llm,
            prompt=prompt
        )

        # Create an agent executor by passing in the agent and tools
        agent_executor = AgentExecutor(
            agent=agent, 
            tools=tools, 
            verbose=True, 
            memory=self.memory,
            return_intermediate_steps=True, 
            handle_parsing_errors=True
        )

        return agent_executor

    def _init_chain(self):
        prompt = ChatPromptTemplate(
            messages=[
                SystemMessagePromptTemplate.from_template(
                    """You are voice assistant Jarvis created by Dennis Vashchuk \
                    If you don't know the answer, just say that you don't know. \
                    Use three sentences maximum and keep the answer concise
                """
                ),
                MessagesPlaceholder(variable_name="chat_history"),
                HumanMessagePromptTemplate.from_template("{input}"),
            ]
        )
        chain = LLMChain(llm=self.llm, prompt=prompt, memory=self.memory, verbose=True)
        return chain

    def ask(self, prompt):      
        response = self.chain.invoke({"input": prompt})
        return response['text'] if 'text' in response else response['output']

Description

I am trying to use Youtube Search agent with gpt-4o-mini, anyway I can tune the LLM or agent? The issue that 4o-mini ignores the specification of the input arguments.

INFO:assistant:recognized: Антонио Вивальди
INFO:assistant:Waiting for the wake word...
INFO:assistant:Speaking...: Сейчас узнаю!


> Entering new AgentExecutor chain...
DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'content': 'Respond to the human as helpfully and accurately as possible. You have access to the following tools:\n\nopen_weather_map - A wrapper around OpenWeatherMap API. Useful for fetching current weather information for a specified location. Input should be a location string (e.g. London,GB)., args: {\'location\': {\'title\': \'Location\', \'type\': \'string\'}}\narxiv - A wrapper around Arxiv.org Useful for when you need to answer questions about Physics, Mathematics, Computer Science, Quantitative Biology, Quantitative Finance, Statistics, Electrical Engineering, and Economics from scientific articles on arxiv.org. Input should be a search query., args: {\'query\': {\'title\': \'Query\', \'description\': \'search query to look up\', \'type\': \'string\'}}\nyoutube_search - search for youtube videos associated with a person. the input to this tool should be a comma separated list, the first part contains a person name and the second a number that is the maximum number of video results to return aka num_results. the second part is optional, args: {\'query\': {\'title\': \'Query\', \'type\': \'string\'}}\n\nUse a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input).\n\nValid "action" values: "Final Answer" or open_weather_map, arxiv, youtube_search\n\nProvide only ONE action per $JSON_BLOB, as shown:\n\n```\n{\n  "action": $TOOL_NAME,\n  "action_input": $INPUT\n}\n```\n\nFollow this format:\n\nQuestion: input question to answer\nThought: consider previous and subsequent steps\nAction:\n```\n$JSON_BLOB\n```\nObservation: action result\n... (repeat Thought/Action/Observation N times)\nThought: I know what to respond\nAction:\n```\n{\n  "action": "Final Answer",\n  "action_input": "Final response to human"\n}\n\nBegin! Reminder to ALWAYS respond with a valid json blob of a single action. Use tools if necessary. Respond directly if appropriate. Format is Action:```$JSON_BLOB```then Observation', 'role': 'system'}, {'content': 'слушаю тебя', 'role': 'user'}, {'content': 'Я готов помочь! Какой у вас вопрос?', 'role': 'assistant'}, {'content': 'Какая сегодня погода в Минске', 'role': 'user'}, {'content': 'Сегодня в Минске погода ясная. Температура составляет 17.86°C, ветер со скоростью 7.77 м/с, влажность 51%, облачность 10%.', 'role': 'assistant'}, {'content': 'Антонио виваль\n\n\n (reminder to respond in a JSON blob no matter what)', 'role': 'user'}], 'model': 'gpt-4o-mini', 'logprobs': False, 'n': 1, 'stop': ['\nObservation'], 'stream': True, 'temperature': 0.7}}
DEBUG:openai._base_client:Sending HTTP Request: POST https://api.openai.com/v1/chat/completions
DEBUG:httpcore.connection:connect_tcp.started host='100.88.150.94' port=8888 local_address=None timeout=None socket_options=None
DEBUG:httpcore.connection:connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x163cc86b0>
DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'CONNECT']>
DEBUG:httpcore.http11:send_request_headers.complete
DEBUG:httpcore.http11:send_request_body.started request=<Request [b'CONNECT']>
DEBUG:httpcore.http11:send_request_body.complete
DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'CONNECT']>
DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.0', 200, b'Connection established', [(b'Proxy-agent', b'tinyproxy/1.11.0')])
DEBUG:httpcore.proxy:start_tls.started ssl_context=<ssl.SSLContext object at 0x15f0b53d0> server_hostname='api.openai.com' timeout=None
DEBUG:httpcore.proxy:start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x163cc8170>
DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_headers.complete
DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:send_request_body.complete
DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Sun, 11 Aug 2024 14:16:01 GMT'), (b'Content-Type', b'text/event-stream; charset=utf-8'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'openai-organization', b'user-zum6n4kvtm5ynbo8eqmuyrwy'), (b'openai-processing-ms', b'390'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15552000; includeSubDomains; preload'), (b'x-ratelimit-limit-requests', b'10000'), (b'x-ratelimit-limit-tokens', b'200000'), (b'x-ratelimit-remaining-requests', b'9999'), (b'x-ratelimit-remaining-tokens', b'199397'), (b'x-ratelimit-reset-requests', b'8.64s'), (b'x-ratelimit-reset-tokens', b'180ms'), (b'x-request-id', b'req_ac39695fa6df451eb13b7f1ba5450ffc'), (b'CF-Cache-Status', b'DYNAMIC'), (b'X-Content-Type-Options', b'nosniff'), (b'Server', b'cloudflare'), (b'CF-RAY', b'8b18d74c6cba3afa-IAD'), (b'alt-svc', b'h3=":443"; ma=86400')])
INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK"
DEBUG:openai._base_client:HTTP Response: POST https://api.openai.com/v1/chat/completions "200 OK" Headers({'date': 'Sun, 11 Aug 2024 14:16:01 GMT', 'content-type': 'text/event-stream; charset=utf-8', 'transfer-encoding': 'chunked', 'connection': 'keep-alive', 'openai-organization': 'user-zum6n4kvtm5ynbo8eqmuyrwy', 'openai-processing-ms': '390', 'openai-version': '2020-10-01', 'strict-transport-security': 'max-age=15552000; includeSubDomains; preload', 'x-ratelimit-limit-requests': '10000', 'x-ratelimit-limit-tokens': '200000', 'x-ratelimit-remaining-requests': '9999', 'x-ratelimit-remaining-tokens': '199397', 'x-ratelimit-reset-requests': '8.64s', 'x-ratelimit-reset-tokens': '180ms', 'x-request-id': 'req_ac39695fa6df451eb13b7f1ba5450ffc', 'cf-cache-status': 'DYNAMIC', 'x-content-type-options': 'nosniff', 'server': 'cloudflare', 'cf-ray': '8b18d74c6cba3afa-IAD', 'alt-svc': 'h3=":443"; ma=86400'})
DEBUG:openai._base_client:request_id: req_ac39695fa6df451eb13b7f1ba5450ffc
DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>
DEBUG:httpcore.http11:receive_response_body.complete
DEBUG:httpcore.http11:response_closed.started
DEBUG:httpcore.http11:response_closed.complete

{
"action": "youtube_search",
"action_input": {
"title": "Антонио Вивальди",
"description": "search for youtube videos associated with Antonio Vivaldi",
"type": "string"
}
}

Traceback (most recent call last):
  File "/opt/homebrew/Cellar/[email protected]/3.12.4/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
    self.run()
  File "/opt/homebrew/Cellar/[email protected]/3.12.4/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py", line 1010, in run
    self._target(*self._args, **self._kwargs)
  File "/Users/engineer/workspace/VoiceAssistant/./src/main.py", line 122, in thinking_process
    text = self.llm.ask(question)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/engineer/workspace/VoiceAssistant/src/llm_langchains.py", line 116, in ask
    response = self.chain.invoke({"input": prompt})
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/engineer/workspace/VoiceAssistant/.venv/lib/python3.12/site-packages/langchain/chains/base.py", line 166, in invoke
    raise e
  File "/Users/engineer/workspace/VoiceAssistant/.venv/lib/python3.12/site-packages/langchain/chains/base.py", line 156, in invoke
    self._call(inputs, run_manager=run_manager)
  File "/Users/engineer/workspace/VoiceAssistant/.venv/lib/python3.12/site-packages/langchain/agents/agent.py", line 1612, in _call
    next_step_output = self._take_next_step(
                       ^^^^^^^^^^^^^^^^^^^^^
  File "/Users/engineer/workspace/VoiceAssistant/.venv/lib/python3.12/site-packages/langchain/agents/agent.py", line 1318, in _take_next_step
    [
  File "/Users/engineer/workspace/VoiceAssistant/.venv/lib/python3.12/site-packages/langchain/agents/agent.py", line 1403, in _iter_next_step
    yield self._perform_agent_action(
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/engineer/workspace/VoiceAssistant/.venv/lib/python3.12/site-packages/langchain/agents/agent.py", line 1425, in _perform_agent_action
    observation = tool.run(
                  ^^^^^^^^^
  File "/Users/engineer/workspace/VoiceAssistant/.venv/lib/python3.12/site-packages/langchain_core/tools.py", line 613, in run
    raise error_to_raise
  File "/Users/engineer/workspace/VoiceAssistant/.venv/lib/python3.12/site-packages/langchain_core/tools.py", line 582, in run
    response = context.run(self._run, *tool_args, **tool_kwargs)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
TypeError: YouTubeSearchTool._run() got an unexpected keyword argument 'title'

Thank you in advance.

System Info

langchain==0.2.12
langchain-community==0.2.11
langchain-core==0.2.29
langchain-google-genai==1.0.8
langchain-openai==0.1.20
langchain-text-splitters==0.2.2
langchainhub==0.1.20

[dosubot[bot]]

Hey @dzianisv! I'm here to help you with any questions or issues you have regarding the repository. If you need assistance, feel free to ask!

To tune the gpt-4o-mini model or agent to correctly handle the input arguments for the YouTube Search tool in your LangChain-based voice assistant, you can follow these steps:

1. Define the Input Argument Model: Create a Pydantic model to define the input arguments for the YouTube Search tool.

2. Set Up the Prompt: Create a prompt that instructs the LLM to generate the correct input arguments for the YouTube Search tool.

3. Bind the Model and Tools: Bind the model and tools to the LLM.

Here’s an example implementation:

from langchain_core.pydantic_v1 import BaseModel, Field
from langchain.output_parsers import PydanticToolsParser
from langchain_core.prompts import ChatPromptTemplate
from langchain_openai import ChatOpenAI

# Define the input argument model for the YouTube Search tool
class YouTubeSearchArgs(BaseModel):
    query: str = Field(..., description="The search query for YouTube.")
    max_results: int = Field(..., description="The maximum number of results to return.")

# Set up the prompt
system = """You are an expert at converting user questions into input arguments for the YouTube Search tool. \
You have access to a YouTube Search API. \
Generate the correct input arguments for the YouTube Search tool based on the user's question.

Return the arguments in the following format:
- query: The search query for YouTube.
- max_results: The maximum number of results to return."""

prompt = ChatPromptTemplate.from_messages(
    [
        ("system", system),
        ("human", "{question}"),
    ]
)

# Initialize the LLM
llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
llm_with_tools = llm.bind_tools([YouTubeSearchArgs])
query_analyzer = prompt | llm_with_tools | PydanticToolsParser(tools=[YouTubeSearchArgs])

# Example usage
response = query_analyzer.invoke(
    {
        "question": "Find the top 5 tutorials on LangChain"
    }
)

print(response)

This setup will help the gpt-4o-mini model to correctly handle the input arguments for the YouTube Search tool by generating the appropriate query and max_results based on the user's question ^[1].

To continue talking to Dosu, mention @dosu.