Add support for calling tools in Open Router (home-assistant#148881)

Author: joostlek

homeassistant/components/open_router/config_flow.py

@@ -16,18 +16,20 @@
     ConfigSubentryFlow,
     SubentryFlowResult,
 )
-from homeassistant.const import CONF_API_KEY, CONF_MODEL
+from homeassistant.const import CONF_API_KEY, CONF_LLM_HASS_API, CONF_MODEL
 from homeassistant.core import callback
+from homeassistant.helpers import llm
 from homeassistant.helpers.aiohttp_client import async_get_clientsession
 from homeassistant.helpers.httpx_client import get_async_client
 from homeassistant.helpers.selector import (
     SelectOptionDict,
     SelectSelector,
     SelectSelectorConfig,
     SelectSelectorMode,
+    TemplateSelector,
 )
 
-from .const import DOMAIN
+from .const import CONF_PROMPT, DOMAIN, RECOMMENDED_CONVERSATION_OPTIONS
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -90,6 +92,8 @@ async def async_step_user(
     ) -> SubentryFlowResult:
         """User flow to create a sensor subentry."""
         if user_input is not None:
+            if not user_input.get(CONF_LLM_HASS_API):
+                user_input.pop(CONF_LLM_HASS_API, None)
             return self.async_create_entry(
                 title=self.options[user_input[CONF_MODEL]], data=user_input
             )
@@ -99,11 +103,17 @@ async def async_step_user(
             api_key=entry.data[CONF_API_KEY],
             http_client=get_async_client(self.hass),
         )
+        hass_apis: list[SelectOptionDict] = [
+            SelectOptionDict(
+                label=api.name,
+                value=api.id,
+            )
+            for api in llm.async_get_apis(self.hass)
+        ]
         options = []
         async for model in client.with_options(timeout=10.0).models.list():
             options.append(SelectOptionDict(value=model.id, label=model.name))  # type: ignore[attr-defined]
             self.options[model.id] = model.name  # type: ignore[attr-defined]
-
         return self.async_show_form(
             step_id="user",
             data_schema=vol.Schema(
@@ -113,6 +123,20 @@ async def async_step_user(
                             options=options, mode=SelectSelectorMode.DROPDOWN, sort=True
                         ),
                     ),
+                    vol.Optional(
+                        CONF_PROMPT,
+                        description={
+                            "suggested_value": RECOMMENDED_CONVERSATION_OPTIONS[
+                                CONF_PROMPT
+                            ]
+                        },
+                    ): TemplateSelector(),
+                    vol.Optional(
+                        CONF_LLM_HASS_API,
+                        default=RECOMMENDED_CONVERSATION_OPTIONS[CONF_LLM_HASS_API],
+                    ): SelectSelector(
+                        SelectSelectorConfig(options=hass_apis, multiple=True)
+                    ),
                 }
             ),
         )

homeassistant/components/open_router/const.py

@@ -2,5 +2,17 @@
 
 import logging
 
+from homeassistant.const import CONF_LLM_HASS_API
+from homeassistant.helpers import llm
+
 DOMAIN = "open_router"
 LOGGER = logging.getLogger(__package__)
+
+CONF_PROMPT = "prompt"
+CONF_RECOMMENDED = "recommended"
+
+RECOMMENDED_CONVERSATION_OPTIONS = {
+    CONF_RECOMMENDED: True,
+    CONF_LLM_HASS_API: [llm.LLM_API_ASSIST],
+    CONF_PROMPT: llm.DEFAULT_INSTRUCTIONS_PROMPT,
+}

homeassistant/components/open_router/conversation.py

@@ -1,25 +1,39 @@
 """Conversation support for OpenRouter."""
 
-from typing import Literal
+from collections.abc import AsyncGenerator, Callable
+import json
+from typing import Any, Literal
 
 import openai
+from openai import NOT_GIVEN
 from openai.types.chat import (
     ChatCompletionAssistantMessageParam,
+    ChatCompletionMessage,
     ChatCompletionMessageParam,
+    ChatCompletionMessageToolCallParam,
     ChatCompletionSystemMessageParam,
+    ChatCompletionToolMessageParam,
+    ChatCompletionToolParam,
     ChatCompletionUserMessageParam,
 )
+from openai.types.chat.chat_completion_message_tool_call_param import Function
+from openai.types.shared_params import FunctionDefinition
+from voluptuous_openapi import convert
 
 from homeassistant.components import conversation
 from homeassistant.config_entries import ConfigSubentry
 from homeassistant.const import CONF_LLM_HASS_API, CONF_MODEL, MATCH_ALL
 from homeassistant.core import HomeAssistant
 from homeassistant.exceptions import HomeAssistantError
+from homeassistant.helpers import llm
 from homeassistant.helpers.device_registry import DeviceEntryType, DeviceInfo
 from homeassistant.helpers.entity_platform import AddConfigEntryEntitiesCallback
 
 from . import OpenRouterConfigEntry
-from .const import DOMAIN, LOGGER
+from .const import CONF_PROMPT, DOMAIN, LOGGER
+
+# Max number of back and forth with the LLM to generate a response
+MAX_TOOL_ITERATIONS = 10
 
 
 async def async_setup_entry(
@@ -35,13 +49,31 @@ async def async_setup_entry(
         )
 
 
+def _format_tool(
+    tool: llm.Tool,
+    custom_serializer: Callable[[Any], Any] | None,
+) -> ChatCompletionToolParam:
+    """Format tool specification."""
+    tool_spec = FunctionDefinition(
+        name=tool.name,
+        parameters=convert(tool.parameters, custom_serializer=custom_serializer),
+    )
+    if tool.description:
+        tool_spec["description"] = tool.description
+    return ChatCompletionToolParam(type="function", function=tool_spec)
+
+
 def _convert_content_to_chat_message(
     content: conversation.Content,
 ) -> ChatCompletionMessageParam | None:
     """Convert any native chat message for this agent to the native format."""
     LOGGER.debug("_convert_content_to_chat_message=%s", content)
     if isinstance(content, conversation.ToolResultContent):
-        return None
+        return ChatCompletionToolMessageParam(
+            role="tool",
+            tool_call_id=content.tool_call_id,
+            content=json.dumps(content.tool_result),
+        )
 
     role: Literal["user", "assistant", "system"] = content.role
     if role == "system" and content.content:
@@ -51,13 +83,55 @@ def _convert_content_to_chat_message(
         return ChatCompletionUserMessageParam(role="user", content=content.content)
 
     if role == "assistant":
-        return ChatCompletionAssistantMessageParam(
-            role="assistant", content=content.content
+        param = ChatCompletionAssistantMessageParam(
+            role="assistant",
+            content=content.content,
         )
+        if isinstance(content, conversation.AssistantContent) and content.tool_calls:
+            param["tool_calls"] = [
+                ChatCompletionMessageToolCallParam(
+                    type="function",
+                    id=tool_call.id,
+                    function=Function(
+                        arguments=json.dumps(tool_call.tool_args),
+                        name=tool_call.tool_name,
+                    ),
+                )
+                for tool_call in content.tool_calls
+            ]
+        return param
     LOGGER.warning("Could not convert message to Completions API: %s", content)
     return None
 
 
+def _decode_tool_arguments(arguments: str) -> Any:
+    """Decode tool call arguments."""
+    try:
+        return json.loads(arguments)
+    except json.JSONDecodeError as err:
+        raise HomeAssistantError(f"Unexpected tool argument response: {err}") from err
+
+
+async def _transform_response(
+    message: ChatCompletionMessage,
+) -> AsyncGenerator[conversation.AssistantContentDeltaDict]:
+    """Transform the OpenRouter message to a ChatLog format."""
+    data: conversation.AssistantContentDeltaDict = {
+        "role": message.role,
+        "content": message.content,
+    }
+    if message.tool_calls:
+        data["tool_calls"] = [
+            llm.ToolInput(
+                id=tool_call.id,
+                tool_name=tool_call.function.name,
+                tool_args=_decode_tool_arguments(tool_call.function.arguments),
+            )
+            for tool_call in message.tool_calls
+        ]
+    yield data
+
+
 class OpenRouterConversationEntity(conversation.ConversationEntity):
     """OpenRouter conversation agent."""
 
@@ -75,6 +149,10 @@ def __init__(self, entry: OpenRouterConfigEntry, subentry: ConfigSubentry) -> No
             name=subentry.title,
             entry_type=DeviceEntryType.SERVICE,
         )
+        if self.subentry.data.get(CONF_LLM_HASS_API):
+            self._attr_supported_features = (
+                conversation.ConversationEntityFeature.CONTROL
+            )
 
     @property
     def supported_languages(self) -> list[str] | Literal["*"]:
@@ -93,12 +171,19 @@ async def _async_handle_message(
             await chat_log.async_provide_llm_data(
                 user_input.as_llm_context(DOMAIN),
                 options.get(CONF_LLM_HASS_API),
-                None,
+                options.get(CONF_PROMPT),
                 user_input.extra_system_prompt,
             )
         except conversation.ConverseError as err:
             return err.as_conversation_result()
 
+        tools: list[ChatCompletionToolParam] | None = None
+        if chat_log.llm_api:
+            tools = [
+                _format_tool(tool, chat_log.llm_api.custom_serializer)
+                for tool in chat_log.llm_api.tools
+            ]
+
         messages = [
             m
             for content in chat_log.content
@@ -107,27 +192,34 @@ async def _async_handle_message(
 
         client = self.entry.runtime_data
 
-        try:
-            result = await client.chat.completions.create(
-                model=self.model,
-                messages=messages,
-                user=chat_log.conversation_id,
-                extra_headers={
-                    "X-Title": "Home Assistant",
-                    "HTTP-Referer": "https://www.home-assistant.io/integrations/open_router",
-                },
+        for _iteration in range(MAX_TOOL_ITERATIONS):
+            try:
+                result = await client.chat.completions.create(
+                    model=self.model,
+                    messages=messages,
+                    tools=tools or NOT_GIVEN,
+                    user=chat_log.conversation_id,
+                    extra_headers={
+                        "X-Title": "Home Assistant",
+                        "HTTP-Referer": "https://www.home-assistant.io/integrations/open_router",
+                    },
+                )
+            except openai.OpenAIError as err:
+                LOGGER.error("Error talking to API: %s", err)
+                raise HomeAssistantError("Error talking to API") from err
+
+            result_message = result.choices[0].message
+
+            messages.extend(
+                [
+                    msg
+                    async for content in chat_log.async_add_delta_content_stream(
+                        user_input.agent_id, _transform_response(result_message)
+                    )
+                    if (msg := _convert_content_to_chat_message(content))
+                ]
             )
-        except openai.OpenAIError as err:
-            LOGGER.error("Error talking to API: %s", err)
-            raise HomeAssistantError("Error talking to API") from err
-
-        result_message = result.choices[0].message
-
-        chat_log.async_add_assistant_content_without_tools(
-            conversation.AssistantContent(
-                agent_id=user_input.agent_id,
-                content=result_message.content,
-            )
-        )
+            if not chat_log.unresponded_tool_results:
+                break
 
         return conversation.async_get_result_from_chat_log(user_input, chat_log)

homeassistant/components/open_router/strings.json

@@ -24,7 +24,13 @@
         "user": {
           "description": "Configure the new conversation agent",
           "data": {
-            "model": "Model"
+            "model": "Model",
+            "prompt": "Instructions",
+            "llm_hass_api": "[%key:common::config_flow::data::llm_hass_api%]"
+          },
+          "data_description": {
+            "model": "The model to use for the conversation agent",
+            "prompt": "Instruct how the LLM should respond. This can be a template."
           }
         }
       },

tests/components/open_router/conftest.py

@@ -2,17 +2,19 @@
 
 from collections.abc import AsyncGenerator, Generator
 from dataclasses import dataclass
+from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
 
 from openai.types import CompletionUsage
 from openai.types.chat import ChatCompletion, ChatCompletionMessage
 from openai.types.chat.chat_completion import Choice
 import pytest
 
-from homeassistant.components.open_router.const import DOMAIN
+from homeassistant.components.open_router.const import CONF_PROMPT, DOMAIN
 from homeassistant.config_entries import ConfigSubentryData
-from homeassistant.const import CONF_API_KEY, CONF_MODEL
+from homeassistant.const import CONF_API_KEY, CONF_LLM_HASS_API, CONF_MODEL
 from homeassistant.core import HomeAssistant
+from homeassistant.helpers import llm
 from homeassistant.setup import async_setup_component
 
 from tests.common import MockConfigEntry
@@ -29,7 +31,27 @@ def mock_setup_entry() -> Generator[AsyncMock]:
 
 
 @pytest.fixture
-def mock_config_entry(hass: HomeAssistant) -> MockConfigEntry:
+def enable_assist() -> bool:
+    """Mock conversation subentry data."""
+    return False
+
+
+@pytest.fixture
+def conversation_subentry_data(enable_assist: bool) -> dict[str, Any]:
+    """Mock conversation subentry data."""
+    res: dict[str, Any] = {
+        CONF_MODEL: "gpt-3.5-turbo",
+        CONF_PROMPT: "You are a helpful assistant.",
+    }
+    if enable_assist:
+        res[CONF_LLM_HASS_API] = [llm.LLM_API_ASSIST]
+    return res
+
+
+@pytest.fixture
+def mock_config_entry(
+    hass: HomeAssistant, conversation_subentry_data: dict[str, Any]
+) -> MockConfigEntry:
     """Mock a config entry."""
     return MockConfigEntry(
         title="OpenRouter",
@@ -39,7 +61,7 @@ def mock_config_entry(hass: HomeAssistant) -> MockConfigEntry:
         },
         subentries_data=[
             ConfigSubentryData(
-                data={CONF_MODEL: "gpt-3.5-turbo"},
+                data=conversation_subentry_data,
                 subentry_id="ABCDEF",
                 subentry_type="conversation",
                 title="GPT-3.5 Turbo",