Default conversation agent to store tool calls in chat log (home-assistant#157377)

Author: balloob

homeassistant/components/conversation/default_agent.py

@@ -66,6 +66,7 @@
     entity_registry as er,
     floor_registry as fr,
     intent,
+    llm,
     start as ha_start,
     template,
     translation,
@@ -76,7 +77,7 @@
 from homeassistant.util.json import JsonObjectType, json_loads_object
 
 from .agent_manager import get_agent_manager
-from .chat_log import AssistantContent, ChatLog
+from .chat_log import AssistantContent, ChatLog, ToolResultContent
 from .const import (
     DOMAIN,
     METADATA_CUSTOM_FILE,
@@ -430,6 +431,8 @@ async def _async_handle_message(
     ) -> ConversationResult:
         """Handle a message."""
         response: intent.IntentResponse | None = None
+        tool_input: llm.ToolInput | None = None
+        tool_result: dict[str, Any] = {}
 
         # Check if a trigger matched
         if trigger_result := await self.async_recognize_sentence_trigger(user_input):
@@ -438,6 +441,16 @@ async def _async_handle_message(
                 trigger_result, user_input
             )
 
+            # Create tool result
+            tool_input = llm.ToolInput(
+                tool_name="trigger_sentence",
+                tool_args={},
+                external=True,
+            )
+            tool_result = {
+                "response": response_text,
+            }
+
             # Convert to conversation result
             response = intent.IntentResponse(
                 language=user_input.language or self.hass.config.language
@@ -447,10 +460,44 @@ async def _async_handle_message(
         if response is None:
             # Match intents
             intent_result = await self.async_recognize_intent(user_input)
+
             response = await self._async_process_intent_result(
                 intent_result, user_input
             )
 
+            if response.response_type != intent.IntentResponseType.ERROR:
+                assert intent_result is not None
+                assert intent_result.intent is not None
+                # Create external tool call for the intent
+                tool_input = llm.ToolInput(
+                    tool_name=intent_result.intent.name,
+                    tool_args={
+                        entity.name: entity.value or entity.text
+                        for entity in intent_result.entities_list
+                    },
+                    external=True,
+                )
+                # Create tool result from intent response
+                tool_result = llm.IntentResponseDict(response)
+
+        # Add tool call and result to chat log if we have one
+        if tool_input is not None:
+            chat_log.async_add_assistant_content_without_tools(
+                AssistantContent(
+                    agent_id=user_input.agent_id,
+                    content=None,
+                    tool_calls=[tool_input],
+                )
+            )
+            chat_log.async_add_assistant_content_without_tools(
+                ToolResultContent(
+                    agent_id=user_input.agent_id,
+                    tool_call_id=tool_input.id,
+                    tool_name=tool_input.tool_name,
+                    tool_result=tool_result,
+                )
+            )
+
         speech: str = response.speech.get("plain", {}).get("speech", "")
         chat_log.async_add_assistant_content_without_tools(
             AssistantContent(

tests/components/conversation/test_default_agent.py

@@ -17,6 +17,11 @@
     default_agent,
     get_agent_manager,
 )
+from homeassistant.components.conversation.chat_log import (
+    AssistantContent,
+    ToolResultContent,
+    async_get_chat_log,
+)
 from homeassistant.components.conversation.default_agent import METADATA_CUSTOM_SENTENCE
 from homeassistant.components.conversation.models import ConversationInput
 from homeassistant.components.conversation.trigger import TriggerDetails
@@ -52,6 +57,7 @@
 )
 from homeassistant.helpers import (
     area_registry as ar,
+    chat_session,
     device_registry as dr,
     entity_registry as er,
     floor_registry as fr,
@@ -3424,3 +3430,149 @@ async def test_fuzzy_matching(
         if slot_name != "preferred_area_id"  # context area
     }
     assert actual_slots == slots
+
+
+@pytest.mark.usefixtures("init_components")
+async def test_intent_tool_call_in_chat_log(hass: HomeAssistant) -> None:
+    """Test that intent tool calls are stored in the chat log."""
+    hass.states.async_set(
+        "light.test_light", "off", attributes={ATTR_FRIENDLY_NAME: "Test Light"}
+    )
+    async_mock_service(hass, "light", "turn_on")
+
+    result = await conversation.async_converse(
+        hass, "turn on test light", None, Context(), None
+    )
+
+    assert result.response.response_type == intent.IntentResponseType.ACTION_DONE
+
+    with (
+        chat_session.async_get_chat_session(hass, result.conversation_id) as session,
+        async_get_chat_log(hass, session) as chat_log,
+    ):
+        pass
+
+    # Find the tool call in the chat log
+    tool_call_content: AssistantContent | None = None
+    tool_result_content: ToolResultContent | None = None
+    assistant_content: AssistantContent | None = None
+
+    for content in chat_log.content:
+        if content.role == "assistant" and content.tool_calls:
+            tool_call_content = content
+        if content.role == "tool_result":
+            tool_result_content = content
+        if content.role == "assistant" and not content.tool_calls:
+            assistant_content = content
+
+    # Verify tool call was stored
+    assert tool_call_content is not None and tool_call_content.tool_calls is not None
+    assert len(tool_call_content.tool_calls) == 1
+    assert tool_call_content.tool_calls[0].tool_name == "HassTurnOn"
+    assert tool_call_content.tool_calls[0].external is True
+    assert tool_call_content.tool_calls[0].tool_args.get("name") == "Test Light"
+
+    # Verify tool result was stored
+    assert tool_result_content is not None
+    assert tool_result_content.tool_name == "HassTurnOn"
+    assert tool_result_content.tool_result["response_type"] == "action_done"
+
+    # Verify final assistant content with speech
+    assert assistant_content is not None
+    assert assistant_content.content is not None
+
+
+@pytest.mark.usefixtures("init_components")
+async def test_trigger_tool_call_in_chat_log(hass: HomeAssistant) -> None:
+    """Test that trigger tool calls are stored in the chat log."""
+    trigger_sentence = "test automation trigger"
+    trigger_response = "Trigger activated!"
+
+    manager = get_agent_manager(hass)
+    callback = AsyncMock(return_value=trigger_response)
+    manager.register_trigger(TriggerDetails([trigger_sentence], callback))
+
+    result = await conversation.async_converse(
+        hass, trigger_sentence, None, Context(), None
+    )
+
+    assert result.response.response_type == intent.IntentResponseType.ACTION_DONE
+
+    with (
+        chat_session.async_get_chat_session(hass, result.conversation_id) as session,
+        async_get_chat_log(hass, session) as chat_log,
+    ):
+        pass
+
+    # Find the tool call in the chat log
+    tool_call_content: AssistantContent | None = None
+    tool_result_content: ToolResultContent | None = None
+
+    for content in chat_log.content:
+        if content.role == "assistant" and content.tool_calls:
+            tool_call_content = content
+        if content.role == "tool_result":
+            tool_result_content = content
+
+    # Verify tool call was stored
+    assert tool_call_content is not None and tool_call_content.tool_calls is not None
+    assert len(tool_call_content.tool_calls) == 1
+    assert tool_call_content.tool_calls[0].tool_name == "trigger_sentence"
+    assert tool_call_content.tool_calls[0].external is True
+    assert tool_call_content.tool_calls[0].tool_args == {}
+
+    # Verify tool result was stored
+    assert tool_result_content is not None
+    assert tool_result_content.tool_name == "trigger_sentence"
+    assert tool_result_content.tool_result["response"] == trigger_response
+
+
+@pytest.mark.usefixtures("init_components")
+async def test_no_tool_call_on_no_intent_match(hass: HomeAssistant) -> None:
+    """Test that no tool call is stored when no intent is matched."""
+    result = await conversation.async_converse(
+        hass, "this is a random sentence that should not match", None, Context(), None
+    )
+
+    assert result.response.response_type == intent.IntentResponseType.ERROR
+
+    with (
+        chat_session.async_get_chat_session(hass, result.conversation_id) as session,
+        async_get_chat_log(hass, session) as chat_log,
+    ):
+        pass
+
+    # Verify no tool call was stored
+    for content in chat_log.content:
+        if content.role == "assistant":
+            assert content.tool_calls is None or len(content.tool_calls) == 0
+            break
+    else:
+        pytest.fail("No assistant content found in chat log")
+
+
+@pytest.mark.usefixtures("init_components")
+async def test_intent_tool_call_with_error_response(hass: HomeAssistant) -> None:
+    """Test that intent tool calls store error information correctly."""
+    # Request to turn on a non-existent device
+    result = await conversation.async_converse(
+        hass, "turn on the non existent device", None, Context(), None
+    )
+
+    assert result.response.response_type == intent.IntentResponseType.ERROR
+    assert result.response.error_code == intent.IntentResponseErrorCode.NO_VALID_TARGETS
+
+    with (
+        chat_session.async_get_chat_session(hass, result.conversation_id) as session,
+        async_get_chat_log(hass, session) as chat_log,
+    ):
+        pass
+
+    # Verify no tool call was stored for unmatched entities
+    tool_call_found = False
+    for content in chat_log.content:
+        if content.role == "assistant" and content.tool_calls:
+            tool_call_found = True
+
+    # No tool call should be stored since the entity could not be matched
+    assert not tool_call_found