Add support for tool use and language model config (#123)

Author: gregorybchris

.vscode/settings.json

@@ -17,5 +17,5 @@
   "pylint.args": ["--rcfile", "pyproject.toml"],
   "python.analysis.extraPaths": ["tests"],
   "python.testing.cwd": "tests",
-  "cSpell.words": ["agen", "cffi", "deepgram", "indata", "samplerate", "simpleaudio", "sounddevice"]
+  "cSpell.words": ["agen", "cffi", "deepgram", "indata", "metas", "samplerate", "simpleaudio", "sounddevice"]
 }

hume/__init__.py

@@ -11,7 +11,16 @@
     TranscriptionConfig,
 )
 from hume._measurement.stream import HumeStreamClient, StreamSocket
-from hume._voice import HumeVoiceClient, MicrophoneInterface, VoiceChat, VoiceConfig, VoiceSocket
+from hume._voice import (
+    HumeVoiceClient,
+    LanguageModelConfig,
+    MicrophoneInterface,
+    VoiceChat,
+    VoiceConfig,
+    VoiceIdentityConfig,
+    VoiceSocket,
+    VoiceTool,
+)
 from hume.error.hume_client_exception import HumeClientException
 
 __version__ = version("hume")
@@ -26,10 +35,13 @@
     "HumeClientException",
     "HumeStreamClient",
     "HumeVoiceClient",
+    "LanguageModelConfig",
     "MicrophoneInterface",
     "StreamSocket",
     "TranscriptionConfig",
     "VoiceChat",
     "VoiceConfig",
+    "VoiceIdentityConfig",
     "VoiceSocket",
+    "VoiceTool",
 ]

hume/_common/client_base.py

@@ -115,7 +115,8 @@ def _request(
             response = self._http_client.send(request)
             response.raise_for_status()
         except Exception as exc:  # pylint: disable=broad-exception-caught
-            raise HumeClientException(str(exc)) from exc
+            response_body = response.json()
+            raise HumeClientException(str(response_body)) from exc
 
         return response
 

hume/_voice/__init__.py

@@ -3,13 +3,17 @@
 from hume._voice.hume_voice_client import HumeVoiceClient
 from hume._voice.microphone.microphone_interface import MicrophoneInterface
 from hume._voice.models.chats_models import VoiceChat
-from hume._voice.models.configs_models import VoiceConfig
+from hume._voice.models.configs_models import LanguageModelConfig, VoiceConfig, VoiceIdentityConfig
+from hume._voice.models.tools_models import VoiceTool
 from hume._voice.voice_socket import VoiceSocket
 
 __all__ = [
     "HumeVoiceClient",
+    "LanguageModelConfig",
     "MicrophoneInterface",
     "VoiceChat",
     "VoiceConfig",
+    "VoiceIdentityConfig",
     "VoiceSocket",
+    "VoiceTool",
 ]

hume/_voice/hume_voice_client.py

@@ -5,9 +5,10 @@
 from hume._voice.mixins.chat_mixin import ChatMixin
 from hume._voice.mixins.chats_mixin import ChatsMixin
 from hume._voice.mixins.configs_mixin import ConfigsMixin
+from hume._voice.mixins.tools_mixin import ToolsMixin
 
 logger = logging.getLogger(__name__)
 
 
-class HumeVoiceClient(ChatMixin, ChatsMixin, ConfigsMixin):
+class HumeVoiceClient(ChatMixin, ChatsMixin, ConfigsMixin, ToolsMixin):
     """Empathic Voice Interface client."""

hume/_voice/microphone/chat_client.py

@@ -44,6 +44,11 @@ def _map_role(cls, role: str) -> str:
             return cls.DEFAULT_ASSISTANT_ROLE_NAME
         return role
 
+    def _print_prompt(self, text: str) -> None:
+        now = datetime.datetime.now(tz=datetime.timezone.utc)
+        now_str = now.strftime("%H:%M:%S")
+        print(f"[{now_str}] {text}")
+
     async def _recv(self, *, socket: VoiceSocket) -> None:
         async for socket_message in socket:
             message = json.loads(socket_message)
@@ -60,14 +65,22 @@ async def _recv(self, *, socket: VoiceSocket) -> None:
                 error_message: str = message["message"]
                 error_code: str = message["code"]
                 raise HumeClientException(f"Error ({error_code}): {error_message}")
+            elif message["type"] == "tool_call":
+                print(
+                    "Warning: EVI is trying to make a tool call. "
+                    "Either remove tool calling from your config or "
+                    "use the VoiceSocket directly without a MicrophoneInterface."
+                )
+                tool_call_id = message["tool_call_id"]
+                if message["response_required"]:
+                    content = "Let's start over"
+                    await self.sender.send_tool_response(socket=socket, tool_call_id=tool_call_id, content=content)
+                continue
             else:
                 message_type = message["type"].upper()
                 text = f"<{message_type}>"
 
-            now = datetime.datetime.now(tz=datetime.timezone.utc)
-            now_str = now.strftime("%H:%M:%S")
-
-            print(f"[{now_str}] {text}")
+            self._print_prompt(text)
 
     async def _play(self) -> None:
         async for byte_str in self.byte_strs:

hume/_voice/microphone/microphone_sender.py

@@ -1,5 +1,6 @@
 """Sender for streaming audio from a microphone."""
 
+import json
 import logging
 from dataclasses import dataclass
 from typing import Protocol
@@ -29,6 +30,16 @@ async def send(self, *, socket: VoiceSocket) -> None:
         """
         raise NotImplementedError()
 
+    async def send_tool_response(self, *, socket: VoiceSocket, tool_call_id: str, content: str) -> None:
+        """Send a tool response over an EVI socket.
+
+        Args:
+            socket (VoiceSocket): EVI socket.
+            tool_call_id (str): Tool call ID.
+            content (str): Tool response content.
+        """
+        raise NotImplementedError()
+
 
 @dataclass
 class MicrophoneSender(Sender):
@@ -65,3 +76,18 @@ async def send(self, *, socket: VoiceSocket) -> None:
         async for byte_str in self.microphone:
             if self.send_audio:
                 await socket.send(byte_str)
+
+    async def send_tool_response(self, *, socket: VoiceSocket, tool_call_id: str, content: str) -> None:
+        """Send a tool response over an EVI socket.
+
+        Args:
+            socket (VoiceSocket): EVI socket.
+            tool_call_id (str): Tool call ID.
+            content (str): Tool response content.
+        """
+        response_message = {
+            "type": "tool_response",
+            "tool_call_id": tool_call_id,
+            "content": content,
+        }
+        await socket.send(json.dumps(response_message).encode("utf-8"))

hume/_voice/mixins/configs_mixin.py

@@ -1,20 +1,22 @@
 """Client operations for managing EVI configurations."""
 
 import logging
-from typing import Iterator, Optional
+from typing import Iterator, List, Optional
 
 from hume._common.client_base import ClientBase
 from hume._common.utilities.paging_utilities import Paging
 from hume._voice.models.configs_models import (
     ConfigResponse,
     ConfigsResponse,
+    LanguageModelConfig,
     PostConfigRequest,
     PostPromptRequest,
     PromptMeta,
     PromptResponse,
     VoiceConfig,
     VoiceIdentityConfig,
 )
+from hume._voice.models.tools_models import ToolMeta, VoiceTool
 from hume.error.hume_client_exception import HumeClientException
 
 logger = logging.getLogger(__name__)
@@ -32,7 +34,9 @@ def create_config(
         name: str,
         prompt: str,
         description: Optional[str] = None,
-        voice_name: Optional[str] = DEFAULT_VOICE_NAME,
+        voice_identity_config: Optional[VoiceIdentityConfig] = None,
+        tools: Optional[List[VoiceTool]] = None,
+        language_model: Optional[LanguageModelConfig] = None,
     ) -> VoiceConfig:
         """Create a new EVI config.
 
@@ -48,11 +52,14 @@ def create_config(
         prompt_response = PromptResponse.model_validate_json(response.text)
         prompt_meta = PromptMeta(id=prompt_response.id, version=prompt_response.version)
 
+        tool_metas = None if tools is None else [ToolMeta(id=tool.id, version=0) for tool in tools]
         post_config_request = PostConfigRequest(
             name=name,
             version_description=description,
             prompt=prompt_meta,
-            voice=VoiceIdentityConfig(name=voice_name),
+            voice=voice_identity_config,
+            tools=tool_metas,
+            language_model=language_model,
         )
         post_config_body = post_config_request.to_json_str()
         endpoint = self._build_endpoint("evi", "configs")

hume/_voice/mixins/tools_mixin.py

@@ -0,0 +1,105 @@
+"""Client operations for managing EVI tools."""
+
+import logging
+from typing import Iterator, Optional
+
+from hume._common.client_base import ClientBase
+from hume._common.utilities.paging_utilities import Paging
+from hume._voice.models.tools_models import PostToolRequest, ToolResponse, ToolsResponse, VoiceTool
+from hume.error.hume_client_exception import HumeClientException
+
+logger = logging.getLogger(__name__)
+
+
+# pylint: disable=redefined-builtin
+class ToolsMixin(ClientBase):
+    """Client operations for managing EVI tools."""
+
+    def create_tool(
+        self,
+        *,
+        name: str,
+        parameters: str,
+        fallback_content: Optional[str] = None,
+        description: Optional[str] = None,
+    ) -> VoiceTool:
+        """Create a new EVI tool.
+
+        Args:
+            name (str): Tool name.
+            parameters (str): Stringified JSON defining the parameters used by the tool.
+            fallback_content (Optional[str]): Text to use if the tool fails to generate content.
+            description (Optional[str]): Tool description.
+        """
+        post_tool_request = PostToolRequest(
+            name=name,
+            description=description,
+            version_description=None,
+            parameters=parameters,
+            fallback_content=fallback_content,
+        )
+        post_tool_body = post_tool_request.to_json_str()
+        endpoint = self._build_endpoint("evi", "tools")
+        response = self._request(endpoint, method="POST", body_json_str=post_tool_body)
+        tool_response = ToolResponse.model_validate_json(response.text)
+
+        return self._tool_from_response(tool_response)
+
+    def get_tool(self, id: str, _version: Optional[int] = None) -> VoiceTool:
+        """Get an EVI tool by its ID.
+
+        Args:
+            id (str): Tool ID.
+        """
+        route = f"tools/{id}" if _version is None else f"tools/{id}/version/{_version}"
+        endpoint = self._build_endpoint("evi", route)
+        response = self._request(endpoint, method="GET")
+        tools_response = ToolsResponse.model_validate_json(response.text)
+        if len(tools_response.tools_page) == 0:
+            raise HumeClientException(f"Tool not found with ID: {id}")
+
+        return self._tool_from_response(tools_response.tools_page[0])
+
+    def _iter_tool_versions(self, id: str) -> Iterator[VoiceTool]:
+        endpoint = self._build_endpoint("evi", f"tools/{id}")
+        for page_number in range(self.PAGING_LIMIT):
+            paging = Paging(page_size=self._page_size, page_number=page_number)
+            response = self._request(endpoint, method="GET", paging=paging)
+            tools_response = ToolsResponse.model_validate_json(response.text)
+            if len(tools_response.tools_page) == 0:
+                break
+            for res in tools_response.tools_page:
+                yield self._tool_from_response(res)
+
+    def _tool_from_response(self, tool_response: ToolResponse) -> VoiceTool:
+        return VoiceTool(
+            id=tool_response.id,
+            name=tool_response.name,
+            created_on=tool_response.created_on,
+            modified_on=tool_response.modified_on,
+            parameters=tool_response.parameters,
+            description=tool_response.description,
+            fallback_content=tool_response.fallback_content,
+        )
+
+    def iter_tools(self) -> Iterator[VoiceTool]:
+        """Iterate over existing EVI tools."""
+        endpoint = self._build_endpoint("evi", "tools")
+        for page_number in range(self.PAGING_LIMIT):
+            paging = Paging(page_size=self._page_size, page_number=page_number)
+            response = self._request(endpoint, method="GET", paging=paging)
+            tools_response = ToolsResponse.model_validate_json(response.text)
+            if len(tools_response.tools_page) == 0:
+                break
+            for res in tools_response.tools_page:
+                yield self._tool_from_response(res)
+
+    def delete_tool(self, id: str, _version: Optional[int] = None) -> None:
+        """Delete an EVI tool.
+
+        Args:
+            id (str): Tool ID.
+        """
+        route = f"tools/{id}" if _version is None else f"tools/{id}/version/{_version}"
+        endpoint = self._build_endpoint("evi", route)
+        self._request(endpoint, method="DELETE")

hume/_voice/models/configs_models.py

@@ -3,6 +3,7 @@
 from typing import List, Optional
 
 from hume._common.utilities.model_utilities import BaseModel
+from hume._voice.models.tools_models import ToolMeta
 
 
 class PromptResponse(BaseModel):
@@ -32,6 +33,14 @@ class PromptMeta(BaseModel):
     version: int
 
 
+class LanguageModelConfig(BaseModel):
+    """Language model configuration for EVI."""
+
+    model_provider: str
+    model_resource: str
+    temperature: Optional[float] = None
+
+
 class PostPromptRequest(BaseModel):
     """Post request model for creating a new EVI prompt."""
 
@@ -67,13 +76,23 @@ class VoiceIdentityConfig(BaseModel):
     name: Optional[str] = None
 
 
+class BuiltinToolConfig(BaseModel):
+    """Configuration for a built-in EVI tool."""
+
+    name: str
+    tool_type: str
+    fallback_content: Optional[str]
+
+
 class PostConfigRequest(BaseModel):
     """Post request model for creating a new EVI configuration."""
 
     name: str
     version_description: Optional[str]
     prompt: PromptMeta
-    voice: VoiceIdentityConfig
+    voice: Optional[VoiceIdentityConfig]
+    language_model: Optional[LanguageModelConfig]
+    tools: Optional[List[ToolMeta]]
 
 
 class ConfigMeta(BaseModel):
@@ -91,4 +110,5 @@ class VoiceConfig(BaseModel):
     description: Optional[str]
     created_on: int
     modified_on: int
+    # TODO: Add tool info
     prompt: Optional[str]