voice_assistant entity and voice stream handler

Author: zehnm

docs/setup.md

@@ -16,3 +16,28 @@ Local installation:
 ```shell
 pip3 install --force-reinstall dist/ucapi-$VERSION-py3-none-any.whl
 ```
+
+## Protobuf
+
+1. Optional (recommended): install the Python plugin toolchain for consistent results:
+   ```bash
+   python3 -m pip install --upgrade grpcio-tools protobuf
+   ```
+2. From the project root, run:
+   ```bash
+   python3 scripts/compile_protos.py
+   ```
+   - This will generate `ucapi/proto/ucr_integration_voice_pb2.py` (and `.pyi` if supported).
+3. Add and commit the generated files to Git:
+   ```bash
+   git add ucapi/proto/ucr_integration_voice_pb2.py ucapi/proto/ucr_integration_voice_pb2.pyi || true
+   git commit -m "Generate protobuf Python modules for voice integration"
+   ```
+
+Notes:
+- The library does not re-generate at build time; we ship the generated code with the package.
+- If you prefer using system `protoc`, ensure it’s on `PATH`; the script will fall back to it automatically.
+- Imports at runtime (if/when needed) will look like:
+  ```python
+  from ucapi.proto import ucr_integration_voice_pb2 as voice_pb2
+  ```

examples/setup_flow.py

@@ -38,7 +38,7 @@ async def handle_driver_setup(
     """
     Start driver setup.
 
-    Initiated by Remote Two to set up the driver.
+    Initiated by Remote Two/3 to set up the driver.
 
     :param msg: value(s) of input fields in the first setup screen.
     :return: the setup action on how to continue

examples/voice.json

@@ -0,0 +1,18 @@
+{
+  "driver_id": "voice_test",
+  "version": "0.0.1",
+  "min_core_api": "0.20.0",
+  "name": { "en": "Voice test" },
+  "icon": "uc:integration",
+  "description": {
+    "en": "Minimal Python integration driver example for voice commands."
+  },
+  "port": 9084,
+  "developer": {
+    "name": "Unfolded Circle ApS",
+    "email": "hello@unfoldedcircle.com",
+    "url": "https://www.unfoldedcircle.com"
+  },
+  "home_page": "https://www.unfoldedcircle.com",
+  "release_date": "2025-12-11"
+}

examples/voice.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""Voice assistant entity integration example. Bare minimum of an integration driver."""
+import asyncio
+import logging
+from asyncio import sleep
+from typing import Any
+
+import ucapi
+from ucapi import AssistantEvent, AssistantEventType, VoiceAssistant
+from ucapi.api_definitions import AssistantTextResponse, AssistantSttResponse
+from ucapi.voice_assistant import (
+    Commands as VACommands,
+    Features as VAFeatures,
+    Attributes as VAAttr,
+    VoiceAssistantEntityOptions,
+    AudioConfiguration,
+    SampleFormat,
+)
+
+loop = asyncio.new_event_loop()
+api = ucapi.IntegrationAPI(loop)
+
+session_id = 0
+
+
+@api.listens_to(ucapi.Events.CONNECT)
+async def on_connect() -> None:
+    # When the remote connects, we just set the device state. We are ready all the time!
+    await api.set_device_state(ucapi.DeviceStates.CONNECTED)
+
+
+@api.listens_to(ucapi.Events.SUBSCRIBE_ENTITIES)
+async def on_subscribe_entities(entity_ids: list[str]) -> None:
+    for entity_id in entity_ids:
+        api.configured_entities.update_attributes(entity_id, {VAAttr.STATE: "ON"})
+
+
+async def on_voice_cmd(
+    entity: ucapi.VoiceAssistant, cmd_id: str, params: dict[str, Any] | None
+) -> ucapi.StatusCodes:
+    """
+    Voice assistant command handler.
+
+    Called by the integration-API if a command is sent to a configured voice_assistant-entity.
+
+    :param entity: voice assistant entity
+    :param cmd_id: command
+    :param params: optional command parameters
+    :return: status of the command
+    """
+    # HACK until core is fixed
+    global session_id
+
+    print(f"Got {entity.id} command request: {cmd_id}")
+    if params is None:
+        return ucapi.StatusCodes.BAD_REQUEST
+
+    session_id = params.get("session_id", 0)
+    if session_id <= 0:
+        return ucapi.StatusCodes.BAD_REQUEST
+
+    if cmd_id == VACommands.VOICE_START:
+        ready_evt = AssistantEvent(
+            type=AssistantEventType.READY,
+            entity_id=entity.id,
+            session_id=session_id,
+        )
+        await api.broadcast_assistant_event(ready_evt)
+
+        # Acknowledge start; binary audio will arrive on the WS binary channel
+        return ucapi.StatusCodes.OK
+    return ucapi.StatusCodes.NOT_IMPLEMENTED
+
+
+async def on_voice_session(session):
+    print(
+        f"Voice stream started: session={session.session_id}, "
+        f"{session.config.channels}ch @ {session.config.sample_rate} Hz"
+    )
+    # HACK until core is fixed
+    global session_id
+
+    total = 0
+    async for frame in session:  # frame is bytes
+        total += len(frame)
+        # feed frame into your voice assistant / LLM here
+        print(f"Got {len(frame)} bytes of audio data")
+    print(f"Voice stream ended: session={session.session_id}, bytes={total}")
+
+    event = AssistantEvent(
+        type=AssistantEventType.STT_RESPONSE,
+        entity_id="va_main",
+        session_id=session_id,
+        data=AssistantSttResponse(
+            text="I'm just a demo and I don't know what you said."
+        ),
+    )
+    await api.broadcast_assistant_event(event)
+
+    await sleep(1)
+    event = AssistantEvent(
+        type=AssistantEventType.TEXT_RESPONSE,
+        entity_id="va_main",
+        session_id=session_id,
+        data=AssistantTextResponse(
+            success=True, text=f"You have sent {total} bytes of audio data"
+        ),
+    )
+    await api.broadcast_assistant_event(event)
+
+    await sleep(1)
+    event = AssistantEvent(
+        type=AssistantEventType.FINISHED,
+        entity_id="va_main",
+        session_id=session_id,
+    )
+    await api.broadcast_assistant_event(event)
+
+
+if __name__ == "__main__":
+    logging.basicConfig()
+
+    entity = VoiceAssistant(
+        identifier="va_main",
+        name={"en": "Demo Voice Assistant"},
+        features=[VAFeatures.TRANSCRIPTION, VAFeatures.RESPONSE_TEXT],
+        attributes={VAAttr.STATE.value: "ON"},
+        options=VoiceAssistantEntityOptions(
+            audio_cfg=AudioConfiguration(
+                channels=1, sample_rate=16000, sample_format=SampleFormat.I16
+            ),
+        ),
+        cmd_handler=on_voice_cmd,
+    )
+
+    api.available_entities.add(entity)
+    api.set_voice_stream_handler(on_voice_session)
+
+    loop.run_until_complete(api.init("voice.json"))
+    loop.run_forever()

requirements.txt

@@ -2,6 +2,7 @@
 # Waiting for: https://github.com/pypa/pip/issues/11440
 # Workaround: use a pre-commit hook with https://github.com/scikit-image/scikit-image/blob/main/tools/generate_requirements.py
 
+protobuf~=6.33.2
 pyee>=9.0
 websockets>=14.0
 zeroconf>=0.120.0

ucapi/__init__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-Integration driver library for Remote Two.
+Integration driver library for Remote Two/3.
 
 :copyright: (c) 2023 by Unfolded Circle ApS.
 :license: MPL-2.0, see LICENSE for more details.
@@ -11,6 +11,10 @@
 
 from .api_definitions import (  # isort:skip # noqa: F401
     AbortDriverSetup,
+    AssistantError,
+    AssistantErrorCode,
+    AssistantEvent,
+    AssistantEventType,
     DeviceStates,
     DriverSetupRequest,
     Events,
@@ -28,6 +32,11 @@
 from .entity import Entity, EntityTypes  # isort:skip # noqa: F401
 from .entities import Entities  # isort:skip # noqa: F401
 from .api import IntegrationAPI  # isort:skip # noqa: F401
+from .voice_stream import (  # isort:skip # noqa: F401
+    AudioConfig,
+    VoiceSession,
+    VoiceStreamHandler,
+)
 
 # Entity types
 from .button import Button  # noqa: F401
@@ -38,6 +47,7 @@
 from .remote import Remote  # noqa: F401
 from .sensor import Sensor  # noqa: F401
 from .switch import Switch  # noqa: F401
+from .voice_assistant import VoiceAssistant  # noqa: F401
 
 try:
     from ._version import version as __version__