improve utf8 split handling

Author: lukasIO

livekit-rtc/livekit/rtc/_utils.py

@@ -17,7 +17,7 @@
 from collections import deque
 import ctypes
 import random
-from typing import Callable, Generic, List, TypeVar
+from typing import Callable, Generator, Generic, List, TypeVar
 
 logger = logging.getLogger("livekit")
 
@@ -133,12 +133,13 @@ def generate_random_base62(length=12):
 
 
 # adapted from https://stackoverflow.com/a/6043797
-def split_utf8(s: str, n: int):
+def split_utf8(s: str, n: int) -> Generator[bytes, None, None]:
     """Split UTF-8 s into chunks of maximum length n."""
-    while len(s) > n:
+    encoded = s.encode()
+    while len(encoded) > n:
         k = n
-        while (ord(s[k]) & 0xC0) == 0x80:
+        while (encoded[k] & 0xC0) == 0x80:
             k -= 1
-        yield s[:k]
-        s = s[k:]
-    yield s
+        yield encoded[:k]
+        encoded = encoded[k:]
+    yield encoded

livekit-rtc/livekit/rtc/data_stream.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 import uuid
 import datetime
 from collections.abc import Callable
@@ -27,11 +28,14 @@
 from ._utils import split_utf8
 from typing import TYPE_CHECKING
 
+
 if TYPE_CHECKING:
     from .participant import LocalParticipant
 
 STREAM_CHUNK_SIZE = 15_000
 
+logger = logging.getLogger("livekit")
+
 
 @dataclass
 class BaseStreamInfo:
@@ -66,6 +70,7 @@ def __init__(
         self._queue: asyncio.Queue[proto_DataStream.Chunk | None] = asyncio.Queue()
 
     async def _on_chunk_update(self, chunk: proto_DataStream.Chunk):
+        logger.info("Received text chunk: %s", chunk.content)
         await self._queue.put(chunk)
 
     async def _on_stream_close(self, trailer: proto_DataStream.Trailer):
@@ -280,7 +285,7 @@ def __init__(
     async def write(self, text: str):
         async with self._write_lock:
             for chunk in split_utf8(text, STREAM_CHUNK_SIZE):
-                content = chunk.encode()
+                content = chunk
                 chunk_index = self._next_chunk_index
                 self._next_chunk_index += 1
                 chunk_msg = proto_DataStream.Chunk(