Improve utf8 text split and add unit test (#1414)

Author: lukasIO

.changeset/grumpy-zebras-itch.md

@@ -0,0 +1,5 @@
+---
+"livekit-client": patch
+---
+
+Improve utf8 text split and add unit test

src/room/participant/LocalParticipant.ts

@@ -1608,10 +1608,10 @@ export default class LocalParticipant extends Participant {
     const writableStream = new WritableStream<string>({
       // Implement the sink
       async write(text) {
-        for (const textChunk of splitUtf8(text, STREAM_CHUNK_SIZE)) {
+        for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE)) {
           await localP.engine.waitForBufferStatusLow(DataPacket_Kind.RELIABLE);
           const chunk = new DataStream_Chunk({
-            content: new TextEncoder().encode(textChunk),
+            content: textByteChunk,
             streamId,
             chunkIndex: numberToBigInt(chunkId),
           });

src/room/utils.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { toWebsocketUrl } from './utils';
+import { splitUtf8, toWebsocketUrl } from './utils';
 
 describe('toWebsocketUrl', () => {
   it('leaves wss urls alone', () => {
@@ -14,3 +14,49 @@ describe('toWebsocketUrl', () => {
     expect(toWebsocketUrl('https://httpsmywebsite.com')).toEqual('wss://httpsmywebsite.com');
   });
 });
+
+describe('splitUtf8', () => {
+  it('splits a string into chunks of the given size', () => {
+    expect(splitUtf8('hello world', 5)).toEqual([
+      new TextEncoder().encode('hello'),
+      new TextEncoder().encode(' worl'),
+      new TextEncoder().encode('d'),
+    ]);
+  });
+
+  it('splits a string with special characters into chunks of the given size', () => {
+    expect(splitUtf8('héllo wörld', 5)).toEqual([
+      new TextEncoder().encode('héll'),
+      new TextEncoder().encode('o wö'),
+      new TextEncoder().encode('rld'),
+    ]);
+  });
+
+  it('splits a string with multi-byte utf8 characters correctly', () => {
+    expect(splitUtf8('こんにちは世界', 5)).toEqual([
+      new TextEncoder().encode('こ'),
+      new TextEncoder().encode('ん'),
+      new TextEncoder().encode('に'),
+      new TextEncoder().encode('ち'),
+      new TextEncoder().encode('は'),
+      new TextEncoder().encode('世'),
+      new TextEncoder().encode('界'),
+    ]);
+  });
+
+  it('handles a string with a single multi-byte utf8 character', () => {
+    expect(splitUtf8('😊', 5)).toEqual([new TextEncoder().encode('😊')]);
+  });
+
+  it('handles a string with mixed single and multi-byte utf8 characters', () => {
+    expect(splitUtf8('a😊b', 4)).toEqual([
+      new TextEncoder().encode('a'),
+      new TextEncoder().encode('😊'),
+      new TextEncoder().encode('b'),
+    ]);
+  });
+
+  it('handles an empty string', () => {
+    expect(splitUtf8('', 5)).toEqual([]);
+  });
+});

src/room/utils.ts

@@ -626,18 +626,27 @@ export function isRemoteParticipant(p: Participant): p is RemoteParticipant {
   return !p.isLocal;
 }
 
-export function splitUtf8(s: string, n: number): string[] {
+export function splitUtf8(s: string, n: number): Uint8Array[] {
+  if (n < 4) {
+    throw new Error('n must be at least 4 due to utf8 encoding rules');
+  }
   // adapted from https://stackoverflow.com/a/6043797
-  const result: string[] = [];
-  while (s.length > n) {
+  const result: Uint8Array[] = [];
+  let encoded = new TextEncoder().encode(s);
+  while (encoded.length > n) {
     let k = n;
-    // Move back to find the start of a UTF-8 character
-    while ((s.charCodeAt(k) & 0xc0) === 0x80) {
+    while (k > 0) {
+      const byte = encoded[k];
+      if (byte !== undefined && (byte & 0xc0) !== 0x80) {
+        break;
+      }
       k--;
     }
-    result.push(s.slice(0, k));
-    s = s.slice(k);
+    result.push(encoded.slice(0, k));
+    encoded = encoded.slice(k);
+  }
+  if (encoded.length > 0) {
+    result.push(encoded);
   }
-  result.push(s);
   return result;
 }