terminal/sagas: fix handling multibyte unicode

Enable the stream option on the text decode for the stdout and uart
stream handlers that pipe data to the terminal. This fixes improperly
printing multibyte unicode characters that get split across multiple
data packets.

A new decode is added for each different stream in case both are used
at the same time.

Fixes: pybricks/support#1743

Author: dlech

CHANGELOG.md

@@ -6,8 +6,11 @@
 
 ### Fixed
 - Fixed Bluetooth firmware updates sometimes failing on macOS. ([support#1787])
+- Fixed multibyte unicode characters not printing correctly in terminal when
+  split across Bluetooth packets. ([support#1743])
 
-[[support#1787]]: https://github.com/pybricks/support/issues/1787
+[support#1743]: https://github.com/pybricks/support/issues/1743
+[support#1787]: https://github.com/pybricks/support/issues/1787
 
 ## [2.3.0-beta.1] - 2023-11-24
 

src/terminal/sagas.test.ts

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: MIT
-// Copyright (c) 2020-2023 The Pybricks Authors
+// Copyright (c) 2020-2024 The Pybricks Authors
 
 import PushStream from 'zen-push';
 import { AsyncSaga, delay } from '../../test';
@@ -86,6 +86,14 @@ describe('receiving stdout from hub', () => {
 
         await expect(saga.take()).resolves.toEqual(sendData(' '));
 
+        // ensure that unicode characters are handled correctly when split
+        // across buffers
+
+        saga.put(didReceiveWriteStdout(new Uint8Array([0xe4]).buffer));
+        await expect(saga.take()).resolves.toEqual(sendData(''));
+        saga.put(didReceiveWriteStdout(new Uint8Array([0xb8, 0xad]).buffer));
+        await expect(saga.take()).resolves.toEqual(sendData('中'));
+
         await saga.end();
     });
 });

src/terminal/sagas.ts

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: MIT
-// Copyright (c) 2020-2023 The Pybricks Authors
+// Copyright (c) 2020-2024 The Pybricks Authors
 
 import { AnyAction } from 'redux';
 import {
@@ -39,7 +39,8 @@ import { receiveData, sendData } from './actions';
 export type TerminalSagaContext = { terminal: TerminalContextValue };
 
 const encoder = new TextEncoder();
-const decoder = new TextDecoder();
+const uartDecoder = new TextDecoder();
+const stdoutDecoder = new TextDecoder();
 
 function* receiveUartData(action: ReturnType<typeof didNotify>): Generator {
     const { runtime: hubState, useLegacyStdio } = yield* select(
@@ -56,14 +57,14 @@ function* receiveUartData(action: ReturnType<typeof didNotify>): Generator {
         return;
     }
 
-    const value = decoder.decode(action.value.buffer);
+    const value = uartDecoder.decode(action.value.buffer, { stream: true });
     yield* put(sendData(value));
 }
 
 function* handleReceiveWriteStdout(
     action: ReturnType<typeof didReceiveWriteStdout>,
 ): Generator {
-    const value = decoder.decode(action.payload);
+    const value = stdoutDecoder.decode(action.payload, { stream: true });
     yield* put(sendData(value));
 }