Add user chat feature with text input field

.gitignore

@@ -25,6 +25,7 @@ docs/_static/audio2face
 # next.js
 /.next/
 /out/
+package-lock.json
 
 # production
 /build

app/components/layout/ChatModeMenu.tsx

@@ -0,0 +1,125 @@
+'use client'
+
+import React, { useState, useEffect, useRef } from 'react'
+import { useTranslation } from 'react-i18next'
+
+/**
+ * Props for the ChatModeMenu component.
+ */
+interface ChatModeMenuProps {
+  /**
+   * Current chat mode.
+   */
+  chatMode: 'text' | 'voice'
+  /**
+   * Callback function called when a chat mode is selected.
+   * @param mode The selected chat mode.
+   */
+  onModeSelect: (mode: 'text' | 'voice') => void
+}
+
+/**
+ * ChatModeMenu
+ *
+ * A menu component for selecting chat modes (Text Chat, Voice Chat).
+ * Displays a button that opens a popup menu with mode options.
+ *
+ * @param chatMode Current chat mode.
+ * @param onModeSelect Callback function called when a mode is selected.
+ * @returns JSX.Element The chat mode menu UI component.
+ */
+export default function ChatModeMenu({ chatMode, onModeSelect }: ChatModeMenuProps) {
+  const { t } = useTranslation('fronted')
+  const [showChatModeMenu, setShowChatModeMenu] = useState(false)
+  const chatModeMenuRef = useRef<HTMLDivElement>(null)
+
+  /**
+   * Close chat mode menu when clicking outside.
+   */
+  useEffect(() => {
+    /**
+     * Handle click outside event to close the menu.
+     *
+     * @param event The mouse event.
+     * @returns void
+     */
+    function handleClickOutside(event: MouseEvent) {
+      if (
+        chatModeMenuRef.current &&
+        !chatModeMenuRef.current.contains(event.target as Node)
+      ) {
+        setShowChatModeMenu(false)
+      }
+    }
+
+    if (showChatModeMenu) {
+      document.addEventListener('mousedown', handleClickOutside)
+      return () => document.removeEventListener('mousedown', handleClickOutside)
+    }
+  }, [showChatModeMenu])
+
+  /**
+   * Get the display text for the current chat mode.
+   *
+   * @returns The translated text for the current chat mode.
+   */
+  const getChatModeText = () => {
+    switch (chatMode) {
+      case 'text':
+        return t('chat.textChat')
+      case 'voice':
+        return t('chat.voiceChat')
+      default:
+        return t('chat.voiceChat')
+    }
+  }
+
+  /**
+   * Handle chat mode selection.
+   *
+   * Calls the onModeSelect callback and closes the menu.
+   *
+   * @param mode The selected chat mode.
+   * @returns void
+   */
+  const handleChatModeSelect = (mode: 'text' | 'voice') => {
+    onModeSelect(mode)
+    setShowChatModeMenu(false)
+  }
+
+  return (
+    <div
+      ref={chatModeMenuRef}
+      style={{
+        position: 'fixed',
+        bottom: '40px',
+        left: '20px',
+        zIndex: 1000,
+      }}
+    >
+      <button
+        className="chatSwitch-btn"
+        onClick={() => setShowChatModeMenu(!showChatModeMenu)}
+      >
+        {getChatModeText()}
+      </button>
+
+      {showChatModeMenu && (
+        <div className="chat-mode-menu">
+          <button
+            className={`chat-mode-menu-item ${chatMode === 'text' ? 'active' : ''}`}
+            onClick={() => handleChatModeSelect('text')}
+          >
+            {t('chat.textChat')}
+          </button>
+          <button
+            className={`chat-mode-menu-item ${chatMode === 'voice' ? 'active' : ''}`}
+            onClick={() => handleChatModeSelect('voice')}
+          >
+            {t('chat.voiceChat')}
+          </button>
+        </div>
+      )}
+    </div>
+  )
+}

app/components/layout/Chatbox.tsx

@@ -0,0 +1,91 @@
+'use client'
+
+import React, { useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import { GlobalState } from '@/library/babylonjs/core'
+import {
+  Conditions,
+  ConditionedMessage,
+} from '@/library/babylonjs/runtime/fsm/conditions'
+
+/**
+ * Props for the Chatbox component.
+ */
+interface ChatboxProps {
+  /**
+   * Global state for accessing BabylonJS scene and runtime.
+   */
+  globalState?: GlobalState
+}
+
+/**
+ * Chatbox
+ *
+ * A text input component for sending chat messages.
+ * Displays a textarea and send button, positioned at the bottom center of the screen.
+ *
+ * @param globalState Global state for accessing BabylonJS scene and runtime.
+ * @returns JSX.Element The chatbox UI component.
+ */
+export default function Chatbox({ globalState }: ChatboxProps) {
+  const { t } = useTranslation('fronted')
+  const [chatMessage, setChatMessage] = useState('')
+
+  /**
+   * Handle sending chat message.
+   *
+   * Sends the current chat message to the state machine. If an animation is currently playing,
+   * it will interrupt the animation before sending the message.
+   *
+   * @returns void
+   */
+  const handleSendMessage = () => {
+    if (chatMessage.trim()) {
+      console.log('[Handle Text Message]:', chatMessage)
+      if (globalState?.runtime?.streamedAnimationPlaying()) {
+        globalState?.stateMachine?.putConditionedMessage(
+          new ConditionedMessage(Conditions.USER_TEXT_INTERRUPT_ANIMATION, null),
+        )
+      }
+      globalState?.stateMachine?.putConditionedMessage(
+        new ConditionedMessage(Conditions.USER_TEXT_INPUT, { message: chatMessage }),
+      )
+      setChatMessage('')
+    }
+  }
+
+  /**
+   * Handle Enter key press in textarea.
+   *
+   * Sends the message when Enter is pressed without Shift. Prevents default behavior
+   * to avoid adding a new line.
+   *
+   * @param e The keyboard event from the textarea.
+   * @returns void
+   */
+  const handleTextareaKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
+    if (e.key === 'Enter' && !e.shiftKey) {
+      e.preventDefault()
+      handleSendMessage()
+    }
+  }
+
+  return (
+    <div className="chatbox">
+      <textarea
+        className="chatbox-textarea"
+        placeholder={t('chat.inputMessagePlaceholder')}
+        value={chatMessage}
+        onChange={e => setChatMessage(e.target.value)}
+        onKeyDown={handleTextareaKeyDown}
+      />
+      <button
+        className="chatbox-send-btn"
+        onClick={handleSendMessage}
+        disabled={!chatMessage.trim()}
+      >
+        {t('chat.send')}
+      </button>
+    </div>
+  )
+}

app/contexts/BabylonJSContext.tsx

@@ -19,6 +19,8 @@ import { AudioRecordState } from '@/data_structures/audioStreamState'
 import { WebSocketConnectionState } from '@/data_structures/webSocketState'
 import * as orchestrator_v4 from '@/library/babylonjs/runtime/io/orchestrator_v4_pb'
 import { uint8Array2ArrayBuffer } from '@/library/babylonjs/utils/array'
+import Chatbox from '@/components/layout/Chatbox'
+import ChatModeMenu from '@/components/layout/ChatModeMenu'
 
 /**
  * React context carrying BabylonJS canvas ref and global state.
@@ -50,6 +52,7 @@ function BabylonJSProvider({
   const [globalState, setGlobalState] = useState<GlobalState>()
   const webSocketState = useWebSocket()
   const [isSceneInitialized, setIsSceneInitialized] = useState(false)
+  const [chatMode, setChatMode] = useState<'text' | 'voice'>('voice')
 
   // Add PCM queue to store audio data when websocket is not ready
   const pcmQueue = useRef<ArrayBuffer[]>([])
@@ -274,9 +277,24 @@ function BabylonJSProvider({
     }
   }, [globalState, audioStreamState])
 
+  /**
+   * Handle chat mode selection.
+   * Updates the local state and the record audio button visibility.
+   */
+  const handleChatModeSelect = (mode: 'text' | 'voice') => {
+    setChatMode(mode)
+    const isVoiceMode = mode === 'voice'
+    if (globalState?.gui?.recordAudioButton) {
+      globalState.gui.recordAudioButton.isVisible = isVoiceMode
+    }
+  }
+
   return (
     <BabylonJSContext.Provider value={{ canvas, globalState }}>
       {children}
+      <ChatModeMenu chatMode={chatMode} onModeSelect={handleChatModeSelect} />
+
+      {chatMode === 'text' && <Chatbox globalState={globalState} />}
     </BabylonJSContext.Provider>
   )
 }

app/i18n/locales/en/fronted.json

@@ -72,7 +72,16 @@
     "chat": "Chat",
     "chatList": "Chat List",
     "loginToChat": "Login to Chat",
-    "myConversations": "My conversations"
+    "myConversations": "My conversations",
+    "textChat": "Text Chat",
+    "voiceChat": "Voice Chat",
+    "inputMessagePlaceholder": "Enter message...",
+    "send": "Send",
+    "autoMode": "Auto Mode",
+    "confirm": "Confirm",
+    "disconnect": "Disconnect",
+    "notConnected": "Not Connected",
+    "connected": "Connected"
   },
   "sidebar": {
     "model": "3D model",

app/i18n/locales/zh/fronted.json

@@ -72,7 +72,16 @@
     "chat": "开始对话",
     "chatList": "对话列表",
     "loginToChat": "登录以开始对话",
-    "myConversations": "我的对话"
+    "myConversations": "我的对话",
+    "textChat": "文本聊天",
+    "voiceChat": "语音聊天",
+    "inputMessagePlaceholder": "输入消息...",
+    "send": "发送",
+    "autoMode": "自动模式",
+    "connect": "连接",
+    "disconnect": "断开连接",
+    "notConnected": "未连接",
+    "connected": "已连接"
   },
   "sidebar": {
     "model": "3D模型",

app/library/babylonjs/runtime/fsm/conditions.ts

@@ -26,11 +26,13 @@ export enum Conditions {
   ALGORITHM_GENERATION_STREAM_TIMEOUT = 16,
   ANIMATION_FINISHED = 17,
   USER_START_GAME = 18,
-  USER_INTERRUPT_ANIMATION = 19,
+  USER_AUDIO_INTERRUPT_ANIMATION = 19,
   JOINT_ANIMATION_FINISHED = 20,
   MORPH_ANIMATION_FINISHED = 21,
   JOINT_STREAM_BROKEN = 22,
   MORPH_STREAM_BROKEN = 23,
+  USER_TEXT_INPUT = 24,
+  USER_TEXT_INTERRUPT_ANIMATION = 25,
 }
 
 /**