Merge pull request #279 from CambrianTech/feature/avatar-emotive-rendering

Emotive avatar rendering: expressions, gestures, bone-based eye gaze

Author: joelteply

src/commands/collaboration/live/export/server/LiveExportServerCommand.ts

@@ -0,0 +1,146 @@
+/**
+ * Live Export Command - Server Implementation
+ *
+ * Exports recent utterances from the active live voice session
+ * to markdown format. Mirrors collaboration/chat/export.
+ */
+
+import { CommandBase, type ICommandDaemon } from '@daemons/command-daemon/shared/CommandBase';
+import type { JTAGContext } from '@system/core/types/JTAGTypes';
+import { transformPayload } from '@system/core/types/JTAGTypes';
+import { getTSVoiceOrchestrator } from '@system/voice/server';
+import { extractSentiment, formatEmotionLabel } from '@system/rag/shared/TextSentiment';
+import type { LiveExportParams, LiveExportResult } from '../shared/LiveExportTypes';
+import * as fs from 'fs';
+import * as path from 'path';
+
+export class LiveExportServerCommand extends CommandBase<LiveExportParams, LiveExportResult> {
+
+  constructor(context: JTAGContext, subpath: string, commander: ICommandDaemon) {
+    super('collaboration/live/export', context, subpath, commander);
+  }
+
+  async execute(params: LiveExportParams): Promise<LiveExportResult> {
+    const orchestrator = getTSVoiceOrchestrator();
+
+    // Auto-discover active session
+    const sessionId = params.callSessionId ?? orchestrator.activeSessionId;
+    if (!sessionId) {
+      return transformPayload(params, {
+        success: false,
+        message: 'No active live session.',
+        utteranceCount: 0,
+        callSessionId: '',
+      });
+    }
+
+    const limit = params.limit ?? 20;
+    const includeEmotions = params.includeEmotions ?? true;
+
+    // Get utterances from VoiceOrchestrator's in-memory session context
+    const utterances = orchestrator.getRecentUtterances(sessionId, limit);
+    const participants = orchestrator.getParticipants(sessionId);
+
+    if (utterances.length === 0) {
+      const markdown = '# Live Call Transcript\n\nNo utterances yet.\n';
+      console.log(markdown);
+      return transformPayload(params, {
+        success: true,
+        message: 'No utterances in session',
+        utteranceCount: 0,
+        markdown,
+        callSessionId: sessionId,
+      });
+    }
+
+    // Generate markdown
+    const markdown = this.generateMarkdown(utterances, participants, includeEmotions, sessionId);
+
+    // Write to file or return as string
+    if (params.output) {
+      const filepath = path.resolve(params.output);
+      const dir = path.dirname(filepath);
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+      }
+      fs.writeFileSync(filepath, markdown, 'utf-8');
+      console.log(`Exported ${utterances.length} utterances to ${filepath}`);
+
+      return transformPayload(params, {
+        success: true,
+        message: `Exported ${utterances.length} utterances to ${filepath}`,
+        utteranceCount: utterances.length,
+        filepath,
+        callSessionId: sessionId,
+      });
+    }
+
+    console.log(markdown);
+    return transformPayload(params, {
+      success: true,
+      message: `Exported ${utterances.length} utterances`,
+      utteranceCount: utterances.length,
+      markdown,
+      callSessionId: sessionId,
+    });
+  }
+
+  private generateMarkdown(
+    utterances: Array<{
+      sessionId: string;
+      speakerId: string;
+      speakerName: string;
+      speakerType: 'human' | 'persona' | 'agent';
+      transcript: string;
+      confidence: number;
+      timestamp: number;
+    }>,
+    participants: Array<{ userId: string; displayName: string; type: string }>,
+    includeEmotions: boolean,
+    sessionId: string,
+  ): string {
+    const lines: string[] = [];
+
+    // Header
+    lines.push('# Live Call Transcript');
+    lines.push('');
+    lines.push(`Session: ${sessionId}`);
+    lines.push(`Exported: ${new Date().toISOString()}`);
+    lines.push(`Utterances: ${utterances.length}`);
+    lines.push(`Participants: ${participants.map(p => p.displayName).join(', ')}`);
+    lines.push('');
+    lines.push('---');
+    lines.push('');
+
+    // Utterances
+    for (const u of utterances) {
+      const timestamp = new Date(u.timestamp).toLocaleString();
+      const speakerLabel = this.getSpeakerLabel(u.speakerType);
+
+      let emotionTag = '';
+      if (includeEmotions) {
+        const sentiment = extractSentiment(u.transcript);
+        const label = formatEmotionLabel(sentiment);
+        if (label) emotionTag = ` (${label})`;
+      }
+
+      lines.push(`**${speakerLabel} ${u.speakerName}${emotionTag}** — *${timestamp}*`);
+      lines.push('');
+      lines.push(u.transcript);
+      lines.push('');
+      lines.push('---');
+      lines.push('');
+    }
+
+    return lines.join('\n');
+  }
+
+  private getSpeakerLabel(type: string): string {
+    switch (type) {
+      case 'human': return '[HUMAN]';
+      case 'persona': return '[AI]';
+      case 'agent': return '[AGENT]';
+      default: return '[UNKNOWN]';
+    }
+  }
+}

src/commands/collaboration/live/export/shared/LiveExportTypes.ts

@@ -0,0 +1,58 @@
+/**
+ * Live Export Command
+ * Export recent utterances from the active live voice session.
+ * Mirrors collaboration/chat/export but for voice sessions.
+ *
+ * Usage:
+ *   ./jtag collaboration/live/export
+ *   ./jtag collaboration/live/export --limit=20
+ *   ./jtag collaboration/live/export --output="/tmp/call-transcript.md"
+ */
+
+import type { CommandParams, CommandResult, CommandInput } from '@system/core/types/JTAGTypes';
+import { Commands } from '@system/core/shared/Commands';
+
+export interface LiveExportParams extends CommandParams {
+  /** Number of utterances to export (default: 20) */
+  limit?: number;
+
+  /** Output file path (optional — prints to stdout if not provided) */
+  output?: string;
+
+  /** Explicit session ID (auto-discovered if not provided) */
+  callSessionId?: string;
+
+  /** Include emotional annotations (default: true) */
+  includeEmotions?: boolean;
+}
+
+export interface LiveExportResult extends CommandResult {
+  success: boolean;
+  message: string;
+
+  /** Number of utterances exported */
+  utteranceCount: number;
+
+  /** Markdown content (if output not specified) */
+  markdown?: string;
+
+  /** Output file path (if output specified) */
+  filepath?: string;
+
+  /** Session ID exported from */
+  callSessionId: string;
+}
+
+/**
+ * LiveExport — Type-safe command executor
+ *
+ * Usage:
+ *   import { LiveExport } from '@commands/collaboration/live/export/shared/LiveExportTypes';
+ *   const result = await LiveExport.execute({ limit: 20 });
+ */
+export const LiveExport = {
+  execute(params?: CommandInput<LiveExportParams>): Promise<LiveExportResult> {
+    return Commands.execute<LiveExportParams, LiveExportResult>('collaboration/live/export', params as Partial<LiveExportParams>);
+  },
+  commandName: 'collaboration/live/export' as const,
+} as const;

src/commands/collaboration/live/join/server/LiveJoinServerCommand.ts

@@ -18,7 +18,7 @@ import { Events } from '@system/core/shared/Events';
 import type { DataListParams, DataListResult } from '@commands/data/list/shared/DataListTypes';
 import type { DataCreateParams, DataCreateResult } from '@commands/data/create/shared/DataCreateTypes';
 import type { DataUpdateParams, DataUpdateResult } from '@commands/data/update/shared/DataUpdateTypes';
-import { getVoiceOrchestrator } from '@system/voice/server';
+import { getVoiceOrchestrator, getTSVoiceOrchestrator } from '@system/voice/server';
 import { LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET } from '@shared/AudioConstants';
 import { getSecret } from '@system/secrets/SecretManager';
 
@@ -66,6 +66,13 @@ export class LiveJoinServerCommand extends LiveJoinCommand {
     // 3. Find or create active call for this room (with retry logic for race conditions)
     const { call, existed } = await this.findOrCreateCall(room, params);
 
+    // 3b. Sync late joiners — add room members who came online after the call was created.
+    // Persona initialization is batched (6 at a time), so some personas come online
+    // after the call exists. Every join sweeps them in. No hard limits.
+    if (existed) {
+      await this.syncLateJoiners(call, room, params);
+    }
+
     // 4. Add current user as participant (if not already in the call)
     const myParticipant = call.addParticipant(
       user.id,
@@ -86,6 +93,11 @@ export class LiveJoinServerCommand extends LiveJoinCommand {
     const allParticipantIds = call.getActiveParticipants().map(p => p.userId);
     await getVoiceOrchestrator().registerSession(call.id, room.id, allParticipantIds);
 
+    // Also register with TS VoiceOrchestrator for session context tracking.
+    // In Rust voice mode, the Rust bridge handles routing but TS tracks utterance
+    // history for RAG context (VoiceConversationSource) and live/export.
+    await getTSVoiceOrchestrator().registerSession(call.id, room.id, allParticipantIds);
+
     // 8. Generate LiveKit access token for WebRTC connection
     const livekitToken = await this.generateLiveKitToken(
       call.id,
@@ -359,6 +371,41 @@ export class LiveJoinServerCommand extends LiveJoinCommand {
     return await token.toJwt();
   }
 
+  /**
+   * Sync late joiners — when joining an existing call, add room members who are
+   * now online but weren't in the call yet.
+   *
+   * Persona initialization is batched, so personas come online over time.
+   * This catches up any that arrived after the call was created.
+   * No hard limits — all online members join.
+   */
+  private async syncLateJoiners(call: CallEntity, room: RoomEntity, params: LiveJoinParams): Promise<void> {
+    if (!room.members || room.members.length === 0) return;
+
+    const memberIds = room.members.map(m => m.userId);
+    const membersInfo = await this.lookupUsers(memberIds, params);
+
+    let addedCount = 0;
+    for (const memberUser of membersInfo) {
+      const isHuman = memberUser.type === 'human';
+      const isOnline = memberUser.status === 'online';
+      const alreadyInCall = call.getActiveParticipants().some(p => p.userId === memberUser.id);
+
+      if (!alreadyInCall && (isHuman || isOnline)) {
+        call.addParticipant(
+          memberUser.id as UUID,
+          memberUser.displayName || memberUser.uniqueId,
+          memberUser.avatar
+        );
+        addedCount++;
+      }
+    }
+
+    if (addedCount > 0) {
+      console.log(`🎙️ LiveJoin: Synced ${addedCount} late joiner(s) to existing call ${call.id.slice(0, 8)}`);
+    }
+  }
+
   /**
    * Look up user info for a list of user IDs
    */

src/commands/collaboration/live/send/server/LiveSendServerCommand.ts

@@ -0,0 +1,85 @@
+/**
+ * Live Send Command - Server Implementation
+ *
+ * Sends a text message into the active live voice session.
+ * Auto-discovers session and speaker identity.
+ * Mirrors collaboration/chat/send but for voice sessions.
+ */
+
+import { CommandBase, type ICommandDaemon } from '@daemons/command-daemon/shared/CommandBase';
+import type { JTAGContext } from '@system/core/types/JTAGTypes';
+import { transformPayload } from '@system/core/types/JTAGTypes';
+import { getTSVoiceOrchestrator } from '@system/voice/server';
+import { CollaborationLiveTranscription } from '../../transcription/shared/CollaborationLiveTranscriptionTypes';
+import type { LiveSendParams, LiveSendResult } from '../shared/LiveSendTypes';
+import { Commands } from '@system/core/shared/Commands';
+import type { DataListResult } from '@commands/data/list/shared/DataListTypes';
+import { DataList } from '@commands/data/list/shared/DataListTypes';
+import type { UserEntity } from '@system/data/entities/UserEntity';
+
+export class LiveSendServerCommand extends CommandBase<LiveSendParams, LiveSendResult> {
+
+  constructor(context: JTAGContext, subpath: string, commander: ICommandDaemon) {
+    super('collaboration/live/send', context, subpath, commander);
+  }
+
+  async execute(params: LiveSendParams): Promise<LiveSendResult> {
+    const orchestrator = getTSVoiceOrchestrator();
+
+    // Auto-discover active session
+    const sessionId = params.callSessionId ?? orchestrator.activeSessionId;
+    if (!sessionId) {
+      return transformPayload(params, {
+        success: false,
+        message: 'No active live session. Start a call first.',
+        callSessionId: '',
+        responderCount: 0,
+      });
+    }
+
+    // Resolve speaker identity
+    const speakerId = params.speakerId ?? params.userId ?? '';
+    let speakerName = params.speakerName ?? '';
+
+    if (!speakerName && speakerId) {
+      try {
+        const result = await DataList.execute<UserEntity>({
+          collection: 'users',
+          filter: { id: speakerId },
+          limit: 1,
+          dbHandle: 'default',
+        });
+        if (result.success && result.items?.length) {
+          speakerName = result.items[0].displayName || result.items[0].uniqueId;
+        }
+      } catch {
+        // Fall through to default
+      }
+    }
+    if (!speakerName) speakerName = 'CLI';
+
+    // Route through the existing transcription command
+    const transcriptionResult = await CollaborationLiveTranscription.execute({
+      callSessionId: sessionId,
+      speakerId,
+      speakerName,
+      transcript: params.message,
+      confidence: params.confidence ?? 1.0,
+      language: 'en',
+      timestamp: Date.now(),
+    });
+
+    const responderCount = transcriptionResult.success
+      ? parseInt(transcriptionResult.message.match(/(\d+) AI/)?.[1] ?? '0', 10)
+      : 0;
+
+    return transformPayload(params, {
+      success: transcriptionResult.success,
+      message: transcriptionResult.success
+        ? `Sent to live session → ${responderCount} AI responders`
+        : `Failed: ${transcriptionResult.message}`,
+      callSessionId: sessionId,
+      responderCount,
+    });
+  }
+}