feat: Enhance GUI for Rank 4/5 testing support

- Fix session_id bug (was hardcoded to 'ui-session')
- Add speaker status panel with recognition indicators
- Add tool execution status display
- Add memory visualization panel with type-colored items
- Add /api/memories endpoint for retrieving memories
- Add /api/speakers endpoint for listing speakers
- Add /api/speakers/{id} endpoint for speaker profiles
- Add /api/context endpoint for conversation context
- Update broker to return speaker_info and tool_info
- Add CSS styling for new UI components

GUI Enhancements:
- Real-time speaker identification display (new vs recognized)
- Tool execution feedback with iteration tracking
- Memory panel with filtering by type (FACT, PREFERENCE, etc.)
- Speaker badge showing recognition status

Also added docs/GUI_ENHANCEMENT_PLAN.md with roadmap for future improvements

Author: John Major

client/gui.py

@@ -1070,6 +1070,203 @@ def _effective_region() -> str:
     )
 
 
+# ─────────────────────────────────────────────────────────────────────────────
+# Memory and Speaker API Endpoints (for Rank 4/5 features)
+# ─────────────────────────────────────────────────────────────────────────────
+
+@app.get("/api/memories")
+async def get_memories(
+    limit: int = 50,
+    speaker_id: Optional[str] = None,
+    kind: Optional[str] = None,
+):
+    """Retrieve memories from the agent state table."""
+    table = _agent_state_table()
+    if not table:
+        return {"memories": [], "error": "No table configured"}
+
+    agent_id = STATE.selected_agent_id or "agent1"
+
+    try:
+        # Query main agent partition
+        pk = f"AGENT#{agent_id}"
+        resp = table.query(
+            KeyConditionExpression=Key("pk").eq(pk),
+            ScanIndexForward=False,
+            Limit=limit * 2,  # Fetch extra for filtering
+        )
+        items = resp.get("Items", [])
+
+        # Also query speaker-specific partition if speaker_id provided
+        if speaker_id:
+            speaker_pk = f"AGENT#{agent_id}#SPEAKER#{speaker_id}"
+            speaker_resp = table.query(
+                KeyConditionExpression=Key("pk").eq(speaker_pk),
+                ScanIndexForward=False,
+                Limit=limit,
+            )
+            items.extend(speaker_resp.get("Items", []))
+
+        # Filter to memories only
+        memories = [i for i in items if "#MEMORY#" in i.get("sk", "") or i.get("item_type") == "MEMORY"]
+
+        # Filter by kind if specified
+        if kind and kind.upper() != "ALL":
+            memories = [m for m in memories if m.get("kind") == kind.upper()]
+
+        # Filter by speaker_id if specified (in meta)
+        if speaker_id:
+            memories = [
+                m for m in memories
+                if m.get("speaker_id") == speaker_id or m.get("meta", {}).get("speaker_id") == speaker_id
+            ]
+
+        # Sort by timestamp and limit
+        memories.sort(key=lambda x: x.get("ts", ""), reverse=True)
+        memories = memories[:limit]
+
+        return {
+            "memories": memories,
+            "total": len(memories),
+            "agent_id": agent_id,
+        }
+
+    except ClientError as e:
+        logging.error("get_memories failed: %s", e)
+        return {"memories": [], "error": str(e)}
+
+
+@app.get("/api/speakers")
+async def list_speakers():
+    """List all known speakers for the current agent."""
+    table = _agent_state_table()
+    if not table:
+        return {"speakers": [], "error": "No table configured"}
+
+    agent_id = STATE.selected_agent_id or "agent1"
+
+    try:
+        # Query VOICE partition for speakers
+        pk = f"AGENT#{agent_id}#VOICE"
+        resp = table.query(
+            KeyConditionExpression=Key("pk").eq(pk),
+            Limit=100,
+        )
+
+        speakers = []
+        for item in resp.get("Items", []):
+            speakers.append({
+                "speaker_id": item.get("sk") or item.get("speaker_id"),
+                "speaker_name": item.get("speaker_name"),
+                "first_seen": item.get("first_seen_ts"),
+                "last_seen": item.get("last_seen_ts"),
+                "interaction_count": item.get("interaction_count", 0),
+                "enrollment_status": item.get("enrollment_status", "unknown"),
+            })
+
+        return {
+            "speakers": speakers,
+            "total": len(speakers),
+            "agent_id": agent_id,
+        }
+
+    except ClientError as e:
+        logging.error("list_speakers failed: %s", e)
+        return {"speakers": [], "error": str(e)}
+
+
+@app.get("/api/speakers/{speaker_id}")
+async def get_speaker_profile(speaker_id: str):
+    """Get detailed profile for a specific speaker."""
+    table = _agent_state_table()
+    if not table:
+        return {"error": "No table configured"}
+
+    agent_id = STATE.selected_agent_id or "agent1"
+
+    try:
+        pk = f"AGENT#{agent_id}#VOICE"
+        resp = table.get_item(Key={"pk": pk, "sk": speaker_id})
+        item = resp.get("Item")
+
+        if not item:
+            return {"error": "Speaker not found", "speaker_id": speaker_id}
+
+        # Also fetch speaker-specific memories
+        memories_pk = f"AGENT#{agent_id}#SPEAKER#{speaker_id}"
+        mem_resp = table.query(
+            KeyConditionExpression=Key("pk").eq(memories_pk),
+            ScanIndexForward=False,
+            Limit=20,
+        )
+
+        return {
+            "profile": {
+                "speaker_id": item.get("sk") or item.get("speaker_id"),
+                "speaker_name": item.get("speaker_name"),
+                "first_seen": item.get("first_seen_ts"),
+                "last_seen": item.get("last_seen_ts"),
+                "interaction_count": item.get("interaction_count", 0),
+                "enrollment_status": item.get("enrollment_status", "unknown"),
+                "notes": item.get("notes"),
+                "preferences": item.get("preferences", {}),
+                "voice_samples": item.get("voice_samples", []),
+            },
+            "memories": mem_resp.get("Items", []),
+            "agent_id": agent_id,
+        }
+
+    except ClientError as e:
+        logging.error("get_speaker_profile failed: %s", e)
+        return {"error": str(e)}
+
+
+@app.get("/api/context")
+async def get_conversation_context(session_id: Optional[str] = None):
+    """Get full conversation context for a session."""
+    table = _agent_state_table()
+    if not table:
+        return {"context": [], "error": "No table configured"}
+
+    agent_id = STATE.selected_agent_id or "agent1"
+    session_id = session_id or STATE.selected_session
+
+    try:
+        pk = f"AGENT#{agent_id}"
+        resp = table.query(
+            KeyConditionExpression=Key("pk").eq(pk),
+            ScanIndexForward=False,
+            Limit=100,
+        )
+
+        items = resp.get("Items", [])
+
+        # Filter by session if specified
+        if session_id:
+            items = [
+                i for i in items
+                if i.get("session_id") == session_id or not i.get("session_id")
+            ]
+
+        # Separate events and memories
+        events = [i for i in items if "#EVENT#" in i.get("sk", "") or i.get("item_type") == "EVENT"]
+        memories = [i for i in items if "#MEMORY#" in i.get("sk", "") or i.get("item_type") == "MEMORY"]
+
+        return {
+            "events": events[:50],
+            "memories": memories[:50],
+            "agent_id": agent_id,
+            "session_id": session_id,
+        }
+
+    except ClientError as e:
+        logging.error("get_conversation_context failed: %s", e)
+        return {"context": [], "error": str(e)}
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+
+
 async def _call_agent_broker(
     *,
     transcript: str,

client/templates/chat.html

@@ -16,8 +16,12 @@
     button:hover { border-color: #888; }
     .row { display:flex; gap: 10px; flex-wrap: wrap; align-items: center; }
     select { padding: 6px; border-radius: 8px; border: 1px solid #bbb; max-width: 360px; }
-    .pill { border: 1px solid #ddd; border-radius: 999px; padding: 4px 9px; }
+    .pill { border: 1px solid #ddd; border-radius: 999px; padding: 4px 9px; font-size: 0.85em; }
     .card { border: 1px solid #ddd; border-radius: 10px; padding: 12px; margin-top: 14px; }
+    #statusPanel { background: #f8f9fa; }
+    #statusPanel strong { color: #495057; }
+    #memoryPanel { background: #fefefe; max-height: 250px; overflow-y: auto; }
+    #toolDetails { background: #f1f3f4; padding: 6px 10px; border-radius: 6px; font-family: monospace; font-size: 0.85em; }
     .debug-log { max-height: 180px; overflow-y: auto; background: #f7f7f7; border: 1px solid #eee; padding: 8px; border-radius: 8px; }
     .debug-tools { display: grid; gap: 10px; margin-top: 10px; }
     .debug-tool { border: 1px dashed #ccc; padding: 8px; border-radius: 8px; }
@@ -96,6 +100,28 @@ <h1>Chat</h1>
     <input type="text" id="speaker_name_input" placeholder="e.g. Major" style="min-width: 260px;" />
   </div>
 
+  <!-- Speaker & Tool Status Panel -->
+  <div class="card" id="statusPanel" style="display: none;">
+    <div class="row" style="gap: 20px;">
+      <div id="speakerStatus">
+        <strong>Speaker:</strong> <span id="currentSpeaker">Unknown</span>
+        <span id="speakerBadge" class="pill" style="display: none;"></span>
+      </div>
+      <div id="toolStatus" style="display: none;">
+        <strong>Tools:</strong> <span id="toolIterations">-</span>
+        <span id="toolProgress" class="pill"></span>
+      </div>
+    </div>
+    <div id="toolDetails" class="meta" style="margin-top: 8px; display: none;"></div>
+  </div>
+
+  <!-- Memory Context Panel -->
+  <div class="card" id="memoryPanel" style="display: none;">
+    <h3 style="margin-top: 0;">Recent Context</h3>
+    <div id="memoryList" style="max-height: 150px; overflow-y: auto;"></div>
+    <button onclick="loadMemories()" class="btn" style="margin-top: 8px;">Refresh Memories</button>
+  </div>
+
   <div class="card" id="debugCard">
     <h2 style="margin-top:0">Debug</h2>
     <div class="meta">Verbose logging is {{ 'enabled' if state.verbose else 'disabled' }} (toggle in <a href="/settings">Settings</a>).</div>
@@ -158,6 +184,114 @@ <h2 style="margin-top:0">Debug</h2>
     debugOutput.textContent = text || '';
   }
 
+  // Speaker and tool status management
+  const statusPanel = document.getElementById('statusPanel');
+  const currentSpeakerEl = document.getElementById('currentSpeaker');
+  const speakerBadgeEl = document.getElementById('speakerBadge');
+  const toolStatusEl = document.getElementById('toolStatus');
+  const toolIterationsEl = document.getElementById('toolIterations');
+  const toolProgressEl = document.getElementById('toolProgress');
+  const toolDetailsEl = document.getElementById('toolDetails');
+  const memoryPanel = document.getElementById('memoryPanel');
+  const memoryListEl = document.getElementById('memoryList');
+
+  function updateSpeakerStatus(speakerInfo) {
+    if (!speakerInfo) return;
+    statusPanel.style.display = 'block';
+
+    const name = speakerInfo.speaker_name || speakerInfo.current_speaker_name || 'Unknown';
+    const voiceId = speakerInfo.voice_id || speakerInfo.current_speaker_voice_id || '';
+    const isNew = speakerInfo.is_new_speaker || speakerInfo.is_new_voice || false;
+
+    currentSpeakerEl.textContent = name;
+
+    if (isNew) {
+      speakerBadgeEl.textContent = 'New Speaker';
+      speakerBadgeEl.style.background = '#ffe066';
+      speakerBadgeEl.style.display = 'inline-block';
+    } else if (voiceId) {
+      speakerBadgeEl.textContent = 'Recognized';
+      speakerBadgeEl.style.background = '#90EE90';
+      speakerBadgeEl.style.display = 'inline-block';
+    } else {
+      speakerBadgeEl.style.display = 'none';
+    }
+  }
+
+  function updateToolStatus(toolInfo) {
+    if (!toolInfo || !toolInfo.tool_calls || toolInfo.tool_calls.length === 0) {
+      toolStatusEl.style.display = 'none';
+      toolDetailsEl.style.display = 'none';
+      return;
+    }
+
+    statusPanel.style.display = 'block';
+    toolStatusEl.style.display = 'block';
+
+    const iterations = toolInfo.iterations || 1;
+    const calls = toolInfo.tool_calls || [];
+
+    toolIterationsEl.textContent = `${iterations} iteration(s)`;
+    toolProgressEl.textContent = `${calls.length} tool(s) executed`;
+
+    // Show tool details
+    if (calls.length > 0) {
+      toolDetailsEl.style.display = 'block';
+      toolDetailsEl.innerHTML = calls.map(c =>
+        `<div>• ${c.name}: ${JSON.stringify(c.input).substring(0, 50)}...</div>`
+      ).join('');
+    }
+  }
+
+  function displayMemoryItem(memory) {
+    const div = document.createElement('div');
+    div.className = 'msg';
+    div.style.fontSize = '0.9em';
+    div.style.padding = '4px 8px';
+
+    const kind = memory.kind || 'UNKNOWN';
+    const text = memory.text || '';
+    const speaker = memory.speaker_id || memory.meta?.speaker_id || '';
+
+    // Color by kind
+    const kindColors = {
+      'FACT': '#e3f2fd',
+      'PREFERENCE': '#fff3e0',
+      'AI_INSIGHT': '#f3e5f5',
+      'RELATIONSHIP': '#e8f5e9',
+      'ACTION': '#fce4ec',
+      'META': '#f5f5f5'
+    };
+    div.style.background = kindColors[kind] || '#fff';
+
+    let html = `<strong>[${kind}]</strong> ${text}`;
+    if (speaker) html += ` <em>(${speaker})</em>`;
+    div.innerHTML = html;
+
+    return div;
+  }
+
+  async function loadMemories() {
+    try {
+      const resp = await fetch('/api/memories?limit=20');
+      if (!resp.ok) throw new Error('Failed to load memories');
+      const data = await resp.json();
+
+      memoryPanel.style.display = 'block';
+      memoryListEl.innerHTML = '';
+
+      (data.memories || []).forEach(m => {
+        memoryListEl.appendChild(displayMemoryItem(m));
+      });
+
+      if (!data.memories || data.memories.length === 0) {
+        memoryListEl.innerHTML = '<div class="meta">No memories found</div>';
+      }
+    } catch (e) {
+      appendMeta('Failed to load memories: ' + e);
+    }
+  }
+
   function setAsrStatus(t) {
     asrStatus.textContent = 'ASR: ' + t;
   }
@@ -470,6 +604,21 @@ <h2 style="margin-top:0">Debug</h2>
           if (data.reply_text) appendMsg('agent', data.reply_text);
           if (data.actions && data.actions.length) appendMeta('actions: ' + JSON.stringify(data.actions));
           if (data.audio) playAgentAudio(data.audio);
+          // Update speaker status from response
+          if (data.speaker_info || data.speaker_name || data.voice_id) {
+            updateSpeakerStatus({
+              speaker_name: data.speaker_name || data.speaker_info?.speaker_name,
+              voice_id: data.voice_id || data.speaker_info?.voice_id,
+              is_new_speaker: data.is_new_voice || data.speaker_info?.is_new_speaker
+            });
+          }
+          // Update tool status from response
+          if (data.tool_info || data.tool_calls) {
+            updateToolStatus({
+              iterations: data.tool_info?.iterations || 1,
+              tool_calls: data.tool_calls || data.tool_info?.tool_calls || []
+            });
+          }
         } else if (msg.type === 'error') {
           appendMeta('ASR error: ' + (msg.error || 'unknown'));
         }
@@ -496,11 +645,13 @@ <h2 style="margin-top:0">Debug</h2>
       const persona = (document.getElementById('persona_input').value || '').trim();
       const speakerName = (document.getElementById('speaker_name_input').value || '').trim();
 
+      // Use actual session from state, not hardcoded value
+      const sessionId = '{{ state.selected_session or "" }}' || 'session-' + Date.now();
       ws.send(JSON.stringify({
         type: 'start',
         language_code: (languageSel.value || 'en-US'),
         sample_rate_hz: 16000,
-        session_id: 'ui-session',
+        session_id: sessionId,
         voice_id: selectedMicDeviceId || null,
         speaker_name: speakerName || null,
         personality_prompt: persona || null,