Merge pull request #1266 from yeyan1996/fix/playback-tools

fix: refactor playback controls and consolidate XML tag constants

Author: markokraemer

frontend/src/app/share/[threadId]/page.tsx

@@ -288,33 +288,6 @@ export default function ThreadPage({
     [],
   );
 
-  // useEffect(() => {
-  //   if (!isPlaying || messages.length === 0) return;
-
-  //   let playbackTimeout: NodeJS.Timeout;
-
-  //   const playbackNextMessage = async () => {
-  //     if (currentMessageIndex >= messages.length) {
-  //       setIsPlaying(false);
-  //       return;
-  //     }
-
-  //     const currentMessage = messages[currentMessageIndex];
-  //     console.log(
-  //       `Playing message ${currentMessageIndex}:`,
-  //       currentMessage.type,
-  //       currentMessage.message_id,
-  //     );
-
-  //     setCurrentMessageIndex((prevIndex) => prevIndex + 1);
-  //   };
-  //   playbackTimeout = setTimeout(playbackNextMessage, 500);
-
-  //   return () => {
-  //     clearTimeout(playbackTimeout);
-  //   };
-  // }, [isPlaying, currentMessageIndex, messages]);
-
   const {
     status: streamHookStatus,
     toolCall: streamingToolCall,

frontend/src/components/thread/content/PlaybackControls.tsx

@@ -11,42 +11,15 @@ import {
 import { UnifiedMessage } from '@/components/thread/types';
 import { safeJsonParse } from '@/components/thread/utils';
 import Link from 'next/link';
-
-// Define the set of tags whose raw XML should be hidden during streaming
-const HIDE_STREAMING_XML_TAGS = new Set([
-  'execute-command',
-  'create-file',
-  'delete-file',
-  'full-file-rewrite',
-  'edit-file',
-  'str-replace',
-  'browser-click-element',
-  'browser-close-tab',
-  'browser-drag-drop',
-  'browser-get-dropdown-options',
-  'browser-go-back',
-  'browser-input-text',
-  'browser-navigate-to',
-  'browser-scroll-down',
-  'browser-scroll-to-text',
-  'browser-scroll-up',
-  'browser-select-dropdown-option',
-  'browser-send-keys',
-  'browser-switch-tab',
-  'browser-wait',
-  'deploy',
-  'ask',
-  'complete',
-  'crawl-webpage',
-  'web-search',
-]);
+import { parseXmlToolCalls } from '../tool-views/xml-parser';
+import { HIDE_STREAMING_XML_TAGS } from '@/components/thread/utils';
 
 export interface PlaybackControlsProps {
   messages: UnifiedMessage[];
   isSidePanelOpen: boolean;
   onToggleSidePanel: () => void;
   toolCalls: any[];
-  setCurrentToolIndex: (index: number) => void;
+  setCurrentToolIndex: React.Dispatch<React.SetStateAction<number>>;
   onFileViewerOpen: () => void;
   projectName?: string;
 }
@@ -58,7 +31,6 @@ export interface PlaybackState {
   streamingText: string;
   isStreamingText: boolean;
   currentToolCall: any | null;
-  toolPlaybackIndex: number;
 }
 
 export interface PlaybackController {
@@ -88,7 +60,6 @@ export const PlaybackControls = ({
     streamingText: '',
     isStreamingText: false,
     currentToolCall: null,
-    toolPlaybackIndex: -1,
   });
 
   // Extract state variables for easier access
@@ -99,10 +70,10 @@ export const PlaybackControls = ({
     streamingText,
     isStreamingText,
     currentToolCall,
-    toolPlaybackIndex,
   } = playbackState;
 
   const playbackTimeout = useRef<NodeJS.Timeout | null>(null);
+  const [isToolInitialized, setIsToolInitialized] = useState(false);
 
   // Helper function to update playback state
   const updatePlaybackState = useCallback((updates: Partial<PlaybackState>) => {
@@ -129,9 +100,9 @@ export const PlaybackControls = ({
       streamingText: '',
       isStreamingText: false,
       currentToolCall: null,
-      toolPlaybackIndex: -1,
     });
-    setCurrentToolIndex(-1);
+    setCurrentToolIndex(0);
+    setIsToolInitialized(false);
 
     if (playbackTimeout.current) {
       clearTimeout(playbackTimeout.current);
@@ -192,7 +163,6 @@ export const PlaybackControls = ({
       streamingText: '',
       isStreamingText: false,
       currentToolCall: null,
-      toolPlaybackIndex: toolCalls.length - 1,
     });
 
     if (toolCalls.length > 0) {
@@ -242,11 +212,9 @@ export const PlaybackControls = ({
         }
 
         // Add the tool call
-        const toolName = match[1] || match[2];
         chunks.push({
           text: match[0],
           isTool: true,
-          toolName,
         });
 
         lastIndex = toolCallRegex.lastIndex;
@@ -306,40 +274,24 @@ export const PlaybackControls = ({
         // If this is a tool call chunk and we're at the start of it
         if (currentChunk.isTool && currentIndex === 0) {
           // For tool calls, check if they should be hidden during streaming
-          if (
-            currentChunk.toolName &&
-            HIDE_STREAMING_XML_TAGS.has(currentChunk.toolName)
-          ) {
-            // Instead of showing the XML, create a tool call object
-            const toolCall = {
-              name: currentChunk.toolName,
-              arguments: currentChunk.text,
-              xml_tag_name: currentChunk.toolName,
-            };
-
-            updatePlaybackState({
-              currentToolCall: toolCall,
-              toolPlaybackIndex: toolPlaybackIndex + 1,
-            });
-
-            if (!isSidePanelOpen) {
-              onToggleSidePanel();
-            }
-
-            setCurrentToolIndex(toolPlaybackIndex + 1);
+          if (isToolInitialized) {
+            // TODO: better to change tool index by uniq tool id
+            setCurrentToolIndex((prev) => prev + 1);
+          } else {
+            setIsToolInitialized(true);
+          }
 
-            // Pause streaming briefly while showing the tool
-            isPaused = true;
-            setTimeout(() => {
-              isPaused = false;
-              updatePlaybackState({ currentToolCall: null });
-              chunkIndex++; // Move to next chunk
-              currentIndex = 0; // Reset index for next chunk
-              processNextCharacter();
-            }, 500); // Reduced from 1500ms to 500ms pause for tool display
+          // Pause streaming briefly while showing the tool
+          isPaused = true;
+          setTimeout(() => {
+            isPaused = false;
+            updatePlaybackState({ currentToolCall: null });
+            chunkIndex++; // Move to next chunk
+            currentIndex = 0; // Reset index for next chunk
+            processNextCharacter();
+          }, 500); // Reduced from 1500ms to 500ms pause for tool display
 
-            return;
-          }
+          return;
         }
 
         // Handle normal text streaming for non-tool chunks or visible tool chunks
@@ -387,7 +339,6 @@ export const PlaybackControls = ({
       isPlaying,
       messages,
       currentMessageIndex,
-      toolPlaybackIndex,
       setCurrentToolIndex,
       isSidePanelOpen,
       onToggleSidePanel,

frontend/src/components/thread/content/ThreadContent.tsx

@@ -16,40 +16,8 @@ import { AgentLoader } from './loader';
 import { parseXmlToolCalls, isNewXmlFormat } from '@/components/thread/tool-views/xml-parser';
 import { ShowToolStream } from './ShowToolStream';
 import { ComposioUrlDetector } from './composio-url-detector';
+import { HIDE_STREAMING_XML_TAGS } from '@/components/thread/utils';
 
-const HIDE_STREAMING_XML_TAGS = new Set([
-    'execute-command',
-    'create-file',
-    'delete-file',
-    'full-file-rewrite',
-    'edit-file',
-    'str-replace',
-    'browser-click-element',
-    'browser-close-tab',
-    'browser-drag-drop',
-    'browser-get-dropdown-options',
-    'browser-go-back',
-    'browser-input-text',
-    'browser-navigate-to',
-    'browser-scroll-down',
-    'browser-scroll-to-text',
-    'browser-scroll-up',
-    'browser-select-dropdown-option',
-    'browser-send-keys',
-    'browser-switch-tab',
-    'browser-wait',
-    'deploy',
-    'ask',
-    'complete',
-    'crawl-webpage',
-    'web-search',
-    'see-image',
-    'execute_data_provider_call',
-    'execute_data_provider_endpoint',
-
-    'execute-data-provider-call',
-    'execute-data-provider-endpoint',
-]);
 
 // Helper function to render attachments (keeping original implementation for now)
 export function renderAttachments(attachments: string[], fileViewerHandler?: (filePath?: string, filePathList?: string[]) => void, sandboxId?: string, project?: Project) {

frontend/src/components/thread/tool-call-side-panel.tsx

@@ -151,10 +151,9 @@ export function ToolCallSidePanel({
   }, [toolCalls, navigationMode, toolCallSnapshots.length, isInitialized]);
 
   React.useEffect(() => {
-    if (isOpen && !isInitialized && toolCallSnapshots.length > 0) {
-      setInternalIndex(Math.min(currentIndex, toolCallSnapshots.length - 1));
-    }
-  }, [isOpen, currentIndex, isInitialized, toolCallSnapshots.length]);
+    // This is used to sync the internal index to the current index
+    setInternalIndex(Math.min(currentIndex, toolCallSnapshots.length - 1));
+  }, [currentIndex, toolCallSnapshots.length]);
 
   const safeInternalIndex = Math.min(internalIndex, Math.max(0, toolCallSnapshots.length - 1));
   const currentSnapshot = toolCallSnapshots[safeInternalIndex];

frontend/src/components/thread/utils.ts

@@ -494,3 +494,38 @@ export function getUserFriendlyToolName(toolName: string): string {
   }
   return TOOL_DISPLAY_NAMES.get(toolName) || toolName;
 }
+
+export const HIDE_STREAMING_XML_TAGS = new Set([
+  'create-tasks',
+  'execute-command',
+  'create-file',
+  'delete-file',
+  'full-file-rewrite',
+  'edit-file',
+  'str-replace',
+  'browser-click-element',
+  'browser-close-tab',
+  'browser-drag-drop',
+  'browser-get-dropdown-options',
+  'browser-go-back',
+  'browser-input-text',
+  'browser-navigate-to',
+  'browser-scroll-down',
+  'browser-scroll-to-text',
+  'browser-scroll-up',
+  'browser-select-dropdown-option',
+  'browser-send-keys',
+  'browser-switch-tab',
+  'browser-wait',
+  'deploy',
+  'ask',
+  'complete',
+  'crawl-webpage',
+  'web-search',
+  'see-image',
+  'execute_data_provider_call',
+  'execute_data_provider_endpoint',
+
+  'execute-data-provider-call',
+  'execute-data-provider-endpoint',
+]);

frontend/src/components/ui/slider.tsx

@@ -45,15 +45,15 @@ function Slider({
         <SliderPrimitive.Range
           data-slot="slider-range"
           className={cn(
-            'bg-primary absolute data-[orientation=horizontal]:h-full data-[orientation=vertical]:w-full',
+            'cursor-pointer bg-primary absolute data-[orientation=horizontal]:h-full data-[orientation=vertical]:w-full',
           )}
         />
       </SliderPrimitive.Track>
       {Array.from({ length: _values.length }, (_, index) => (
         <SliderPrimitive.Thumb
           data-slot="slider-thumb"
           key={index}
-          className="border-primary bg-background ring-ring/50 block size-4 shrink-0 rounded-full border shadow-sm transition-[color,box-shadow] hover:ring-4 focus-visible:ring-4 focus-visible:outline-hidden disabled:pointer-events-none disabled:opacity-50"
+          className="cursor-pointer border-primary bg-background ring-ring/50 block size-4 shrink-0 rounded-full border shadow-sm transition-[color,box-shadow] hover:ring-4 focus-visible:ring-4 focus-visible:outline-hidden disabled:pointer-events-none disabled:opacity-50"
         />
       ))}
     </SliderPrimitive.Root>