✨ Multi modal agent.

Add tooltips

Author: Zhi-a

backend/agents/create_agent_info.py

@@ -315,12 +315,11 @@ async def join_minio_file_description_to_query(minio_files, query):
         file_descriptions = []
         for file in minio_files:
             if isinstance(file, dict) and "url" in file and file["url"] and "name" in file and file["name"]:
-                file_descriptions.append("File S3 URL: " + "s3:/" + file["url"] + ", file name:" + file["name"])
-
+                file_descriptions.append(f"File name: {file['name']}, S3 URL: s3:/{file['url']}")
         if file_descriptions:
-            final_query = "User provided some reference files:\n"
+            final_query = "User uploaded files. The file information is as follows:\n"
             final_query += "\n".join(file_descriptions) + "\n\n"
-            final_query += f"User wants to answer questions based on the above information: {query}"
+            final_query += f"User wants to answer questions based on the information in the above files: {query}"
     return final_query
 
 

frontend/app/[locale]/chat/internal/chatInterface.tsx

@@ -451,218 +451,23 @@ export function ChatInterface() {
         ],
       }));
 
-      // If there are attachment files, preprocess first
+      // If there are attachment files, skip preprocessing (no API call, no UI prompts)
       let finalQuery = userMessage.content;
       // Declare a variable to save file description information
       let fileDescriptionsMap: Record<string, string> = {};
 
       if (attachments.length > 0) {
-        // Attachment preprocessing step, as independent step in assistant steps
-        setSessionMessages((prev) => ({
-          ...prev,
-          [currentConversationId]: [
-            ...(prev[currentConversationId] || []),
-            {
-              id: uuidv4(),
-              role: ROLE_ASSISTANT,
-              content: "",
-              timestamp: new Date(),
-              isComplete: false,
-              steps: [
-                {
-                  id: `preprocess-${Date.now()}`,
-                  title: t("chatInterface.filePreprocessing"),
-                  content: "",
-                  expanded: true,
-                  metrics: "",
-                  thinking: { content: "", expanded: false },
-                  code: { content: "", expanded: false },
-                  output: { content: "", expanded: false },
-                  contents: [
-                    {
-                      id: `preprocess-content-${Date.now()}`,
-                      type: chatConfig.contentTypes.PREPROCESS,
-                      content: t("chatInterface.parsingFile"),
-                      expanded: false,
-                      timestamp: Date.now(),
-                    },
-                  ],
-                },
-              ],
-            },
-          ],
-        }));
-
-        // Buffer for truncation messages with deduplication
-        const truncationBuffer: any[] = [];
-        const processedTruncationIds = new Set<string>(); // Track processed truncation messages to avoid duplicates
-
-        // Use extracted preprocessing function to process attachments
+        // Skip preprocessing - directly use original content
+        // No preprocessing UI will be shown
         const result = await preprocessAttachments(
           userMessage.content,
           attachments,
           currentController.signal,
-          (jsonData) => {
-            setSessionMessages((prev) => {
-              const newMessages = { ...prev };
-              const lastMsg =
-                newMessages[currentConversationId]?.[
-                  newMessages[currentConversationId].length - 1
-                ];
-              if (lastMsg && lastMsg.role === ROLE_ASSISTANT) {
-                if (!lastMsg.steps) lastMsg.steps = [];
-                // Find the latest preprocessing step
-                let step = lastMsg.steps.find(
-                  (s) => s.title === t("chatInterface.filePreprocessing")
-                );
-                if (!step) {
-                  step = {
-                    id: `preprocess-${Date.now()}`,
-                    title: t("chatInterface.filePreprocessing"),
-                    content: "",
-                    expanded: true,
-                    metrics: "",
-                    thinking: { content: "", expanded: false },
-                    code: { content: "", expanded: false },
-                    output: { content: "", expanded: false },
-                    contents: [
-                      {
-                        id: `preprocess-content-${Date.now()}`,
-                        type: chatConfig.contentTypes.PREPROCESS,
-                        content: t("chatInterface.parsingFile"),
-                        expanded: false,
-                        timestamp: Date.now(),
-                      },
-                    ],
-                  };
-                  lastMsg.steps.push(step);
-                }
-
-                // Handle truncation messages - buffer them instead of updating immediately
-                if (jsonData.type === "truncation") {
-                  // Create a unique ID for this truncation message to avoid duplicates
-                  const truncationId = `${jsonData.filename || "unknown"}_${
-                    jsonData.message || ""
-                  }`;
-
-                  // Only add if not already processed
-                  if (!processedTruncationIds.has(truncationId)) {
-                    truncationBuffer.push(jsonData);
-                    processedTruncationIds.add(truncationId);
-                  }
-                  return newMessages; // Don't update stepContent for truncation
-                }
-
-                let stepContent = "";
-                switch (jsonData.type) {
-                  case "progress":
-                    if (jsonData.message_data) {
-                      const i18nKey = getI18nKeyByType(jsonData.type);
-                      stepContent = String(
-                        t(i18nKey, jsonData.message_data.params)
-                      );
-                    } else {
-                      stepContent = jsonData.message || "";
-                    }
-                    break;
-                  case "error":
-                    stepContent = t("chatInterface.parseFileFailed", {
-                      filename: jsonData.filename,
-                      message: jsonData.message,
-                    });
-                    break;
-                  case "file_processed":
-                    stepContent = t("chatInterface.fileParsed", {
-                      filename: jsonData.filename,
-                    });
-                    break;
-                  case "complete":
-                    // When complete, process all buffered truncation messages
-                    if (truncationBuffer.length > 0) {
-                      // Process truncation messages using internationalization
-                      const truncationInfo = truncationBuffer
-                        .map((truncation) => {
-                          if (truncation.message_data) {
-                            const i18nKey = getI18nKeyByType(truncation.type);
-                            return String(
-                              t(i18nKey, truncation.message_data.params)
-                            );
-                          } else {
-                            return truncation.message;
-                          }
-                        })
-                        .join(String(t("chatInterface.truncationSeparator")));
-
-                      stepContent = t(
-                        "chatInterface.fileParsingCompleteWithTruncation",
-                        {
-                          truncationInfo: truncationInfo,
-                        }
-                      );
-                    } else {
-                      stepContent = t("chatInterface.fileParsingComplete");
-                    }
-                    break;
-                  default:
-                    stepContent = jsonData.message || "";
-                }
-                // Only update the first content, don't add new ones
-                if (step && step.contents && step.contents.length > 0) {
-                  step.contents[0].content = stepContent;
-                  step.contents[0].timestamp = Date.now();
-                }
-              }
-              return newMessages;
-            });
-          },
+          () => {}, // Empty progress callback - won't be called
           t,
           currentConversationId
         );
 
-        // Handle preprocessing result
-        if (!result.success) {
-          // Reset button states immediately when preprocessing fails
-          setIsLoading(false);
-          setIsStreaming(false);
-            
-          // Remove from streaming conversations (both new and existing conversations)
-          if (currentConversationId) {
-            setStreamingConversations((prev) => {
-              const newSet = new Set(prev);
-              newSet.delete(currentConversationId);
-              return newSet;
-            });
-          }
-          
-          setSessionMessages((prev) => {
-            const newMessages = { ...prev };
-            const lastMsg =
-              newMessages[currentConversationId]?.[
-                newMessages[currentConversationId].length - 1
-              ];
-            
-            if (lastMsg && lastMsg.role === ROLE_ASSISTANT) {
-              // Handle error codes with internationalization
-              let errorMessage;
-              if (result.error === 'REQUEST_ENTITY_TOO_LARGE') {
-                errorMessage = t("chatInterface.fileSizeExceeded");
-              } else if (result.error === 'FILE_PARSING_FAILED') {
-                errorMessage = t("chatInterface.fileParsingFailed");
-              } else {
-                // For any other error, show a simple message
-                errorMessage = t("chatInterface.fileProcessingStopped");
-              }
-              
-              lastMsg.content = errorMessage;
-              lastMsg.isComplete = true;
-            }
-            
-            return newMessages;
-          });
-          shouldResetButtonStates = false; // Don't reset again in finally block
-          return;
-        }
-
         finalQuery = result.finalQuery;
         fileDescriptionsMap = result.fileDescriptions || {};
       }
@@ -1433,25 +1238,6 @@ export function ChatInterface() {
         if (lastMsg && lastMsg.role === ROLE_ASSISTANT) {
           lastMsg.isComplete = true;
           lastMsg.thinking = undefined; // Explicitly clear thinking state
-
-          // If this was a preprocess step, mark it as stopped
-          if (lastMsg.steps && lastMsg.steps.length > 0) {
-            const preprocessStep = lastMsg.steps.find(
-              (step) => step.title === t("chatInterface.filePreprocessing")
-            );
-            if (preprocessStep) {
-              const stoppedMessage =
-                (t("chatInterface.fileProcessingStopped") as string) ||
-                "File preprocessing stopped";
-              preprocessStep.content = stoppedMessage;
-              if (
-                preprocessStep.contents &&
-                preprocessStep.contents.length > 0
-              ) {
-                preprocessStep.contents[0].content = stoppedMessage;
-              }
-            }
-          }
         }
         return newMessages;
       });

frontend/app/[locale]/chat/internal/chatPreprocess.tsx

@@ -82,6 +82,12 @@ export const preprocessAttachments = async (
     return { finalQuery: content, success: true };
   }
 
+  // Skip preprocessing API call - return original content directly
+  // If you want to re-enable preprocessing, uncomment the code below
+  return { finalQuery: content, success: true };
+
+  /* 
+  // Original preprocessing code (disabled)
   try {
     // Call file preprocessing interface
     const preProcessReader = await conversationService.preprocessFiles(
@@ -152,6 +158,7 @@ export const preprocessAttachments = async (
       error: error instanceof Error ? (error as Error).message : String(error),
     };
   }
+  */
 };
 
 /**

test/backend/agents/test_create_agent_info.py

@@ -1254,7 +1254,7 @@ async def test_join_minio_file_description_to_query_with_files(self):
 
         result = await join_minio_file_description_to_query(minio_files, query)
 
-        expected = "User provided some reference files:\nFile S3 URL: s3://nexent/1.pdf, file name:1.pdf\nFile S3 URL: s3://nexent/2.pdf, file name:2.pdf\n\nUser wants to answer questions based on the above information: test query"
+        expected = "User uploaded files. The file information is as follows:\nFile name: 1.pdf, S3 URL: s3://nexent/1.pdf\nFile name: 2.pdf, S3 URL: s3://nexent/2.pdf\n\nUser wants to answer questions based on the information in the above files: test query"
         assert result == expected
 
     @pytest.mark.asyncio