SearchApi
diff --git a/‎api/app/clients/BaseClient.js‎
Lines changed: 41 additions & 14 deletions b/‎api/app/clients/BaseClient.js‎
Lines changed: 41 additions & 14 deletions
diff --git a/‎api/app/clients/OpenAIClient.js‎
Lines changed: 10 additions & 9 deletions b/‎api/app/clients/OpenAIClient.js‎
Lines changed: 10 additions & 9 deletions
diff --git a/‎api/app/clients/llm/createLLM.js‎
Lines changed: 1 addition & 1 deletion b/‎api/app/clients/llm/createLLM.js‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎api/app/clients/prompts/formatMessages.js‎
Lines changed: 73 additions & 5 deletions b/‎api/app/clients/prompts/formatMessages.js‎
Lines changed: 73 additions & 5 deletions
diff --git a/‎api/app/clients/tools/structured/TavilySearch.js‎
Lines changed: 78 additions & 0 deletions b/‎api/app/clients/tools/structured/TavilySearch.js‎
Lines changed: 78 additions & 0 deletions
diff --git a/‎api/models/Action.js‎
Lines changed: 3 additions & 3 deletions b/‎api/models/Action.js‎
Lines changed: 3 additions & 3 deletions
@@ -34,6 +34,12 @@ class BaseClient {
     this.userMessagePromise;
     /** @type {ClientDatabaseSavePromise} */
     this.responsePromise;
+    /** @type {string} */
+    this.user;
+    /** @type {string} */
+    this.conversationId;
+    /** @type {string} */
+    this.responseMessageId;
   }
 
   setOptions() {
@@ -161,6 +167,8 @@ class BaseClient {
       this.currentMessages[this.currentMessages.length - 1].messageId = head;
     }
 
+    this.responseMessageId = responseMessageId;
+
     return {
       ...opts,
       user,
@@ -347,7 +355,12 @@ class BaseClient {
     };
   }
 
-  async handleContextStrategy({ instructions, orderedMessages, formattedMessages }) {
+  async handleContextStrategy({
+    instructions,
+    orderedMessages,
+    formattedMessages,
+    buildTokenMap = true,
+  }) {
     let _instructions;
     let tokenCount;
 
@@ -417,19 +430,23 @@ class BaseClient {
       maxContextTokens: this.maxContextTokens,
     });
 
-    let tokenCountMap = orderedWithInstructions.reduce((map, message, index) => {
-      const { messageId } = message;
-      if (!messageId) {
-        return map;
-      }
+    /** @type {Record<string, number> | undefined} */
+    let tokenCountMap;
+    if (buildTokenMap) {
+      tokenCountMap = orderedWithInstructions.reduce((map, message, index) => {
+        const { messageId } = message;
+        if (!messageId) {
+          return map;
+        }
 
-      if (shouldSummarize && index === summaryIndex && !usePrevSummary) {
-        map.summaryMessage = { ...summaryMessage, messageId, tokenCount: summaryTokenCount };
-      }
+        if (shouldSummarize && index === summaryIndex && !usePrevSummary) {
+          map.summaryMessage = { ...summaryMessage, messageId, tokenCount: summaryTokenCount };
+        }
 
-      map[messageId] = orderedWithInstructions[index].tokenCount;
-      return map;
-    }, {});
+        map[messageId] = orderedWithInstructions[index].tokenCount;
+        return map;
+      }, {});
+    }
 
     const promptTokens = this.maxContextTokens - remainingContextTokens;
 
@@ -542,13 +559,19 @@ class BaseClient {
       isEdited,
       model: this.modelOptions.model,
       sender: this.sender,
-      text: addSpaceIfNeeded(generation) + completion,
       promptTokens,
       iconURL: this.options.iconURL,
       endpoint: this.options.endpoint,
       ...(this.metadata ?? {}),
     };
 
+    if (typeof completion === 'string') {
+      responseMessage.text = addSpaceIfNeeded(generation) + completion;
+    } else if (completion) {
+      responseMessage.text = '';
+      responseMessage.content = completion;
+    }
+
     if (
       tokenCountMap &&
       this.recordTokenUsage &&
@@ -868,8 +891,12 @@ class BaseClient {
 
           processValue(nestedValue);
         }
-      } else {
+      } else if (typeof value === 'string') {
         numTokens += this.getTokenCount(value);
+      } else if (typeof value === 'number') {
+        numTokens += this.getTokenCount(value.toString());
+      } else if (typeof value === 'boolean') {
+        numTokens += this.getTokenCount(value.toString());
       }
     };
 
 
@@ -1023,7 +1023,7 @@ ${convo}
   async chatCompletion({ payload, onProgress, abortController = null }) {
     let error = null;
     const errorCallback = (err) => (error = err);
-    let intermediateReply = '';
+    const intermediateReply = [];
     try {
       if (!abortController) {
         abortController = new AbortController();
@@ -1217,19 +1217,19 @@ ${convo}
             }
 
             if (typeof finalMessage.content !== 'string' || finalMessage.content.trim() === '') {
-              finalChatCompletion.choices[0].message.content = intermediateReply;
+              finalChatCompletion.choices[0].message.content = intermediateReply.join('');
             }
           })
           .on('finalMessage', (message) => {
             if (message?.role !== 'assistant') {
-              stream.messages.push({ role: 'assistant', content: intermediateReply });
+              stream.messages.push({ role: 'assistant', content: intermediateReply.join('') });
               UnexpectedRoleError = true;
             }
           });
 
         for await (const chunk of stream) {
           const token = chunk.choices[0]?.delta?.content || '';
-          intermediateReply += token;
+          intermediateReply.push(token);
           onProgress(token);
           if (abortController.signal.aborted) {
             stream.controller.abort();
@@ -1285,11 +1285,12 @@ ${convo}
       }
 
       if (typeof message.content !== 'string' || message.content.trim() === '') {
+        const reply = intermediateReply.join('');
         logger.debug(
           '[OpenAIClient] chatCompletion: using intermediateReply due to empty message.content',
-          { intermediateReply },
+          { intermediateReply: reply },
         );
-        return intermediateReply;
+        return reply;
       }
 
       return message.content;
@@ -1298,7 +1299,7 @@ ${convo}
         err?.message?.includes('abort') ||
         (err instanceof OpenAI.APIError && err?.message?.includes('abort'))
       ) {
-        return intermediateReply;
+        return intermediateReply.join('');
       }
       if (
         err?.message?.includes(
@@ -1313,10 +1314,10 @@ ${convo}
         (err instanceof OpenAI.OpenAIError && err?.message?.includes('missing finish_reason'))
       ) {
         logger.error('[OpenAIClient] Known OpenAI error:', err);
-        return intermediateReply;
+        return intermediateReply.join('');
       } else if (err instanceof OpenAI.APIError) {
         if (intermediateReply) {
-          return intermediateReply;
+          return intermediateReply.join('');
         } else {
           throw err;
         }
 
@@ -8,7 +8,7 @@ const { isEnabled } = require('~/server/utils');
  * @param {Object} options - The options for creating the LLM.
  * @param {ModelOptions} options.modelOptions - The options specific to the model, including modelName, temperature, presence_penalty, frequency_penalty, and other model-related settings.
  * @param {ConfigOptions} options.configOptions - Configuration options for the API requests, including proxy settings and custom headers.
- * @param {Callbacks} options.callbacks - Callback functions for managing the lifecycle of the LLM, including token buffers, context, and initial message count.
+ * @param {Callbacks} [options.callbacks] - Callback functions for managing the lifecycle of the LLM, including token buffers, context, and initial message count.
  * @param {boolean} [options.streaming=false] - Determines if the LLM should operate in streaming mode.
  * @param {string} options.openAIApiKey - The API key for OpenAI, used for authentication.
  * @param {AzureOptions} [options.azure={}] - Optional Azure-specific configurations. If provided, Azure configurations take precedence over OpenAI configurations.
 
@@ -1,4 +1,5 @@
-const { EModelEndpoint } = require('librechat-data-provider');
+const { ToolMessage } = require('@langchain/core/messages');
+const { EModelEndpoint, ContentTypes } = require('librechat-data-provider');
 const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
 
 /**
@@ -14,11 +15,11 @@ const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema');
  */
 const formatVisionMessage = ({ message, image_urls, endpoint }) => {
   if (endpoint === EModelEndpoint.anthropic) {
-    message.content = [...image_urls, { type: 'text', text: message.content }];
+    message.content = [...image_urls, { type: ContentTypes.TEXT, text: message.content }];
     return message;
   }
 
-  message.content = [{ type: 'text', text: message.content }, ...image_urls];
+  message.content = [{ type: ContentTypes.TEXT, text: message.content }, ...image_urls];
 
   return message;
 };
@@ -51,7 +52,7 @@ const formatMessage = ({ message, userName, assistantName, endpoint, langChain =
     _role = roleMapping[lc_id[2]];
   }
   const role = _role ?? (sender && sender?.toLowerCase() === 'user' ? 'user' : 'assistant');
-  const content = text ?? _content ?? '';
+  const content = _content ?? text ?? '';
   const formattedMessage = {
     role,
     content,
@@ -131,4 +132,71 @@ const formatFromLangChain = (message) => {
   };
 };
 
-module.exports = { formatMessage, formatLangChainMessages, formatFromLangChain };
+/**
+ * Formats an array of messages for LangChain, handling tool calls and creating ToolMessage instances.
+ *
+ * @param {Array<Partial<TMessage>>} payload - The array of messages to format.
+ * @returns {Array<(HumanMessage|AIMessage|SystemMessage|ToolMessage)>} - The array of formatted LangChain messages, including ToolMessages for tool calls.
+ */
+const formatAgentMessages = (payload) => {
+  const messages = [];
+
+  for (const message of payload) {
+    if (message.role !== 'assistant') {
+      messages.push(formatMessage({ message, langChain: true }));
+      continue;
+    }
+
+    let currentContent = [];
+    let lastAIMessage = null;
+
+    for (const part of message.content) {
+      if (part.type === ContentTypes.TEXT && part.tool_call_ids) {
+        // If there's pending content, add it as an AIMessage
+        if (currentContent.length > 0) {
+          messages.push(new AIMessage({ content: currentContent }));
+          currentContent = [];
+        }
+
+        // Create a new AIMessage with this text and prepare for tool calls
+        lastAIMessage = new AIMessage({
+          content: part.text || '',
+        });
+
+        messages.push(lastAIMessage);
+      } else if (part.type === ContentTypes.TOOL_CALL) {
+        if (!lastAIMessage) {
+          throw new Error('Invalid tool call structure: No preceding AIMessage with tool_call_ids');
+        }
+
+        // Note: `tool_calls` list is defined when constructed by `AIMessage` class, and outputs should be excluded from it
+        const { output, ...tool_call } = part.tool_call;
+        lastAIMessage.tool_calls.push(tool_call);
+
+        // Add the corresponding ToolMessage
+        messages.push(
+          new ToolMessage({
+            tool_call_id: tool_call.id,
+            name: tool_call.name,
+            content: output,
+          }),
+        );
+      } else {
+        currentContent.push(part);
+      }
+    }
+
+    if (currentContent.length > 0) {
+      messages.push(new AIMessage({ content: currentContent }));
+    }
+  }
+
+  return messages;
+};
+
+module.exports = {
+  formatMessage,
+  formatFromLangChain,
+  formatAgentMessages,
+  formatLangChainMessages,
+};
@@ -0,0 +1,78 @@
+const { z } = require('zod');
+const { tool } = require('@langchain/core/tools');
+const { getEnvironmentVariable } = require('@langchain/core/utils/env');
+
+function createTavilySearchTool(fields = {}) {
+  const envVar = 'TAVILY_API_KEY';
+  const override = fields.override ?? false;
+  const apiKey = fields.apiKey ?? getApiKey(envVar, override);
+  const kwargs = fields?.kwargs ?? {};
+
+  function getApiKey(envVar, override) {
+    const key = getEnvironmentVariable(envVar);
+    if (!key && !override) {
+      throw new Error(`Missing ${envVar} environment variable.`);
+    }
+    return key;
+  }
+
+  return tool(
+    async (input) => {
+      const { query, ...rest } = input;
+
+      const requestBody = {
+        api_key: apiKey,
+        query,
+        ...rest,
+        ...kwargs,
+      };
+
+      const response = await fetch('https://api.tavily.com/search', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(requestBody),
+      });
+
+      const json = await response.json();
+      if (!response.ok) {
+        throw new Error(`Request failed with status ${response.status}: ${json.error}`);
+      }
+
+      return JSON.stringify(json);
+    },
+    {
+      name: 'tavily_search_results_json',
+      description:
+        'A search engine optimized for comprehensive, accurate, and trusted results. Useful for when you need to answer questions about current events.',
+      schema: z.object({
+        query: z.string().min(1).describe('The search query string.'),
+        max_results: z
+          .number()
+          .min(1)
+          .max(10)
+          .optional()
+          .describe('The maximum number of search results to return. Defaults to 5.'),
+        search_depth: z
+          .enum(['basic', 'advanced'])
+          .optional()
+          .describe(
+            'The depth of the search, affecting result quality and response time (`basic` or `advanced`). Default is basic for quick results and advanced for indepth high quality results but longer response time. Advanced calls equals 2 requests.',
+          ),
+        include_images: z
+          .boolean()
+          .optional()
+          .describe(
+            'Whether to include a list of query-related images in the response. Default is False.',
+          ),
+        include_answer: z
+          .boolean()
+          .optional()
+          .describe('Whether to include answers in the search results. Default is False.'),
+      }),
+    },
+  );
+}
+
+module.exports = createTavilySearchTool;
@@ -12,7 +12,7 @@ const Action = mongoose.model('action', actionSchema);
  * @param {string} searchParams.user - The user ID of the action's author.
  * @param {Object} updateData - An object containing the properties to update.
  * @param {mongoose.ClientSession} [session] - The transaction session to use.
- * @returns {Promise<Object>} The updated or newly created action document as a plain object.
+ * @returns {Promise<Action>} The updated or newly created action document as a plain object.
  */
 const updateAction = async (searchParams, updateData, session = null) => {
   const options = { new: true, upsert: true, session };
@@ -24,7 +24,7 @@ const updateAction = async (searchParams, updateData, session = null) => {
  *
  * @param {Object} searchParams - The search parameters to find matching actions.
  * @param {boolean} includeSensitive - Flag to include sensitive data in the metadata.
- * @returns {Promise<Array<Object>>} A promise that resolves to an array of action documents as plain objects.
+ * @returns {Promise<Array<Action>>} A promise that resolves to an array of action documents as plain objects.
  */
 const getActions = async (searchParams, includeSensitive = false) => {
   const actions = await Action.find(searchParams).lean();
@@ -55,7 +55,7 @@ const getActions = async (searchParams, includeSensitive = false) => {
  * @param {string} searchParams.action_id - The ID of the action to delete.
  * @param {string} searchParams.user - The user ID of the action's author.
  * @param {mongoose.ClientSession} [session] - The transaction session to use (optional).
- * @returns {Promise<Object>} A promise that resolves to the deleted action document as a plain object, or null if no document was found.
+ * @returns {Promise<Action>} A promise that resolves to the deleted action document as a plain object, or null if no document was found.
  */
 const deleteAction = async (searchParams, session = null) => {
   const options = session ? { session } : {};