Merge pull request #71 from OpenAgentPlatform/gemini

feat: support google gemini & mistral ai models settings

Author: johnfunmula

electron/main/ipc/llm.ts

@@ -2,6 +2,7 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import { ipcMain } from "electron"
 import { Ollama } from "ollama"
 import OpenAI from "openai"
+import { Mistral } from "@mistralai/mistralai"
 
 export function ipcLlmHandler() {
   ipcMain.handle("llm:openaiModelList", async (_, apiKey: string) => {
@@ -43,4 +44,25 @@ export function ipcLlmHandler() {
       return []
     }
   })
-}
+
+  ipcMain.handle("llm:googleGenaiModelList", async (_, apiKey: string) => {
+    try {
+      const url = `https://generativelanguage.googleapis.com/v1beta/models?key=${apiKey}`
+      const response = await fetch(url)
+      const data = await response.json() as { models: { name: string }[] }
+      return data.models.map((model) => model.name)
+    } catch (error) {
+      return []
+    }
+  })
+
+  ipcMain.handle("llm:mistralaiModelList", async (_, apiKey: string) => {
+    try {
+      const client = new Mistral({ apiKey })
+      const models = await client.models.list()
+      return models.data?.map((model) => model.id) ?? []
+    } catch (error) {
+      return []
+    }
+  })
+}

electron/preload/index.ts

@@ -30,6 +30,8 @@ contextBridge.exposeInMainWorld("ipcRenderer", {
   openaiCompatibleModelList: (apiKey: string, baseURL: string) => ipcRenderer.invoke("llm:openaiCompatibleModelList", apiKey, baseURL),
   anthropicModelList: (apiKey: string, baseURL: string) => ipcRenderer.invoke("llm:anthropicModelList", apiKey, baseURL),
   ollamaModelList: (baseURL: string) => ipcRenderer.invoke("llm:ollamaModelList", baseURL),
+  googleGenaiModelList: (apiKey: string) => ipcRenderer.invoke("llm:googleGenaiModelList", apiKey),
+  mistralaiModelList: (apiKey: string) => ipcRenderer.invoke("llm:mistralaiModelList", apiKey),
 
   // context menu
   showSelectionContextMenu: () => ipcRenderer.invoke("show-selection-context-menu"),

package-lock.json

@@ -56,8 +56,11 @@
     "@langchain/anthropic": "^0.3.12",
     "@langchain/community": "^0.3.0",
     "@langchain/core": "^0.3.0",
+    "@langchain/google-genai": "^0.1.10",
+    "@langchain/mistralai": "^0.2.0",
     "@langchain/ollama": "^0.1.4",
     "@langchain/openai": "^0.3.17",
+    "@mistralai/mistralai": "^1.5.1",
     "@modelcontextprotocol/inspector": "^0.4.1",
     "@modelcontextprotocol/sdk": "^1.4.1",
     "@radix-ui/react-dropdown-menu": "^2.1.6",

package.json

@@ -11,7 +11,9 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { ModelManager } from "./models/index.js";
 import { imageToBase64 } from "./utils/image.js";
 import logger from "./utils/logger.js";
-import { iQueryInput, iStreamMessage } from "./utils/types.js";
+import { iQueryInput, iStreamMessage, ModelSettings } from "./utils/types.js";
+import { openAIConvertToGeminiTools } from "./utils/toolHandler.js";
+import { ToolDefinition } from "@langchain/core/language_models/base";
 
 // Map to store abort controllers
 export const abortControllerMap = new Map<string, AbortController>();
@@ -31,7 +33,7 @@ interface TokenUsage {
 
 export async function handleProcessQuery(
   toolToClientMap: Map<string, Client>,
-  availableTools: BindToolsInput[],
+  availableTools: ToolDefinition[],
   model: BaseChatModel | null,
   input: string | iQueryInput,
   history: BaseMessage[],
@@ -121,12 +123,16 @@ export async function handleProcessQuery(
 
     let hasToolCalls = true;
 
-    const runModel = modelManager.enableTools ? model.bindTools?.(availableTools) || model : model;
+    const tools = currentModelSettings?.modelProvider === "google-genai" ? openAIConvertToGeminiTools(availableTools) : availableTools;
+
+    const runModel = modelManager.enableTools ? model.bindTools?.(tools) || model : model;
 
     const isOllama = currentModelSettings?.modelProvider === "ollama";
     const isDeepseek =
       currentModelSettings?.configuration?.baseURL?.toLowerCase().includes("deepseek") ||
       currentModelSettings?.model?.toLowerCase().includes("deepseek");
+    const isMistralai = currentModelSettings?.modelProvider === "mistralai";
+    const isBedrock = currentModelSettings?.modelProvider === "bedrock";
 
     logger.debug(`[${chatId}] Start to process LLM query`);
 
@@ -141,7 +147,7 @@ export async function handleProcessQuery(
       try {
         // Track token usage if available
         for await (const chunk of stream) {
-          caculateTokenUsage(tokenUsage, chunk, currentModelSettings!.modelProvider!);
+          caculateTokenUsage(tokenUsage, chunk, currentModelSettings!);
 
           if (chunk.content) {
             let chunkMessage = "";
@@ -235,6 +241,8 @@ export async function handleProcessQuery(
         throw error;
       }
 
+      logger.debug(`[${chatId}] Chunk collected`);
+
       // filter empty tool calls
       toolCalls = toolCalls.filter((call) => call);
 
@@ -247,6 +255,7 @@ export async function handleProcessQuery(
         break;
       }
 
+      logger.debug(`[${chatId}] Tool calls: ${JSON.stringify(toolCalls, null, 2)}`);
       // support anthropic multiple tool calls version but other not sure
       messages.push(
         new AIMessage({
@@ -257,14 +266,23 @@ export async function handleProcessQuery(
               text: currentContent || ".",
             },
             // Deepseek will recursive when tool_use exist in content
-            ...(isDeepseek
+            ...(isDeepseek || isMistralai || isBedrock
               ? []
-              : toolCalls.map((toolCall) => ({
+              : toolCalls.map((toolCall) => {
+                let parsedArgs = {}
+                try {
+                  parsedArgs = toolCall.function.arguments === "" ? {} : JSON.parse(toolCall.function.arguments);
+                } catch (error) {
+                  toolCall.function.arguments = "{}";
+                  logger.error(`[${chatId}] Error parsing tool call ${toolCall.function.name} args: ${error}`);
+                }
+                return {
                   type: "tool_use",
                   id: toolCall.id,
                   name: toolCall.function.name,
-                  input: toolCall.function.arguments === "" ? {} : JSON.parse(toolCall.function.arguments),
-                }))),
+                  input: parsedArgs,
+                }
+              })),
           ],
           additional_kwargs: {
             tool_calls: toolCalls.map((toolCall) => ({
@@ -297,6 +315,8 @@ export async function handleProcessQuery(
         );
       }
 
+      logger.debug(`[${chatId}] Tool calls collected`);
+
       // Execute all tool calls in parallel
       const toolResults = await Promise.all(
         toolCalls.map(async (toolCall) => {
@@ -407,10 +427,14 @@ export async function handleProcessQuery(
         })
       );
 
+      logger.debug(`[${chatId}] Tool results collected`);
+
       // Add tool results to conversation
       if (toolResults.length > 0) {
         messages.push(...toolResults.map((result) => new ToolMessage(result)));
       }
+
+      logger.debug(`[${chatId}] Messages collected and ready to next round`);
     }
 
     // Log token usage at the end of processing
@@ -437,8 +461,20 @@ export async function handleProcessQuery(
   }
 }
 
-function caculateTokenUsage(tokenUsage: TokenUsage, chunk: AIMessageChunk, currentModelProvider: string) {
-  switch (currentModelProvider) {
+function caculateTokenUsage(tokenUsage: TokenUsage, chunk: AIMessageChunk, currentModelSettings: ModelSettings) {
+  if (!currentModelSettings) {
+    return;
+  }
+
+  if (currentModelSettings.configuration?.baseURL?.toLowerCase().includes("silicon")) {
+    const usage = chunk.response_metadata.usage;
+    tokenUsage.totalInputTokens = usage?.prompt_tokens || 0;
+    tokenUsage.totalOutputTokens = usage?.completion_tokens || 0;
+    tokenUsage.totalTokens = usage?.total_tokens || 0;
+    return;
+  }
+
+  switch (currentModelSettings.modelProvider) {
     case "openai":
       if (chunk.response_metadata?.usage) {
         const usage = chunk.response_metadata.usage;
@@ -455,7 +491,8 @@ function caculateTokenUsage(tokenUsage: TokenUsage, chunk: AIMessageChunk, curre
         tokenUsage.totalOutputTokens += usage?.output_tokens || 0;
         tokenUsage.totalTokens += usage?.total_tokens || 0;
       }
+      break;
     default:
       break;
   }
-}
+}

public/image/model_gemini.svg

@@ -74,6 +74,7 @@ class ModelVerificationService {
         result: fullResponse,
       };
     } catch (error) {
+      logger.error(`[ModelVerify] error: ${JSON.stringify(error, null, 2)}`);
       if ((error as { name?: string })?.name === "AbortError") {
         logger.warn(`Model tools test aborted for ${modelName}`);
       }
@@ -246,30 +247,25 @@ const testTools = [
           page: {
             type: "number",
             description: "Page number (default 1)",
-            default: 1,
           },
           language: {
             type: "string",
             description: "Search language code (e.g. 'en', 'zh', 'jp', 'all')",
-            default: "all",
           },
           categories: {
             type: "array",
             items: {
               type: "string",
               enum: ["general", "news", "science", "files", "images", "videos", "music", "social media", "it"],
             },
-            default: ["general"],
           },
           time_range: {
             type: "string",
-            enum: ["", "day", "week", "month", "year"],
-            default: "",
+            enum: ["none", "day", "week", "month", "year"],
           },
           safesearch: {
             type: "number",
             description: "0: None, 1: Moderate, 2: Strict",
-            default: 1,
           },
         },
         required: ["query"],