Gemini is responding with empty text content as well as tool_call

[DevDeepakBhattarai]

import { env } from "@/env";
import { ChatAnthropic } from "@langchain/anthropic";
import { ChatGoogleGenerativeAI } from "@langchain/google-genai";
import { ChatOpenAI, type ChatOpenAICallOptions } from "@langchain/openai";
import { z } from "zod";
type Model =
  | ChatOpenAI<ChatOpenAICallOptions>
  | ChatAnthropic
  | ChatGoogleGenerativeAI;

export const AvailableModels = z.enum(["gpt", "claude", "gemini"]);

export function modelPicker(
  model: z.infer<typeof AvailableModels>,
  stream?: boolean,
  modelName = "gpt-4o",
) {
  let modelObject: Model;
  switch (model) {
    case "gpt": {
      modelObject = new ChatOpenAI({
        model: modelName,
        apiKey: env.OPENAI_API_KEY,
        streaming: stream,
        modelKwargs: {
          parallel_tool_calls: false,
        },
      });

      break;
    }

    case "claude": {
      modelObject = new ChatAnthropic({
        model: "claude-3-sonnet-20240229",
        modelName: "claude-3-sonnet-20240229",
        apiKey: env.ANTHROPIC_API_KEY,
        streaming: true,
      });
      // modelObject.streaming = true;
      break;
    }

    case "gemini": {
      modelObject = new ChatGoogleGenerativeAI({
        model: "gemini-pro",
        modelName: "gemini-1.5-flash-latest",
        apiKey: env.GOOGLE_AI_API_KEY,
        // streaming: stream,
      });
      break;
    }
  }
  return modelObject;
}


import {
  AIMessage,
  type BaseMessage,
  ToolMessage,
  HumanMessage,
} from "@langchain/core/messages";
import { type RunnableConfig } from "@langchain/core/runnables";
import { StateGraph, START, END } from "@langchain/langgraph";
import {
  ChatPromptTemplate,
  MessagesPlaceholder,
} from "@langchain/core/prompts";
import { ChatOpenAI } from "@langchain/openai";
import { type DynamicStructuredTool } from "@langchain/core/tools";
import { search_tool } from "./tools/search";
import { modelPicker } from "@/lib/modelPicker";

export interface AgentExecutorState {
  input: string;
  model: "gpt" | "claude" | "gemini";
  chat_history: BaseMessage[];
  /**
   * The plain text result of the LLM if
   * no tool was used.
   */
  result?: string;
  /**
   * The parsed tool result that was called.
   */
  toolCall?: {
    name: string;
    parameters: Record<string, unknown>;
    id: string;
  };
  /**
   * The result of a tool.
   */
  toolResult?: Record<string, unknown>;
}

const invokeModel = async (
  state: AgentExecutorState,
  config?: RunnableConfig,
): Promise<Partial<AgentExecutorState>> => {
  const initialPrompt = ChatPromptTemplate.fromMessages([
    [
      "system",
      `You are a helpful assistant. You're provided a list of tools, and an input from the user.\n
      Your job is to determine whether or not you have a tools which can handle the users input, or respond with plain text.`,
    ],
    ["human", "{input}"],
    new MessagesPlaceholder({
      variableName: "chat_history",
      optional: true,
    }),
  ]);

  const tools = [search_tool];
  console.log(state.model);
  const llm = modelPicker(state.model, true).bind({ tools: tools });
  const chain = initialPrompt.pipe(llm);
  const result = await chain.invoke(state, config);
  if (result.tool_calls && result.tool_calls.length > 0) {
    const toolCall = {
      toolCall: {
        name: result.tool_calls[0]!.name,
        parameters: result?.tool_calls[0]!.args,
        id: result?.tool_calls[0]!.id ?? "",
      },
    };
    state.chat_history.push(result);

    return toolCall;
  }

  return {
    result: result.content as string,
    toolCall: undefined,
  };
};

const invokeToolsOrReturn = (state: AgentExecutorState) => {
  if (state.toolCall) {
    return "invokeTools";
  }

  if (state.result) {
    return END;
  }
  throw new Error("No tool call or result found.");
};

const invokeTools = async (
  state: AgentExecutorState,
  config?: RunnableConfig,
): Promise<Partial<AgentExecutorState>> => {
  if (!state.toolCall) {
    throw new Error("No tool call found.");
  }
  const toolMap: Record<string, DynamicStructuredTool> = {
    [search_tool.name]: search_tool,
  };

  const selectedTool = toolMap[state.toolCall.name];
  if (!selectedTool) {
    throw new Error("No tool found in tool map.");
  }
  const toolResult = await selectedTool.invoke(
    state.toolCall.parameters,
    config,
  );

  state.chat_history.push(
    new ToolMessage({ tool_call_id: state.toolCall.id, content: toolResult }),
  );

  return {
    toolResult: JSON.parse(toolResult),
  };
};

export function agentExecutor() {
  const workflow = new StateGraph<AgentExecutorState>({
    channels: {
      input: null,
      chat_history: null,
      result: null,
      toolCall: null,
      toolResult: null,
      model: null,
    },
  })
    .addNode("invokeModel", invokeModel)
    .addNode("invokeTools", invokeTools)
    .addConditionalEdges("invokeModel", invokeToolsOrReturn)
    .addEdge(START, "invokeModel")
    .addEdge("invokeTools", "invokeModel");

  const graph = workflow.compile();
  return graph;
}

I have the above code to select models based on user request. When ever I try to use gemini, it does not give me any text response as well as any tool call . The entire output is empty when ever I ask it to do anything related to the given tool.

If I ask it question like "Hello"
It responds normally, but as soon as I say tell me "today's news" or "what is the current weather" where it should be using the search tool. It doesn't , it just gives me empty response.

It this a bug with Langchain or Google ? I don't even know what is happening.

I am using the free api key that google provides. Does that have to do anything with this ? I didn't find any information about that in google's documentation.