specs

Author: SamSaffron

lib/ai_bot/bot.rb

@@ -8,7 +8,7 @@ class Bot
       BOT_NOT_FOUND = Class.new(StandardError)
 
       # the future is agentic, allow for more turns
-      MAX_COMPLETIONS = 2
+      MAX_COMPLETIONS = 8
 
       # limit is arbitrary, but 5 which was used in the past was too low
       MAX_TOOLS = 20

lib/completions/dialects/dialect.rb

@@ -66,9 +66,13 @@ def tool_choice
           prompt.tool_choice
         end
 
-        def no_more_tool_calls_text
+        def self.no_more_tool_calls_text
           # note, Anthropic must never prefill with an ending whitespace
-          "Since you explicitly asked me not to use tools any more I will not call tools any more.\nHere is the best, complete, answer I can come up with given the information I know:"
+          "I WILL NOT USE TOOLS IN THIS REPLY, user expressed they wanted to stop using tool calls.\nHere is the best, complete, answer I can come up with given the information I have."
+        end
+
+        def no_more_tool_calls_text
+          self.class.no_more_tool_calls_text
         end
 
         def translate
@@ -80,7 +84,23 @@ def translate
             messages.pop
           end
 
-          trim_messages(messages).map { |msg| send("#{msg[:type]}_msg", msg) }.compact
+          translated = trim_messages(messages).map { |msg| send("#{msg[:type]}_msg", msg) }.compact
+
+          if !native_tool_support?
+            if prompt.tools.present? && prompt.tool_choice.present?
+              if prompt.tool_choice == :none
+                translated << model_msg(role: "assistant", content: no_more_tool_calls_text)
+              else
+                translated << model_msg(
+                  role: "assistant",
+                  content:
+                    "User required I call the tool: #{prompt.tool_choice} I will makes sure I use it now:",
+                )
+              end
+            end
+          end
+
+          translated
         end
 
         def conversation_context

spec/lib/completions/endpoints/anthropic_spec.rb

@@ -714,4 +714,59 @@
       expect(parsed_body[:max_tokens]).to eq(500)
     end
   end
+
+  describe "disabled tool use" do
+    it "can properly disable tool use with :none" do
+      prompt =
+        DiscourseAi::Completions::Prompt.new(
+          "You are a bot",
+          messages: [type: :user, id: "user1", content: "don't use any tools please"],
+          tools: [echo_tool],
+          tool_choice: :none,
+        )
+
+      response_body = {
+        id: "msg_01RdJkxCbsEj9VFyFYAkfy2S",
+        type: "message",
+        role: "assistant",
+        model: "claude-3-haiku-20240307",
+        content: [
+          { type: "text", text: "I won't use any tools. Here's a direct response instead." },
+        ],
+        stop_reason: "end_turn",
+        stop_sequence: nil,
+        usage: {
+          input_tokens: 345,
+          output_tokens: 65,
+        },
+      }.to_json
+
+      parsed_body = nil
+      stub_request(:post, url).with(
+        body:
+          proc do |req_body|
+            parsed_body = JSON.parse(req_body, symbolize_names: true)
+            true
+          end,
+      ).to_return(status: 200, body: response_body)
+
+      result = llm.generate(prompt, user: Discourse.system_user)
+
+      # Verify that tool_choice is set to { type: "none" }
+      expect(parsed_body[:tool_choice]).to eq({ type: "none" })
+
+      # Verify that an assistant message with no_more_tool_calls_text was added
+      messages = parsed_body[:messages]
+      expect(messages.length).to eq(2) # user message + added assistant message
+
+      last_message = messages.last
+      expect(last_message[:role]).to eq("assistant")
+
+      expect(last_message[:content]).to eq(
+        DiscourseAi::Completions::Dialects::Dialect.no_more_tool_calls_text,
+      )
+
+      expect(result).to eq("I won't use any tools. Here's a direct response instead.")
+    end
+  end
 end

spec/lib/completions/endpoints/aws_bedrock_spec.rb

@@ -484,4 +484,66 @@ def encode_message(message)
       expect(request_body["max_tokens"]).to eq(500)
     end
   end
+
+  describe "disabled tool use" do
+    it "handles tool_choice: :none by adding a prefill message instead of using tool_choice param" do
+      proxy = DiscourseAi::Completions::Llm.proxy("custom:#{model.id}")
+      request = nil
+
+      # Create a prompt with tool_choice: :none
+      prompt =
+        DiscourseAi::Completions::Prompt.new(
+          "You are a helpful assistant",
+          messages: [{ type: :user, content: "don't use any tools please" }],
+          tools: [
+            {
+              name: "echo",
+              description: "echo something",
+              parameters: [
+                { name: "text", type: "string", description: "text to echo", required: true },
+              ],
+            },
+          ],
+          tool_choice: :none,
+        )
+
+      # Mock response from Bedrock
+      content = {
+        content: [text: "I won't use any tools. Here's a direct response instead."],
+        usage: {
+          input_tokens: 25,
+          output_tokens: 15,
+        },
+      }.to_json
+
+      stub_request(
+        :post,
+        "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet-20240229-v1:0/invoke",
+      )
+        .with do |inner_request|
+          request = inner_request
+          true
+        end
+        .to_return(status: 200, body: content)
+
+      proxy.generate(prompt, user: user)
+
+      # Parse the request body
+      request_body = JSON.parse(request.body)
+
+      # Verify that tool_choice is NOT present (not supported in Bedrock)
+      expect(request_body).not_to have_key("tool_choice")
+
+      # Verify that an assistant message was added with no_more_tool_calls_text
+      messages = request_body["messages"]
+      expect(messages.length).to eq(2) # user message + added assistant message
+
+      last_message = messages.last
+      expect(last_message["role"]).to eq("assistant")
+
+      expect(last_message["content"]).to eq(
+        DiscourseAi::Completions::Dialects::Dialect.no_more_tool_calls_text,
+      )
+    end
+  end
 end

spec/lib/completions/endpoints/gemini_spec.rb

@@ -377,4 +377,60 @@ def tool_response
 
     expect(output.join).to eq("Hello World Sam")
   end
+
+  it "can properly disable tool use with :none" do
+    prompt = DiscourseAi::Completions::Prompt.new("Hello", tools: [echo_tool], tool_choice: :none)
+
+    response = gemini_mock.response("I won't use any tools").to_json
+
+    req_body = nil
+
+    llm = DiscourseAi::Completions::Llm.proxy("custom:#{model.id}")
+    url = "#{model.url}:generateContent?key=123"
+
+    stub_request(:post, url).with(
+      body:
+        proc do |_req_body|
+          req_body = _req_body
+          true
+        end,
+    ).to_return(status: 200, body: response)
+
+    response = llm.generate(prompt, user: user)
+
+    expect(response).to eq("I won't use any tools")
+
+    parsed = JSON.parse(req_body, symbolize_names: true)
+
+    # Verify that function_calling_config mode is set to "NONE"
+    expect(parsed[:tool_config]).to eq({ function_calling_config: { mode: "NONE" } })
+  end
+
+  it "can properly force specific tool use" do
+    prompt = DiscourseAi::Completions::Prompt.new("Hello", tools: [echo_tool], tool_choice: "echo")
+
+    response = gemini_mock.response("World").to_json
+
+    req_body = nil
+
+    llm = DiscourseAi::Completions::Llm.proxy("custom:#{model.id}")
+    url = "#{model.url}:generateContent?key=123"
+
+    stub_request(:post, url).with(
+      body:
+        proc do |_req_body|
+          req_body = _req_body
+          true
+        end,
+    ).to_return(status: 200, body: response)
+
+    response = llm.generate(prompt, user: user)
+
+    parsed = JSON.parse(req_body, symbolize_names: true)
+
+    # Verify that function_calling_config is correctly set to ANY mode with the specified tool
+    expect(parsed[:tool_config]).to eq(
+      { function_calling_config: { mode: "ANY", allowed_function_names: ["echo"] } },
+    )
+  end
 end

spec/lib/completions/endpoints/open_ai_spec.rb

@@ -395,6 +395,65 @@ def request_body(prompt, stream: false, tool_call: false)
     end
   end
 
+  describe "disabled tool use" do
+    it "can properly disable tool use with :none" do
+      llm = DiscourseAi::Completions::Llm.proxy("custom:#{model.id}")
+
+      tools = [
+        {
+          name: "echo",
+          description: "echo something",
+          parameters: [
+            { name: "text", type: "string", description: "text to echo", required: true },
+          ],
+        },
+      ]
+
+      prompt =
+        DiscourseAi::Completions::Prompt.new(
+          "You are a bot",
+          messages: [type: :user, id: "user1", content: "don't use any tools please"],
+          tools: tools,
+          tool_choice: :none,
+        )
+
+      response = {
+        id: "chatcmpl-9JxkAzzaeO4DSV3omWvok9TKhCjBH",
+        object: "chat.completion",
+        created: 1_714_544_914,
+        model: "gpt-4-turbo-2024-04-09",
+        choices: [
+          {
+            index: 0,
+            message: {
+              role: "assistant",
+              content: "I won't use any tools. Here's a direct response instead.",
+            },
+            logprobs: nil,
+            finish_reason: "stop",
+          },
+        ],
+        usage: {
+          prompt_tokens: 55,
+          completion_tokens: 13,
+          total_tokens: 68,
+        },
+        system_fingerprint: "fp_ea6eb70039",
+      }.to_json
+
+      body_json = nil
+      stub_request(:post, "https://api.openai.com/v1/chat/completions").with(
+        body: proc { |body| body_json = JSON.parse(body, symbolize_names: true) },
+      ).to_return(body: response)
+
+      result = llm.generate(prompt, user: user)
+
+      # Verify that tool_choice is set to "none" in the request
+      expect(body_json[:tool_choice]).to eq("none")
+      expect(result).to eq("I won't use any tools. Here's a direct response instead.")
+    end
+  end
+
   describe "parameter disabling" do
     it "excludes disabled parameters from the request" do
       model.update!(provider_params: { disable_top_p: true, disable_temperature: true })