Feat Support image URLs in tool outputs for Langchain::Assistant (#894)

* Feat handle image_url from tools output

* Update assistant.rb

* better handling of image_url

* add specs

* add ToolResponse class helper and use it

* Add helpers for response

* update response type

* Update tool_response.rb

* update specs

* Update assistant.rb

* Update assistant_spec.rb

* Update assistant.rb

* reset llm provider changes

* reset default llm provider specs

* clear and add helper documentation

* Update assistant_spec.rb

* Update assistant_spec.rb

* Merge Langchain::ToolHelpers module into Langchain::ToolDefinition

* Fixing specs

* CHANGELOG + README entries

---------

Co-authored-by: Andrei Bondarev <[email protected]>

Author: Eth3rnit3

.tool-versions

@@ -1 +1 @@
-ruby 3.3.0
+ruby 3.4

CHANGELOG.md

@@ -10,6 +10,7 @@
 - [SECURITY]: A change which fixes a security vulnerability.
 
 ## [Unreleased]
+- [BREAKING] [https://github.com/patterns-ai-core/langchainrb/pull/894] Tools can now output image_urls, and all tool output must be wrapped by a tool_response() method
 
 ## [0.19.3] - 2025-01-13
 - [BUGFIX] [https://github.com/patterns-ai-core/langchainrb/pull/900] Empty text content should not be set when content is nil when using AnthropicMessage

README.md

@@ -580,11 +580,11 @@ class MovieInfoTool
   end
 
   def search_movie(query:)
-    ...
+    tool_response(...)
   end
 
   def get_movie_details(movie_id:)
-    ...
+    tool_response(...)
   end
 end
 ```

lib/langchain/assistant.rb

@@ -371,9 +371,15 @@ def run_tool(tool_call)
 
       # Call the callback if set
       tool_execution_callback.call(tool_call_id, tool_name, method_name, tool_arguments) if tool_execution_callback # rubocop:disable Style/SafeNavigation
+
       output = tool_instance.send(method_name, **tool_arguments)
 
-      submit_tool_output(tool_call_id: tool_call_id, output: output)
+      # Handle both ToolResponse and legacy return values
+      if output.is_a?(ToolResponse)
+        add_message(role: @llm_adapter.tool_role, content: output.content, image_url: output.image_url, tool_call_id: tool_call_id)
+      else
+        submit_tool_output(tool_call_id: tool_call_id, output: output)
+      end
     end
 
     # Build a message

lib/langchain/tool/calculator.rb

@@ -26,13 +26,14 @@ def initialize
     # Evaluates a pure math expression or if equation contains non-math characters (e.g.: "12F in Celsius") then it uses the google search calculator to evaluate the expression
     #
     # @param input [String] math expression
-    # @return [String] Answer
+    # @return [Langchain::Tool::Response] Answer
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
-      Eqn::Calculator.calc(input)
+      result = Eqn::Calculator.calc(input)
+      tool_response(content: result)
     rescue Eqn::ParseError, Eqn::NoVariableValueError
-      "\"#{input}\" is an invalid mathematical expression"
+      tool_response(content: "\"#{input}\" is an invalid mathematical expression")
     end
   end
 end

lib/langchain/tool/database.rb

@@ -49,58 +49,61 @@ def initialize(connection_string:, tables: [], exclude_tables: [])
 
     # Database Tool: Returns a list of tables in the database
     #
-    # @return [Array<Symbol>] List of tables in the database
+    # @return [Langchain::Tool::Response] List of tables in the database
     def list_tables
-      db.tables
+      tool_response(content: db.tables)
     end
 
     # Database Tool: Returns the schema for a list of tables
     #
     # @param tables [Array<String>] The tables to describe.
-    # @return [String] The schema for the tables
+    # @return [Langchain::Tool::Response] The schema for the tables
     def describe_tables(tables: [])
       return "No tables specified" if tables.empty?
 
       Langchain.logger.debug("#{self.class} - Describing tables: #{tables}")
 
-      tables
+      result = tables
         .map do |table|
           describe_table(table)
         end
         .join("\n")
+
+      tool_response(content: result)
     end
 
     # Database Tool: Returns the database schema
     #
-    # @return [String] Database schema
+    # @return [Langchain::Tool::Response] Database schema
     def dump_schema
       Langchain.logger.debug("#{self.class} - Dumping schema tables and keys")
 
       schemas = db.tables.map do |table|
         describe_table(table)
       end
-      schemas.join("\n")
+
+      tool_response(content: schemas.join("\n"))
     end
 
     # Database Tool: Executes a SQL query and returns the results
     #
     # @param input [String] SQL query to be executed
-    # @return [Array] Results from the SQL query
+    # @return [Langchain::Tool::Response] Results from the SQL query
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
-      db[input].to_a
+      tool_response(content: db[input].to_a)
     rescue Sequel::DatabaseError => e
       Langchain.logger.error("#{self.class} - #{e.message}")
-      e.message # Return error to LLM
+      tool_response(content: e.message)
     end
 
     private
 
     # Describes a table and its schema
     #
     # @param table [String] The table to describe
-    # @return [String] The schema for the table
+    # @return [Langchain::Tool::Response] The schema for the table
     def describe_table(table)
       # TODO: There's probably a clear way to do all of this below
 
@@ -127,6 +130,8 @@ def describe_table(table)
         schema << ",\n" unless fk == db.foreign_key_list(table).last
       end
       schema << ");\n"
+
+      tool_response(content: schema)
     end
   end
 end

lib/langchain/tool/file_system.rb

@@ -24,21 +24,22 @@ class FileSystem
     end
 
     def list_directory(directory_path:)
-      Dir.entries(directory_path)
+      tool_response(content: Dir.entries(directory_path))
     rescue Errno::ENOENT
-      "No such directory: #{directory_path}"
+      tool_response(content: "No such directory: #{directory_path}")
     end
 
     def read_file(file_path:)
-      File.read(file_path)
+      tool_response(content: File.read(file_path))
     rescue Errno::ENOENT
-      "No such file: #{file_path}"
+      tool_response(content: "No such file: #{file_path}")
     end
 
     def write_to_file(file_path:, content:)
       File.write(file_path, content)
+      tool_response(content: "File written successfully")
     rescue Errno::EACCES
-      "Permission denied: #{file_path}"
+      tool_response(content: "Permission denied: #{file_path}")
     end
   end
 end

lib/langchain/tool/google_search.rb

@@ -36,39 +36,39 @@ def initialize(api_key:)
     # Executes Google Search and returns the result
     #
     # @param input [String] search query
-    # @return [String] Answer
+    # @return [Langchain::Tool::Response] Answer
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
       results = execute_search(input: input)
 
       answer_box = results[:answer_box_list] ? results[:answer_box_list].first : results[:answer_box]
       if answer_box
-        return answer_box[:result] ||
+        return tool_response(content: answer_box[:result] ||
             answer_box[:answer] ||
             answer_box[:snippet] ||
             answer_box[:snippet_highlighted_words] ||
-            answer_box.reject { |_k, v| v.is_a?(Hash) || v.is_a?(Array) || v.start_with?("http") }
+            answer_box.reject { |_k, v| v.is_a?(Hash) || v.is_a?(Array) || v.start_with?("http") })
       elsif (events_results = results[:events_results])
-        return events_results.take(10)
+        return tool_response(content: events_results.take(10))
       elsif (sports_results = results[:sports_results])
-        return sports_results
+        return tool_response(content: sports_results)
       elsif (top_stories = results[:top_stories])
-        return top_stories
+        return tool_response(content: top_stories)
       elsif (news_results = results[:news_results])
-        return news_results
+        return tool_response(content: news_results)
       elsif (jobs_results = results.dig(:jobs_results, :jobs))
-        return jobs_results
+        return tool_response(content: jobs_results)
       elsif (shopping_results = results[:shopping_results]) && shopping_results.first.key?(:title)
-        return shopping_results.take(3)
+        return tool_response(content: shopping_results.take(3))
       elsif (questions_and_answers = results[:questions_and_answers])
-        return questions_and_answers
+        return tool_response(content: questions_and_answers)
       elsif (popular_destinations = results.dig(:popular_destinations, :destinations))
-        return popular_destinations
+        return tool_response(content: popular_destinations)
       elsif (top_sights = results.dig(:top_sights, :sights))
-        return top_sights
+        return tool_response(content: top_sights)
       elsif (images_results = results[:images_results]) && images_results.first.key?(:thumbnail)
-        return images_results.map { |h| h[:thumbnail] }.take(10)
+        return tool_response(content: images_results.map { |h| h[:thumbnail] }.take(10))
       end
 
       snippets = []
@@ -110,8 +110,8 @@ def execute(input:)
         snippets << local_results
       end
 
-      return "No good search result found" if snippets.empty?
-      snippets
+      return tool_response(content: "No good search result found") if snippets.empty?
+      tool_response(content: snippets)
     end
 
     #

lib/langchain/tool/news_retriever.rb

@@ -57,7 +57,7 @@ def initialize(api_key: ENV["NEWS_API_KEY"])
     # @param page_size [Integer] The number of results to return per page. 20 is the API's default, 100 is the maximum. Our default is 5.
     # @param page [Integer] Use this to page through the results.
     #
-    # @return [String] JSON response
+    # @return [Langchain::Tool::Response] JSON response
     def get_everything(
       q: nil,
       search_in: nil,
@@ -86,7 +86,8 @@ def get_everything(
       params[:pageSize] = page_size if page_size
       params[:page] = page if page
 
-      send_request(path: "everything", params: params)
+      response = send_request(path: "everything", params: params)
+      tool_response(content: response)
     end
 
     # Retrieve top headlines
@@ -98,7 +99,7 @@ def get_everything(
     # @param page_size [Integer] The number of results to return per page. 20 is the API's default, 100 is the maximum. Our default is 5.
     # @param page [Integer] Use this to page through the results.
     #
-    # @return [String] JSON response
+    # @return [Langchain::Tool::Response] JSON response
     def get_top_headlines(
       country: nil,
       category: nil,
@@ -117,7 +118,8 @@ def get_top_headlines(
       params[:pageSize] = page_size if page_size
       params[:page] = page if page
 
-      send_request(path: "top-headlines", params: params)
+      response = send_request(path: "top-headlines", params: params)
+      tool_response(content: response)
     end
 
     # Retrieve news sources
@@ -126,7 +128,7 @@ def get_top_headlines(
     # @param language [String] The 2-letter ISO-639-1 code of the language you want to get headlines for. Possible options: ar, de, en, es, fr, he, it, nl, no, pt, ru, se, ud, zh.
     # @param country [String] The 2-letter ISO 3166-1 code of the country you want to get headlines for. Possible options: ae, ar, at, au, be, bg, br, ca, ch, cn, co, cu, cz, de, eg, fr, gb, gr, hk, hu, id, ie, il, in, it, jp, kr, lt, lv, ma, mx, my, ng, nl, no, nz, ph, pl, pt, ro, rs, ru, sa, se, sg, si, sk, th, tr, tw, ua, us, ve, za.
     #
-    # @return [String] JSON response
+    # @return [Langchain::Tool::Response] JSON response
     def get_sources(
       category: nil,
       language: nil,
@@ -139,7 +141,8 @@ def get_sources(
       params[:category] = category if category
       params[:language] = language if language
 
-      send_request(path: "top-headlines/sources", params: params)
+      response = send_request(path: "top-headlines/sources", params: params)
+      tool_response(content: response)
     end
 
     private

lib/langchain/tool/ruby_code_interpreter.rb

@@ -27,11 +27,11 @@ def initialize(timeout: 30)
     # Executes Ruby code in a sandboxes environment.
     #
     # @param input [String] ruby code expression
-    # @return [String] Answer
+    # @return [Langchain::Tool::Response] Answer
     def execute(input:)
       Langchain.logger.debug("#{self.class} - Executing \"#{input}\"")
 
-      safe_eval(input)
+      tool_response(content: safe_eval(input))
     end
 
     def safe_eval(code)