add image tool

Author: 404Wolf

src/helpers/beta/zod.ts

@@ -1,7 +1,7 @@
 import type { infer as zodInfer, ZodType } from 'zod/v4';
 import * as z from 'zod/v4';
 import type { BetaRunnableTool, Promisable } from '../../lib/beta/BetaRunnableTool';
-import type { FunctionTool } from '../../resources/beta';
+import type { ChatCompletionContentPart } from '../../resources';
 
 /**
  * Creates a tool using the provided Zod schema that can be passed
@@ -13,7 +13,7 @@ export function betaZodFunctionTool<InputSchema extends ZodType>(options: {
   name: string;
   parameters: InputSchema;
   description: string;
-  run: (args: zodInfer<InputSchema>) => Promisable<string | Array<FunctionTool>>; // TODO: I changed this but double check
+  run: (args: zodInfer<InputSchema>) => Promisable<string | ChatCompletionContentPart[]>;
 }): BetaRunnableTool<zodInfer<InputSchema>> {
   const jsonSchema = z.toJSONSchema(options.parameters, { reused: 'ref' });
 

src/lib/beta/BetaRunnableTool.ts

@@ -1,11 +1,10 @@
-import { FunctionTool } from '../../resources/beta';
-import type { ChatCompletionTool } from '../../resources';
+import type { ChatCompletionContentPart, ChatCompletionTool } from '../../resources';
 
 export type Promisable<T> = T | Promise<T>;
 
 // this type is just an extension of BetaTool with a run and parse method
 // that will be called by `toolRunner()` helpers
 export type BetaRunnableTool<Input = any> = ChatCompletionTool & {
-  run: (args: Input) => Promisable<string | Array<FunctionTool>>;
+  run: (args: Input) => Promisable<string | ChatCompletionContentPart[]>;
   parse: (content: unknown) => Input;
 };

tests/lib/tools/BetaToolRunner.test.ts

@@ -11,6 +11,7 @@ import type {
 } from 'openai/resources';
 import type { Fetch } from 'openai/internal/builtin-types';
 import type { BetaToolRunnerParams } from 'openai/lib/beta/BetaToolRunner';
+import { betaZodFunctionTool } from 'openai/helpers/beta/zod';
 
 const weatherTool: BetaRunnableTool<{ location: string }> = {
   type: 'function',
@@ -1007,6 +1008,63 @@ describe('ToolRunner', () => {
       expect(runner.params.messages[2]).toMatchObject(getWeatherToolResult('Paris', 'tool_1'));
       await expectDone(iterator);
     });
+
+    it('allows you to use non-string returning custom tools', async () => {
+      const customTool: BetaRunnableTool<{ location: string }> = {
+        type: 'function',
+        function: {
+          name: 'getWeather',
+          description: 'Get the weather in a given location',
+          parameters: {
+            type: 'object',
+            properties: {
+              location: { type: 'string' },
+            },
+          },
+        },
+        run: async ({ location }) => {
+          return [
+            {
+              type: 'image_url' as const,
+              image_url: {
+                url: `https://example.com/weather-${location}.jpg`,
+              },
+            },
+          ];
+        },
+        parse: (input: unknown) => input as { location: string },
+      };
+
+      const { runner, handleAssistantMessage } = setupTest({
+        messages: [{ role: 'user', content: 'Test done method' }],
+        tools: [customTool],
+      });
+
+      const iterator = runner[Symbol.asyncIterator]();
+
+      // Assistant requests the custom tool
+      handleAssistantMessage([getWeatherToolUse('Paris')]);
+      await expectEvent(iterator, (message) => {
+        expect(message?.choices[0]?.message?.tool_calls).toMatchObject([getWeatherToolUse('Paris')]);
+      });
+
+      // Verify generateToolResponse returns the custom tool result
+      const toolResponse = await runner.generateToolResponse();
+      expect(toolResponse).toMatchObject([
+        {
+          role: 'tool',
+          tool_call_id: 'tool_1',
+          content: JSON.stringify([
+            {
+              type: 'image_url',
+              image_url: {
+                url: 'https://example.com/weather-Paris.jpg',
+              },
+            },
+          ]),
+        },
+      ]);
+    });
   });
 
   describe('.runUntilDone()', () => {

tests/lib/tools/BetaToolRunnerE2E.test.ts

@@ -1,9 +1,10 @@
 import { OpenAI } from '../../../src';
 import { betaZodFunctionTool } from '../../../src/helpers/beta/zod';
-import * as z from 'zod';
+import { z } from 'zod/v4';
 import nock from 'nock';
 import { gunzipSync } from 'zlib';
 import { RequestInfo } from 'openai/internal/builtin-types';
+import * as fs from 'node:fs/promises';
 
 describe('toolRunner integration tests', () => {
   let client: OpenAI;
@@ -272,4 +273,47 @@ describe('toolRunner integration tests', () => {
       expect(params.messages).toEqual([{ role: 'user', content: 'Updated message' }]);
     });
   });
+
+  describe('Non string returning tools', () => {
+    it('should handle non-string returning tools', async () => {
+      const exampleImageBuffer = await fs.readFile(__dirname + '/logo.png');
+      const exampleImageBase64 = exampleImageBuffer.toString('base64');
+      const exampleImageUrl = `data:image/png;base64,${exampleImageBase64}`;
+
+      const tool = betaZodFunctionTool({
+        name: 'cool_logo_getter_tool',
+        description: 'query for a company logo',
+        parameters: z.object({
+          name: z.string().min(1).max(100).describe('the name of the company whose logo you want'),
+        }),
+        run: async () => {
+          return [
+            {
+              type: 'image_url' as const,
+              image_url: {
+                url: exampleImageUrl,
+              },
+            },
+          ];
+        },
+      });
+
+      const runner = client.beta.chat.completions.toolRunner({
+        model: 'gpt-4o',
+        max_tokens: 1000,
+        messages: [
+          {
+            role: 'user',
+            content:
+              'what is the dominant colour of the logo of the company "Stainless"? One word response nothing else',
+          },
+        ],
+        tools: [tool],
+      });
+
+      const finalMessage = await runner.runUntilDone();
+      const color = finalMessage.content?.toLowerCase();
+      expect(['blue', 'black', 'gray', 'grey']).toContain(color); // ai is bad at colours apparently
+    });
+  });
 });