feat: #313 Enable tools to return image/file data to an Agent

Author: seratch

.changeset/bumpy-papayas-drum.md

@@ -0,0 +1,7 @@
+---
+'@openai/agents-extensions': minor
+'@openai/agents-openai': minor
+'@openai/agents-core': minor
+---
+
+feat: #313 Enable tools to return image/file data to an Agent

examples/basic/README.md

@@ -35,6 +35,14 @@ Run them with `pnpm` using the commands shown below.
   ```bash
   pnpm -F basic start:local-image
   ```
+- `image-tool-output.ts` – Return an image from a tool and let the agent describe it.
+  ```bash
+  pnpm -F basic start:image-tool-output
+  ```
+- `file-tool-output.ts` – Return a file from a tool and have the agent summarize it.
+  ```bash
+  pnpm -F basic start:file-tool-output
+  ```
 - `remote-image.ts` – Send an image URL to the agent.
   ```bash
   pnpm -F basic start:remote-image

examples/basic/file-tool-output.ts

@@ -0,0 +1,47 @@
+import { Agent, run, tool, ToolOutputFileContent } from '@openai/agents';
+import fs from 'node:fs';
+import path from 'node:path';
+import { z } from 'zod';
+
+const fetchSystemCard = tool({
+  name: 'fetch_system_card',
+  description: 'Fetch the system card for the given topic.',
+  parameters: z.object({ topic: z.string() }),
+  execute: async ({ topic }): Promise<ToolOutputFileContent> => {
+    console.log('[tool] Fetching system card for topic:', topic);
+    const pdfPath = path.join(
+      __dirname,
+      'media',
+      'partial_o3-and-o4-mini-system-card.pdf',
+    );
+    return {
+      type: 'file',
+      file: {
+        data: fs.readFileSync(pdfPath),
+        mediaType: 'application/pdf',
+        filename: 'partial_o3-and-o4-mini-system-card.pdf',
+      },
+    };
+  },
+});
+
+const agent = new Agent({
+  name: 'System Card Agent',
+  instructions:
+    "You are a helpful assistant who can fetch system cards. When you cannot find the answer in the data from tools, you must not guess anything. Just say you don't know.",
+  tools: [fetchSystemCard],
+});
+
+async function main() {
+  const result = await run(
+    agent,
+    'Call fetch_system_card and let me know what version of Preparedness Framework was used?',
+  );
+
+  console.log(result.finalOutput);
+  // The version of the Preparedness Framework used is Version 2.
+}
+
+if (require.main === module) {
+  main().catch(console.error);
+}

examples/basic/image-tool-output.ts

@@ -0,0 +1,37 @@
+import { Agent, run, tool, ToolOutputImage } from '@openai/agents';
+import { z } from 'zod';
+
+const fetchRandomImage = tool({
+  name: 'fetch_random_image',
+  description: 'Return a sample image for the model to describe.',
+  parameters: z.object({}),
+  execute: async (): Promise<ToolOutputImage> => {
+    console.log('[tool] Returning a publicly accessible URL for the image ...');
+    return {
+      type: 'image',
+      image:
+        'https://upload.wikimedia.org/wikipedia/commons/0/0c/GoldenGateBridge-001.jpg',
+      detail: 'auto',
+    };
+  },
+});
+
+const agent = new Agent({
+  name: 'Assistant',
+  instructions: 'You are a helpful assistant.',
+  tools: [fetchRandomImage],
+});
+
+async function main() {
+  const result = await run(
+    agent,
+    'Call fetch_random_image and describe what you see in the picture.',
+  );
+
+  console.log(result.finalOutput);
+  // The image shows a large, iconic suspension bridge painted in a bright reddish-orange color. The bridge spans over a large body of water, connecting two landmasses. The weather is clear, with a blue sky and soft clouds in the background. Vehicles can be seen traveling along the bridge, and there is some greenery in the foreground. The overall atmosphere is serene and scenic.
+}
+
+if (require.main === module) {
+  main().catch(console.error);
+}

examples/basic/package.json

@@ -15,6 +15,8 @@
     "start:hello-world-gpt-oss": "tsx hello-world-gpt-oss.ts",
     "start:lifecycle-example": "tsx lifecycle-example.ts",
     "start:local-image": "tsx local-image.ts",
+    "start:image-tool-output": "tsx image-tool-output.ts",
+    "start:file-tool-output": "tsx file-tool-output.ts",
     "start:previous-response-id": "tsx previous-response-id.ts",
     "start:prompt": "tsx prompt-id.ts",
     "start:remote-image": "tsx remote-image.ts",

examples/nextjs/src/components/History.tsx

@@ -40,12 +40,20 @@ function processItems(items: AgentInputItem[]): ProcessedItem[] {
       );
 
       if (index !== -1 && processedItems[index].type === 'function_call') {
+        const outputValue = item.output as
+          | string
+          | { type: 'text'; text: string }
+          | { type: 'image'; data?: string }
+          | undefined;
+
         processedItems[index].output =
-          item.output.type === 'text'
-            ? item.output.text
-            : item.output.type === 'image'
-              ? item.output.data
-              : '';
+          typeof outputValue === 'string'
+            ? outputValue
+            : outputValue?.type === 'text'
+              ? outputValue.text
+              : outputValue?.type === 'image'
+                ? (outputValue.data ?? '')
+                : '';
         processedItems[index].status = 'completed';
       }
     }

packages/agents-core/src/index.ts

@@ -127,6 +127,13 @@ export {
   ToolExecuteArgument,
   ToolEnabledFunction,
 } from './tool';
+export type {
+  ToolOutputText,
+  ToolOutputImage,
+  ToolOutputFileContent,
+  ToolCallStructuredOutput,
+  ToolCallOutputContent,
+} from './types/protocol';
 export * from './tracing';
 export { getGlobalTraceProvider, TraceProvider } from './tracing/provider';
 /* only export the types not the parsers */