make sure to resize images coming from tool calls (microsoft#257785)

justschen · web-flow · commit df984eb56f52 · 2025-07-28T18:00:38.000Z
* make sure to resize images coming from tool calls

* try/catch, resize in the right areas

* new image service

* some cleanup

* clean up service usage

* no more electron main service

* app.ts cleanup

* address comments

* wrap in try catch and revert

* proper revert
diff --git a/src/vs/platform/imageResize/browser/imageResizeService.ts b/src/vs/platform/imageResize/browser/imageResizeService.ts
@@ -0,0 +1,167 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { decodeBase64, VSBuffer } from '../../../base/common/buffer.js';
+import { joinPath } from '../../../base/common/resources.js';
+import { URI } from '../../../base/common/uri.js';
+import { IFileService } from '../../files/common/files.js';
+import { InstantiationType, registerSingleton } from '../../instantiation/common/extensions.js';
+import { ILogService } from '../../log/common/log.js';
+import { IImageResizeService } from '../common/imageResizeService.js';
+
+
+export class ImageResizeService implements IImageResizeService {
+
+	declare readonly _serviceBrand: undefined;
+
+	/**
+	 * Resizes an image provided as a UInt8Array string. Resizing is based on Open AI's algorithm for tokenzing images.
+	 * https://platform.openai.com/docs/guides/vision#calculating-costs
+	 * @param data - The UInt8Array string of the image to resize.
+	 * @returns A promise that resolves to the UInt8Array string of the resized image.
+	 */
+
+	async resizeImage(data: Uint8Array | string, mimeType?: string): Promise<Uint8Array> {
+		const isGif = mimeType === 'image/gif';
+
+		if (typeof data === 'string') {
+			data = this.convertStringToUInt8Array(data);
+		}
+
+		return new Promise((resolve, reject) => {
+			const blob = new Blob([data as Uint8Array<ArrayBuffer>], { type: mimeType });
+			const img = new Image();
+			const url = URL.createObjectURL(blob);
+			img.src = url;
+
+			img.onload = () => {
+				URL.revokeObjectURL(url);
+				let { width, height } = img;
+
+				if ((width <= 768 || height <= 768) && !isGif) {
+					resolve(data);
+					return;
+				}
+
+				// Calculate the new dimensions while maintaining the aspect ratio
+				if (width > 2048 || height > 2048) {
+					const scaleFactor = 2048 / Math.max(width, height);
+					width = Math.round(width * scaleFactor);
+					height = Math.round(height * scaleFactor);
+				}
+
+				const scaleFactor = 768 / Math.min(width, height);
+				width = Math.round(width * scaleFactor);
+				height = Math.round(height * scaleFactor);
+
+				const canvas = document.createElement('canvas');
+				canvas.width = width;
+				canvas.height = height;
+				const ctx = canvas.getContext('2d');
+				if (ctx) {
+					ctx.drawImage(img, 0, 0, width, height);
+					canvas.toBlob((blob) => {
+						if (blob) {
+							const reader = new FileReader();
+							reader.onload = () => {
+								resolve(new Uint8Array(reader.result as ArrayBuffer));
+							};
+							reader.onerror = (error) => reject(error);
+							reader.readAsArrayBuffer(blob);
+						} else {
+							reject(new Error('Failed to create blob from canvas'));
+						}
+					}, mimeType || 'image/png');
+				} else {
+					reject(new Error('Failed to get canvas context'));
+				}
+			};
+			img.onerror = (error) => {
+				URL.revokeObjectURL(url);
+				reject(error);
+			};
+		});
+	}
+
+	convertStringToUInt8Array(data: string): Uint8Array {
+		const base64Data = data.includes(',') ? data.split(',')[1] : data;
+		if (this.isValidBase64(base64Data)) {
+			return decodeBase64(base64Data).buffer;
+		}
+		return new TextEncoder().encode(data);
+	}
+
+	// Only used for URLs
+	convertUint8ArrayToString(data: Uint8Array): string {
+		try {
+			const decoder = new TextDecoder();
+			const decodedString = decoder.decode(data);
+			return decodedString;
+		} catch {
+			return '';
+		}
+	}
+
+	isValidBase64(str: string): boolean {
+		try {
+			decodeBase64(str);
+			return true;
+		} catch {
+			return false;
+		}
+	}
+
+	async createFileForMedia(fileService: IFileService, imagesFolder: URI, dataTransfer: Uint8Array, mimeType: string): Promise<URI | undefined> {
+		const exists = await fileService.exists(imagesFolder);
+		if (!exists) {
+			await fileService.createFolder(imagesFolder);
+		}
+
+		const ext = mimeType.split('/')[1] || 'png';
+		const filename = `image-${Date.now()}.${ext}`;
+		const fileUri = joinPath(imagesFolder, filename);
+
+		const buffer = VSBuffer.wrap(dataTransfer);
+		await fileService.writeFile(fileUri, buffer);
+
+		return fileUri;
+	}
+
+	async cleanupOldImages(fileService: IFileService, logService: ILogService, imagesFolder: URI): Promise<void> {
+		const exists = await fileService.exists(imagesFolder);
+		if (!exists) {
+			return;
+		}
+
+		const duration = 7 * 24 * 60 * 60 * 1000; // 7 days
+		const files = await fileService.resolve(imagesFolder);
+		if (!files.children) {
+			return;
+		}
+
+		await Promise.all(files.children.map(async (file) => {
+			try {
+				const timestamp = this.getTimestampFromFilename(file.name);
+				if (timestamp && (Date.now() - timestamp > duration)) {
+					await fileService.del(file.resource);
+				}
+			} catch (err) {
+				logService.error('Failed to clean up old images', err);
+			}
+		}));
+	}
+
+	getTimestampFromFilename(filename: string): number | undefined {
+		const match = filename.match(/image-(\d+)\./);
+		if (match) {
+			return parseInt(match[1], 10);
+		}
+		return undefined;
+	}
+
+
+}
+
+registerSingleton(IImageResizeService, ImageResizeService, InstantiationType.Delayed);
diff --git a/src/vs/platform/imageResize/common/imageResizeService.ts b/src/vs/platform/imageResize/common/imageResizeService.ts
@@ -0,0 +1,19 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { createDecorator } from '../../instantiation/common/instantiation.js';
+
+export const IImageResizeService = createDecorator<IImageResizeService>('imageResizeMainService');
+
+
+export interface IImageResizeService {
+
+	readonly _serviceBrand: undefined;
+
+	/**
+	 * Resizes an image to a maximum dimension of 768px while maintaining aspect ratio.
+	 */
+	resizeImage(data: Uint8Array | string, mimeType?: string): Promise<Uint8Array>;
+}
diff --git a/src/vs/workbench/contrib/chat/browser/contrib/chatInputCompletions.ts b/src/vs/workbench/contrib/chat/browser/contrib/chatInputCompletions.ts
@@ -56,6 +56,7 @@ import { ToolSet } from '../../common/languageModelToolsService.js';
 import { IPromptsService } from '../../common/promptSyntax/service/promptsService.js';
 import { ChatSubmitAction } from '../actions/chatExecuteActions.js';
 import { IChatWidget, IChatWidgetService } from '../chat.js';
+import { resizeImage } from '../imageUtils.js';
 import { ChatDynamicVariableModel } from './chatDynamicVariables.js';
 
 class SlashCommandCompletions extends Disposable {
@@ -647,11 +648,13 @@ class StartParameterizedPromptAction extends Action2 {
 						});
 					}
 				} else if (mimeType && getAttachableImageExtension(mimeType)) {
+					const resized = await resizeImage(contents)
+						.catch(() => decodeBase64(contents).buffer);
 					chatWidget.attachmentModel.addContext({
 						id: generateUuid(),
 						name: localize('mcp.prompt.image', 'Prompt Image'),
 						fullName: localize('mcp.prompt.image', 'Prompt Image'),
-						value: decodeBase64(contents).buffer,
+						value: resized,
 						kind: 'image',
 						references: validURI && [{ reference: validURI, kind: 'reference' }],
 					});
diff --git a/src/vs/workbench/contrib/mcp/common/mcpLanguageModelToolContribution.ts b/src/vs/workbench/contrib/mcp/common/mcpLanguageModelToolContribution.ts
@@ -15,6 +15,7 @@ import { basename } from '../../../../base/common/resources.js';
 import { URI } from '../../../../base/common/uri.js';
 import { localize } from '../../../../nls.js';
 import { IFileService } from '../../../../platform/files/common/files.js';
+import { IImageResizeService } from '../../../../platform/imageResize/common/imageResizeService.js';
 import { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js';
 import { IProductService } from '../../../../platform/product/common/productService.js';
 import { StorageScope } from '../../../../platform/storage/common/storage.js';
@@ -163,6 +164,7 @@ class McpToolImplementation implements IToolImpl {
 		private readonly _server: IMcpServer,
 		@IProductService private readonly _productService: IProductService,
 		@IFileService private readonly _fileService: IFileService,
+		@IImageResizeService private readonly _imageResizeService: IImageResizeService,
 	) { }
 
 	async prepareToolInvocation(context: IToolInvocationPreparationContext): Promise<IPreparedToolInvocation> {
@@ -222,14 +224,18 @@ class McpToolImplementation implements IToolImpl {
 				}
 			}
 
-			// Rewrite image rsources to images so they are inlined nicely
-			const addAsInlineData = (mimeType: string, value: string, uri?: URI) => {
+			// Rewrite image resources to images so they are inlined nicely
+			const addAsInlineData = async (mimeType: string, value: string, uri?: URI): Promise<VSBuffer | void> => {
 				details.output.push({ type: 'embed', mimeType, value, uri });
 				if (isForModel) {
-					result.content.push({
-						kind: 'data',
-						value: { mimeType, data: decodeBase64(value) }
-					});
+					let finalData: VSBuffer;
+					try {
+						const resized = await this._imageResizeService.resizeImage(decodeBase64(value).buffer, mimeType);
+						finalData = VSBuffer.wrap(resized);
+					} catch {
+						finalData = decodeBase64(value);
+					}
+					result.content.push({ kind: 'data', value: { mimeType, data: finalData } });
 				}
 			};
 
@@ -246,7 +252,7 @@ class McpToolImplementation implements IToolImpl {
 				}
 			} else if (item.type === 'image' || item.type === 'audio') {
 				// default to some image type if not given to hint
-				addAsInlineData(item.mimeType || 'image/png', item.data);
+				await addAsInlineData(item.mimeType || 'image/png', item.data);
 			} else if (item.type === 'resource_link') {
 				const uri = McpResourceURI.fromServer(this._server.definition, item.uri);
 				details.output.push({
@@ -274,7 +280,7 @@ class McpToolImplementation implements IToolImpl {
 			} else if (item.type === 'resource') {
 				const uri = McpResourceURI.fromServer(this._server.definition, item.resource.uri);
 				if (item.resource.mimeType && getAttachableImageExtension(item.resource.mimeType) && 'blob' in item.resource) {
-					addAsInlineData(item.resource.mimeType, item.resource.blob, uri);
+					await addAsInlineData(item.resource.mimeType, item.resource.blob, uri);
 				} else {
 					details.output.push({
 						type: 'embed',
@@ -305,4 +311,5 @@ class McpToolImplementation implements IToolImpl {
 		result.toolResultDetails = details;
 		return result;
 	}
+
 }
diff --git a/src/vs/workbench/services/imageResize/browser/imageResizeService.ts b/src/vs/workbench/services/imageResize/browser/imageResizeService.ts
@@ -0,0 +1,10 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { IImageResizeService } from '../../../../platform/imageResize/common/imageResizeService.js';
+import { ImageResizeService } from '../../../../platform/imageResize/browser/imageResizeService.js';
+import { InstantiationType, registerSingleton } from '../../../../platform/instantiation/common/extensions.js';
+
+registerSingleton(IImageResizeService, ImageResizeService, InstantiationType.Delayed);
diff --git a/src/vs/workbench/services/imageResize/electron-browser/imageResizeService.ts b/src/vs/workbench/services/imageResize/electron-browser/imageResizeService.ts
@@ -0,0 +1,10 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { IImageResizeService } from '../../../../platform/imageResize/common/imageResizeService.js';
+import { ImageResizeService } from '../../../../platform/imageResize/browser/imageResizeService.js';
+import { InstantiationType, registerSingleton } from '../../../../platform/instantiation/common/extensions.js';
+
+registerSingleton(IImageResizeService, ImageResizeService, InstantiationType.Delayed);
diff --git a/src/vs/workbench/workbench.desktop.main.ts b/src/vs/workbench/workbench.desktop.main.ts
@@ -55,6 +55,7 @@ import './services/themes/electron-browser/nativeHostColorSchemeService.js';
 import './services/extensionManagement/electron-browser/extensionManagementService.js';
 import './services/mcp/electron-browser/mcpWorkbenchManagementService.js';
 import './services/encryption/electron-browser/encryptionService.js';
+import './services/imageResize/electron-browser/imageResizeService.js';
 import './services/browserElements/electron-browser/browserElementsService.js';
 import './services/secrets/electron-browser/secretStorageService.js';
 import './services/localization/electron-browser/languagePackService.js';
diff --git a/src/vs/workbench/workbench.web.main.internal.ts b/src/vs/workbench/workbench.web.main.internal.ts
@@ -58,6 +58,7 @@ import './services/localization/browser/localeService.js';
 import './services/path/browser/pathService.js';
 import './services/themes/browser/browserHostColorSchemeService.js';
 import './services/encryption/browser/encryptionService.js';
+import './services/imageResize/browser/imageResizeService.js';
 import './services/secrets/browser/secretStorageService.js';
 import './services/workingCopy/browser/workingCopyBackupService.js';
 import './services/tunnel/browser/tunnelService.js';
