use image uploader in chat (#499)

* image uploader in chat extension

* fix comment

* cleanup

* use capi client + extract into own service

* capi version bump + injected service

* cleanup

* add service in promptrenderer

* add null image service for tets

* add null service in vscode node too

* trying to remove unsued service in prompt renderer

* cleanup spacing

* use urlREquestMetaData instead

* remove vendor

* remove whitespace

* some more cleanup

* more robust mimetype check

* add exp setting

Author: justschen

package-lock.json

@@ -3685,7 +3685,7 @@
 		"@humanwhocodes/gitignore-to-minimatch": "1.0.2",
 		"@microsoft/tiktokenizer": "^1.0.10",
 		"@roamhq/mac-ca": "^1.0.7",
-		"@vscode/copilot-api": "^0.1.3",
+		"@vscode/copilot-api": "^0.1.4",
 		"@vscode/extension-telemetry": "^1.0.0",
 		"@vscode/l10n": "^0.0.18",
 		"@vscode/prompt-tsx": "^0.4.0-alpha.5",

package.json

@@ -20,7 +20,7 @@ export type Props = PromptElementProps<{
 }>;
 
 export class LanguageModelAccessPrompt extends PromptElement<Props> {
-	render() {
+	async render() {
 
 		const systemMessages: string[] = [];
 		const chatMessages: (UserMessage | AssistantMessage)[] = [];
@@ -44,19 +44,20 @@ export class LanguageModelAccessPrompt extends PromptElement<Props> {
 				const statefulMarkerElement = statefulMarker && <StatefulMarkerContainer statefulMarker={statefulMarker} />;
 				chatMessages.push(<AssistantMessage name={message.name} toolCalls={toolCalls.map(tc => ({ id: tc.callId, type: 'function', function: { name: tc.name, arguments: JSON.stringify(tc.input) } }))}>{statefulMarkerElement}{content?.value}</AssistantMessage>);
 			} else if (message.role === vscode.LanguageModelChatMessageRole.User) {
-				message.content.forEach(part => {
+				for (const part of message.content) {
 					if (part instanceof vscode.LanguageModelToolResultPart2 || part instanceof vscode.LanguageModelToolResultPart) {
 						chatMessages.push(
 							<ToolMessage toolCallId={part.callId}>
 								<ToolResult content={part.content} />
 							</ToolMessage>
 						);
 					} else if (isImageDataPart(part)) {
-						chatMessages.push(<UserMessage priority={0}>{imageDataPartToTSX(part)}</UserMessage>);
+						const imageElement = await imageDataPartToTSX(part);
+						chatMessages.push(<UserMessage priority={0}>{imageElement}</UserMessage>);
 					} else if (part instanceof vscode.LanguageModelTextPart) {
 						chatMessages.push(<UserMessage name={message.name}>{part.value}</UserMessage>);
 					}
-				});
+				}
 			}
 		}
 

src/extension/conversation/vscode-node/languageModelAccessPrompt.tsx

@@ -28,6 +28,8 @@ import { IGithubRepositoryService } from '../../../platform/github/common/github
 import { GithubRepositoryService } from '../../../platform/github/node/githubRepositoryService';
 import { IIgnoreService } from '../../../platform/ignore/common/ignoreService';
 import { VsCodeIgnoreService } from '../../../platform/ignore/vscode-node/ignoreService';
+import { IImageService } from '../../../platform/image/common/imageService';
+import { ImageServiceImpl } from '../../../platform/image/node/imageServiceImpl';
 import { ILanguageContextService } from '../../../platform/languageServer/common/languageContextService';
 import { ICompletionsFetchService } from '../../../platform/nesFetch/common/completionsFetchService';
 import { CompletionsFetchService } from '../../../platform/nesFetch/node/completionsFetchServiceImpl';
@@ -121,6 +123,7 @@ export function registerServices(builder: IInstantiationServiceBuilder, extensio
 	builder.define(IFetcherService, new SyncDescriptor(FetcherService, [undefined]));
 	builder.define(IDomainService, new SyncDescriptor(DomainService));
 	builder.define(ICAPIClientService, new SyncDescriptor(CAPIClientImpl));
+	builder.define(IImageService, new SyncDescriptor(ImageServiceImpl));
 
 	builder.define(ITelemetryUserConfig, new SyncDescriptor(TelemetryUserConfigImpl, [undefined, undefined]));
 	const internalAIKey = extensionContext.extension.packageJSON.internalAIKey ?? '';

src/extension/extension/vscode-node/services.ts

@@ -3,8 +3,15 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import { RequestType } from '@vscode/copilot-api';
 import * as l10n from '@vscode/l10n';
-import { BasePromptElementProps, ChatResponseReferencePartStatusKind, PromptElement, PromptReference, PromptSizing, UserMessage, Image as BaseImage } from '@vscode/prompt-tsx';
+import { Image as BaseImage, BasePromptElementProps, ChatResponseReferencePartStatusKind, PromptElement, PromptReference, PromptSizing, UserMessage } from '@vscode/prompt-tsx';
+import { IAuthenticationService } from '../../../../platform/authentication/common/authentication';
+import { ConfigKey, IConfigurationService } from '../../../../platform/configuration/common/configurationService';
+import { IImageService } from '../../../../platform/image/common/imageService';
+import { ILogService } from '../../../../platform/log/common/logService';
+import { IExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService';
+import { getMimeType } from '../../../../util/common/imageUtils';
 import { Uri } from '../../../../vscodeTypes';
 import { IPromptEndpoint } from '../base/promptRenderer';
 
@@ -18,7 +25,12 @@ export interface ImageProps extends BasePromptElementProps {
 export class Image extends PromptElement<ImageProps, unknown> {
 	constructor(
 		props: ImageProps,
-		@IPromptEndpoint private readonly promptEndpoint: IPromptEndpoint
+		@IPromptEndpoint private readonly promptEndpoint: IPromptEndpoint,
+		@IAuthenticationService private readonly authService: IAuthenticationService,
+		@ILogService private readonly logService: ILogService,
+		@IImageService private readonly imageService: IImageService,
+		@IConfigurationService private readonly configurationService: IConfigurationService,
+		@IExperimentationService private readonly experimentationService: IExperimentationService
 	) {
 		super(props);
 	}
@@ -41,16 +53,24 @@ export class Image extends PromptElement<ImageProps, unknown> {
 				);
 			}
 			const variable = await this.props.variableValue;
-			let decoded = Buffer.from(variable).toString('base64');
-			const decoder = new TextDecoder();
-			const decodedString = decoder.decode(variable);
-			if (/^https?:\/\/.+/.test(decodedString)) {
-				decoded = decodedString;
+			let imageSource = Buffer.from(variable).toString('base64');
+			const isChatCompletions = typeof this.promptEndpoint.urlOrRequestMetadata !== 'string' && this.promptEndpoint.urlOrRequestMetadata.type === RequestType.ChatCompletions;
+			const enabled = this.configurationService.getExperimentBasedConfig(ConfigKey.Internal.EnableChatImageUpload, this.experimentationService);
+			if (isChatCompletions && enabled) {
+				try {
+					const githubToken = (await this.authService.getAnyGitHubSession())?.accessToken;
+					const uri = await this.imageService.uploadChatImageAttachment(variable, this.props.variableName, getMimeType(imageSource) ?? 'image/png', githubToken);
+					if (uri) {
+						imageSource = uri.toString();
+					}
+				} catch (error) {
+					this.logService.warn(`Image upload failed, using base64 fallback: ${error}`);
+				}
 			}
 
 			return (
 				<UserMessage priority={0}>
-					<BaseImage src={decoded} detail='high' />
+					<BaseImage src={imageSource} detail='high' />
 					{this.props.reference && (
 						<references value={[new PromptReference(this.props.variableName ? { variableName: this.props.variableName, value: fillerUri } : fillerUri, undefined)]} />
 					)}

src/extension/prompts/node/panel/image.tsx

@@ -3,12 +3,17 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import { RequestMetadata, RequestType } from '@vscode/copilot-api';
 import { AssistantMessage, BasePromptElementProps, PromptRenderer as BasePromptRenderer, Chunk, IfEmpty, Image, JSONTree, PromptElement, PromptElementProps, PromptMetadata, PromptPiece, PromptSizing, TokenLimit, ToolCall, ToolMessage, useKeepWith, UserMessage } from '@vscode/prompt-tsx';
 import type { ChatParticipantToolToken, LanguageModelToolResult2, LanguageModelToolTokenizationOptions } from 'vscode';
+import { IAuthenticationService } from '../../../../platform/authentication/common/authentication';
+import { ConfigKey, IConfigurationService } from '../../../../platform/configuration/common/configurationService';
 import { IEndpointProvider } from '../../../../platform/endpoint/common/endpointProvider';
 import { CacheType } from '../../../../platform/endpoint/common/endpointTypes';
 import { StatefulMarkerContainer } from '../../../../platform/endpoint/common/statefulMarkerContainer';
+import { IImageService } from '../../../../platform/image/common/imageService';
 import { ILogService } from '../../../../platform/log/common/logService';
+import { IExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService';
 import { ITelemetryService } from '../../../../platform/telemetry/common/telemetry';
 import { ITokenizer } from '../../../../util/common/tokenizer';
 import { CancellationToken } from '../../../../util/vs/base/common/cancellation';
@@ -297,10 +302,25 @@ enum ToolInvocationOutcome {
 	Cancelled = 'cancelled',
 }
 
-export function imageDataPartToTSX(part: LanguageModelDataPart) {
+export async function imageDataPartToTSX(part: LanguageModelDataPart, githubToken?: string, urlOrRequestMetadata?: string | RequestMetadata, logService?: ILogService, imageService?: IImageService) {
 	if (isImageDataPart(part)) {
 		const base64 = Buffer.from(part.data).toString('base64');
-		return <Image src={`data:${part.mimeType};base64,${base64}`} />;
+		let imageSource = `data:${part.mimeType};base64,${base64}`;
+		const isChatCompletions = typeof urlOrRequestMetadata !== 'string' && urlOrRequestMetadata?.type === RequestType.ChatCompletions;
+		if (githubToken && isChatCompletions && imageService) {
+			try {
+				const uri = await imageService.uploadChatImageAttachment(part.data, 'tool-result-image', part.mimeType ?? 'image/png', githubToken);
+				if (uri) {
+					imageSource = uri.toString();
+				}
+			} catch (error) {
+				if (logService) {
+					logService.warn(`Image upload failed, using base64 fallback: ${error}`);
+				}
+			}
+		}
+
+		return <Image src={imageSource} />;
 	}
 }
 
@@ -340,6 +360,19 @@ interface IPrimitiveToolResultProps extends BasePromptElementProps {
 }
 
 class PrimitiveToolResult<T extends IPrimitiveToolResultProps> extends PromptElement<T> {
+
+	constructor(
+		props: T,
+		@IPromptEndpoint protected readonly endpoint: IPromptEndpoint,
+		@IAuthenticationService private readonly authService: IAuthenticationService,
+		@ILogService private readonly logService?: ILogService,
+		@IImageService private readonly imageService?: IImageService,
+		@IConfigurationService private readonly configurationService?: IConfigurationService,
+		@IExperimentationService private readonly experimentationService?: IExperimentationService
+	) {
+		super(props);
+	}
+
 	async render(): Promise<PromptPiece | undefined> {
 		return (
 			<>
@@ -368,8 +401,13 @@ class PrimitiveToolResult<T extends IPrimitiveToolResultProps> extends PromptEle
 		return part.audience.includes(LanguageModelPartAudience.Assistant);
 	}
 
-	protected onData(part: LanguageModelDataPart) {
-		return Promise.resolve(imageDataPartToTSX(part));
+	protected async onData(part: LanguageModelDataPart) {
+		const githubToken = (await this.authService.getAnyGitHubSession())?.accessToken;
+		const uploadsEnabled = this.configurationService && this.experimentationService
+			? this.configurationService.getExperimentBasedConfig(ConfigKey.Internal.EnableChatImageUpload, this.experimentationService)
+			: false;
+		const effectiveToken = uploadsEnabled ? githubToken : undefined;
+		return Promise.resolve(imageDataPartToTSX(part, effectiveToken, this.endpoint.urlOrRequestMetadata, this.logService, this.imageService));
 	}
 
 	protected onTSX(part: JSONTree.PromptElementJSON) {
@@ -396,9 +434,14 @@ export interface IToolResultProps extends IPrimitiveToolResultProps {
 export class ToolResult extends PrimitiveToolResult<IToolResultProps> {
 	constructor(
 		props: PromptElementProps<IToolResultProps>,
-		@IPromptEndpoint private readonly endpoint: IPromptEndpoint,
+		@IPromptEndpoint endpoint: IPromptEndpoint,
+		@IAuthenticationService authService: IAuthenticationService,
+		@ILogService logService: ILogService,
+		@IImageService imageService: IImageService,
+		@IConfigurationService configurationService: IConfigurationService,
+		@IExperimentationService experimentationService: IExperimentationService
 	) {
-		super(props);
+		super(props, endpoint, authService, logService, imageService, configurationService, experimentationService);
 	}
 
 	protected override async onTSX(part: JSONTree.PromptElementJSON): Promise<any> {

src/extension/prompts/node/panel/toolCalling.tsx

@@ -102,6 +102,7 @@ import { IToolsService, NullToolsService } from '../../tools/common/toolsService
 import { ToolGroupingService } from '../../tools/common/virtualTools/toolGroupingService';
 import { ToolGroupingCache } from '../../tools/common/virtualTools/virtualToolGroupCache';
 import { IToolGroupingCache, IToolGroupingService } from '../../tools/common/virtualTools/virtualToolTypes';
+import { IImageService, nullImageService } from '../../../platform/image/common/imageService';
 
 /**
  * A default context for VSCode extension testing, building on general one in `lib`.
@@ -127,6 +128,7 @@ export function createExtensionTestingServices(): TestingServiceCollection {
 	testingServiceCollection.define(IWorkspaceService, new SyncDescriptor(ExtensionTextDocumentManager));
 	testingServiceCollection.define(IExtensionsService, new SyncDescriptor(VSCodeExtensionsService));
 	testingServiceCollection.define(IChatMLFetcher, new SyncDescriptor(ChatMLFetcherImpl));
+	testingServiceCollection.define(IImageService, nullImageService);
 	testingServiceCollection.define(ITabsAndEditorsService, new SyncDescriptor(TabsAndEditorsServiceImpl));
 	testingServiceCollection.define(IEmbeddingsComputer, new SyncDescriptor(RemoteEmbeddingsComputer));
 	testingServiceCollection.define(ITelemetryService, new SyncDescriptor(NullTelemetryService));

src/extension/test/vscode-node/services.ts

@@ -704,6 +704,8 @@ export namespace ConfigKey {
 		export const AgentHistorySummarizationForceGpt41 = defineExpSetting<boolean | undefined>('chat.advanced.agentHistorySummarizationForceGpt41', false, INTERNAL_RESTRICTED);
 		export const UseResponsesApiTruncation = defineSetting<boolean | undefined>('chat.advanced.useResponsesApiTruncation', false, INTERNAL_RESTRICTED);
 
+		export const EnableChatImageUpload = defineExpSetting<boolean>('chat.advanced.imageUpload', false, INTERNAL);
+
 		export const EnableReadFileV2 = defineExpSetting<boolean>('chat.advanced.enableReadFileV2', isPreRelease, INTERNAL_RESTRICTED);
 		export const AskAgent = defineExpSetting<boolean>('chat.advanced.enableAskAgent', { defaultValue: false, teamDefaultValue: true, internalDefaultValue: true }, INTERNAL_RESTRICTED);
 		export const VerifyTextDocumentChanges = defineExpSetting<boolean>('chat.advanced.inlineEdits.verifyTextDocumentChanges', true, INTERNAL_RESTRICTED);

src/platform/configuration/common/configurationService.ts

@@ -0,0 +1,30 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { createServiceIdentifier } from '../../../util/common/services';
+import { URI } from '../../../util/vs/base/common/uri';
+
+export const IImageService = createServiceIdentifier<IImageService>('IImageService');
+
+export interface IImageService {
+	readonly _serviceBrand: undefined;
+
+	/**
+	 * Upload image data to GitHub Copilot chat attachments endpoint
+	 * @param binaryData The image binary data as Uint8Array
+	 * @param name The name for the uploaded file
+	 * @param mimeType The MIME type of the image
+	 * @param token The authentication token for GitHub API
+	 * @returns Promise<URI> The URI of the uploaded image
+	 */
+	uploadChatImageAttachment(binaryData: Uint8Array, name: string, mimeType: string | undefined, token: string | undefined): Promise<URI>;
+}
+
+export const nullImageService: IImageService = {
+	_serviceBrand: undefined,
+	async uploadChatImageAttachment(): Promise<URI> {
+		throw new Error('Image service not implemented');
+	}
+};

src/platform/image/common/imageService.ts

@@ -0,0 +1,53 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { RequestType } from '@vscode/copilot-api';
+import { URI } from '../../../util/vs/base/common/uri';
+import { ICAPIClientService } from '../../endpoint/common/capiClient';
+import { IImageService } from '../common/imageService';
+
+export class ImageServiceImpl implements IImageService {
+	declare readonly _serviceBrand: undefined;
+
+	constructor(
+		@ICAPIClientService private readonly capiClient: ICAPIClientService,
+	) { }
+
+	async uploadChatImageAttachment(binaryData: Uint8Array, name: string, mimeType: string | undefined, token: string | undefined): Promise<URI> {
+		if (!mimeType || !token) {
+			throw new Error('Missing required mimeType or token for image upload');
+		}
+
+		const sanitizedName = name.replace(/[^a-zA-Z0-9._-]/g, '');
+		let uploadName = sanitizedName;
+
+		// can catch unexpected types like "IMAGE/JPEG", "image/svg+xml", or "image/png; charset=UTF-8"
+		const subtypeMatch = mimeType.toLowerCase().match(/^[^\/]+\/([^+;]+)/);
+		const subtype = subtypeMatch?.[1];
+
+		// add the extension if it is missing.
+		if (subtype && !uploadName.toLowerCase().endsWith(`.${subtype}`)) {
+			uploadName = `${uploadName}.${subtype}`;
+		}
+
+		try {
+			const response = await this.capiClient.makeRequest<Response>({
+				method: 'POST',
+				body: binaryData,
+				headers: {
+					'Content-Type': 'application/octet-stream',
+					Authorization: `Bearer ${token}`,
+				}
+			}, { type: RequestType.ChatAttachmentUpload, uploadName, mimeType });
+			if (!response.ok) {
+				throw new Error(`Image upload failed: ${response.status} ${response.statusText}`);
+			}
+			const result = await response.json() as { url: string };
+			return URI.parse(result.url);
+		} catch (error) {
+			throw new Error(`Error uploading image: ${error}`);
+		}
+	}
+}