use image uploader in chat

src/extension/extension/vscode-node/services.ts

@@ -28,6 +28,8 @@ import { IGithubRepositoryService } from '../../../platform/github/common/github
 import { GithubRepositoryService } from '../../../platform/github/node/githubRepositoryService';
 import { IIgnoreService } from '../../../platform/ignore/common/ignoreService';
 import { VsCodeIgnoreService } from '../../../platform/ignore/vscode-node/ignoreService';
+import { IImageService } from '../../../platform/image/common/imageService';
+import { ImageServiceImpl } from '../../../platform/image/node/imageServiceImpl';
 import { ILanguageContextService } from '../../../platform/languageServer/common/languageContextService';
 import { ICompletionsFetchService } from '../../../platform/nesFetch/common/completionsFetchService';
 import { CompletionsFetchService } from '../../../platform/nesFetch/node/completionsFetchServiceImpl';
@@ -121,6 +123,7 @@ export function registerServices(builder: IInstantiationServiceBuilder, extensio
 	builder.define(IFetcherService, new SyncDescriptor(FetcherService, [undefined]));
 	builder.define(IDomainService, new SyncDescriptor(DomainService));
 	builder.define(ICAPIClientService, new SyncDescriptor(CAPIClientImpl));
+	builder.define(IImageService, new SyncDescriptor(ImageServiceImpl));
 
 	builder.define(ITelemetryUserConfig, new SyncDescriptor(TelemetryUserConfigImpl, [undefined, undefined]));
 	const internalAIKey = extensionContext.extension.packageJSON.internalAIKey ?? '';

src/extension/prompts/node/panel/image.tsx

@@ -4,7 +4,11 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as l10n from '@vscode/l10n';
-import { BasePromptElementProps, ChatResponseReferencePartStatusKind, PromptElement, PromptReference, PromptSizing, UserMessage, Image as BaseImage } from '@vscode/prompt-tsx';
+import { Image as BaseImage, BasePromptElementProps, ChatResponseReferencePartStatusKind, PromptElement, PromptReference, PromptSizing, UserMessage } from '@vscode/prompt-tsx';
+import { IAuthenticationService } from '../../../../platform/authentication/common/authentication';
+import { IImageService } from '../../../../platform/image/common/imageService';
+import { ILogService } from '../../../../platform/log/common/logService';
+import { getMimeType } from '../../../../util/common/imageUtils';
 import { Uri } from '../../../../vscodeTypes';
 import { IPromptEndpoint } from '../base/promptRenderer';
 
@@ -18,7 +22,10 @@ export interface ImageProps extends BasePromptElementProps {
 export class Image extends PromptElement<ImageProps, unknown> {
 	constructor(
 		props: ImageProps,
-		@IPromptEndpoint private readonly promptEndpoint: IPromptEndpoint
+		@IPromptEndpoint private readonly promptEndpoint: IPromptEndpoint,
+		@IAuthenticationService private readonly authService: IAuthenticationService,
+		@ILogService private readonly logService: ILogService,
+		@IImageService private readonly imageService: IImageService
 	) {
 		super(props);
 	}
@@ -41,16 +48,22 @@ export class Image extends PromptElement<ImageProps, unknown> {
 				);
 			}
 			const variable = await this.props.variableValue;
-			let decoded = Buffer.from(variable).toString('base64');
-			const decoder = new TextDecoder();
-			const decodedString = decoder.decode(variable);
-			if (/^https?:\/\/.+/.test(decodedString)) {
-				decoded = decodedString;
+			let imageSource = Buffer.from(variable).toString('base64');
+			if (this.promptEndpoint.vendor === 'copilot') {
+				try {
+					const githubToken = (await this.authService.getAnyGitHubSession())?.accessToken;
+					const uri = await this.imageService.uploadChatImageAttachment(variable, this.props.variableName, getMimeType(imageSource) ?? 'image/png', githubToken);
+					if (uri) {
+						imageSource = uri.toString();
+					}
+				} catch (error) {
+					this.logService.warn(`Image upload failed, using base64 fallback: ${error}`);
+				}
 			}
 
 			return (
 				<UserMessage priority={0}>
-					<BaseImage src={decoded} detail='high' />
+					<BaseImage src={imageSource} detail='high' />
 					{this.props.reference && (
 						<references value={[new PromptReference(this.props.variableName ? { variableName: this.props.variableName, value: fillerUri } : fillerUri, undefined)]} />
 					)}

src/extension/prompts/node/panel/toolCalling.tsx

@@ -5,9 +5,11 @@
 
 import { AssistantMessage, BasePromptElementProps, PromptRenderer as BasePromptRenderer, Chunk, IfEmpty, Image, JSONTree, PromptElement, PromptElementProps, PromptMetadata, PromptPiece, PromptSizing, TokenLimit, ToolCall, ToolMessage, useKeepWith, UserMessage } from '@vscode/prompt-tsx';
 import type { ChatParticipantToolToken, LanguageModelToolResult2, LanguageModelToolTokenizationOptions } from 'vscode';
+import { IAuthenticationService } from '../../../../platform/authentication/common/authentication';
 import { IEndpointProvider } from '../../../../platform/endpoint/common/endpointProvider';
 import { CacheType } from '../../../../platform/endpoint/common/endpointTypes';
 import { StatefulMarkerContainer } from '../../../../platform/endpoint/common/statefulMarkerContainer';
+import { IImageService } from '../../../../platform/image/common/imageService';
 import { ILogService } from '../../../../platform/log/common/logService';
 import { ITelemetryService } from '../../../../platform/telemetry/common/telemetry';
 import { ITokenizer } from '../../../../util/common/tokenizer';
@@ -297,10 +299,25 @@ enum ToolInvocationOutcome {
 	Cancelled = 'cancelled',
 }
 
-export function imageDataPartToTSX(part: LanguageModelDataPart) {
+export async function imageDataPartToTSX(part: LanguageModelDataPart, githubToken?: string, vendor?: string, logService?: ILogService, imageService?: IImageService) {
 	if (isImageDataPart(part)) {
 		const base64 = Buffer.from(part.data).toString('base64');
-		return <Image src={`data:${part.mimeType};base64,${base64}`} />;
+		let imageSource = `data:${part.mimeType};base64,${base64}`;
+
+		if (githubToken && vendor === 'copilot' && imageService) {
+			try {
+				const uri = await imageService.uploadChatImageAttachment(part.data, 'mcp-image', part.mimeType ?? 'image/png', githubToken);
+				if (uri) {
+					imageSource = uri.toString();
+				}
+			} catch (error) {
+				if (logService) {
+					logService.warn(`Image upload failed, using base64 fallback: ${error}`);
+				}
+			}
+		}
+
+		return <Image src={imageSource} />;
 	}
 }
 
@@ -340,6 +357,17 @@ interface IPrimitiveToolResultProps extends BasePromptElementProps {
 }
 
 class PrimitiveToolResult<T extends IPrimitiveToolResultProps> extends PromptElement<T> {
+
+	constructor(
+		props: T,
+		@IPromptEndpoint protected readonly endpoint: IPromptEndpoint,
+		@IAuthenticationService private readonly authService: IAuthenticationService,
+		@ILogService private readonly logService?: ILogService,
+		@IImageService private readonly imageService?: IImageService
+	) {
+		super(props);
+	}
+
 	async render(): Promise<PromptPiece | undefined> {
 		return (
 			<>
@@ -368,8 +396,9 @@ class PrimitiveToolResult<T extends IPrimitiveToolResultProps> extends PromptEle
 		return part.audience.includes(LanguageModelPartAudience.Assistant);
 	}
 
-	protected onData(part: LanguageModelDataPart) {
-		return Promise.resolve(imageDataPartToTSX(part));
+	protected async onData(part: LanguageModelDataPart) {
+		const githubToken = (await this.authService.getAnyGitHubSession())?.accessToken;
+		return Promise.resolve(imageDataPartToTSX(part, githubToken, this.endpoint.vendor, this.logService, this.imageService));
 	}
 
 	protected onTSX(part: JSONTree.PromptElementJSON) {
@@ -396,9 +425,11 @@ export interface IToolResultProps extends IPrimitiveToolResultProps {
 export class ToolResult extends PrimitiveToolResult<IToolResultProps> {
 	constructor(
 		props: PromptElementProps<IToolResultProps>,
-		@IPromptEndpoint private readonly endpoint: IPromptEndpoint,
+		@IPromptEndpoint endpoint: IPromptEndpoint,
+		@IAuthenticationService authService: IAuthenticationService,
+		@ILogService logService: ILogService
 	) {
-		super(props);
+		super(props, endpoint, authService, logService);
 	}
 
 	protected override async onTSX(part: JSONTree.PromptElementJSON): Promise<any> {

src/platform/endpoint/common/autoChatEndpoint.ts

@@ -45,6 +45,7 @@ export class AutoChatEndpoint implements IChatEndpoint {
 	version: string = this._wrappedEndpoint.version;
 	family: string = this._wrappedEndpoint.family;
 	tokenizer: TokenizerType = this._wrappedEndpoint.tokenizer;
+	vendor?: string = this._wrappedEndpoint.vendor;
 
 	constructor(
 		private readonly _wrappedEndpoint: IChatEndpoint,

src/platform/endpoint/common/endpointProvider.ts

@@ -55,6 +55,7 @@ export interface IModelAPIResponse {
 	name: string;
 	policy?: ModelPolicy;
 	model_picker_enabled: boolean;
+	vendor?: string;
 	preview?: boolean;
 	is_chat_default: boolean;
 	is_chat_fallback: boolean;

src/platform/endpoint/node/chatEndpoint.ts

@@ -155,6 +155,7 @@ export class ChatEndpoint implements IChatEndpoint {
 	public readonly supportsStatefulResponses: boolean;
 	public readonly isPremium?: boolean | undefined;
 	public readonly multiplier?: number | undefined;
+	public readonly vendor?: string;
 	public readonly restrictedToSkus?: string[] | undefined;
 	private readonly _supportsStreaming: boolean;
 	private _policyDetails: ModelPolicy | undefined;
@@ -193,6 +194,7 @@ export class ChatEndpoint implements IChatEndpoint {
 		this._supportsStreaming = !!_modelMetadata.capabilities.supports.streaming;
 		this.supportsStatefulResponses = !!_modelMetadata.capabilities.supports.statefulResponses;
 		this._policyDetails = _modelMetadata.policy;
+		this.vendor = _modelMetadata.vendor;
 	}
 
 	public get modelMaxPromptTokens(): number {

src/platform/endpoint/node/copilotChatEndpoint.ts

@@ -34,8 +34,14 @@ export class CopilotChatEndpoint extends ChatEndpoint {
 		@IInstantiationService readonly instantiationService: IInstantiationService,
 		@IThinkingDataService readonly thinkingDataService: IThinkingDataService,
 	) {
+
+		const modifiedMetadata = {
+			...modelMetadata,
+			vendor: 'copilot'
+		};
+
 		super(
-			modelMetadata,
+			modifiedMetadata,
 			domainService,
 			capiClientService,
 			fetcherService,

src/platform/endpoint/node/proxyExperimentEndpoint.ts

@@ -102,6 +102,10 @@ export class ProxyExperimentEndpoint implements IChatEndpoint {
 		return this.selectedEndpoint.version;
 	}
 
+	get vendor(): string | undefined {
+		return this.selectedEndpoint.vendor;
+	}
+
 	get tokenizer(): TokenizerType {
 		return this.selectedEndpoint.tokenizer;
 	}

src/platform/endpoint/vscode-node/extChatEndpoint.ts

@@ -100,6 +100,10 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint {
 		return 'enabled';
 	}
 
+	get vendor(): string | undefined {
+		return this.languageModel.vendor;
+	}
+
 	async processResponseFromChatEndpoint(
 		telemetryService: ITelemetryService,
 		logService: ILogService,

src/platform/image/common/imageService.ts

@@ -0,0 +1,23 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { createServiceIdentifier } from '../../../util/common/services';
+import { URI } from '../../../util/vs/base/common/uri';
+
+export const IImageService = createServiceIdentifier<IImageService>('IImageService');
+
+export interface IImageService {
+	readonly _serviceBrand: undefined;
+
+	/**
+	 * Upload image data to GitHub Copilot chat attachments endpoint
+	 * @param binaryData The image binary data as Uint8Array
+	 * @param name The name for the uploaded file
+	 * @param mimeType The MIME type of the image
+	 * @param token The authentication token for GitHub API
+	 * @returns Promise<URI> The URI of the uploaded image
+	 */
+	uploadChatImageAttachment(binaryData: Uint8Array, name: string, mimeType: string | undefined, token: string | undefined): Promise<URI>;
+}

src/platform/image/node/imageServiceImpl.ts

@@ -0,0 +1,48 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { RequestType } from '@vscode/copilot-api';
+import { URI } from '../../../util/vs/base/common/uri';
+import { ICAPIClientService } from '../../endpoint/common/capiClient';
+import { IImageService } from '../common/imageService';
+
+export class ImageServiceImpl implements IImageService {
+	declare readonly _serviceBrand: undefined;
+
+	constructor(
+		@ICAPIClientService private readonly capiClient: ICAPIClientService,
+	) { }
+
+	async uploadChatImageAttachment(binaryData: Uint8Array, name: string, mimeType: string | undefined, token: string | undefined): Promise<URI> {
+		if (!mimeType || !token) {
+			throw new Error('Missing required mimeType or token for image upload');
+		}
+
+		const sanitizedName = name.replace(/\s+/g, '').replace(/%20/g, '');
+		let uploadName = sanitizedName;
+		const subtype = mimeType.split('/')[1].split('+')[0].toLowerCase();
+		if (!uploadName.toLowerCase().endsWith(`.${subtype}`)) {
+			uploadName = `${uploadName}.${subtype}`;
+		}
+
+		try {
+			const response = await this.capiClient.makeRequest<Response>({
+				method: 'POST',
+				body: binaryData,
+				headers: {
+					'Content-Type': 'application/octet-stream',
+					Authorization: `Bearer ${token}`,
+				}
+			}, { type: RequestType.ChatAttachmentUpload, uploadName, mimeType });
+			if (!response.ok) {
+				throw new Error(`Invalid GitHub URL provided: ${response.status} ${response.statusText}`);
+			}
+			const result = await response.json() as { url: string };
+			return URI.parse(result.url);
+		} catch (error) {
+			throw new Error(`Error uploading image: ${error}`);
+		}
+	}
+}

src/platform/networking/common/networking.ts

@@ -171,6 +171,7 @@ export interface IChatEndpoint extends IEndpoint {
 	readonly isDefault: boolean;
 	readonly isFallback: boolean;
 	readonly policy: 'enabled' | { terms: string };
+	readonly vendor?: string;
 	/**
 	 * Handles processing of responses from a chat endpoint. Each endpoint can have different response formats.
 	 * @param telemetryService The telemetry service

src/util/common/imageUtils.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 
+
 export function getImageDimensions(base64: string) {
 	if (!base64.startsWith('data:image/')) {
 		throw new Error("Could not read image: invalid base64 image string");
@@ -100,7 +101,7 @@ export function getWebPDimensions(base64String: string) {
 	}
 }
 
-function getMimeType(base64String: string): string | undefined {
+export function getMimeType(base64String: string): string | undefined {
 	const mimeTypes: { [key: string]: string } = {
 		'/9j/': 'image/jpeg',
 		'iVBOR': 'image/png',