diff --git a/package-lock.json b/package-lock.json index 57c9fb64b..7b7958c8a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@humanwhocodes/gitignore-to-minimatch": "1.0.2", "@microsoft/tiktokenizer": "^1.0.10", "@roamhq/mac-ca": "^1.0.7", - "@vscode/copilot-api": "^0.1.3", + "@vscode/copilot-api": "^0.1.4", "@vscode/extension-telemetry": "^1.0.0", "@vscode/l10n": "^0.0.18", "@vscode/prompt-tsx": "^0.4.0-alpha.5", @@ -6034,9 +6034,9 @@ } }, "node_modules/@vscode/copilot-api": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@vscode/copilot-api/-/copilot-api-0.1.3.tgz", - "integrity": "sha512-tcxlkIO/gzSBwdGWXlB6egFzwM9s80Nq0Hqc4HpwrFYshNzcdhzuUL7ox2XlwbIsMahm0AYNlxW45Osnb5720A==", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@vscode/copilot-api/-/copilot-api-0.1.4.tgz", + "integrity": "sha512-qSd3oKF7lb7RaTwL537LBCv7Q7q7k7sGsx8doqIFQpslwGQ6yvUsqwCdXQY1EanYuNg5UYXX2aa36BOBoe8gvg==", "license": "SEE LICENSE" }, "node_modules/@vscode/dts": { diff --git a/package.json b/package.json index 4c0be6d72..763812adf 100644 --- a/package.json +++ b/package.json @@ -3685,7 +3685,7 @@ "@humanwhocodes/gitignore-to-minimatch": "1.0.2", "@microsoft/tiktokenizer": "^1.0.10", "@roamhq/mac-ca": "^1.0.7", - "@vscode/copilot-api": "^0.1.3", + "@vscode/copilot-api": "^0.1.4", "@vscode/extension-telemetry": "^1.0.0", "@vscode/l10n": "^0.0.18", "@vscode/prompt-tsx": "^0.4.0-alpha.5", diff --git a/src/extension/conversation/vscode-node/languageModelAccessPrompt.tsx b/src/extension/conversation/vscode-node/languageModelAccessPrompt.tsx index cbaae3b12..caee663d2 100644 --- a/src/extension/conversation/vscode-node/languageModelAccessPrompt.tsx +++ b/src/extension/conversation/vscode-node/languageModelAccessPrompt.tsx @@ -20,7 +20,7 @@ export type Props = PromptElementProps<{ }>; export class LanguageModelAccessPrompt extends PromptElement { - render() { + async render() { const systemMessages: string[] = []; const chatMessages: (UserMessage | AssistantMessage)[] = []; @@ -44,7 +44,7 @@ export class LanguageModelAccessPrompt extends PromptElement { const statefulMarkerElement = statefulMarker && ; chatMessages.push( ({ id: tc.callId, type: 'function', function: { name: tc.name, arguments: JSON.stringify(tc.input) } }))}>{statefulMarkerElement}{content?.value}); } else if (message.role === vscode.LanguageModelChatMessageRole.User) { - message.content.forEach(part => { + for (const part of message.content) { if (part instanceof vscode.LanguageModelToolResultPart2 || part instanceof vscode.LanguageModelToolResultPart) { chatMessages.push( @@ -52,11 +52,12 @@ export class LanguageModelAccessPrompt extends PromptElement { ); } else if (isImageDataPart(part)) { - chatMessages.push({imageDataPartToTSX(part)}); + const imageElement = await imageDataPartToTSX(part); + chatMessages.push({imageElement}); } else if (part instanceof vscode.LanguageModelTextPart) { chatMessages.push({part.value}); } - }); + } } } diff --git a/src/extension/extension/vscode-node/services.ts b/src/extension/extension/vscode-node/services.ts index f872ad2b9..117ae54c8 100644 --- a/src/extension/extension/vscode-node/services.ts +++ b/src/extension/extension/vscode-node/services.ts @@ -28,6 +28,8 @@ import { IGithubRepositoryService } from '../../../platform/github/common/github import { GithubRepositoryService } from '../../../platform/github/node/githubRepositoryService'; import { IIgnoreService } from '../../../platform/ignore/common/ignoreService'; import { VsCodeIgnoreService } from '../../../platform/ignore/vscode-node/ignoreService'; +import { IImageService } from '../../../platform/image/common/imageService'; +import { ImageServiceImpl } from '../../../platform/image/node/imageServiceImpl'; import { ILanguageContextService } from '../../../platform/languageServer/common/languageContextService'; import { ICompletionsFetchService } from '../../../platform/nesFetch/common/completionsFetchService'; import { CompletionsFetchService } from '../../../platform/nesFetch/node/completionsFetchServiceImpl'; @@ -121,6 +123,7 @@ export function registerServices(builder: IInstantiationServiceBuilder, extensio builder.define(IFetcherService, new SyncDescriptor(FetcherService, [undefined])); builder.define(IDomainService, new SyncDescriptor(DomainService)); builder.define(ICAPIClientService, new SyncDescriptor(CAPIClientImpl)); + builder.define(IImageService, new SyncDescriptor(ImageServiceImpl)); builder.define(ITelemetryUserConfig, new SyncDescriptor(TelemetryUserConfigImpl, [undefined, undefined])); const internalAIKey = extensionContext.extension.packageJSON.internalAIKey ?? ''; diff --git a/src/extension/prompts/node/panel/image.tsx b/src/extension/prompts/node/panel/image.tsx index 137ad58a3..eb461ee15 100644 --- a/src/extension/prompts/node/panel/image.tsx +++ b/src/extension/prompts/node/panel/image.tsx @@ -3,8 +3,15 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { RequestType } from '@vscode/copilot-api'; import * as l10n from '@vscode/l10n'; -import { BasePromptElementProps, ChatResponseReferencePartStatusKind, PromptElement, PromptReference, PromptSizing, UserMessage, Image as BaseImage } from '@vscode/prompt-tsx'; +import { Image as BaseImage, BasePromptElementProps, ChatResponseReferencePartStatusKind, PromptElement, PromptReference, PromptSizing, UserMessage } from '@vscode/prompt-tsx'; +import { IAuthenticationService } from '../../../../platform/authentication/common/authentication'; +import { ConfigKey, IConfigurationService } from '../../../../platform/configuration/common/configurationService'; +import { IImageService } from '../../../../platform/image/common/imageService'; +import { ILogService } from '../../../../platform/log/common/logService'; +import { IExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService'; +import { getMimeType } from '../../../../util/common/imageUtils'; import { Uri } from '../../../../vscodeTypes'; import { IPromptEndpoint } from '../base/promptRenderer'; @@ -18,7 +25,12 @@ export interface ImageProps extends BasePromptElementProps { export class Image extends PromptElement { constructor( props: ImageProps, - @IPromptEndpoint private readonly promptEndpoint: IPromptEndpoint + @IPromptEndpoint private readonly promptEndpoint: IPromptEndpoint, + @IAuthenticationService private readonly authService: IAuthenticationService, + @ILogService private readonly logService: ILogService, + @IImageService private readonly imageService: IImageService, + @IConfigurationService private readonly configurationService: IConfigurationService, + @IExperimentationService private readonly experimentationService: IExperimentationService ) { super(props); } @@ -41,16 +53,24 @@ export class Image extends PromptElement { ); } const variable = await this.props.variableValue; - let decoded = Buffer.from(variable).toString('base64'); - const decoder = new TextDecoder(); - const decodedString = decoder.decode(variable); - if (/^https?:\/\/.+/.test(decodedString)) { - decoded = decodedString; + let imageSource = Buffer.from(variable).toString('base64'); + const isChatCompletions = typeof this.promptEndpoint.urlOrRequestMetadata !== 'string' && this.promptEndpoint.urlOrRequestMetadata.type === RequestType.ChatCompletions; + const enabled = this.configurationService.getExperimentBasedConfig(ConfigKey.Internal.EnableChatImageUpload, this.experimentationService); + if (isChatCompletions && enabled) { + try { + const githubToken = (await this.authService.getAnyGitHubSession())?.accessToken; + const uri = await this.imageService.uploadChatImageAttachment(variable, this.props.variableName, getMimeType(imageSource) ?? 'image/png', githubToken); + if (uri) { + imageSource = uri.toString(); + } + } catch (error) { + this.logService.warn(`Image upload failed, using base64 fallback: ${error}`); + } } return ( - + {this.props.reference && ( )} diff --git a/src/extension/prompts/node/panel/toolCalling.tsx b/src/extension/prompts/node/panel/toolCalling.tsx index 700beb483..a57d9e15f 100644 --- a/src/extension/prompts/node/panel/toolCalling.tsx +++ b/src/extension/prompts/node/panel/toolCalling.tsx @@ -3,12 +3,17 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { RequestMetadata, RequestType } from '@vscode/copilot-api'; import { AssistantMessage, BasePromptElementProps, PromptRenderer as BasePromptRenderer, Chunk, IfEmpty, Image, JSONTree, PromptElement, PromptElementProps, PromptMetadata, PromptPiece, PromptSizing, TokenLimit, ToolCall, ToolMessage, useKeepWith, UserMessage } from '@vscode/prompt-tsx'; import type { ChatParticipantToolToken, LanguageModelToolResult2, LanguageModelToolTokenizationOptions } from 'vscode'; +import { IAuthenticationService } from '../../../../platform/authentication/common/authentication'; +import { ConfigKey, IConfigurationService } from '../../../../platform/configuration/common/configurationService'; import { IEndpointProvider } from '../../../../platform/endpoint/common/endpointProvider'; import { CacheType } from '../../../../platform/endpoint/common/endpointTypes'; import { StatefulMarkerContainer } from '../../../../platform/endpoint/common/statefulMarkerContainer'; +import { IImageService } from '../../../../platform/image/common/imageService'; import { ILogService } from '../../../../platform/log/common/logService'; +import { IExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService'; import { ITelemetryService } from '../../../../platform/telemetry/common/telemetry'; import { ITokenizer } from '../../../../util/common/tokenizer'; import { CancellationToken } from '../../../../util/vs/base/common/cancellation'; @@ -297,10 +302,25 @@ enum ToolInvocationOutcome { Cancelled = 'cancelled', } -export function imageDataPartToTSX(part: LanguageModelDataPart) { +export async function imageDataPartToTSX(part: LanguageModelDataPart, githubToken?: string, urlOrRequestMetadata?: string | RequestMetadata, logService?: ILogService, imageService?: IImageService) { if (isImageDataPart(part)) { const base64 = Buffer.from(part.data).toString('base64'); - return ; + let imageSource = `data:${part.mimeType};base64,${base64}`; + const isChatCompletions = typeof urlOrRequestMetadata !== 'string' && urlOrRequestMetadata?.type === RequestType.ChatCompletions; + if (githubToken && isChatCompletions && imageService) { + try { + const uri = await imageService.uploadChatImageAttachment(part.data, 'tool-result-image', part.mimeType ?? 'image/png', githubToken); + if (uri) { + imageSource = uri.toString(); + } + } catch (error) { + if (logService) { + logService.warn(`Image upload failed, using base64 fallback: ${error}`); + } + } + } + + return ; } } @@ -340,6 +360,19 @@ interface IPrimitiveToolResultProps extends BasePromptElementProps { } class PrimitiveToolResult extends PromptElement { + + constructor( + props: T, + @IPromptEndpoint protected readonly endpoint: IPromptEndpoint, + @IAuthenticationService private readonly authService: IAuthenticationService, + @ILogService private readonly logService?: ILogService, + @IImageService private readonly imageService?: IImageService, + @IConfigurationService private readonly configurationService?: IConfigurationService, + @IExperimentationService private readonly experimentationService?: IExperimentationService + ) { + super(props); + } + async render(): Promise { return ( <> @@ -368,8 +401,13 @@ class PrimitiveToolResult extends PromptEle return part.audience.includes(LanguageModelPartAudience.Assistant); } - protected onData(part: LanguageModelDataPart) { - return Promise.resolve(imageDataPartToTSX(part)); + protected async onData(part: LanguageModelDataPart) { + const githubToken = (await this.authService.getAnyGitHubSession())?.accessToken; + const uploadsEnabled = this.configurationService && this.experimentationService + ? this.configurationService.getExperimentBasedConfig(ConfigKey.Internal.EnableChatImageUpload, this.experimentationService) + : false; + const effectiveToken = uploadsEnabled ? githubToken : undefined; + return Promise.resolve(imageDataPartToTSX(part, effectiveToken, this.endpoint.urlOrRequestMetadata, this.logService, this.imageService)); } protected onTSX(part: JSONTree.PromptElementJSON) { @@ -396,9 +434,14 @@ export interface IToolResultProps extends IPrimitiveToolResultProps { export class ToolResult extends PrimitiveToolResult { constructor( props: PromptElementProps, - @IPromptEndpoint private readonly endpoint: IPromptEndpoint, + @IPromptEndpoint endpoint: IPromptEndpoint, + @IAuthenticationService authService: IAuthenticationService, + @ILogService logService: ILogService, + @IImageService imageService: IImageService, + @IConfigurationService configurationService: IConfigurationService, + @IExperimentationService experimentationService: IExperimentationService ) { - super(props); + super(props, endpoint, authService, logService, imageService, configurationService, experimentationService); } protected override async onTSX(part: JSONTree.PromptElementJSON): Promise { diff --git a/src/extension/test/vscode-node/services.ts b/src/extension/test/vscode-node/services.ts index 6df2091c8..e8c84864d 100644 --- a/src/extension/test/vscode-node/services.ts +++ b/src/extension/test/vscode-node/services.ts @@ -102,6 +102,7 @@ import { IToolsService, NullToolsService } from '../../tools/common/toolsService import { ToolGroupingService } from '../../tools/common/virtualTools/toolGroupingService'; import { ToolGroupingCache } from '../../tools/common/virtualTools/virtualToolGroupCache'; import { IToolGroupingCache, IToolGroupingService } from '../../tools/common/virtualTools/virtualToolTypes'; +import { IImageService, nullImageService } from '../../../platform/image/common/imageService'; /** * A default context for VSCode extension testing, building on general one in `lib`. @@ -127,6 +128,7 @@ export function createExtensionTestingServices(): TestingServiceCollection { testingServiceCollection.define(IWorkspaceService, new SyncDescriptor(ExtensionTextDocumentManager)); testingServiceCollection.define(IExtensionsService, new SyncDescriptor(VSCodeExtensionsService)); testingServiceCollection.define(IChatMLFetcher, new SyncDescriptor(ChatMLFetcherImpl)); + testingServiceCollection.define(IImageService, nullImageService); testingServiceCollection.define(ITabsAndEditorsService, new SyncDescriptor(TabsAndEditorsServiceImpl)); testingServiceCollection.define(IEmbeddingsComputer, new SyncDescriptor(RemoteEmbeddingsComputer)); testingServiceCollection.define(ITelemetryService, new SyncDescriptor(NullTelemetryService)); diff --git a/src/platform/configuration/common/configurationService.ts b/src/platform/configuration/common/configurationService.ts index f3849e9ef..53298dada 100644 --- a/src/platform/configuration/common/configurationService.ts +++ b/src/platform/configuration/common/configurationService.ts @@ -704,6 +704,8 @@ export namespace ConfigKey { export const AgentHistorySummarizationForceGpt41 = defineExpSetting('chat.advanced.agentHistorySummarizationForceGpt41', false, INTERNAL_RESTRICTED); export const UseResponsesApiTruncation = defineSetting('chat.advanced.useResponsesApiTruncation', false, INTERNAL_RESTRICTED); + export const EnableChatImageUpload = defineExpSetting('chat.advanced.imageUpload', false, INTERNAL); + export const EnableReadFileV2 = defineExpSetting('chat.advanced.enableReadFileV2', isPreRelease, INTERNAL_RESTRICTED); export const AskAgent = defineExpSetting('chat.advanced.enableAskAgent', { defaultValue: false, teamDefaultValue: true, internalDefaultValue: true }, INTERNAL_RESTRICTED); export const VerifyTextDocumentChanges = defineExpSetting('chat.advanced.inlineEdits.verifyTextDocumentChanges', true, INTERNAL_RESTRICTED); diff --git a/src/platform/image/common/imageService.ts b/src/platform/image/common/imageService.ts new file mode 100644 index 000000000..8c1d40575 --- /dev/null +++ b/src/platform/image/common/imageService.ts @@ -0,0 +1,30 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createServiceIdentifier } from '../../../util/common/services'; +import { URI } from '../../../util/vs/base/common/uri'; + +export const IImageService = createServiceIdentifier('IImageService'); + +export interface IImageService { + readonly _serviceBrand: undefined; + + /** + * Upload image data to GitHub Copilot chat attachments endpoint + * @param binaryData The image binary data as Uint8Array + * @param name The name for the uploaded file + * @param mimeType The MIME type of the image + * @param token The authentication token for GitHub API + * @returns Promise The URI of the uploaded image + */ + uploadChatImageAttachment(binaryData: Uint8Array, name: string, mimeType: string | undefined, token: string | undefined): Promise; +} + +export const nullImageService: IImageService = { + _serviceBrand: undefined, + async uploadChatImageAttachment(): Promise { + throw new Error('Image service not implemented'); + } +}; diff --git a/src/platform/image/node/imageServiceImpl.ts b/src/platform/image/node/imageServiceImpl.ts new file mode 100644 index 000000000..fe189f25e --- /dev/null +++ b/src/platform/image/node/imageServiceImpl.ts @@ -0,0 +1,53 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { RequestType } from '@vscode/copilot-api'; +import { URI } from '../../../util/vs/base/common/uri'; +import { ICAPIClientService } from '../../endpoint/common/capiClient'; +import { IImageService } from '../common/imageService'; + +export class ImageServiceImpl implements IImageService { + declare readonly _serviceBrand: undefined; + + constructor( + @ICAPIClientService private readonly capiClient: ICAPIClientService, + ) { } + + async uploadChatImageAttachment(binaryData: Uint8Array, name: string, mimeType: string | undefined, token: string | undefined): Promise { + if (!mimeType || !token) { + throw new Error('Missing required mimeType or token for image upload'); + } + + const sanitizedName = name.replace(/[^a-zA-Z0-9._-]/g, ''); + let uploadName = sanitizedName; + + // can catch unexpected types like "IMAGE/JPEG", "image/svg+xml", or "image/png; charset=UTF-8" + const subtypeMatch = mimeType.toLowerCase().match(/^[^\/]+\/([^+;]+)/); + const subtype = subtypeMatch?.[1]; + + // add the extension if it is missing. + if (subtype && !uploadName.toLowerCase().endsWith(`.${subtype}`)) { + uploadName = `${uploadName}.${subtype}`; + } + + try { + const response = await this.capiClient.makeRequest({ + method: 'POST', + body: binaryData, + headers: { + 'Content-Type': 'application/octet-stream', + Authorization: `Bearer ${token}`, + } + }, { type: RequestType.ChatAttachmentUpload, uploadName, mimeType }); + if (!response.ok) { + throw new Error(`Image upload failed: ${response.status} ${response.statusText}`); + } + const result = await response.json() as { url: string }; + return URI.parse(result.url); + } catch (error) { + throw new Error(`Error uploading image: ${error}`); + } + } +} diff --git a/src/platform/test/node/services.ts b/src/platform/test/node/services.ts index 8ec0245f9..46c6498e0 100644 --- a/src/platform/test/node/services.ts +++ b/src/platform/test/node/services.ts @@ -96,6 +96,7 @@ import { TestAuthenticationService } from './testAuthenticationService'; import { TestChatAgentService } from './testChatAgentService'; import { TestWorkbenchService } from './testWorkbenchService'; import { TestWorkspaceService } from './testWorkspaceService'; +import { IImageService, nullImageService } from '../../image/common/imageService'; /** * Collects descriptors for services to use in testing. @@ -255,6 +256,7 @@ export function createPlatformServices(): TestingServiceCollection { testingServiceCollection.define(IRunCommandExecutionService, new SyncDescriptor(MockRunCommandExecutionService)); testingServiceCollection.define(INaiveChunkingService, new SyncDescriptor(NaiveChunkingService)); testingServiceCollection.define(IHeatmapService, nullHeatmapService); + testingServiceCollection.define(IImageService, nullImageService); testingServiceCollection.define(ILanguageContextService, NullLanguageContextService); testingServiceCollection.define(ILanguageContextProviderService, new SyncDescriptor(NullLanguageContextProviderService)); testingServiceCollection.define(ILanguageDiagnosticsService, new SyncDescriptor(TestLanguageDiagnosticsService)); diff --git a/src/util/common/imageUtils.ts b/src/util/common/imageUtils.ts index 726e49de7..531197c74 100644 --- a/src/util/common/imageUtils.ts +++ b/src/util/common/imageUtils.ts @@ -100,7 +100,7 @@ export function getWebPDimensions(base64String: string) { } } -function getMimeType(base64String: string): string | undefined { +export function getMimeType(base64String: string): string | undefined { const mimeTypes: { [key: string]: string } = { '/9j/': 'image/jpeg', 'iVBOR': 'image/png',