Skip to content

Commit bd21c70

Browse files
connor4312connor4312
authored
mcp: use tool embedded resource data for 'permalinks' (microsoft#250329)
* mcp: use tool embedded resource data for 'permalinks' Previously we kind of normalized on MCP resource URIs in tool responses. This means we basically ignored the embedded content and _required_ servers implement resources to make it work. In this PR, I register a read-only filesystem that serves contents from the chat tool response. This avoids extra resources calls and might be more predictable as well. * hygenie
1 parent 1417ec8 commit bd21c70

File tree

11 files changed

+233
-81
lines changed

11 files changed

+233
-81
lines changed

‎src/vs/workbench/contrib/chat/browser/chat.contribution.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ import { ChatViewsWelcomeHandler } from './viewsWelcome/chatViewsWelcomeHandler.
110110
import { registerAction2 } from '../../../../platform/actions/common/actions.js';
111111
import product from '../../../../platform/product/common/product.js';
112112
import { ChatModeService, IChatModeService } from '../common/chatModes.js';
113+
import { ChatResponseResourceFileSystemProvider } from '../common/chatResponseResourceFileSystemProvider.js';
113114

114115
// Register configuration
115116
const configurationRegistry = Registry.as<IConfigurationRegistry>(ConfigurationExtensions.Configuration);
@@ -708,6 +709,7 @@ registerWorkbenchContribution2(SimpleBrowserOverlay.ID, SimpleBrowserOverlay, Wo
708709
registerWorkbenchContribution2(ChatEditingEditorContextKeys.ID, ChatEditingEditorContextKeys, WorkbenchPhase.AfterRestored);
709710
registerWorkbenchContribution2(ChatTransferContribution.ID, ChatTransferContribution, WorkbenchPhase.BlockRestore);
710711
registerWorkbenchContribution2(ChatContextContributions.ID, ChatContextContributions, WorkbenchPhase.AfterRestored);
712+
registerWorkbenchContribution2(ChatResponseResourceFileSystemProvider.ID, ChatResponseResourceFileSystemProvider, WorkbenchPhase.AfterRestored);
711713

712714
registerChatActions();
713715
registerChatCopyActions();

‎src/vs/workbench/contrib/chat/browser/chatAttachmentWidgets.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ export class ImageAttachmentWidget extends AbstractChatAttachmentWidget {
247247
@IHoverService private readonly hoverService: IHoverService,
248248
@ILanguageModelsService private readonly languageModelsService: ILanguageModelsService,
249249
@ITelemetryService private readonly telemetryService: ITelemetryService,
250+
@IInstantiationService instantiationService: IInstantiationService,
250251
) {
251252
super(attachment, options, container, contextResourceLabels, hoverDelegate, currentLanguageModel, commandService, openerService);
252253

@@ -290,6 +291,9 @@ export class ImageAttachmentWidget extends AbstractChatAttachmentWidget {
290291

291292
if (resource) {
292293
this.addResourceOpenHandlers(resource, undefined);
294+
instantiationService.invokeFunction(accessor => {
295+
this._register(hookUpResourceAttachmentDragAndContextMenu(accessor, this.element, resource));
296+
});
293297
}
294298

295299
this.attachClearButton();

‎src/vs/workbench/contrib/chat/browser/chatContentParts/chatAttachmentsContentPart.ts

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ export class ChatAttachmentsContentPart extends Disposable {
2828
private readonly _contextResourceLabels: ResourceLabels;
2929

3030
public contextMenuHandler?: (attachment: IChatRequestVariableEntry, event: MouseEvent) => void;
31-
public dragStartHandler?: (attachment: IChatRequestVariableEntry, event: DragEvent, element: HTMLElement) => void;
3231

3332
constructor(
3433
private readonly variables: IChatRequestVariableEntry[],
@@ -94,7 +93,6 @@ export class ChatAttachmentsContentPart extends Disposable {
9493
}
9594

9695
this._register(dom.addDisposableListener(widget.element, 'contextmenu', e => this.contextMenuHandler?.(attachment, e)));
97-
this._register(dom.addDisposableListener(widget.element, 'dragstart', e => this.dragStartHandler?.(attachment, e, widget.element)));
9896

9997
if (this.attachedContextDisposables.isDisposed) {
10098
widget.dispose();

‎src/vs/workbench/contrib/chat/browser/chatContentParts/chatToolInputOutputContentPart.ts

Lines changed: 15 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,13 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6-
import { DataTransfers } from '../../../../../base/browser/dnd.js';
76
import * as dom from '../../../../../base/browser/dom.js';
87
import { ButtonWithIcon } from '../../../../../base/browser/ui/button/button.js';
9-
import { applyDragImage } from '../../../../../base/browser/ui/dnd/dnd.js';
10-
import { assertNever } from '../../../../../base/common/assert.js';
118
import { VSBuffer } from '../../../../../base/common/buffer.js';
129
import { Codicon } from '../../../../../base/common/codicons.js';
1310
import { Emitter } from '../../../../../base/common/event.js';
1411
import { IMarkdownString } from '../../../../../base/common/htmlContent.js';
1512
import { Disposable } from '../../../../../base/common/lifecycle.js';
16-
import { getExtensionForMimeType } from '../../../../../base/common/mime.js';
1713
import { autorun, ISettableObservable, observableValue } from '../../../../../base/common/observable.js';
1814
import { basename, joinPath } from '../../../../../base/common/resources.js';
1915
import { ThemeIcon } from '../../../../../base/common/themables.js';
@@ -34,9 +30,8 @@ import { ILabelService } from '../../../../../platform/label/common/label.js';
3430
import { INotificationService } from '../../../../../platform/notification/common/notification.js';
3531
import { IProgressService, ProgressLocation } from '../../../../../platform/progress/common/progress.js';
3632
import { IWorkspaceContextService } from '../../../../../platform/workspace/common/workspace.js';
37-
import { fillEditorsDragData } from '../../../../browser/dnd.js';
3833
import { REVEAL_IN_EXPLORER_COMMAND_ID } from '../../../files/browser/fileConstants.js';
39-
import { getAttachableImageExtension, IChatRequestVariableEntry, OmittedState } from '../../common/chatModel.js';
34+
import { getAttachableImageExtension, IChatRequestVariableEntry } from '../../common/chatModel.js';
4035
import { IChatRendererContent } from '../../common/chatViewModel.js';
4136
import { ChatTreeItem, IChatCodeBlockInfo } from '../chat.js';
4237
import { CodeBlockPart, ICodeBlockData, ICodeBlockRenderOptions } from '../codeBlockPart.js';
@@ -57,16 +52,11 @@ export interface IChatCollapsibleIOCodePart {
5752
export interface IChatCollapsibleIODataPart {
5853
kind: 'data';
5954
value: Uint8Array;
60-
mimeType: string;
61-
uri?: URI;
62-
}
63-
64-
export interface IChatCollapsibleIOResourcePart {
65-
kind: 'resource';
55+
mimeType: string | undefined;
6656
uri: URI;
6757
}
6858

69-
export type ChatCollapsibleIOPart = IChatCollapsibleIOCodePart | IChatCollapsibleIODataPart | IChatCollapsibleIOResourcePart;
59+
export type ChatCollapsibleIOPart = IChatCollapsibleIOCodePart | IChatCollapsibleIODataPart;
7060

7161
export interface IChatCollapsibleInputData extends IChatCollapsibleIOCodePart { }
7262
export interface IChatCollapsibleOutputData {
@@ -192,11 +182,11 @@ export class ChatCollapsibleInputOutputContentPart extends Disposable {
192182
continue;
193183
}
194184

195-
const group: (IChatCollapsibleIODataPart | IChatCollapsibleIOResourcePart)[] = [];
185+
const group: IChatCollapsibleIODataPart[] = [];
196186
for (let k = i; k < output.parts.length; k++) {
197187
const part = output.parts[k];
198-
if (!(part.kind === 'data' || part.kind === 'resource')) {
199-
continue;
188+
if (part.kind !== 'data') {
189+
break;
200190
}
201191
group.push(part);
202192
}
@@ -209,21 +199,17 @@ export class ChatCollapsibleInputOutputContentPart extends Disposable {
209199
return contents.root;
210200
}
211201

212-
private addResourceGroup(parts: (IChatCollapsibleIODataPart | IChatCollapsibleIOResourcePart)[], container: HTMLElement) {
202+
private addResourceGroup(parts: IChatCollapsibleIODataPart[], container: HTMLElement) {
213203
const el = dom.h('.chat-collapsible-io-resource-group', [
214204
dom.h('.chat-collapsible-io-resource-items@items'),
215205
dom.h('.chat-collapsible-io-resource-actions@actions'),
216206
]);
217207

218208
const entries = parts.map((part): IChatRequestVariableEntry => {
219-
if (part.kind === 'data' && getAttachableImageExtension(part.mimeType)) {
220-
return { kind: 'image', id: generateUuid(), name: part.uri ? basename(part.uri) : `image.${getAttachableImageExtension(part.mimeType)}`, value: part.value, mimeType: part.mimeType, isURL: false };
221-
} else if (part.kind === 'resource') {
222-
return { kind: 'file', id: generateUuid(), name: basename(part.uri), fullName: part.uri.path, value: part.uri };
223-
} else if (part.kind === 'data') {
224-
return { kind: 'generic', id: generateUuid(), name: localize('chat.unknownData', "Unknown Data"), value: part.value, fullName: localize('chat.unknownData.full', "Unknown Data with MIME type {0}", part.mimeType), omittedState: OmittedState.Full };
209+
if (part.mimeType && getAttachableImageExtension(part.mimeType)) {
210+
return { kind: 'image', id: generateUuid(), name: basename(part.uri), value: part.value, mimeType: part.mimeType, isURL: false, references: [{ kind: 'reference', reference: part.uri }] };
225211
} else {
226-
assertNever(part);
212+
return { kind: 'file', id: generateUuid(), name: basename(part.uri), fullName: part.uri.path, value: part.uri };
227213
}
228214
});
229215

@@ -250,23 +236,6 @@ export class ChatCollapsibleInputOutputContentPart extends Disposable {
250236
}
251237
};
252238

253-
attachments.dragStartHandler = (attachment, event, element) => {
254-
if (!event.dataTransfer) {
255-
return;
256-
}
257-
258-
const index = entries.indexOf(attachment);
259-
const part = parts[index];
260-
if (!part.uri) {
261-
return;
262-
}
263-
264-
applyDragImage(event, element, attachment.name);
265-
event.dataTransfer.effectAllowed = 'copy';
266-
event.dataTransfer.setData(DataTransfers.TEXT, part.uri.toString());
267-
this._instantiationService.invokeFunction(accessor => fillEditorsDragData(accessor, [part.uri!], event));
268-
};
269-
270239
el.items.appendChild(attachments.domNode!);
271240

272241
const toolbar = this._register(this._instantiationService.createInstance(MenuWorkbenchToolBar, el.actions, MenuId.ChatToolOutputResourceToolbar, {
@@ -309,7 +278,7 @@ export class ChatCollapsibleInputOutputContentPart extends Disposable {
309278
}
310279

311280
interface IChatToolOutputResourceToolbarContext {
312-
parts: (IChatCollapsibleIODataPart | IChatCollapsibleIOResourcePart)[];
281+
parts: IChatCollapsibleIODataPart[];
313282
}
314283

315284
class SaveResourcesAction extends Action2 {
@@ -339,11 +308,8 @@ class SaveResourcesAction extends Action2 {
339308
const labelService = accessor.get(ILabelService);
340309
const defaultFilepath = await fileDialog.defaultFilePath();
341310

342-
const partBasename = (part: IChatCollapsibleIODataPart | IChatCollapsibleIOResourcePart) =>
343-
(part.kind === 'resource' || part.uri) ? basename(part.uri!) : ('file' + (getExtensionForMimeType(part.mimeType) || ''));
344-
345-
const savePart = async (part: IChatCollapsibleIODataPart | IChatCollapsibleIOResourcePart, isFolder: boolean, uri: URI) => {
346-
const target = isFolder ? joinPath(uri, partBasename(part)) : uri;
311+
const savePart = async (part: IChatCollapsibleIODataPart, isFolder: boolean, uri: URI) => {
312+
const target = isFolder ? joinPath(uri, basename(part.uri)) : uri;
347313
try {
348314
if (part.kind === 'data') {
349315
await fileService.writeFile(target, VSBuffer.wrap(part.value));
@@ -353,7 +319,7 @@ class SaveResourcesAction extends Action2 {
353319
await fileService.writeFile(target, contents.value);
354320
}
355321
} catch (e) {
356-
notificationService.error(localize('chat.saveResources.error', "Failed to save {0}: {1}", partBasename(part), e));
322+
notificationService.error(localize('chat.saveResources.error', "Failed to save {0}: {1}", basename(part.uri), e));
357323
}
358324
};
359325

@@ -378,7 +344,7 @@ class SaveResourcesAction extends Action2 {
378344

379345
if (context.parts.length === 1) {
380346
const part = context.parts[0];
381-
const uri = await fileDialog.pickFileToSave(joinPath(defaultFilepath, partBasename(part)));
347+
const uri = await fileDialog.pickFileToSave(joinPath(defaultFilepath, basename(part.uri)));
382348
if (!uri) {
383349
return;
384350
}

‎src/vs/workbench/contrib/chat/browser/chatContentParts/toolInvocationParts/chatInputOutputMarkdownProgressPart.ts

Lines changed: 27 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,18 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6-
import { assertNever } from '../../../../../../base/common/assert.js';
76
import { decodeBase64 } from '../../../../../../base/common/buffer.js';
87
import { IMarkdownString } from '../../../../../../base/common/htmlContent.js';
98
import { toDisposable } from '../../../../../../base/common/lifecycle.js';
9+
import { getExtensionForMimeType } from '../../../../../../base/common/mime.js';
1010
import { autorun } from '../../../../../../base/common/observable.js';
11+
import { basename } from '../../../../../../base/common/resources.js';
1112
import { ILanguageService } from '../../../../../../editor/common/languages/language.js';
1213
import { IModelService } from '../../../../../../editor/common/services/model.js';
13-
import { localize } from '../../../../../../nls.js';
1414
import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js';
15-
import { getAttachableImageExtension } from '../../../common/chatModel.js';
15+
import { ChatResponseResource } from '../../../common/chatModel.js';
1616
import { IChatToolInvocation, IChatToolInvocationSerialized } from '../../../common/chatService.js';
17+
import { isResponseVM } from '../../../common/chatViewModel.js';
1718
import { IToolResultInputOutputDetails } from '../../../common/languageModelToolsService.js';
1819
import { IChatCodeBlockInfo } from '../../chat.js';
1920
import { IChatContentPartRenderContext } from '../chatContentParts.js';
@@ -87,9 +88,10 @@ export class ChatInputOutputMarkdownProgressPart extends BaseChatToolInvocationS
8788

8889
let processedOutput = output;
8990
if (typeof output === 'string') { // back compat with older stored versions
90-
processedOutput = [{ type: 'text', value: output }];
91+
processedOutput = [{ value: output, isText: true }];
9192
}
9293

94+
const requestId = isResponseVM(context.element) ? context.element.requestId : context.element.id;
9395
const collapsibleListPart = this._register(instantiationService.createInstance(
9496
ChatCollapsibleInputOutputContentPart,
9597
message,
@@ -98,20 +100,29 @@ export class ChatInputOutputMarkdownProgressPart extends BaseChatToolInvocationS
98100
editorPool,
99101
toCodePart(input),
100102
processedOutput && {
101-
parts: processedOutput.map((o): ChatCollapsibleIOPart => {
102-
if (o.type === 'data') {
103-
const decoded = decodeBase64(o.value64).buffer;
104-
if (getAttachableImageExtension(o.mimeType)) {
105-
return { kind: 'data', value: decoded, mimeType: o.mimeType, uri: o.uri };
106-
} else {
107-
return toCodePart(localize('toolResultData', "Data of type {0} ({1} bytes)", o.mimeType, decoded.byteLength));
108-
}
109-
} else if (o.type === 'text') {
103+
parts: processedOutput.map((o, i): ChatCollapsibleIOPart => {
104+
const permalinkBasename = o.uri
105+
? basename(o.uri)
106+
: o.mimeType && getExtensionForMimeType(o.mimeType)
107+
? `file${getExtensionForMimeType(o.mimeType)}`
108+
: 'file' + (o.isText ? '.txt' : '.bin');
109+
110+
const permalinkUri = ChatResponseResource.createUri(context.element.sessionId, requestId, toolInvocation.toolCallId, i, permalinkBasename);
111+
112+
if (o.isText && !o.asResource) {
110113
return toCodePart(o.value);
111-
} else if (o.type === 'resource') {
112-
return { kind: 'resource', uri: o.uri };
113114
} else {
114-
assertNever(o);
115+
let decoded: Uint8Array | undefined;
116+
try {
117+
if (!o.isText) {
118+
decoded = decodeBase64(o.value).buffer;
119+
}
120+
} catch {
121+
// ignored
122+
}
123+
124+
// Fall back to text if it's not valid base64
125+
return { kind: 'data', value: decoded || new TextEncoder().encode(o.value), mimeType: o.mimeType, uri: permalinkUri };
115126
}
116127
}),
117128
},

‎src/vs/workbench/contrib/chat/browser/languageModelToolsService.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,7 @@ export class LanguageModelToolsService extends Disposable implements ILanguageMo
340340
toolResult ??= { content: [] };
341341
toolResult.toolResultError = err instanceof Error ? err.message : String(err);
342342
if (tool.data.alwaysDisplayInputOutput) {
343-
toolResult.toolResultDetails = { input: this.formatToolInput(dto), output: [{ type: 'text', value: String(err) }], isError: true };
343+
toolResult.toolResultDetails = { input: this.formatToolInput(dto), output: [{ isText: true, value: String(err) }], isError: true };
344344
}
345345

346346
throw err;
@@ -402,11 +402,11 @@ export class LanguageModelToolsService extends Disposable implements ILanguageMo
402402
private toolResultToIO(toolResult: IToolResult): IToolResultInputOutputDetails['output'] {
403403
return toolResult.content.map(part => {
404404
if (part.kind === 'text') {
405-
return { type: 'text', value: part.value };
405+
return { isText: true, value: part.value };
406406
} else if (part.kind === 'promptTsx') {
407-
return { type: 'text', value: stringifyPromptTsxPart(part) };
407+
return { isText: true, value: stringifyPromptTsxPart(part) };
408408
} else if (part.kind === 'data') {
409-
return { type: 'data', value64: encodeBase64(part.value.data), mimeType: part.value.mimeType };
409+
return { value: encodeBase64(part.value.data), mimeType: part.value.mimeType };
410410
} else {
411411
assertNever(part);
412412
}

‎src/vs/workbench/contrib/chat/common/chatModel.ts

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1946,3 +1946,39 @@ export interface IChatAgentEditedFileEvent {
19461946
readonly uri: URI;
19471947
readonly eventKind: ChatRequestEditedFileEventKind;
19481948
}
1949+
1950+
/** URI for a resource embedded in a chat request/response */
1951+
export namespace ChatResponseResource {
1952+
export const scheme = 'vscode-chat-response-resource';
1953+
1954+
export function createUri(sessionId: string, requestId: string, toolCallId: string, index: number, basename?: string): URI {
1955+
return URI.from({
1956+
scheme: ChatResponseResource.scheme,
1957+
authority: sessionId,
1958+
path: `/tool/${requestId}/${toolCallId}/${index}` + (basename ? `/${basename}` : ''),
1959+
});
1960+
}
1961+
1962+
export function parseUri(uri: URI): undefined | { sessionId: string; requestId: string; toolCallId: string; index: number } {
1963+
if (uri.scheme !== ChatResponseResource.scheme) {
1964+
return undefined;
1965+
}
1966+
1967+
const parts = uri.path.split('/');
1968+
if (parts.length < 5) {
1969+
return undefined;
1970+
}
1971+
1972+
const [, kind, requestId, toolCallId, index] = parts;
1973+
if (kind !== 'tool') {
1974+
return undefined;
1975+
}
1976+
1977+
return {
1978+
sessionId: uri.authority,
1979+
requestId: requestId,
1980+
toolCallId: toolCallId,
1981+
index: Number(index),
1982+
};
1983+
}
1984+
}

0 commit comments

Comments
 (0)