Skip to content

Commit 06785d0

Browse files
mjbvzmjbvz
authored
Merge pull request microsoft#255000 from mjbvz/quiet-leopon
Add basic API for extensible rendering inside of chat output
2 parents fe04362 + e74e451 commit 06785d0

19 files changed

+501
-18
lines changed

‎src/vs/platform/extensions/common/extensionsApiProposals.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ const _allApiProposals = {
3737
chatEditing: {
3838
proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatEditing.d.ts',
3939
},
40+
chatOutputRenderer: {
41+
proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatOutputRenderer.d.ts',
42+
},
4043
chatParticipantAdditions: {
4144
proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatParticipantAdditions.d.ts',
4245
},

‎src/vs/workbench/api/browser/extensionHost.contribution.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ import './mainThreadAiEmbeddingVector.js';
9191
import './mainThreadAiSettingsSearch.js';
9292
import './mainThreadMcp.js';
9393
import './mainThreadChatStatus.js';
94+
import './mainThreadChatOutputRenderer.js';
9495
import './mainThreadChatSessions.js';
9596
import './mainThreadDataChannels.js';
9697

‎src/vs/workbench/api/browser/mainThreadChatOutputRenderer.ts

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import { VSBuffer } from '../../../base/common/buffer.js';
7+
import { Disposable, IDisposable } from '../../../base/common/lifecycle.js';
8+
import { URI, UriComponents } from '../../../base/common/uri.js';
9+
import { ExtensionIdentifier } from '../../../platform/extensions/common/extensions.js';
10+
import { IChatOutputRendererService } from '../../contrib/chat/browser/chatOutputItemRenderer.js';
11+
import { IExtHostContext } from '../../services/extensions/common/extHostCustomers.js';
12+
import { ExtHostChatOutputRendererShape, ExtHostContext, MainThreadChatOutputRendererShape } from '../common/extHost.protocol.js';
13+
import { MainThreadWebviews } from './mainThreadWebviews.js';
14+
15+
export class MainThreadChatOutputRenderer extends Disposable implements MainThreadChatOutputRendererShape {
16+
17+
private readonly _proxy: ExtHostChatOutputRendererShape;
18+
19+
private _webviewHandlePool = 0;
20+
21+
private readonly registeredRenderers = new Map<string, IDisposable>();
22+
23+
constructor(
24+
extHostContext: IExtHostContext,
25+
private readonly _mainThreadWebview: MainThreadWebviews,
26+
@IChatOutputRendererService private readonly _rendererService: IChatOutputRendererService,
27+
) {
28+
super();
29+
this._proxy = extHostContext.getProxy(ExtHostContext.ExtHostChatOutputRenderer);
30+
}
31+
32+
override dispose(): void {
33+
super.dispose();
34+
35+
this.registeredRenderers.forEach(disposable => disposable.dispose());
36+
this.registeredRenderers.clear();
37+
}
38+
39+
$registerChatOutputRenderer(mime: string, extensionId: ExtensionIdentifier, extensionLocation: UriComponents): void {
40+
this._rendererService.registerRenderer(mime, {
41+
renderOutputPart: async (mime, data, webview, token) => {
42+
const webviewHandle = `chat-output-${++this._webviewHandlePool}`;
43+
44+
this._mainThreadWebview.addWebview(webviewHandle, webview, {
45+
serializeBuffersForPostMessage: true,
46+
});
47+
48+
this._proxy.$renderChatOutput(mime, VSBuffer.wrap(data), webviewHandle, token);
49+
},
50+
}, {
51+
extension: { id: extensionId, location: URI.revive(extensionLocation) }
52+
});
53+
}
54+
55+
$unregisterChatOutputRenderer(mime: string): void {
56+
this.registeredRenderers.get(mime)?.dispose();
57+
}
58+
}

‎src/vs/workbench/api/browser/mainThreadWebviewManager.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import { MainThreadWebviews } from './mainThreadWebviews.js';
1111
import { MainThreadWebviewsViews } from './mainThreadWebviewViews.js';
1212
import * as extHostProtocol from '../common/extHost.protocol.js';
1313
import { extHostCustomer, IExtHostContext } from '../../services/extensions/common/extHostCustomers.js';
14+
import { MainThreadChatOutputRenderer } from './mainThreadChatOutputRenderer.js';
1415

1516
@extHostCustomer
1617
export class MainThreadWebviewManager extends Disposable {
@@ -31,5 +32,8 @@ export class MainThreadWebviewManager extends Disposable {
3132

3233
const webviewViews = this._register(instantiationService.createInstance(MainThreadWebviewsViews, context, webviews));
3334
context.set(extHostProtocol.MainContext.MainThreadWebviewViews, webviewViews);
35+
36+
const chatOutputRenderers = this._register(instantiationService.createInstance(MainThreadChatOutputRenderer, context, webviews));
37+
context.set(extHostProtocol.MainContext.MainThreadChatOutputRenderer, chatOutputRenderers);
3438
}
3539
}

‎src/vs/workbench/api/browser/mainThreadWebviews.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,11 @@ import { localize } from '../../../nls.js';
1313
import { ExtensionIdentifier } from '../../../platform/extensions/common/extensions.js';
1414
import { IOpenerService } from '../../../platform/opener/common/opener.js';
1515
import { IProductService } from '../../../platform/product/common/productService.js';
16-
import * as extHostProtocol from '../common/extHost.protocol.js';
17-
import { deserializeWebviewMessage, serializeWebviewMessage } from '../common/extHostWebviewMessaging.js';
18-
import { IOverlayWebview, IWebview, WebviewContentOptions, WebviewExtensionDescription } from '../../contrib/webview/browser/webview.js';
16+
import { IWebview, WebviewContentOptions, WebviewExtensionDescription } from '../../contrib/webview/browser/webview.js';
1917
import { IExtHostContext } from '../../services/extensions/common/extHostCustomers.js';
2018
import { SerializableObjectWithBuffers } from '../../services/extensions/common/proxyIdentifier.js';
19+
import * as extHostProtocol from '../common/extHost.protocol.js';
20+
import { deserializeWebviewMessage, serializeWebviewMessage } from '../common/extHostWebviewMessaging.js';
2121

2222
export class MainThreadWebviews extends Disposable implements extHostProtocol.MainThreadWebviewsShape {
2323

@@ -43,7 +43,7 @@ export class MainThreadWebviews extends Disposable implements extHostProtocol.Ma
4343
this._proxy = context.getProxy(extHostProtocol.ExtHostContext.ExtHostWebviews);
4444
}
4545

46-
public addWebview(handle: extHostProtocol.WebviewHandle, webview: IOverlayWebview, options: { serializeBuffersForPostMessage: boolean }): void {
46+
public addWebview(handle: extHostProtocol.WebviewHandle, webview: IWebview, options: { serializeBuffersForPostMessage: boolean }): void {
4747
if (this._webviews.has(handle)) {
4848
throw new Error('Webview already registered');
4949
}
@@ -72,7 +72,7 @@ export class MainThreadWebviews extends Disposable implements extHostProtocol.Ma
7272
return webview.postMessage(message, arrayBuffers);
7373
}
7474

75-
private hookupWebviewEventDelegate(handle: extHostProtocol.WebviewHandle, webview: IOverlayWebview, options: { serializeBuffersForPostMessage: boolean }) {
75+
private hookupWebviewEventDelegate(handle: extHostProtocol.WebviewHandle, webview: IWebview, options: { serializeBuffersForPostMessage: boolean }) {
7676
const disposables = new DisposableStore();
7777

7878
disposables.add(webview.onDidClickLink((uri) => this.onDidClickLink(handle, uri)));

‎src/vs/workbench/api/common/extHost.api.impl.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ import { IExtHostWindow } from './extHostWindow.js';
113113
import { IExtHostWorkspace } from './extHostWorkspace.js';
114114
import { ExtHostAiSettingsSearch } from './extHostAiSettingsSearch.js';
115115
import { ExtHostChatSessions } from './extHostChatSessions.js';
116+
import { ExtHostChatOutputRenderer } from './extHostChatOutputRenderer.js';
116117

117118
export interface IExtensionRegistries {
118119
mine: ExtensionDescriptionRegistry;
@@ -220,6 +221,7 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
220221
const extHostTesting = rpcProtocol.set(ExtHostContext.ExtHostTesting, accessor.get(IExtHostTesting));
221222
const extHostUriOpeners = rpcProtocol.set(ExtHostContext.ExtHostUriOpeners, new ExtHostUriOpeners(rpcProtocol));
222223
const extHostProfileContentHandlers = rpcProtocol.set(ExtHostContext.ExtHostProfileContentHandlers, new ExtHostProfileContentHandlers(rpcProtocol));
224+
const extHostChatOutputRenderer = rpcProtocol.set(ExtHostContext.ExtHostChatOutputRenderer, new ExtHostChatOutputRenderer(rpcProtocol, extHostWebviews));
223225
rpcProtocol.set(ExtHostContext.ExtHostInteractive, new ExtHostInteractive(rpcProtocol, extHostNotebook, extHostDocumentsAndEditors, extHostCommands, extHostLogService));
224226
const extHostLanguageModelTools = rpcProtocol.set(ExtHostContext.ExtHostLanguageModelTools, new ExtHostLanguageModelTools(rpcProtocol, extHostLanguageModels));
225227
const extHostChatAgents2 = rpcProtocol.set(ExtHostContext.ExtHostChatAgents2, new ExtHostChatAgents2(rpcProtocol, extHostLogService, extHostCommands, extHostDocuments, extHostLanguageModels, extHostDiagnostics, extHostLanguageModelTools));
@@ -1505,10 +1507,14 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
15051507
checkProposedApiEnabled(extension, 'chatParticipantPrivate');
15061508
return _asExtensionEvent(extHostChatAgents2.onDidDisposeChatSession)(listeners, thisArgs, disposables);
15071509
},
1508-
registerChatSessionItemProvider(chatSessionType: string, provider: vscode.ChatSessionItemProvider) {
1510+
registerChatSessionItemProvider: (chatSessionType: string, provider: vscode.ChatSessionItemProvider) => {
15091511
checkProposedApiEnabled(extension, 'chatSessionsProvider');
15101512
return extHostChatSessions.registerChatSessionItemProvider(chatSessionType, provider);
15111513
},
1514+
registerChatOutputRenderer: (mime: string, renderer: vscode.ChatOutputRenderer) => {
1515+
checkProposedApiEnabled(extension, 'chatOutputRenderer');
1516+
return extHostChatOutputRenderer.registerChatOutputRenderer(extension, mime, renderer);
1517+
},
15121518
};
15131519

15141520
// namespace: lm

‎src/vs/workbench/api/common/extHost.protocol.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1452,6 +1452,15 @@ export interface ExtHostUriOpenersShape {
14521452
$openUri(id: string, context: { resolvedUri: UriComponents; sourceUri: UriComponents }, token: CancellationToken): Promise<void>;
14531453
}
14541454

1455+
export interface MainThreadChatOutputRendererShape extends IDisposable {
1456+
$registerChatOutputRenderer(mime: string, extensionId: ExtensionIdentifier, extensionLocation: UriComponents): void;
1457+
$unregisterChatOutputRenderer(mime: string): void;
1458+
}
1459+
1460+
export interface ExtHostChatOutputRendererShape {
1461+
$renderChatOutput(mime: string, valueData: VSBuffer, webviewHandle: string, token: CancellationToken): Promise<void>;
1462+
}
1463+
14551464
export interface MainThreadProfileContentHandlersShape {
14561465
$registerProfileContentHandler(id: string, name: string, description: string | undefined, extensionId: string): Promise<void>;
14571466
$unregisterProfileContentHandler(id: string): Promise<void>;
@@ -3188,6 +3197,7 @@ export const MainContext = {
31883197
MainThreadAiSettingsSearch: createProxyIdentifier<MainThreadAiSettingsSearchShape>('MainThreadAiSettingsSearch'),
31893198
MainThreadDataChannels: createProxyIdentifier<MainThreadDataChannelsShape>('MainThreadDataChannels'),
31903199
MainThreadChatSessions: createProxyIdentifier<MainThreadChatSessionsShape>('MainThreadChatSessions'),
3200+
MainThreadChatOutputRenderer: createProxyIdentifier<MainThreadChatOutputRendererShape>('MainThreadChatOutputRenderer'),
31913201
};
31923202

31933203
export const ExtHostContext = {
@@ -3233,6 +3243,7 @@ export const ExtHostContext = {
32333243
ExtHostStorage: createProxyIdentifier<ExtHostStorageShape>('ExtHostStorage'),
32343244
ExtHostUrls: createProxyIdentifier<ExtHostUrlsShape>('ExtHostUrls'),
32353245
ExtHostUriOpeners: createProxyIdentifier<ExtHostUriOpenersShape>('ExtHostUriOpeners'),
3246+
ExtHostChatOutputRenderer: createProxyIdentifier<ExtHostChatOutputRendererShape>('ExtHostChatOutputRenderer'),
32363247
ExtHostProfileContentHandlers: createProxyIdentifier<ExtHostProfileContentHandlersShape>('ExtHostProfileContentHandlers'),
32373248
ExtHostOutputService: createProxyIdentifier<ExtHostOutputServiceShape>('ExtHostOutputService'),
32383249
ExtHostLabelService: createProxyIdentifier<ExtHostLabelServiceShape>('ExtHostLabelService'),

‎src/vs/workbench/api/common/extHostChatOutputRenderer.ts

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import type * as vscode from 'vscode';
7+
import { CancellationToken } from '../../../base/common/cancellation.js';
8+
import { ExtHostChatOutputRendererShape, IMainContext, MainContext, MainThreadChatOutputRendererShape } from './extHost.protocol.js';
9+
import { Disposable } from './extHostTypes.js';
10+
import { ExtHostWebviews } from './extHostWebview.js';
11+
import { IExtensionDescription } from '../../../platform/extensions/common/extensions.js';
12+
import { VSBuffer } from '../../../base/common/buffer.js';
13+
14+
export class ExtHostChatOutputRenderer implements ExtHostChatOutputRendererShape {
15+
16+
private readonly _proxy: MainThreadChatOutputRendererShape;
17+
18+
private readonly _renderers = new Map</*mime*/ string, {
19+
readonly renderer: vscode.ChatOutputRenderer;
20+
readonly extension: IExtensionDescription;
21+
}>();
22+
23+
constructor(
24+
mainContext: IMainContext,
25+
private readonly webviews: ExtHostWebviews,
26+
) {
27+
this._proxy = mainContext.getProxy(MainContext.MainThreadChatOutputRenderer);
28+
}
29+
30+
registerChatOutputRenderer(extension: IExtensionDescription, mime: string, renderer: vscode.ChatOutputRenderer): vscode.Disposable {
31+
if (this._renderers.has(mime)) {
32+
throw new Error(`Chat output renderer already registered for mime type: ${mime}`);
33+
}
34+
35+
this._renderers.set(mime, { extension, renderer });
36+
this._proxy.$registerChatOutputRenderer(mime, extension.identifier, extension.extensionLocation);
37+
38+
return new Disposable(() => {
39+
this._renderers.delete(mime);
40+
this._proxy.$unregisterChatOutputRenderer(mime);
41+
});
42+
}
43+
44+
async $renderChatOutput(mime: string, valueData: VSBuffer, webviewHandle: string, token: CancellationToken): Promise<void> {
45+
const entry = this._renderers.get(mime);
46+
if (!entry) {
47+
throw new Error(`No chat output renderer registered for mime type: ${mime}`);
48+
}
49+
50+
const webview = this.webviews.createNewWebview(webviewHandle, {}, entry.extension);
51+
return entry.renderer.renderChatOutput(valueData.buffer, webview, {}, token);
52+
}
53+
}

‎src/vs/workbench/api/common/extHostTypeConverters.ts

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ import { IChatAgentRequest, IChatAgentResult } from '../../contrib/chat/common/c
4242
import { IChatRequestDraft } from '../../contrib/chat/common/chatEditingService.js';
4343
import { IChatRequestVariableEntry, isImageVariableEntry } from '../../contrib/chat/common/chatVariableEntries.js';
4444
import { IChatAgentMarkdownContentWithVulnerability, IChatCodeCitation, IChatCommandButton, IChatConfirmation, IChatContentInlineReference, IChatContentReference, IChatExtensionsContent, IChatFollowup, IChatMarkdownContent, IChatMoveMessage, IChatPrepareToolInvocationPart, IChatProgressMessage, IChatResponseCodeblockUriPart, IChatTaskDto, IChatTaskResult, IChatTextEdit, IChatTreeData, IChatUserActionEvent, IChatWarningMessage } from '../../contrib/chat/common/chatService.js';
45-
import { IToolResult, ToolDataSource } from '../../contrib/chat/common/languageModelToolsService.js';
45+
import { IToolResult, IToolResultInputOutputDetails, IToolResultOutputDetails, ToolDataSource } from '../../contrib/chat/common/languageModelToolsService.js';
4646
import * as chatProvider from '../../contrib/chat/common/languageModels.js';
4747
import { IChatMessageDataPart, IChatResponseDataPart, IChatResponsePromptTsxPart, IChatResponseTextPart } from '../../contrib/chat/common/languageModels.js';
4848
import { DebugTreeItemCollapsibleState, IDebugVisualizationTreeItem } from '../../contrib/debug/common/debug.js';
@@ -3346,12 +3346,30 @@ export namespace LanguageModelToolResult2 {
33463346
}));
33473347
}
33483348

3349-
export function from(result: vscode.ExtendedLanguageModelToolResult, extension: IExtensionDescription): Dto<IToolResult> | SerializableObjectWithBuffers<Dto<IToolResult>> {
3349+
export function from(result: vscode.ExtendedLanguageModelToolResult2, extension: IExtensionDescription): Dto<IToolResult> | SerializableObjectWithBuffers<Dto<IToolResult>> {
33503350
if (result.toolResultMessage) {
33513351
checkProposedApiEnabled(extension, 'chatParticipantPrivate');
33523352
}
33533353

33543354
let hasBuffers = false;
3355+
let detailsDto: Dto<Array<URI | types.Location> | IToolResultInputOutputDetails | IToolResultOutputDetails | undefined> = undefined;
3356+
if (Array.isArray(result.toolResultDetails)) {
3357+
detailsDto = result.toolResultDetails?.map(detail => {
3358+
return URI.isUri(detail) ? detail : Location.from(detail as vscode.Location);
3359+
});
3360+
} else {
3361+
if (result.toolResultDetails2) {
3362+
detailsDto = {
3363+
output: {
3364+
type: 'data',
3365+
mimeType: (result.toolResultDetails2 as vscode.ToolResultDataOutput).mime,
3366+
value: VSBuffer.wrap((result.toolResultDetails2 as vscode.ToolResultDataOutput).value),
3367+
}
3368+
} satisfies IToolResultOutputDetails;
3369+
hasBuffers = true;
3370+
}
3371+
}
3372+
33553373
const dto: Dto<IToolResult> = {
33563374
content: result.content.map(item => {
33573375
if (item instanceof types.LanguageModelTextPart) {
@@ -3378,7 +3396,7 @@ export namespace LanguageModelToolResult2 {
33783396
}
33793397
}),
33803398
toolResultMessage: MarkdownString.fromStrict(result.toolResultMessage),
3381-
toolResultDetails: result.toolResultDetails?.map(detail => URI.isUri(detail) ? detail : Location.from(detail as vscode.Location)),
3399+
toolResultDetails: detailsDto,
33823400
};
33833401

33843402
return hasBuffers ? new SerializableObjectWithBuffers(dto) : dto;

‎src/vs/workbench/contrib/chat/browser/chat.contribution.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ import { ChatAttachmentResolveService, IChatAttachmentResolveService } from './c
115115
import { registerLanguageModelActions } from './actions/chatLanguageModelActions.js';
116116
import { PromptUrlHandler } from './promptSyntax/promptUrlHandler.js';
117117
import { ChatTaskServiceImpl, IChatTasksService } from '../common/chatTasksService.js';
118+
import { ChatOutputRendererService, IChatOutputRendererService } from './chatOutputItemRenderer.js';
118119

119120
// Register configuration
120121
const configurationRegistry = Registry.as<IConfigurationRegistry>(ConfigurationExtensions.Configuration);
@@ -803,6 +804,7 @@ registerSingleton(IChatContextPickService, ChatContextPickService, Instantiation
803804
registerSingleton(IChatModeService, ChatModeService, InstantiationType.Delayed);
804805
registerSingleton(IChatAttachmentResolveService, ChatAttachmentResolveService, InstantiationType.Delayed);
805806
registerSingleton(IChatTasksService, ChatTaskServiceImpl, InstantiationType.Delayed);
807+
registerSingleton(IChatOutputRendererService, ChatOutputRendererService, InstantiationType.Delayed);
806808

807809
registerWorkbenchContribution2(ChatEditingNotebookFileSystemProviderContrib.ID, ChatEditingNotebookFileSystemProviderContrib, WorkbenchPhase.BlockStartup);
808810

0 commit comments

Comments
 (0)