Change ChatAgent history to include real requests, response data, variables, etc. (microsoft#202541)

* Change ChatAgent history to include real requests, response data, variables, etc.
For microsoft#199908

* Tweak

* Update snapshots

* Clean up

* Update addCompleteRequest

* Filter out other progress types

* Fix chat slash command data in history

* Don't include slashCommand in history request at all, since it's deprecated anyway

* undefined

Author: roblourens

src/vs/workbench/api/browser/mainThreadChatAgents2.ts

@@ -86,7 +86,7 @@ export class MainThreadChatAgents2 extends Disposable implements MainThreadChatA
 			invoke: async (request, progress, history, token) => {
 				this._pendingProgress.set(request.requestId, progress);
 				try {
-					return await this._proxy.$invokeAgent(handle, request.sessionId, request.requestId, request, { history }, token) ?? {};
+					return await this._proxy.$invokeAgent(handle, request, { history }, token) ?? {};
 				} finally {
 					this._pendingProgress.delete(request.requestId);
 				}

src/vs/workbench/api/common/extHost.protocol.ts

@@ -51,6 +51,7 @@ import { SaveReason } from 'vs/workbench/common/editor';
 import { IRevealOptions, ITreeItem, IViewBadge } from 'vs/workbench/common/views';
 import { CallHierarchyItem } from 'vs/workbench/contrib/callHierarchy/common/callHierarchy';
 import { IChatAgentCommand, IChatAgentMetadata, IChatAgentRequest, IChatAgentResult } from 'vs/workbench/contrib/chat/common/chatAgents';
+import { IChatProgressResponseContent } from 'vs/workbench/contrib/chat/common/chatModel';
 import { IChatMessage, IChatResponseFragment, IChatResponseProviderMetadata } from 'vs/workbench/contrib/chat/common/chatProvider';
 import { IChatAsyncContent, IChatDynamicRequest, IChatFollowup, IChatProgress, IChatReplyFollowup, IChatResponseErrorDetails, IChatUserActionEvent, InteractiveSessionVoteDirection } from 'vs/workbench/contrib/chat/common/chatService';
 import { IChatRequestVariableValue, IChatVariableData } from 'vs/workbench/contrib/chat/common/chatVariables';
@@ -1201,8 +1202,18 @@ export interface IChatAgentCompletionItem {
 	documentation?: string | IMarkdownString;
 }
 
+export type IChatContentProgressDto =
+	| Dto<Exclude<IChatProgressResponseContent, IChatAsyncContent>>
+	| IChatAsyncContentDto;
+
+export type IChatAgentHistoryEntryDto = {
+	request: IChatAgentRequest;
+	response: ReadonlyArray<IChatContentProgressDto>;
+	result: IChatAgentResult;
+};
+
 export interface ExtHostChatAgentsShape2 {
-	$invokeAgent(handle: number, sessionId: string, requestId: string, request: IChatAgentRequest, context: { history: IChatMessage[] }, token: CancellationToken): Promise<IChatAgentResult | undefined>;
+	$invokeAgent(handle: number, request: IChatAgentRequest, context: { history: IChatAgentHistoryEntryDto[] }, token: CancellationToken): Promise<IChatAgentResult | undefined>;
 	$provideSlashCommands(handle: number, token: CancellationToken): Promise<IChatAgentCommand[]>;
 	$provideFollowups(handle: number, sessionId: string, token: CancellationToken): Promise<IChatFollowup[]>;
 	$acceptFeedback(handle: number, sessionId: string, requestId: string, vote: InteractiveSessionVoteDirection, reportIssue?: boolean): void;

src/vs/workbench/api/common/extHostChatAgents2.ts

@@ -3,6 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import { coalesce } from 'vs/base/common/arrays';
 import { DeferredPromise, raceCancellation } from 'vs/base/common/async';
 import { CancellationToken } from 'vs/base/common/cancellation';
 import { toErrorMessage } from 'vs/base/common/errorMessage';
@@ -14,12 +15,11 @@ import { localize } from 'vs/nls';
 import { IExtensionDescription } from 'vs/platform/extensions/common/extensions';
 import { ILogService } from 'vs/platform/log/common/log';
 import { Progress } from 'vs/platform/progress/common/progress';
-import { ExtHostChatAgentsShape2, IChatAgentCompletionItem, IMainContext, MainContext, MainThreadChatAgentsShape2 } from 'vs/workbench/api/common/extHost.protocol';
+import { ExtHostChatAgentsShape2, IChatAgentCompletionItem, IChatAgentHistoryEntryDto, IMainContext, MainContext, MainThreadChatAgentsShape2 } from 'vs/workbench/api/common/extHost.protocol';
 import { ExtHostChatProvider } from 'vs/workbench/api/common/extHostChatProvider';
 import * as typeConvert from 'vs/workbench/api/common/extHostTypeConverters';
 import * as extHostTypes from 'vs/workbench/api/common/extHostTypes';
 import { IChatAgentCommand, IChatAgentRequest, IChatAgentResult } from 'vs/workbench/contrib/chat/common/chatAgents';
-import { IChatMessage } from 'vs/workbench/contrib/chat/common/chatProvider';
 import { IChatFollowup, IChatUserActionEvent, InteractiveSessionVoteDirection } from 'vs/workbench/contrib/chat/common/chatService';
 import { checkProposedApiEnabled, isProposedApiEnabled } from 'vs/workbench/services/extensions/common/extensions';
 import type * as vscode from 'vscode';
@@ -51,10 +51,10 @@ export class ExtHostChatAgents2 implements ExtHostChatAgentsShape2 {
 		return agent.apiAgent;
 	}
 
-	async $invokeAgent(handle: number, sessionId: string, requestId: string, request: IChatAgentRequest, context: { history: IChatMessage[] }, token: CancellationToken): Promise<IChatAgentResult | undefined> {
+	async $invokeAgent(handle: number, request: IChatAgentRequest, context: { history: IChatAgentHistoryEntryDto[] }, token: CancellationToken): Promise<IChatAgentResult | undefined> {
 		// Clear the previous result so that $acceptFeedback or $acceptAction during a request will be ignored.
 		// We may want to support sending those during a request.
-		this._previousResultMap.delete(sessionId);
+		this._previousResultMap.delete(request.sessionId);
 
 		const agent = this._agents.get(handle);
 		if (!agent) {
@@ -79,13 +79,10 @@ export class ExtHostChatAgents2 implements ExtHostChatAgentsShape2 {
 		const stopWatch = StopWatch.create(false);
 		let firstProgress: number | undefined;
 		try {
+			const convertedHistory = await this.prepareHistory(agent, request, context);
 			const task = agent.invoke(
-				{
-					prompt: request.message,
-					variables: typeConvert.ChatVariable.objectTo(request.variables),
-					slashCommand
-				},
-				{ history: context.history.map(typeConvert.ChatMessage.to) },
+				typeConvert.ChatAgentRequest.to(request, slashCommand),
+				{ history: convertedHistory },
 				new Progress<vscode.ChatAgentExtendedProgress>(progress => {
 					throwIfDone();
 
@@ -101,7 +98,7 @@ export class ExtHostChatAgents2 implements ExtHostChatAgentsShape2 {
 					}
 
 					if ('placeholder' in progress && 'resolvedContent' in progress) {
-						const resolvedContent = Promise.all([this._proxy.$handleProgressChunk(requestId, convertedProgress), progress.resolvedContent]);
+						const resolvedContent = Promise.all([this._proxy.$handleProgressChunk(request.requestId, convertedProgress), progress.resolvedContent]);
 						raceCancellation(resolvedContent, token).then(res => {
 							if (!res) {
 								return; /* Cancelled */
@@ -113,29 +110,29 @@ export class ExtHostChatAgents2 implements ExtHostChatAgentsShape2 {
 								return;
 							}
 
-							this._proxy.$handleProgressChunk(requestId, convertedContent, progressHandle ?? undefined);
+							this._proxy.$handleProgressChunk(request.requestId, convertedContent, progressHandle ?? undefined);
 						});
 					} else {
-						this._proxy.$handleProgressChunk(requestId, convertedProgress);
+						this._proxy.$handleProgressChunk(request.requestId, convertedProgress);
 					}
 				}),
 				token
 			);
 
 			return await raceCancellation(Promise.resolve(task).then((result) => {
 				if (result) {
-					this._previousResultMap.set(sessionId, result);
-					let sessionResults = this._resultsBySessionAndRequestId.get(sessionId);
+					this._previousResultMap.set(request.sessionId, result);
+					let sessionResults = this._resultsBySessionAndRequestId.get(request.sessionId);
 					if (!sessionResults) {
 						sessionResults = new Map();
-						this._resultsBySessionAndRequestId.set(sessionId, sessionResults);
+						this._resultsBySessionAndRequestId.set(request.sessionId, sessionResults);
 					}
-					sessionResults.set(requestId, result);
+					sessionResults.set(request.requestId, result);
 
 					const timings = { firstProgress: firstProgress, totalElapsed: stopWatch.elapsed() };
 					return { errorDetails: result.errorDetails, timings };
 				} else {
-					this._previousResultMap.delete(sessionId);
+					this._previousResultMap.delete(request.sessionId);
 				}
 
 				return undefined;
@@ -151,6 +148,19 @@ export class ExtHostChatAgents2 implements ExtHostChatAgentsShape2 {
 		}
 	}
 
+	private async prepareHistory<T extends vscode.ChatAgentResult2>(agent: ExtHostChatAgent<T>, request: IChatAgentRequest, context: { history: IChatAgentHistoryEntryDto[] }): Promise<vscode.ChatAgentHistoryEntry[]> {
+		return coalesce(await Promise.all(context.history
+			.map(async h => {
+				const result = request.agentId === h.request.agentId && this._resultsBySessionAndRequestId.get(request.sessionId)?.get(h.request.requestId)
+					|| h.result;
+				return {
+					request: typeConvert.ChatAgentRequest.to(h.request, undefined),
+					response: coalesce(h.response.map(r => typeConvert.ChatResponseProgress.toProgressContent(r))),
+					result
+				} satisfies vscode.ChatAgentHistoryEntry;
+			})));
+	}
+
 	$releaseSession(sessionId: string): void {
 		this._previousResultMap.delete(sessionId);
 		this._resultsBySessionAndRequestId.delete(sessionId);
@@ -248,7 +258,7 @@ class ExtHostChatAgent<TResult extends vscode.ChatAgentResult2> {
 
 	constructor(
 		public readonly extension: IExtensionDescription,
-		private readonly _id: string,
+		public readonly id: string,
 		private readonly _proxy: MainThreadChatAgentsShape2,
 		private readonly _handle: number,
 		private readonly _callback: vscode.ChatAgentExtendedHandler,
@@ -352,7 +362,7 @@ class ExtHostChatAgent<TResult extends vscode.ChatAgentResult2> {
 		const that = this;
 		return {
 			get name() {
-				return that._id;
+				return that.id;
 			},
 			get description() {
 				return that._description ?? '';

src/vs/workbench/api/common/extHostTypeConverters.ts

@@ -11,7 +11,7 @@ import * as htmlContent from 'vs/base/common/htmlContent';
 import { DisposableStore } from 'vs/base/common/lifecycle';
 import { ResourceMap, ResourceSet } from 'vs/base/common/map';
 import { marked } from 'vs/base/common/marked/marked';
-import { parse } from 'vs/base/common/marshalling';
+import { parse, revive } from 'vs/base/common/marshalling';
 import { Mimes } from 'vs/base/common/mime';
 import { cloneAndChange } from 'vs/base/common/objects';
 import { isEmptyObject, isNumber, isString, isUndefinedOrNull } from 'vs/base/common/types';
@@ -34,6 +34,7 @@ import * as extHostProtocol from 'vs/workbench/api/common/extHost.protocol';
 import { getPrivateApiFor } from 'vs/workbench/api/common/extHostTestingPrivateApi';
 import { DEFAULT_EDITOR_ASSOCIATION, SaveReason } from 'vs/workbench/common/editor';
 import { IViewBadge } from 'vs/workbench/common/views';
+import { IChatAgentRequest } from 'vs/workbench/contrib/chat/common/chatAgents';
 import * as chatProvider from 'vs/workbench/contrib/chat/common/chatProvider';
 import { IChatFollowup, IChatReplyFollowup, IChatResponseCommandFollowup } from 'vs/workbench/contrib/chat/common/chatService';
 import { IChatRequestVariableValue } from 'vs/workbench/contrib/chat/common/chatVariables';
@@ -46,6 +47,7 @@ import { CoverageDetails, DetailType, ICoveredCount, IFileCoverage, ISerializedT
 import { EditorGroupColumn } from 'vs/workbench/services/editor/common/editorGroupColumn';
 import { ACTIVE_GROUP, SIDE_GROUP } from 'vs/workbench/services/editor/common/editorService';
 import { checkProposedApiEnabled } from 'vs/workbench/services/extensions/common/extensions';
+import { Dto } from 'vs/workbench/services/extensions/common/proxyIdentifier';
 import type * as vscode from 'vscode';
 import * as types from './extHostTypes';
 
@@ -120,6 +122,12 @@ export namespace Range {
 	}
 }
 
+export namespace Location {
+	export function to(location: Dto<languages.Location>): vscode.Location {
+		return new types.Location(URI.revive(location.uri), Range.to(location.range));
+	}
+}
+
 export namespace TokenType {
 	export function to(type: encodedTokenAttributes.StandardTokenType): types.StandardTokenType {
 		switch (type) {
@@ -2368,6 +2376,69 @@ export namespace ChatResponseProgress {
 			return undefined;
 		}
 	}
+
+	export function to(progress: extHostProtocol.IChatProgressDto): vscode.ChatAgentProgress | undefined {
+		switch (progress.kind) {
+			case 'markdownContent':
+			case 'inlineReference':
+			case 'treeData':
+				return ChatResponseProgress.to(progress);
+			case 'content':
+				return { content: progress.content };
+			case 'usedContext':
+				return { documents: progress.documents.map(d => ({ uri: URI.revive(d.uri), version: d.version, ranges: d.ranges.map(r => Range.to(r)) })) };
+			case 'reference':
+				return {
+					reference:
+						isUriComponents(progress.reference) ?
+							URI.revive(progress.reference) :
+							Location.to(progress.reference)
+				};
+			case 'agentDetection':
+				// For simplicity, don't sent back the 'extended' types
+				return undefined;
+			case 'progressMessage':
+				return { message: progress.content.value };
+			case 'vulnerability':
+				return { content: progress.content, vulnerabilities: progress.vulnerabilities };
+			default:
+				// Unknown type, eg something in history that was removed? Ignore
+				return undefined;
+		}
+	}
+
+	export function toProgressContent(progress: extHostProtocol.IChatContentProgressDto): vscode.ChatAgentContentProgress | undefined {
+		switch (progress.kind) {
+			case 'markdownContent':
+				// For simplicity, don't sent back the 'extended' types, so downgrade markdown to just some text
+				return { content: progress.content.value };
+			case 'inlineReference':
+				return {
+					inlineReference:
+						isUriComponents(progress.inlineReference) ?
+							URI.revive(progress.inlineReference) :
+							Location.to(progress.inlineReference),
+					title: progress.name
+				};
+			case 'treeData':
+				return { treeData: revive(progress.treeData) };
+			default:
+				// Unknown type, eg something in history that was removed? Ignore
+				return undefined;
+		}
+	}
+}
+
+export namespace ChatAgentRequest {
+	export function to(request: IChatAgentRequest, slashCommand: vscode.ChatAgentSlashCommand | undefined): vscode.ChatAgentRequest {
+		return {
+			prompt: request.message,
+			variables: ChatVariable.objectTo(request.variables),
+			slashCommand,
+			subCommand: request.command,
+			agentId: request.agentId,
+		};
+	}
 }
 
 export namespace ChatAgentCompletionItem {

src/vs/workbench/contrib/chat/browser/chatQuick.ts

@@ -292,6 +292,7 @@ class QuickChat extends Disposable {
 			if (request.response?.response.value || request.response?.errorDetails) {
 				this.chatService.addCompleteRequest(widget.viewModel.sessionId,
 					request.message as IParsedChatRequest,
+					request.variableData,
 					{
 						message: request.response.response.value,
 						errorDetails: request.response.errorDetails,

src/vs/workbench/contrib/chat/browser/chatVariables.ts

@@ -9,9 +9,9 @@ import { Iterable } from 'vs/base/common/iterator';
 import { IDisposable, toDisposable } from 'vs/base/common/lifecycle';
 import { IChatWidgetService } from 'vs/workbench/contrib/chat/browser/chat';
 import { ChatDynamicVariableModel } from 'vs/workbench/contrib/chat/browser/contrib/chatDynamicVariables';
-import { IChatModel } from 'vs/workbench/contrib/chat/common/chatModel';
+import { IChatModel, IChatRequestVariableData } from 'vs/workbench/contrib/chat/common/chatModel';
 import { IParsedChatRequest, ChatRequestVariablePart, ChatRequestDynamicVariablePart } from 'vs/workbench/contrib/chat/common/chatParserTypes';
-import { IChatVariablesService, IChatRequestVariableValue, IChatVariableData, IChatVariableResolver, IChatVariableResolveResult, IDynamicVariable } from 'vs/workbench/contrib/chat/common/chatVariables';
+import { IChatVariablesService, IChatRequestVariableValue, IChatVariableData, IChatVariableResolver, IDynamicVariable } from 'vs/workbench/contrib/chat/common/chatVariables';
 
 interface IChatData {
 	data: IChatVariableData;
@@ -28,7 +28,7 @@ export class ChatVariablesService implements IChatVariablesService {
 	) {
 	}
 
-	async resolveVariables(prompt: IParsedChatRequest, model: IChatModel, token: CancellationToken): Promise<IChatVariableResolveResult> {
+	async resolveVariables(prompt: IParsedChatRequest, model: IChatModel, token: CancellationToken): Promise<IChatRequestVariableData> {
 		const resolvedVariables: Record<string, IChatRequestVariableValue[]> = {};
 		const jobs: Promise<any>[] = [];
 
@@ -62,7 +62,7 @@ export class ChatVariablesService implements IChatVariablesService {
 
 		return {
 			variables: resolvedVariables,
-			prompt: parsedPrompt.join('').trim()
+			message: parsedPrompt.join('').trim()
 		};
 	}
 

src/vs/workbench/contrib/chat/browser/chatWidget.ts

@@ -539,7 +539,7 @@ export class ChatWidget extends Disposable implements IChatWidget {
 			const input = !opts ? editorValue :
 				'query' in opts ? opts.query :
 					`${opts.prefix} ${editorValue}`;
-			const isUserQuery = !opts || 'query' in opts;
+			const isUserQuery = !opts || 'prefix' in opts;
 			const result = await this.chatService.sendRequest(this.viewModel.sessionId, input);
 
 			if (result) {

src/vs/workbench/contrib/chat/common/chatAgents.ts

@@ -11,7 +11,7 @@ import { Disposable, IDisposable, toDisposable } from 'vs/base/common/lifecycle'
 import { ThemeIcon } from 'vs/base/common/themables';
 import { URI } from 'vs/base/common/uri';
 import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
-import { IChatMessage } from 'vs/workbench/contrib/chat/common/chatProvider';
+import { IChatProgressResponseContent } from 'vs/workbench/contrib/chat/common/chatModel';
 import { IChatFollowup, IChatProgress, IChatResponseErrorDetails } from 'vs/workbench/contrib/chat/common/chatService';
 import { IChatRequestVariableValue } from 'vs/workbench/contrib/chat/common/chatVariables';
 
@@ -22,8 +22,14 @@ export interface IChatAgentData {
 	metadata: IChatAgentMetadata;
 }
 
+export interface IChatAgentHistoryEntry {
+	request: IChatAgentRequest;
+	response: ReadonlyArray<IChatProgressResponseContent>;
+	result: IChatAgentResult;
+}
+
 export interface IChatAgent extends IChatAgentData {
-	invoke(request: IChatAgentRequest, progress: (part: IChatProgress) => void, history: IChatMessage[], token: CancellationToken): Promise<IChatAgentResult>;
+	invoke(request: IChatAgentRequest, progress: (part: IChatProgress) => void, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<IChatAgentResult>;
 	provideFollowups?(sessionId: string, token: CancellationToken): Promise<IChatFollowup[]>;
 	provideSlashCommands(token: CancellationToken): Promise<IChatAgentCommand[]>;
 }
@@ -72,6 +78,7 @@ export interface IChatAgentMetadata {
 export interface IChatAgentRequest {
 	sessionId: string;
 	requestId: string;
+	agentId: string;
 	command?: string;
 	message: string;
 	variables: Record<string, IChatRequestVariableValue[]>;
@@ -93,7 +100,7 @@ export interface IChatAgentService {
 	_serviceBrand: undefined;
 	readonly onDidChangeAgents: Event<void>;
 	registerAgent(agent: IChatAgent): IDisposable;
-	invokeAgent(id: string, request: IChatAgentRequest, progress: (part: IChatProgress) => void, history: IChatMessage[], token: CancellationToken): Promise<IChatAgentResult>;
+	invokeAgent(id: string, request: IChatAgentRequest, progress: (part: IChatProgress) => void, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<IChatAgentResult>;
 	getFollowups(id: string, sessionId: string, token: CancellationToken): Promise<IChatFollowup[]>;
 	getAgents(): Array<IChatAgent>;
 	getAgent(id: string): IChatAgent | undefined;
@@ -163,7 +170,7 @@ export class ChatAgentService extends Disposable implements IChatAgentService {
 		return data?.agent;
 	}
 
-	async invokeAgent(id: string, request: IChatAgentRequest, progress: (part: IChatProgress) => void, history: IChatMessage[], token: CancellationToken): Promise<IChatAgentResult> {
+	async invokeAgent(id: string, request: IChatAgentRequest, progress: (part: IChatProgress) => void, history: IChatAgentHistoryEntry[], token: CancellationToken): Promise<IChatAgentResult> {
 		const data = this._agents.get(id);
 		if (!data) {
 			throw new Error(`No agent with id ${id}`);