tools: improve string_replace and add healing (ft. Gemini) (#251)

* test implementation of gemini-inspired model based healing

* rationalize types in string replace tools

* cleanup and add telemetry for string_replace healing

* prompting improvements

* add failsafe for string-replace healing

* update snapshot

Author: connor4312

package.json

@@ -935,7 +935,7 @@
 				"name": "copilot_replaceString",
 				"toolReferenceName": "replaceString",
 				"displayName": "%copilot.tools.replaceString.name%",
-				"modelDescription": "This is a tool for making edits in an existing file in the workspace. For moving or renaming files, use run in terminal tool with the 'mv' command instead. For larger edits, split them into smaller edits and call the edit tool multiple times to ensure accuracy. Before editing, always ensure you have the context to understand the file's contents and context. To edit a file, provide: 1) filePath (absolute path), 2) oldString (must exactly match, including whitespace and indentation, uniquely identifying a single occurrence), and 3) newString (replacement text). Each use of this tool replaces exactly ONE occurrence of oldString. CRITICAL REQUIREMENTS: ensure oldString uniquely identifies the change by including at least 3-5 lines of context both before and after the target text, preserving whitespace and indentation exactly. Never use ...existing code... comments in the oldString or newString. Edits must result in valid, idiomatic code and not leave the file broken!",
+				"modelDescription": "This is a tool for making edits in an existing file in the workspace. For moving or renaming files, use run in terminal tool with the 'mv' command instead. For larger edits, split them into smaller edits and call the edit tool multiple times to ensure accuracy. Before editing, always ensure you have the context to understand the file's contents and context. To edit a file, provide: 1) filePath (absolute path), 2) oldString (MUST be the exact literal text to replace including all whitespace, indentation, newlines, and surrounding code etc), and 3) newString (MUST be the exact literal text to replace \\`oldString\\` with (also including all whitespace, indentation, newlines, and surrounding code etc.). Ensure the resulting code is correct and idiomatic.). Each use of this tool replaces exactly ONE occurrence of oldString.\n\nCRITICAL for \\`oldString\\`: Must uniquely identify the single instance to change. Include at least 3 lines of context BEFORE and AFTER the target text, matching whitespace and indentation precisely. If this string matches multiple locations, or does not match exactly, the tool will fail. Never use ...existing code... comments in the oldString or newString.",
 				"when": "!config.github.copilot.chat.disableReplaceTool",
 				"inputSchema": {
 					"type": "object",
@@ -946,11 +946,11 @@
 						},
 						"oldString": {
 							"type": "string",
-							"description": "The string to be replaced in the file. Never use ...existing code... comments in the oldString."
+							"description": "The exact literal text to replace, preferably unescaped. For single replacements (default), include at least 3 lines of context BEFORE and AFTER the target text, matching whitespace and indentation precisely. For multiple replacements, specify expected_replacements parameter. If this string is not the exact literal text (i.e. you escaped it) or does not match exactly, the tool will fail."
 						},
 						"newString": {
 							"type": "string",
-							"description": "The replacement string. Can be empty to delete oldString."
+							"description": "The exact literal text to replace `old_string` with, preferably unescaped. Provide the EXACT text. Ensure the resulting code is correct and idiomatic."
 						}
 					},
 					"required": [

src/extension/prompts/node/agent/agentInstructions.tsx

@@ -75,18 +75,18 @@ export class DefaultAgentPrompt extends PromptElement<DefaultAgentPromptProps> {
 				{hasReplaceStringTool ?
 					<>
 						Before you edit an existing file, make sure you either already have it in the provided context, or read it with the {ToolName.ReadFile} tool, so that you can make proper changes.<br />
-						Use the {ToolName.ReplaceString} tool to replace a string in a file, but only if you are sure that the string is unique enough to not cause any issues. You can use this tool multiple times per file.<br />
-						Use the {ToolName.EditFile} tool to insert code into a file.<br />
+						Use the {ToolName.ReplaceString} tool to edit files, paying attention to context to ensure your replacement is unique. You can use this tool multiple times per file.<br />
+						Use the {ToolName.EditFile} tool to insert code into a file ONLY if {ToolName.ReplaceString} has failed.<br />
 						When editing files, group your changes by file.<br />
 						NEVER show the changes to the user, just call the tool, and the edits will be applied and shown to the user.<br />
-						NEVER print a codeblock that represents a change to a file, use {ToolName.EditFile} or {ToolName.ReplaceString} instead.<br />
+						NEVER print a codeblock that represents a change to a file, use {ToolName.ReplaceString} or {ToolName.EditFile} instead.<br />
 						For each file, give a short description of what needs to be changed, then use the {ToolName.ReplaceString} or {ToolName.EditFile} tools. You can use any tool multiple times in a response, and you can keep writing text after using a tool.<br /></> :
 					<>
 						Don't try to edit an existing file without reading it first, so you can make changes properly.<br />
-						Use the {ToolName.EditFile} tool to edit files. When editing files, group your changes by file.<br />
+						Use the {ToolName.ReplaceString} tool to edit files. When editing files, group your changes by file.<br />
 						NEVER show the changes to the user, just call the tool, and the edits will be applied and shown to the user.<br />
-						NEVER print a codeblock that represents a change to a file, use {ToolName.EditFile} instead.<br />
-						For each file, give a short description of what needs to be changed, then use the {ToolName.EditFile} tool. You can use any tool multiple times in a response, and you can keep writing text after using a tool.<br />
+						NEVER print a codeblock that represents a change to a file, use {ToolName.ReplaceString} instead.<br />
+						For each file, give a short description of what needs to be changed, then use the {ToolName.ReplaceString} tool. You can use any tool multiple times in a response, and you can keep writing text after using a tool.<br />
 					</>}
 				<GenericEditingTips {...this.props} />
 				The {ToolName.EditFile} tool is very smart and can understand how to apply your edits to the user's files, you just need to provide minimal hints.<br />

src/extension/prompts/node/agent/agentPrompt.tsx

@@ -8,6 +8,7 @@ import { isDefined } from '@vscode/test-electron/out/util';
 import type { ChatRequestEditedFileEvent, LanguageModelToolInformation, NotebookEditor, TaskDefinition, TextEditor } from 'vscode';
 import { ChatLocation } from '../../../../platform/chat/common/commonTypes';
 import { ConfigKey, IConfigurationService } from '../../../../platform/configuration/common/configurationService';
+import { modelNeedsStrongReplaceStringHint } from '../../../../platform/endpoint/common/chatModelCapabilities';
 import { CacheType } from '../../../../platform/endpoint/common/endpointTypes';
 import { IEnvService, OperatingSystem } from '../../../../platform/env/common/envService';
 import { getGitHubRepoInfoFromContext, IGitService } from '../../../../platform/git/common/gitService';
@@ -285,7 +286,7 @@ export class AgentUserMessage extends PromptElement<AgentUserMessageProps> {
 					<Tag name='reminderInstructions'>
 						{/* Critical reminders that are effective when repeated right next to the user message */}
 						{getKeepGoingReminder(this.props.endpoint.family)}
-						{getEditingReminder(hasEditFileTool, hasReplaceStringTool)}
+						{getEditingReminder(hasEditFileTool, hasReplaceStringTool, modelNeedsStrongReplaceStringHint(this.props.endpoint))}
 						<NotebookReminderInstructions chatVariables={this.props.chatVariables} query={this.props.request} />
 					</Tag>
 					{query && <Tag name='userRequest' priority={900} flexGrow={7}>{query + attachmentHint}</Tag>}
@@ -596,15 +597,21 @@ class AgentTasksInstructions extends PromptElement {
 	}
 }
 
-export function getEditingReminder(hasEditFileTool: boolean, hasReplaceStringTool: boolean) {
+export function getEditingReminder(hasEditFileTool: boolean, hasReplaceStringTool: boolean, useStrongReplaceStringHint: boolean) {
 	const lines = [];
 	if (hasEditFileTool) {
 		lines.push(<>When using the {ToolName.EditFile} tool, avoid repeating existing code, instead use a line comment with \`{EXISTING_CODE_MARKER}\` to represent regions of unchanged code.<br /></>);
-
 	}
 	if (hasReplaceStringTool) {
 		lines.push(<>When using the {ToolName.ReplaceString} tool, include 3-5 lines of unchanged code before and after the string you want to replace, to make it unambiguous which part of the file should be edited.<br /></>);
 	}
+	if (hasEditFileTool && hasReplaceStringTool) {
+		if (useStrongReplaceStringHint) {
+			lines.push(<>You must always try making file edits using {ToolName.ReplaceString} tool. NEVER use {ToolName.EditFile} unless told to by the user or by a tool.</>);
+		} else {
+			lines.push(<>It is much faster to edit using the {ToolName.ReplaceString} tool. Prefer {ToolName.ReplaceString} for making edits and only fall back to {ToolName.EditFile} if it fails.</>);
+		}
+	}
 
 	return lines;
 }

src/extension/prompts/node/agent/test/__snapshots__/agentPrompt.spec.tsx.snap

@@ -175,11 +175,11 @@ Tools can be disabled by the user. You may see tools used previously in the conv
 </toolUseInstructions>
 <editFileInstructions>
 Before you edit an existing file, make sure you either already have it in the provided context, or read it with the read_file tool, so that you can make proper changes.
-Use the replace_string_in_file tool to replace a string in a file, but only if you are sure that the string is unique enough to not cause any issues. You can use this tool multiple times per file.
-Use the insert_edit_into_file tool to insert code into a file.
+Use the replace_string_in_file tool to edit files, paying attention to context to ensure your replacement is unique. You can use this tool multiple times per file.
+Use the insert_edit_into_file tool to insert code into a file ONLY if replace_string_in_file has failed.
 When editing files, group your changes by file.
 NEVER show the changes to the user, just call the tool, and the edits will be applied and shown to the user.
-NEVER print a codeblock that represents a change to a file, use insert_edit_into_file or replace_string_in_file instead.
+NEVER print a codeblock that represents a change to a file, use replace_string_in_file or insert_edit_into_file instead.
 For each file, give a short description of what needs to be changed, then use the replace_string_in_file or insert_edit_into_file tools. You can use any tool multiple times in a response, and you can keep writing text after using a tool.
 Follow best practices when editing files. If a popular external library exists to solve a problem, use it and properly install the package e.g. creating a "requirements.txt".
 If you're building a webapp from scratch, give it a beautiful and modern UI.
@@ -259,7 +259,7 @@ copilot_cache_control: {"type":"ephemeral"}
 <reminderInstructions>
 When using the insert_edit_into_file tool, avoid repeating existing code, instead use a line comment with /\`...existing code.../\` to represent regions of unchanged code.
 When using the replace_string_in_file tool, include 3-5 lines of unchanged code before and after the string you want to replace, to make it unambiguous which part of the file should be edited.
-
+It is much faster to edit using the replace_string_in_file tool. Prefer replace_string_in_file for making edits and only fall back to insert_edit_into_file if it fails.
 </reminderInstructions>
 <userRequest>
 hello

src/extension/prompts/node/panel/editCodePrompt2.tsx

@@ -5,7 +5,7 @@
 
 import { PromptElement, PromptSizing, SystemMessage, UserMessage } from '@vscode/prompt-tsx';
 import { ConfigKey, IConfigurationService } from '../../../../platform/configuration/common/configurationService';
-import { modelPrefersInstructionsAfterHistory } from '../../../../platform/endpoint/common/chatModelCapabilities';
+import { modelNeedsStrongReplaceStringHint, modelPrefersInstructionsAfterHistory } from '../../../../platform/endpoint/common/chatModelCapabilities';
 import { IExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService';
 import { isLocation, isUri } from '../../../../util/common/types';
 import { ToolName } from '../../../tools/common/toolNames';
@@ -148,7 +148,7 @@ export class EditCode2UserMessage extends PromptElement<AgentPromptProps> {
 					<ChatToolReferences flexGrow={4} priority={898} promptContext={this.props.promptContext} documentContext={this.props.documentContext} />
 					<ChatVariables flexGrow={3} priority={898} chatVariables={chatVariables} />
 					<Tag name='reminder'>
-						{getEditingReminder(hasEditFileTool, hasReplaceStringTool)}
+						{getEditingReminder(hasEditFileTool, hasReplaceStringTool, modelNeedsStrongReplaceStringHint(this.props.endpoint))}
 						<NotebookReminderInstructions chatVariables={chatVariables} query={query} />
 						<NewFilesLocationHint />
 					</Tag>