Improve list files, search files, add some diff normalization

Remove a todo

Revert unrelated changes

Author: nissa-seru

src/core/Cline.ts

@@ -0,0 +1,32 @@
+/**
+ * Code normalization functions for improving diff accuracy in JS/TS code.
+ */
+
+import crypto from 'crypto';
+
+/**
+ * Normalizes whitespace in JS/TS code to improve diff matching.
+ * @param code The JS/TS code string.
+ * @returns The code with normalized whitespace.
+ */
+export function normalizeWhitespace(code: string): string {
+  return code.replace(/[\s\t\n]*/g, ' ');
+}
+
+/**
+ * Hashes template literals in JS/TS code to simplify diff matching.
+ * @param code The JS/TS code string.
+ * @returns The code with template literals replaced by hashes and a mapping of hashes to original literals.
+ */
+export function hashTemplateLiterals(code: string): { code: string; literalMap: { [hash: string]: string } } {
+  const literalMap: { [hash: string]: string } = {};
+  let hashCounter = 0;
+
+  const hashedCode = code.replace(/`([^`]*)`/g, (match, literalContent) => {
+    const hash = `__TEMPLATE_LITERAL_HASH_${hashCounter++}__`;
+    literalMap[hash] = match;
+    return hash;
+  });
+
+  return { code: hashedCode, literalMap };
+}

src/core/diff/code-normalization.ts

@@ -2,6 +2,10 @@ import { Diff, Hunk, Change } from "./types"
 import { findBestMatch, prepareSearchString } from "./search-strategies"
 import { applyEdit } from "./edit-strategies"
 import { DiffResult, DiffStrategy } from "../../types"
+import { normalizeWhitespace, hashTemplateLiterals } from "../../code-normalization" // Import normalization functions
+import { loadRequiredLanguageParsers } from "../../../../services/tree-sitter/languageParser"
+
+const JS_TS_EXTENSIONS = /\.(js|jsx|ts|tsx)$/i; // Regex for JS/TS file extensions
 
 export class NewUnifiedDiffStrategy implements DiffStrategy {
 	private readonly confidenceThreshold: number
@@ -173,7 +177,7 @@ Usage:
 <diff>
 Your diff here
 </diff>
-</apply_diff>`
+		</apply_diff>`
 	}
 
 	// Helper function to split a hunk into smaller hunks based on contiguous changes
@@ -235,8 +239,28 @@ Your diff here
 		startLine?: number,
 		endLine?: number,
 	): Promise<DiffResult> {
-		const parsedDiff = this.parseUnifiedDiff(diffContent)
-		const originalLines = originalContent.split("\n")
+		// Check if the file is a JS/TS file
+		const isJSTSFile = JS_TS_EXTENSIONS.test(arguments[0]); // Assuming path is the first argument
+
+		let normalizedOriginalContent = originalContent;
+		let normalizedDiffContent = diffContent;
+		let literalMap: { [hash: string]: string } = {};
+
+		if (isJSTSFile) {
+			// Apply normalization for JS/TS files
+			normalizedOriginalContent = normalizeWhitespace(originalContent);
+			normalizedDiffContent = normalizeWhitespace(diffContent);
+			const templateLiteralResult = hashTemplateLiterals(normalizedOriginalContent);
+			normalizedOriginalContent = templateLiteralResult.code;
+			literalMap = templateLiteralResult.literalMap;
+			const diffTemplateLiteralResult = hashTemplateLiterals(normalizedDiffContent);
+			normalizedDiffContent = diffTemplateLiteralResult.code;
+			// TODO: Consider merging literalMaps if needed, or handle diffContent literals separately
+		}
+
+
+		const parsedDiff = this.parseUnifiedDiff(normalizedDiffContent)
+		const originalLines = normalizedOriginalContent.split("\n")
 		let result = [...originalLines]
 
 		if (!parsedDiff.hunks.length) {
@@ -345,6 +369,35 @@ Your diff here
 			}
 		}
 
-		return { success: true, content: result.join("\n") }
+		let finalContent = result.join("\n");
+
+		if (isJSTSFile) {
+			// Reverse template literal hashing
+			Object.keys(literalMap).forEach(hash => {
+				finalContent = finalContent.replace(hash, literalMap[hash]);
+			});
+		}
+
+
+		if (isJSTSFile) {
+			// Perform syntax check for JS/TS files
+			try {
+				const parsers = await loadRequiredLanguageParsers(["temp.ts"]); // Dummy file for parser loading
+				const parser = parsers["ts"]?.parser;
+				if (parser) {
+					parser.parse(finalContent); // Attempt to parse
+				} else {
+					console.warn("TypeScript parser not loaded, skipping syntax check.");
+				}
+			} catch (e) {
+				console.error("JS/TS parsing error after applying diff:", e);
+				return {
+					success: false,
+					error: "JS/TS syntax error introduced by diff. Rollback recommended.",
+				};
+			}
+		}
+
+		return { success: true, content: finalContent }
 	}
 }

src/core/diff/strategies/new-unified/index.ts

@@ -136,7 +136,9 @@ Your search/replace content here
 		endLine?: number,
 	): Promise<DiffResult> {
 		// Extract the search and replace blocks
-		const match = diffContent.match(/<<<<<<< SEARCH\n([\s\S]*?)\n?=======\n([\s\S]*?)\n?>>>>>>> REPLACE/)
+		const match = diffContent.match(
+			/<<<<<<< SEARCH(?:\r?\n)([\s\S]*?)(?:\r?\n)?=======(?:\r?\n)([\s\S]*?)(?:\r?\n)?>>>>>>> REPLACE/,
+		)
 		if (!match) {
 			return {
 				success: false,
@@ -266,7 +268,6 @@ Your search/replace content here
 			const bestMatchSection = bestMatchContent
 				? `\n\nBest Match Found:\n${addLineNumbers(bestMatchContent, matchIndex + 1)}`
 				: `\n\nBest Match Found:\n(no match)`
-
 			const lineRange =
 				startLine || endLine
 					? ` at ${startLine ? `start: ${startLine}` : "start"} to ${endLine ? `end: ${endLine}` : "end"}`

src/core/diff/strategies/search-replace.ts

@@ -20,9 +20,9 @@ ${toolUseInstructionsReminder}
 
 # Next Steps
 
-If you have completed the user's task, use the attempt_completion tool. 
-If you require additional information from the user, use the ask_followup_question tool. 
-Otherwise, if you have not completed the task and do not need additional information, then proceed with the next step of the task. 
+If you have completed the user's task, use the attempt_completion tool.
+If you require additional information from the user, use the ask_followup_question tool.
+Otherwise, if you have not completed the task and do not need additional information, then proceed with the next step of the task.
 (This is an automated message, so do not respond to it conversationally.)`,
 
 	tooManyMistakes: (feedback?: string) =>

src/core/prompts/responses.ts

@@ -1,20 +1,95 @@
 import { ToolArgs } from "./types"
 
 export function getListFilesDescription(args: ToolArgs): string {
-	return `## list_files
-Description: Request to list files and directories within the specified directory. If recursive is true, it will list all files and directories recursively. If recursive is false or not provided, it will only list the top-level contents. Do not use this tool to confirm the existence of files you may have created, as the user will let you know if the files were created successfully or not.
-Parameters:
-- path: (required) The path of the directory to list contents for (relative to the current working directory ${args.cwd})
-- recursive: (optional) Whether to list files recursively. Use true for recursive listing, false or omit for top-level only.
-Usage:
+	return `<tool_definition>
+    <name>list_files</name>
+    <description>Lists files and directories in the specified path, returning a JSON structure with file metadata.</description>
+
+    <output_format>
+        <json_structure>
+            <field>
+                <name>name</name>
+                <description>File or directory name</description>
+            </field>
+            <field>
+                <name>type</name>
+                <description>Either "file" or "directory"</description>
+            </field>
+            <field>
+                <name>extension</name>
+                <description>File extension (files only)</description>
+            </field>
+            <field>
+                <name>children</name>
+                <description>Nested files/directories (directories only)</description>
+            </field>
+            <field>
+                <name>hasMore</name>
+                <description>Indicates if listing was truncated</description>
+            </field>
+        </json_structure>
+    </output_format>
+
+    <parameters>
+        <parameter>
+            <name>path</name>
+            <required>true</required>
+            <description>Directory path relative to ${args.cwd}</description>
+        </parameter>
+        <parameter>
+            <name>recursive</name>
+            <required>false</required>
+            <description>When true, lists contents recursively</description>
+        </parameter>
+        <parameter>
+            <name>format</name>
+            <required>false</required>
+            <description>Format of the output. Can be "flat" or "tree". Default is "flat"</description>
+        </parameter>
+    </parameters>
+
+    <syntax_template>
 <list_files>
 <path>Directory path here</path>
 <recursive>true or false (optional)</recursive>
+<format>"flat" or "tree" (optional)</format>
 </list_files>
+    </syntax_template>
 
-Example: Requesting to list all files in the current directory
+    <example>
 <list_files>
 <path>.</path>
 <recursive>false</recursive>
-</list_files>`
+<format>tree</format>
+</list_files>
+    </example>
+
+    <example_output>
+{
+  "root": {
+    "name": "project",
+    "type": "directory",
+    "children": {
+      "src": {
+        "name": "src",
+        "type": "directory",
+        "children": {
+          "index.ts": {
+            "name": "index",
+            "type": "file",
+            "extension": ".ts",
+          }
+        }
+      },
+      "package.json": {
+        "name": "package",
+        "type": "file",
+        "extension": ".json",
+      }
+    }
+  },
+  "hasMore": false
+}
+    </example_output>
+</tool_definition>`
 }

src/core/prompts/tools/list-files.ts

@@ -23,7 +23,7 @@ class WorkspaceTracker {
 		if (!cwd) {
 			return
 		}
-		const [files, _] = await listFiles(cwd, true, MAX_INITIAL_FILES)
+		const [files, _] = await listFiles(cwd, { path: cwd, recursive: true, limit: MAX_INITIAL_FILES, format: "flat" })
 		files.slice(0, MAX_INITIAL_FILES).forEach((file) => this.filePaths.add(this.normalizeFilePath(file)))
 		this.workspaceDidUpdate()
 	}