fix: Remove HTML entity unescaping for non-Claude models in apply_diff (#4077)

Author: MuriloFP

src/core/tools/applyDiffTool.ts

@@ -10,7 +10,6 @@ import { ToolUse, RemoveClosingTag, AskApproval, HandleError, PushToolResult } f
 import { formatResponse } from "../prompts/responses"
 import { fileExistsAtPath } from "../../utils/fs"
 import { RecordSource } from "../context-tracking/FileContextTrackerTypes"
-import { unescapeHtmlEntities } from "../../utils/text-normalization"
 
 export async function applyDiffToolLegacy(
 	cline: Task,
@@ -23,10 +22,6 @@ export async function applyDiffToolLegacy(
 	const relPath: string | undefined = block.params.path
 	let diffContent: string | undefined = block.params.diff
 
-	if (diffContent && !cline.api.getModel().id.includes("claude")) {
-		diffContent = unescapeHtmlEntities(diffContent)
-	}
-
 	const sharedMessageProps: ClineSayTool = {
 		tool: "appliedDiff",
 		path: getReadablePath(cline.cwd, removeClosingTag("path", relPath)),

src/core/tools/multiApplyDiffTool.ts

@@ -10,7 +10,6 @@ import { ToolUse, RemoveClosingTag, AskApproval, HandleError, PushToolResult } f
 import { formatResponse } from "../prompts/responses"
 import { fileExistsAtPath } from "../../utils/fs"
 import { RecordSource } from "../context-tracking/FileContextTrackerTypes"
-import { unescapeHtmlEntities } from "../../utils/text-normalization"
 import { parseXml } from "../../utils/xml"
 import { EXPERIMENT_IDS, experiments } from "../../shared/experiments"
 import { applyDiffToolLegacy } from "./applyDiffTool"
@@ -410,16 +409,8 @@ Original error: ${errorMessage}`
 				let successCount = 0
 				let formattedError = ""
 
-				// Pre-process all diff items for HTML entity unescaping if needed
-				const processedDiffItems = !cline.api.getModel().id.includes("claude")
-					? diffItems.map((item) => ({
-							...item,
-							content: item.content ? unescapeHtmlEntities(item.content) : item.content,
-						}))
-					: diffItems
-
 				// Apply all diffs at once with the array-based method
-				const diffResult = (await cline.diffStrategy?.applyDiff(originalContent, processedDiffItems)) ?? {
+				const diffResult = (await cline.diffStrategy?.applyDiff(originalContent, diffItems)) ?? {
 					success: false,
 					error: "No diff strategy available - please ensure a valid diff strategy is configured",
 				}

src/utils/__tests__/text-normalization.spec.ts

@@ -75,5 +75,55 @@ describe("Text normalization utilities", () => {
 			expect(unescapeHtmlEntities("")).toBe("")
 			expect(unescapeHtmlEntities(undefined as unknown as string)).toBe(undefined)
 		})
+
+		// Issue #4077 - HTML entity handling verification
+		it("should demonstrate that unescapeHtmlEntities converts & to &", () => {
+			const input = "// Step 5 & 6: Verify the data"
+			const result = unescapeHtmlEntities(input)
+
+			expect(result).toBe("// Step 5 & 6: Verify the data")
+			expect(result).not.toBe(input)
+			expect(result).not.toContain("&")
+			expect(result).toContain("&")
+		})
+
+		it("should show the exact problem from issue #4077", () => {
+			// The user's SEARCH block content (what they see in their file)
+			const searchContent = "// Adım 5 & 6: İşaret edilen verinin bölümünü bul ve doğrula"
+
+			// After unescaping (what happens for non-Claude models)
+			const unescapedSearch = unescapeHtmlEntities(searchContent)
+
+			// The actual file content (contains HTML entity)
+			const fileContent = "// Adım 5 & 6: İşaret edilen verinin bölümünü bul ve doğrula"
+
+			// This demonstrates the mismatch
+			expect(unescapedSearch).toBe("// Adım 5 & 6: İşaret edilen verinin bölümünü bul ve doğrula")
+			expect(unescapedSearch).not.toBe(fileContent)
+
+			// The key issue: when searching for the unescaped version in the file content
+			// it won't find a match because the file has & but we're searching for &
+			const willFindMatch = fileContent.includes(unescapedSearch)
+			expect(willFindMatch).toBe(false)
+
+			// But if we search for the original (not unescaped), it would work
+			const wouldFindMatchWithoutUnescape = fileContent.includes(searchContent)
+			expect(wouldFindMatchWithoutUnescape).toBe(true)
+		})
+
+		it("should verify all HTML entities are unescaped", () => {
+			const testCases = [
+				{ input: "&lt;", expected: "<" },
+				{ input: "&gt;", expected: ">" },
+				{ input: "&quot;", expected: '"' },
+				{ input: "&#39;", expected: "'" },
+				{ input: "&apos;", expected: "'" },
+				{ input: "&amp;", expected: "&" },
+			]
+
+			testCases.forEach(({ input, expected }) => {
+				expect(unescapeHtmlEntities(input)).toBe(expected)
+			})
+		})
 	})
 })