Fix Non-UTF-8 File Handling: Improve Encoding Detection to Prevent Garbled Text and Binary Misclassification (RooCodeInc#2347)

* Fix Non-UTF-8 File Handling: Improve Encoding Detection to Prevent Garbled Text and Binary Misclassification

* update package-lock.json

* update

* update

* fix

* fix

* fix

Author: yt3trees

.changeset/shiny-dingos-float.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Fix Non-UTF-8 File Handling: Improve Encoding Detection to Prevent Garbled Text and Binary Misclassification

package-lock.json

@@ -386,8 +386,10 @@
 		"fzf": "^0.5.2",
 		"get-folder-size": "^5.0.0",
 		"globby": "^14.0.2",
+		"iconv-lite": "^0.6.3",
 		"ignore": "^7.0.3",
 		"isbinaryfile": "^5.0.2",
+		"jschardet": "^3.1.4",
 		"mammoth": "^1.8.0",
 		"monaco-vscode-textmate-theme-converter": "^0.1.7",
 		"ollama": "^0.5.13",

package.json

@@ -7,6 +7,8 @@ import { formatResponse } from "../../core/prompts/responses"
 import { DecorationController } from "./DecorationController"
 import * as diff from "diff"
 import { diagnosticsToProblemsString, getNewDiagnostics } from "../diagnostics"
+import { detectEncoding } from "../misc/extract-text"
+import * as iconv from "iconv-lite"
 
 export const DIFF_VIEW_URI_SCHEME = "cline-diff"
 
@@ -23,6 +25,7 @@ export class DiffViewProvider {
 	private activeLineController?: DecorationController
 	private streamedLines: string[] = []
 	private preDiagnostics: [vscode.Uri, vscode.Diagnostic[]][] = []
+	private fileEncoding: string = "utf8"
 
 	constructor(private cwd: string) {}
 
@@ -43,9 +46,12 @@ export class DiffViewProvider {
 		this.preDiagnostics = vscode.languages.getDiagnostics()
 
 		if (fileExists) {
-			this.originalContent = await fs.readFile(absolutePath, "utf-8")
+			const fileBuffer = await fs.readFile(absolutePath)
+			this.fileEncoding = await detectEncoding(fileBuffer)
+			this.originalContent = iconv.decode(fileBuffer, this.fileEncoding)
 		} else {
 			this.originalContent = ""
+			this.fileEncoding = "utf8"
 		}
 		// for new files, create any necessary directories and keep track of new directories to delete if the user denies the operation
 		this.createdDirs = await createDirectoriesForFile(absolutePath)

src/integrations/editor/DiffViewProvider.ts

@@ -5,6 +5,25 @@ import mammoth from "mammoth"
 import fs from "fs/promises"
 import { isBinaryFile } from "isbinaryfile"
 import { getFileSizeInKB } from "../../utils/fs"
+import * as chardet from "jschardet"
+import * as iconv from "iconv-lite"
+
+export async function detectEncoding(fileBuffer: Buffer, fileExtension?: string): Promise<string> {
+	const detected = chardet.detect(fileBuffer)
+	if (typeof detected === "string") {
+		return detected
+	} else if (detected && (detected as any).encoding) {
+		return (detected as any).encoding
+	} else {
+		if (fileExtension) {
+			const isBinary = await isBinaryFile(fileBuffer).catch(() => false)
+			if (isBinary) {
+				throw new Error(`Cannot read text for file type: ${fileExtension}`)
+			}
+		}
+		return "utf8"
+	}
+}
 
 export async function extractTextFromFile(filePath: string): Promise<string> {
 	try {
@@ -21,17 +40,12 @@ export async function extractTextFromFile(filePath: string): Promise<string> {
 		case ".ipynb":
 			return extractTextFromIPYNB(filePath)
 		default:
-			const isBinary = await isBinaryFile(filePath).catch(() => false)
-			if (!isBinary) {
-				// If file is over 300KB, throw an error
-				const fileSizeInKB = await getFileSizeInKB(filePath)
-				if (fileSizeInKB > 300) {
-					throw new Error(`File is too large to read into context.`)
-				}
-				return await fs.readFile(filePath, "utf8")
-			} else {
-				throw new Error(`Cannot read text for file type: ${fileExtension}`)
+			const fileBuffer = await fs.readFile(filePath)
+			if (fileBuffer.byteLength > 300 * 1024) {
+				throw new Error(`File is too large to read into context.`)
 			}
+			const encoding = await detectEncoding(fileBuffer, fileExtension)
+			return iconv.decode(fileBuffer, encoding)
 	}
 }
 
@@ -47,7 +61,9 @@ async function extractTextFromDOCX(filePath: string): Promise<string> {
 }
 
 async function extractTextFromIPYNB(filePath: string): Promise<string> {
-	const data = await fs.readFile(filePath, "utf8")
+	const fileBuffer = await fs.readFile(filePath)
+	const encoding = await detectEncoding(fileBuffer)
+	const data = iconv.decode(fileBuffer, encoding)
 	const notebook = JSON.parse(data)
 	let extractedText = ""