Revert "Git - remove jschardet (microsoft#241256)" (microsoft#242469) (microsoft#242539)

This reverts commit 6fdb419.

Author: lszomoru

extensions/git/package-lock.json

@@ -35,7 +35,6 @@
     "statusBarItemTooltip",
     "tabInputMultiDiff",
     "tabInputTextMerge",
-    "textDocumentEncoding",
     "textEditorDiffInformation",
     "timeline"
   ],
@@ -3567,6 +3566,7 @@
     "@vscode/iconv-lite-umd": "0.7.0",
     "byline": "^5.0.0",
     "file-type": "16.5.4",
+    "jschardet": "3.1.4",
     "picomatch": "2.3.1",
     "vscode-uri": "^2.0.0",
     "which": "4.0.0"

extensions/git/package.json

@@ -1628,29 +1628,19 @@ export class CommandCenter {
 		}
 
 		let modifiedUri = changes.modifiedUri;
-		let modifiedDocument: TextDocument | undefined;
-
 		if (!modifiedUri) {
 			const textEditor = window.activeTextEditor;
 			if (!textEditor) {
 				return;
 			}
-
-			modifiedDocument = textEditor.document;
+			const modifiedDocument = textEditor.document;
 			modifiedUri = modifiedDocument.uri;
 		}
-
 		if (modifiedUri.scheme !== 'file') {
 			return;
 		}
-
-		if (!modifiedDocument) {
-			modifiedDocument = await workspace.openTextDocument(modifiedUri);
-		}
-
 		const result = changes.originalWithModifiedChanges;
-		await this.runByRepository(modifiedUri, async (repository, resource) =>
-			await repository.stage(resource, result, modifiedDocument.encoding));
+		await this.runByRepository(modifiedUri, async (repository, resource) => await repository.stage(resource, result));
 	}
 
 	@command('git.stageSelectedRanges', { diff: true })
@@ -1827,8 +1817,7 @@ export class CommandCenter {
 		const originalDocument = await workspace.openTextDocument(originalUri);
 		const result = applyLineChanges(originalDocument, modifiedDocument, changes);
 
-		await this.runByRepository(modifiedUri, async (repository, resource) =>
-			await repository.stage(resource, result, modifiedDocument.encoding));
+		await this.runByRepository(modifiedUri, async (repository, resource) => await repository.stage(resource, result));
 	}
 
 	@command('git.revertChange')
@@ -2000,7 +1989,7 @@ export class CommandCenter {
 		this.logger.trace(`[CommandCenter][unstageSelectedRanges] invertedDiffs: ${JSON.stringify(invertedDiffs)}`);
 
 		const result = applyLineChanges(modifiedDocument, originalDocument, invertedDiffs);
-		await repository.stage(modifiedUri, result, modifiedDocument.encoding);
+		await repository.stage(modifiedUri, result);
 	}
 
 	@command('git.unstageFile')

extensions/git/src/commands.ts

@@ -0,0 +1,100 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as jschardet from 'jschardet';
+
+function detectEncodingByBOM(buffer: Buffer): string | null {
+	if (!buffer || buffer.length < 2) {
+		return null;
+	}
+
+	const b0 = buffer.readUInt8(0);
+	const b1 = buffer.readUInt8(1);
+
+	// UTF-16 BE
+	if (b0 === 0xFE && b1 === 0xFF) {
+		return 'utf16be';
+	}
+
+	// UTF-16 LE
+	if (b0 === 0xFF && b1 === 0xFE) {
+		return 'utf16le';
+	}
+
+	if (buffer.length < 3) {
+		return null;
+	}
+
+	const b2 = buffer.readUInt8(2);
+
+	// UTF-8
+	if (b0 === 0xEF && b1 === 0xBB && b2 === 0xBF) {
+		return 'utf8';
+	}
+
+	return null;
+}
+
+const IGNORE_ENCODINGS = [
+	'ascii',
+	'utf-8',
+	'utf-16',
+	'utf-32'
+];
+
+const JSCHARDET_TO_ICONV_ENCODINGS: { [name: string]: string } = {
+	'ibm866': 'cp866',
+	'big5': 'cp950'
+};
+
+const MAP_CANDIDATE_GUESS_ENCODING_TO_JSCHARDET: { [key: string]: string } = {
+	utf8: 'UTF-8',
+	utf16le: 'UTF-16LE',
+	utf16be: 'UTF-16BE',
+	windows1252: 'windows-1252',
+	windows1250: 'windows-1250',
+	iso88592: 'ISO-8859-2',
+	windows1251: 'windows-1251',
+	cp866: 'IBM866',
+	iso88595: 'ISO-8859-5',
+	koi8r: 'KOI8-R',
+	windows1253: 'windows-1253',
+	iso88597: 'ISO-8859-7',
+	windows1255: 'windows-1255',
+	iso88598: 'ISO-8859-8',
+	cp950: 'Big5',
+	shiftjis: 'SHIFT_JIS',
+	eucjp: 'EUC-JP',
+	euckr: 'EUC-KR',
+	gb2312: 'GB2312'
+};
+
+export function detectEncoding(buffer: Buffer, candidateGuessEncodings: string[]): string | null {
+	const result = detectEncodingByBOM(buffer);
+
+	if (result) {
+		return result;
+	}
+
+	candidateGuessEncodings = candidateGuessEncodings.map(e => MAP_CANDIDATE_GUESS_ENCODING_TO_JSCHARDET[e]).filter(e => !!e);
+
+	const detected = jschardet.detect(buffer, candidateGuessEncodings.length > 0 ? { detectEncodings: candidateGuessEncodings } : undefined);
+	if (!detected || !detected.encoding) {
+		return null;
+	}
+
+	const encoding = detected.encoding;
+
+	// Ignore encodings that cannot guess correctly
+	// (http://chardet.readthedocs.io/en/latest/supported-encodings.html)
+	if (0 <= IGNORE_ENCODINGS.indexOf(encoding.toLowerCase())) {
+		return null;
+	}
+
+	const normalizedEncodingName = encoding.replace(/[^a-zA-Z0-9]/g, '').toLowerCase();
+	const mapped = JSCHARDET_TO_ICONV_ENCODINGS[normalizedEncodingName];
+
+	return mapped || normalizedEncodingName;
+}

extensions/git/src/encoding.ts

@@ -14,6 +14,7 @@ import * as iconv from '@vscode/iconv-lite-umd';
 import * as filetype from 'file-type';
 import { assign, groupBy, IDisposable, toDisposable, dispose, mkdirp, readBytes, detectUnicodeEncoding, Encoding, onceEvent, splitInChunks, Limiter, Versions, isWindows, pathEquals, isMacintosh, isDescendant } from './util';
 import { CancellationError, CancellationToken, ConfigurationChangeEvent, LogOutputChannel, Progress, Uri, workspace } from 'vscode';
+import { detectEncoding } from './encoding';
 import { Ref, RefType, Branch, Remote, ForcePushMode, GitErrorCodes, LogOptions, Change, Status, CommitOptions, RefQuery, InitOptions } from './api/git';
 import * as byline from 'byline';
 import { StringDecoder } from 'string_decoder';
@@ -1329,6 +1330,18 @@ export class Repository {
 			.filter(entry => !!entry);
 	}
 
+	async bufferString(object: string, encoding: string = 'utf8', autoGuessEncoding = false, candidateGuessEncodings: string[] = []): Promise<string> {
+		const stdout = await this.buffer(object);
+
+		if (autoGuessEncoding) {
+			encoding = detectEncoding(stdout, candidateGuessEncodings) || encoding;
+		}
+
+		encoding = iconv.encodingExists(encoding) ? encoding : 'utf8';
+
+		return iconv.decode(stdout, encoding);
+	}
+
 	async buffer(object: string): Promise<Buffer> {
 		const child = this.stream(['show', '--textconv', object]);
 

extensions/git/src/git.ts

@@ -7,6 +7,7 @@ import TelemetryReporter from '@vscode/extension-telemetry';
 import * as fs from 'fs';
 import * as path from 'path';
 import picomatch from 'picomatch';
+import * as iconv from '@vscode/iconv-lite-umd';
 import { CancellationError, CancellationToken, CancellationTokenSource, Command, commands, Disposable, Event, EventEmitter, FileDecoration, l10n, LogLevel, LogOutputChannel, Memento, ProgressLocation, ProgressOptions, QuickDiffProvider, RelativePattern, scm, SourceControl, SourceControlInputBox, SourceControlInputBoxValidation, SourceControlInputBoxValidationType, SourceControlResourceDecorations, SourceControlResourceGroup, SourceControlResourceState, TabInputNotebookDiff, TabInputTextDiff, TabInputTextMultiDiff, ThemeColor, Uri, window, workspace, WorkspaceEdit } from 'vscode';
 import { ActionButton } from './actionButton';
 import { ApiRepository } from './api/api1';
@@ -24,6 +25,7 @@ import { StatusBarCommands } from './statusbar';
 import { toGitUri } from './uri';
 import { anyEvent, combinedDisposable, debounceEvent, dispose, EmptyDisposable, eventToPromise, filterEvent, find, getCommitShortHash, IDisposable, isDescendant, isLinuxSnap, isRemote, Limiter, onceEvent, pathEquals, relativePath } from './util';
 import { IFileWatcher, watch } from './watch';
+import { detectEncoding } from './encoding';
 import { ISourceControlHistoryItemDetailsProviderRegistry } from './historyItemDetailsProvider';
 
 const timeout = (millis: number) => new Promise(c => setTimeout(c, millis));
@@ -1220,9 +1222,19 @@ export class Repository implements Disposable {
 		await this.run(Operation.Remove, () => this.repository.rm(resources.map(r => r.fsPath)));
 	}
 
-	async stage(resource: Uri, contents: string, encoding: string): Promise<void> {
+	async stage(resource: Uri, contents: string): Promise<void> {
+		const path = relativePath(this.repository.root, resource.fsPath).replace(/\\/g, '/');
 		await this.run(Operation.Stage, async () => {
-			const path = relativePath(this.repository.root, resource.fsPath).replace(/\\/g, '/');
+			const configFiles = workspace.getConfiguration('files', Uri.file(resource.fsPath));
+			let encoding = configFiles.get<string>('encoding') ?? 'utf8';
+			const autoGuessEncoding = configFiles.get<boolean>('autoGuessEncoding') === true;
+			const candidateGuessEncodings = configFiles.get<string[]>('candidateGuessEncodings') ?? [];
+
+			if (autoGuessEncoding) {
+				encoding = detectEncoding(Buffer.from(contents), candidateGuessEncodings) ?? encoding;
+			}
+
+			encoding = iconv.encodingExists(encoding) ? encoding : 'utf8';
 			await this.repository.stage(path, contents, encoding);
 
 			this._onDidChangeOriginalResource.fire(resource);
@@ -1968,15 +1980,17 @@ export class Repository implements Disposable {
 	async show(ref: string, filePath: string): Promise<string> {
 		return await this.run(Operation.Show, async () => {
 			const path = relativePath(this.repository.root, filePath).replace(/\\/g, '/');
+			const configFiles = workspace.getConfiguration('files', Uri.file(filePath));
+			const defaultEncoding = configFiles.get<string>('encoding');
+			const autoGuessEncoding = configFiles.get<boolean>('autoGuessEncoding');
+			const candidateGuessEncodings = configFiles.get<string[]>('candidateGuessEncodings');
 
 			try {
-				const content = await this.repository.buffer(`${ref}:${path}`);
-				return await workspace.decode(content, Uri.file(filePath));
+				return await this.repository.bufferString(`${ref}:${path}`, defaultEncoding, autoGuessEncoding, candidateGuessEncodings);
 			} catch (err) {
 				if (err.gitErrorCode === GitErrorCodes.WrongCase) {
 					const gitRelativePath = await this.repository.getGitRelativePath(ref, path);
-					const content = await this.repository.buffer(`${ref}:${gitRelativePath}`);
-					return await workspace.decode(content, Uri.file(filePath));
+					return await this.repository.bufferString(`${ref}:${gitRelativePath}`, defaultEncoding, autoGuessEncoding, candidateGuessEncodings);
 				}
 
 				throw err;

extensions/git/src/repository.ts

@@ -25,7 +25,6 @@
 		"../../src/vscode-dts/vscode.proposed.statusBarItemTooltip.d.ts",
 		"../../src/vscode-dts/vscode.proposed.tabInputMultiDiff.d.ts",
 		"../../src/vscode-dts/vscode.proposed.tabInputTextMerge.d.ts",
-		"../../src/vscode-dts/vscode.proposed.textDocumentEncoding.d.ts",
 		"../../src/vscode-dts/vscode.proposed.textEditorDiffInformation.d.ts",
 		"../../src/vscode-dts/vscode.proposed.timeline.d.ts",
 		"../types/lib.textEncoder.d.ts"