Skip to content

Commit ee6f694

Browse files
mhegazymhegazy
committed
Merge pull request #377 from Microsoft/bom
add a new compiler command line switch for generating utf-8 BOM in the output
2 parents 702b27b + 9e6cacb commit ee6f694

File tree

18 files changed

+126
-84
lines changed

18 files changed

+126
-84
lines changed

‎src/compiler/commandLineParser.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ module ts {
2323
name: "diagnostics",
2424
type: "boolean",
2525
},
26+
{
27+
name: "emitBOM",
28+
type: "boolean"
29+
},
2630
{
2731
name: "help",
2832
shortName: "h",

‎src/compiler/emitter.ts

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,8 @@ module ts {
157157
return text.substring(skipTrivia(text, node.pos), node.end);
158158
}
159159

160-
function writeFile(filename: string, data: string) {
161-
compilerHost.writeFile(filename, data, hostErrorMessage => {
160+
function writeFile(filename: string, data: string, writeByteOrderMark: boolean) {
161+
compilerHost.writeFile(filename, data, writeByteOrderMark, hostErrorMessage => {
162162
diagnostics.push(createCompilerDiagnostic(Diagnostics.Could_not_write_file_0_Colon_1, filename, hostErrorMessage));
163163
});
164164
}
@@ -426,7 +426,7 @@ module ts {
426426
sourceMapNameIndices.pop();
427427
};
428428

429-
function writeJavaScriptAndSourceMapFile(emitOutput: string) {
429+
function writeJavaScriptAndSourceMapFile(emitOutput: string, writeByteOrderMark: boolean) {
430430
// Write source map file
431431
encodeLastRecordedSourceMapSpan();
432432
writeFile(sourceMapData.sourceMapFilePath, JSON.stringify({
@@ -436,11 +436,11 @@ module ts {
436436
sources: sourceMapData.sourceMapSources,
437437
names: sourceMapData.sourceMapNames,
438438
mappings: sourceMapData.sourceMapMappings
439-
}));
439+
}), /*writeByteOrderMark*/ false);
440440
sourceMapDataList.push(sourceMapData);
441441

442442
// Write sourcemap url to the js file and write the js file
443-
writeJavaScriptFile(emitOutput + "//# sourceMappingURL=" + sourceMapData.jsSourceMappingURL);
443+
writeJavaScriptFile(emitOutput + "//# sourceMappingURL=" + sourceMapData.jsSourceMappingURL, writeByteOrderMark);
444444
}
445445

446446
// Initialize source map data
@@ -513,8 +513,8 @@ module ts {
513513
scopeEmitEnd = recordScopeNameEnd;
514514
}
515515

516-
function writeJavaScriptFile(emitOutput: string) {
517-
writeFile(jsFilePath, emitOutput);
516+
function writeJavaScriptFile(emitOutput: string, writeByteOrderMark: boolean) {
517+
writeFile(jsFilePath, emitOutput, writeByteOrderMark);
518518
}
519519

520520
function emitTokenText(tokenKind: SyntaxKind, startPos: number, emitFn?: () => void) {
@@ -1854,7 +1854,7 @@ module ts {
18541854
}
18551855

18561856
writeLine();
1857-
writeEmittedFiles(writer.getText());
1857+
writeEmittedFiles(writer.getText(), /*writeByteOrderMark*/ compilerOptions.emitBOM);
18581858
}
18591859

18601860
function emitDeclarations(jsFilePath: string, root?: SourceFile) {
@@ -2448,7 +2448,7 @@ module ts {
24482448
// TODO(shkamat): Should we not write any declaration file if any of them can produce error,
24492449
// or should we just not write this file like we are doing now
24502450
if (!reportedDeclarationError) {
2451-
writeFile(getModuleNameFromFilename(jsFilePath) + ".d.ts", referencePathsOutput + writer.getText());
2451+
writeFile(getModuleNameFromFilename(jsFilePath) + ".d.ts", referencePathsOutput + writer.getText(), compilerOptions.emitBOM);
24522452
}
24532453
}
24542454

‎src/compiler/parser.ts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,7 @@ module ts {
417417
nodeIsNestedInLabel(label: Identifier, requireIterationStatement: boolean, stopAtFunctionBoundary: boolean): ControlBlockContext;
418418
}
419419

420-
export function createSourceFile(filename: string, sourceText: string, languageVersion: ScriptTarget, byteOrderMark: ByteOrderMark, version: number = 0, isOpen: boolean = false): SourceFile {
420+
export function createSourceFile(filename: string, sourceText: string, languageVersion: ScriptTarget, version: number = 0, isOpen: boolean = false): SourceFile {
421421
var file: SourceFile;
422422
var scanner: Scanner;
423423
var token: SyntaxKind;
@@ -3535,7 +3535,6 @@ module ts {
35353535
file.nodeCount = nodeCount;
35363536
file.identifierCount = identifierCount;
35373537
file.version = version;
3538-
file.byteOrderMark = byteOrderMark;
35393538
file.isOpen = isOpen;
35403539
file.languageVersion = languageVersion;
35413540
return file;

‎src/compiler/sys.ts

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ interface System {
66
useCaseSensitiveFileNames: boolean;
77
write(s: string): void;
88
readFile(fileName: string, encoding?: string): string;
9-
writeFile(fileName: string, data: string): void;
9+
writeFile(fileName: string, data: string, writeByteOrderMark?: boolean): void;
1010
watchFile?(fileName: string, callback: (fileName: string) => void): FileWatcher;
1111
resolvePath(path: string): string;
1212
fileExists(path: string): boolean;
@@ -75,15 +75,21 @@ var sys: System = (function () {
7575
}
7676
}
7777

78-
function writeFile(fileName: string, data: string): void {
78+
function writeFile(fileName: string, data: string, writeByteOrderMark?: boolean): void {
7979
fileStream.Open();
8080
binaryStream.Open();
8181
try {
8282
// Write characters in UTF-8 encoding
8383
fileStream.Charset = "utf-8";
8484
fileStream.WriteText(data);
85-
// Skip byte order mark and copy remaining data to binary stream
86-
fileStream.Position = 3;
85+
// If we don't want the BOM, then skip it by setting the starting location to 3 (size of BOM).
86+
// If not, start from position 0, as the BOM will be added automatically when charset==utf8.
87+
if (writeByteOrderMark) {
88+
fileStream.Position = 0;
89+
}
90+
else {
91+
fileStream.Position = 3;
92+
}
8793
fileStream.CopyTo(binaryStream);
8894
binaryStream.SaveToFile(fileName, 2 /*overwrite*/);
8995
}
@@ -175,7 +181,12 @@ var sys: System = (function () {
175181
return buffer.toString("utf8");
176182
}
177183

178-
function writeFile(fileName: string, data: string): void {
184+
function writeFile(fileName: string, data: string, writeByteOrderMark?: boolean): void {
185+
// If a BOM is required, emit one
186+
if (writeByteOrderMark) {
187+
data = '\uFEFF' + data;
188+
}
189+
179190
_fs.writeFileSync(fileName, data, "utf8");
180191
}
181192

‎src/compiler/tc.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -142,10 +142,10 @@ module ts {
142142
}
143143
text = "";
144144
}
145-
return text !== undefined ? createSourceFile(filename, text, languageVersion, ByteOrderMark.None) : undefined;
145+
return text !== undefined ? createSourceFile(filename, text, languageVersion) : undefined;
146146
}
147147

148-
function writeFile(fileName: string, data: string, onError?: (message: string) => void) {
148+
function writeFile(fileName: string, data: string, writeByteOrderMark: boolean, onError?: (message: string) => void) {
149149

150150
function directoryExists(directoryPath: string): boolean {
151151
if (hasProperty(existingDirectories, directoryPath)) {
@@ -168,7 +168,7 @@ module ts {
168168

169169
try {
170170
ensureDirectoriesExist(getDirectoryPath(normalizePath(fileName)));
171-
sys.writeFile(fileName, data);
171+
sys.writeFile(fileName, data, writeByteOrderMark);
172172
}
173173
catch (e) {
174174
if (onError) onError(e.message);

‎src/compiler/types.ts

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -525,7 +525,6 @@ module ts {
525525
nodeCount: number;
526526
identifierCount: number;
527527
symbolCount: number;
528-
byteOrderMark: ByteOrderMark;
529528
isOpen: boolean;
530529
version: number;
531530
languageVersion: ScriptTarget;
@@ -940,6 +939,7 @@ module ts {
940939
codepage?: number;
941940
declaration?: boolean;
942941
diagnostics?: boolean;
942+
emitBOM?: boolean;
943943
help?: boolean;
944944
locale?: string;
945945
mapRoot?: string;
@@ -1131,17 +1131,10 @@ module ts {
11311131
getSourceFile(filename: string, languageVersion: ScriptTarget, onError?: (message: string) => void): SourceFile;
11321132
getDefaultLibFilename(): string;
11331133
getCancellationToken? (): CancellationToken;
1134-
writeFile(filename: string, data: string, onError?: (message: string) => void): void;
1134+
writeFile(filename: string, data: string, writeByteOrderMark: boolean, onError?: (message: string) => void): void;
11351135
getCurrentDirectory(): string;
11361136
getCanonicalFileName(fileName: string): string;
11371137
useCaseSensitiveFileNames(): boolean;
11381138
getNewLine(): string;
11391139
}
1140-
1141-
export enum ByteOrderMark {
1142-
None = 0,
1143-
Utf8 = 1,
1144-
Utf16BigEndian = 2,
1145-
Utf16LittleEndian = 3,
1146-
}
11471140
}

‎src/harness/compilerRunner.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,10 @@ class CompilerBaselineRunner extends RunnerBase {
118118
}
119119
});
120120

121+
function getByteOrderMarkText(file: Harness.Compiler.GeneratedFile): string {
122+
return file.writeByteOrderMark ? "\u00EF\u00BB\u00BF" : "";
123+
}
124+
121125
function getErrorBaseline(toBeCompiled: { unitName: string; content: string }[],
122126
otherFiles: { unitName: string; content: string }[],
123127
result: Harness.Compiler.CompilerResult
@@ -282,6 +286,7 @@ class CompilerBaselineRunner extends RunnerBase {
282286
var jsCode = '';
283287
for (var i = 0; i < result.files.length; i++) {
284288
jsCode += '//// [' + Harness.Path.getFileName(result.files[i].fileName) + ']\r\n';
289+
jsCode += getByteOrderMarkText(result.files[i]);
285290
jsCode += result.files[i].code;
286291
// Re-enable this if we want to do another comparison of old vs new compiler baselines
287292
// jsCode += SyntacticCleaner.clean(result.files[i].code);
@@ -291,6 +296,7 @@ class CompilerBaselineRunner extends RunnerBase {
291296
jsCode += '\r\n\r\n';
292297
for (var i = 0; i < result.files.length; i++) {
293298
jsCode += '//// [' + Harness.Path.getFileName(result.declFilesCode[i].fileName) + ']\r\n';
299+
jsCode += getByteOrderMarkText(result.declFilesCode[i]);
294300
jsCode += result.declFilesCode[i].code;
295301
}
296302
}
@@ -320,6 +326,7 @@ class CompilerBaselineRunner extends RunnerBase {
320326
var sourceMapCode = '';
321327
for (var i = 0; i < result.sourceMaps.length; i++) {
322328
sourceMapCode += '//// [' + Harness.Path.getFileName(result.sourceMaps[i].fileName) + ']\r\n';
329+
sourceMapCode += getByteOrderMarkText(result.sourceMaps[i]);
323330
sourceMapCode += result.sourceMaps[i].code;
324331
}
325332

‎src/harness/fourslash.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1895,12 +1895,12 @@ module FourSlash {
18951895
var result = '';
18961896
var fourslashFilename = 'fourslash.ts';
18971897
var tsFn = 'tests/cases/fourslash/' + fourslashFilename;
1898-
fourslashSourceFile = fourslashSourceFile || ts.createSourceFile(tsFn, Harness.IO.readFile(tsFn), ts.ScriptTarget.ES5, ts.ByteOrderMark.None, /*version*/ 0, /*isOpen*/ false);
1899-
libdtsSourceFile = libdtsSourceFile || ts.createSourceFile('lib.d.ts', Harness.Compiler.libTextMinimal, ts.ScriptTarget.ES3, ts.ByteOrderMark.None, /*version*/ 0, /*isOpen*/ false);
1898+
fourslashSourceFile = fourslashSourceFile || ts.createSourceFile(tsFn, Harness.IO.readFile(tsFn), ts.ScriptTarget.ES5, /*version*/ 0, /*isOpen*/ false);
1899+
libdtsSourceFile = libdtsSourceFile || ts.createSourceFile('lib.d.ts', Harness.Compiler.libTextMinimal, ts.ScriptTarget.ES3, /*version*/ 0, /*isOpen*/ false);
19001900

19011901
var files: { [filename: string]: ts.SourceFile; } = {};
19021902
files[fourslashFilename] = fourslashSourceFile;
1903-
files[fileName] = ts.createSourceFile(fileName, content, ts.ScriptTarget.ES5, ts.ByteOrderMark.None, /*version*/ 0, /*isOpen*/ false);
1903+
files[fileName] = ts.createSourceFile(fileName, content, ts.ScriptTarget.ES5, /*version*/ 0, /*isOpen*/ false);
19041904
files['lib.d.ts'] = libdtsSourceFile;
19051905

19061906
var host = Harness.Compiler.createCompilerHost(files, (fn, contents) => result = contents);

‎src/harness/harness.ts

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -534,7 +534,7 @@ module Harness {
534534
export var libText = IO.readFile(libFolder + "lib.d.ts");
535535
export var libTextMinimal = IO.readFile('bin/lib.core.d.ts');
536536

537-
export function createCompilerHost(filemap: { [filename: string]: ts.SourceFile; }, writeFile: (fn: string, contents: string) => void): ts.CompilerHost {
537+
export function createCompilerHost(filemap: { [filename: string]: ts.SourceFile; }, writeFile: (fn: string, contents: string, writeByteOrderMark:boolean) => void): ts.CompilerHost {
538538
return {
539539
getCurrentDirectory: sys.getCurrentDirectory,
540540
getCancellationToken: (): any => undefined,
@@ -544,7 +544,7 @@ module Harness {
544544
} else {
545545
var lib = 'lib.d.ts';
546546
if (fn.substr(fn.length - lib.length) === lib) {
547-
return filemap[fn] = ts.createSourceFile('lib.d.ts', libTextMinimal, languageVersion, ts.ByteOrderMark.None);
547+
return filemap[fn] = ts.createSourceFile('lib.d.ts', libTextMinimal, languageVersion);
548548
}
549549
// Don't throw here -- the compiler might be looking for a test that actually doesn't exist as part of the TC
550550
return null;
@@ -712,6 +712,10 @@ module Harness {
712712
// Not supported yet
713713
break;
714714

715+
case 'emitbom':
716+
options.emitBOM = !!setting.value;
717+
break;
718+
715719
default:
716720
throw new Error('Unsupported compiler setting ' + setting.flag);
717721
}
@@ -720,18 +724,15 @@ module Harness {
720724
var filemap: { [name: string]: ts.SourceFile; } = {};
721725
var register = (file: { unitName: string; content: string; }) => {
722726
var filename = Path.switchToForwardSlashes(file.unitName);
723-
filemap[filename] = ts.createSourceFile(filename, file.content, options.target, ts.ByteOrderMark.None);
727+
filemap[filename] = ts.createSourceFile(filename, file.content, options.target);
724728
};
725729
inputFiles.forEach(register);
726730
otherFiles.forEach(register);
727731

728-
var fileOutputs: {
729-
fileName: string;
730-
file: string;
731-
}[] = [];
732+
var fileOutputs: GeneratedFile[] = [];
732733

733734
var programFiles = inputFiles.map(file => file.unitName);
734-
var program = ts.createProgram(programFiles, options, createCompilerHost(filemap, (fn, contents) => fileOutputs.push({ fileName: fn, file: contents })));
735+
var program = ts.createProgram(programFiles, options, createCompilerHost(filemap, (fn, contents, writeByteOrderMark) => fileOutputs.push({ fileName: fn, code: contents, writeByteOrderMark: writeByteOrderMark })));
735736

736737
var hadParseErrors = program.getDiagnostics().length > 0;
737738

@@ -810,6 +811,7 @@ module Harness {
810811
export interface GeneratedFile {
811812
fileName: string;
812813
code: string;
814+
writeByteOrderMark: boolean;
813815
}
814816

815817
function stringEndsWith(str: string, end: string) {
@@ -837,19 +839,18 @@ module Harness {
837839
public sourceMapRecord: string;
838840

839841
/** @param fileResults an array of strings for the fileName and an ITextWriter with its code */
840-
constructor(fileResults: { fileName: string; file: string; }[], errors: MinimalDiagnostic[], sourceMapRecordLines: string[]) {
842+
constructor(fileResults: GeneratedFile[], errors: MinimalDiagnostic[], sourceMapRecordLines: string[]) {
841843
var lines: string[] = [];
842844

843845
fileResults.forEach(emittedFile => {
844-
var fileObj = { fileName: emittedFile.fileName, code: emittedFile.file };
845846
if (isDTS(emittedFile.fileName)) {
846847
// .d.ts file, add to declFiles emit
847-
this.declFilesCode.push(fileObj);
848+
this.declFilesCode.push(emittedFile);
848849
} else if (isJS(emittedFile.fileName)) {
849850
// .js file, add to files
850-
this.files.push(fileObj);
851+
this.files.push(emittedFile);
851852
} else if (isJSMap(emittedFile.fileName)) {
852-
this.sourceMaps.push(fileObj);
853+
this.sourceMaps.push(emittedFile);
853854
} else {
854855
throw new Error('Unrecognized file extension for file ' + emittedFile.fileName);
855856
}
@@ -896,7 +897,7 @@ module Harness {
896897
var optionRegex = /^[\/]{2}\s*@(\w+)\s*:\s*(\S*)/gm; // multiple matches on multiple lines
897898

898899
// List of allowed metadata names
899-
var fileMetadataNames = ["filename", "comments", "declaration", "module", "nolib", "sourcemap", "target", "out", "outDir", "noimplicitany", "noresolve", "newline", "newlines"];
900+
var fileMetadataNames = ["filename", "comments", "declaration", "module", "nolib", "sourcemap", "target", "out", "outDir", "noimplicitany", "noresolve", "newline", "newlines", "emitbom"];
900901

901902
function extractCompilerSettings(content: string): CompilerSetting[] {
902903

0 commit comments

Comments
 (0)