Merge pull request #7513 from zhengbli/projectSizeLimitFor18-2

Port #7486 to release 1.8

Author: zhengbli

src/compiler/commandLineParser.ts

@@ -305,6 +305,10 @@ namespace ts {
             name: "noCustomAsyncPromise",
             type: "boolean",
             experimental: true
+        },
+        {
+            name: "disableSizeLimit",
+            type: "boolean"
         }
     ];
 
@@ -534,7 +538,7 @@ namespace ts {
                 }
                 else {
                     // by default exclude node_modules, and any specificied output directory
-                    exclude = ["node_modules"];
+                    exclude = ["node_modules", "bower_components"];
                     const outDir = json["compilerOptions"] && json["compilerOptions"]["outDir"];
                     if (outDir) {
                         exclude.push(outDir);

src/compiler/diagnosticMessages.json

@@ -2661,5 +2661,9 @@
     "Unknown typing option '{0}'.": {
         "category": "Error",
         "code": 17010
+    },
+    "Too many JavaScript files in the project. Consider specifying the 'exclude' setting in project configuration to limit included source folders. The likely folder to exclude is '{0}'. To disable the project size limit, set the 'disableSizeLimit' compiler option to 'true'.": {
+        "category": "Error",
+        "code": 17012
     }
 }

src/compiler/program.ts

@@ -7,6 +7,7 @@ namespace ts {
     /* @internal */ export let emitTime = 0;
     /* @internal */ export let ioReadTime = 0;
     /* @internal */ export let ioWriteTime = 0;
+    /* @internal */ export const maxProgramSizeForNonTsFiles = 20 * 1024 * 1024;
 
     /** The version of the TypeScript compiler release */
 
@@ -348,6 +349,8 @@ namespace ts {
         let diagnosticsProducingTypeChecker: TypeChecker;
         let noDiagnosticsTypeChecker: TypeChecker;
         let classifiableNames: Map<string>;
+        const programSizeLimitExceeded = -1;
+        let programSizeForNonTsFiles = 0;
 
         let skipDefaultLib = options.noLib;
         const supportedExtensions = getSupportedExtensions(options);
@@ -394,7 +397,8 @@ namespace ts {
                 (oldOptions.target !== options.target) ||
                 (oldOptions.noLib !== options.noLib) ||
                 (oldOptions.jsx !== options.jsx) ||
-                (oldOptions.allowJs !== options.allowJs)) {
+                (oldOptions.allowJs !== options.allowJs) ||
+                (oldOptions.disableSizeLimit !== options.disableSizeLimit)) {
                 oldProgram = undefined;
             }
         }
@@ -442,6 +446,10 @@ namespace ts {
 
         return program;
 
+        function exceedProgramSizeLimit() {
+            return !options.disableSizeLimit && programSizeForNonTsFiles === programSizeLimitExceeded;
+        }
+
         function getCommonSourceDirectory() {
             if (typeof commonSourceDirectory === "undefined") {
                 if (options.rootDir && checkSourceFilesBelongToPath(files, options.rootDir)) {
@@ -1054,7 +1062,7 @@ namespace ts {
                 }
             }
 
-            if (diagnostic) {
+            if (diagnostic && !exceedProgramSizeLimit()) {
                 if (refFile !== undefined && refEnd !== undefined && refPos !== undefined) {
                     fileProcessingDiagnostics.add(createFileDiagnostic(refFile, refPos, refEnd - refPos, diagnostic, ...diagnosticArgument));
                 }
@@ -1087,6 +1095,11 @@ namespace ts {
                 return file;
             }
 
+            const isNonTsFile = !hasTypeScriptFileExtension(fileName);
+            if (isNonTsFile && exceedProgramSizeLimit()) {
+                return undefined;
+            }
+
             // We haven't looked for this file, do so now and cache result
             const file = host.getSourceFile(fileName, options.target, hostErrorMessage => {
                 if (refFile !== undefined && refPos !== undefined && refEnd !== undefined) {
@@ -1098,6 +1111,25 @@ namespace ts {
                 }
             });
 
+            if (isNonTsFile && !options.disableSizeLimit && file && file.text) {
+                programSizeForNonTsFiles += file.text.length;
+                if (programSizeForNonTsFiles > maxProgramSizeForNonTsFiles) {
+                    // If the program size limit was reached when processing a file, this file is
+                    // likely in the problematic folder than contains too many files.
+                    // Normally the folder is one level down from the commonSourceDirectory, for example,
+                    // if the commonSourceDirectory is "/src/", and the last processed path was "/src/node_modules/a/b.js",
+                    // we should show in the error message "/src/node_modules/".
+                    const commonSourceDirectory = getCommonSourceDirectory();
+                    let rootLevelDirectory = path.substring(0, Math.max(commonSourceDirectory.length, path.indexOf(directorySeparator, commonSourceDirectory.length)));
+                    if (rootLevelDirectory[rootLevelDirectory.length - 1] !== directorySeparator) {
+                        rootLevelDirectory += directorySeparator;
+                    }
+                    programDiagnostics.add(createCompilerDiagnostic(Diagnostics.Too_many_JavaScript_files_in_the_project_Consider_specifying_the_exclude_setting_in_project_configuration_to_limit_included_source_folders_The_likely_folder_to_exclude_is_0_To_disable_the_project_size_limit_set_the_disableSizeLimit_compiler_option_to_true, rootLevelDirectory));
+                    programSizeForNonTsFiles = programSizeLimitExceeded;
+                    return undefined;
+                }
+            }
+
             filesByName.set(path, file);
             if (file) {
                 file.path = path;

src/compiler/sys.ts

@@ -74,7 +74,7 @@ namespace ts {
         watchDirectory?(path: string, callback: DirectoryWatcherCallback, recursive?: boolean): FileWatcher;
     };
 
-    export var sys: System = (function () {
+    export var sys: System = (function() {
 
         function getWScriptSystem(): System {
 
@@ -407,8 +407,8 @@ namespace ts {
             const watchedFileSet = createWatchedFileSet();
 
             function isNode4OrLater(): boolean {
-                 return parseInt(process.version.charAt(1)) >= 4;
-             }
+                return parseInt(process.version.charAt(1)) >= 4;
+            }
 
             const platform: string = _os.platform();
             // win32\win64 are case insensitive platforms, MacOS (darwin) by default is also case insensitive
@@ -477,15 +477,20 @@ namespace ts {
                     for (const current of files) {
                         const name = combinePaths(path, current);
                         if (!contains(exclude, getCanonicalPath(name))) {
-                            const stat = _fs.statSync(name);
-                            if (stat.isFile()) {
-                                if (!extension || fileExtensionIs(name, extension)) {
-                                    result.push(name);
+                            // fs.statSync would throw an exception if the file is a symlink
+                            // whose linked file doesn't exist.
+                            try {
+                                const stat = _fs.statSync(name);
+                                if (stat.isFile()) {
+                                    if (!extension || fileExtensionIs(name, extension)) {
+                                        result.push(name);
+                                    }
+                                }
+                                else if (stat.isDirectory()) {
+                                    directories.push(name);
                                 }
                             }
-                            else if (stat.isDirectory()) {
-                                directories.push(name);
-                            }
+                            catch (e) { }
                         }
                     }
                     for (const current of directories) {
@@ -509,7 +514,7 @@ namespace ts {
                     // and https://github.com/Microsoft/TypeScript/issues/4643), therefore
                     // if the current node.js version is newer than 4, use `fs.watch` instead.
                     const watchSet = isNode4OrLater() ? watchedFileSet : pollingWatchedFileSet;
-                    const watchedFile =  watchSet.addFile(filePath, callback);
+                    const watchedFile = watchSet.addFile(filePath, callback);
                     return {
                         close: () => watchSet.removeFile(watchedFile)
                     };
@@ -539,7 +544,7 @@ namespace ts {
                         }
                     );
                 },
-                resolvePath: function (path: string): string {
+                resolvePath: function(path: string): string {
                     return _path.resolve(path);
                 },
                 fileExists(path: string): boolean {

src/compiler/types.ts

@@ -2437,6 +2437,7 @@ namespace ts {
         allowSyntheticDefaultImports?: boolean;
         allowJs?: boolean;
         noImplicitUseStrict?: boolean;
+        disableSizeLimit?: boolean;
         /* @internal */ stripInternal?: boolean;
 
         // Skip checking lib.d.ts to help speed up tests.

src/compiler/utilities.ts

@@ -2476,6 +2476,10 @@ namespace ts {
         return forEach(supportedJavascriptExtensions, extension => fileExtensionIs(fileName, extension));
     }
 
+    export function hasTypeScriptFileExtension(fileName: string) {
+        return forEach(supportedTypeScriptExtensions, extension => fileExtensionIs(fileName, extension));
+    }
+
     /**
      * Replace each instance of non-ascii characters by one, two, three, or four escape sequences
      * representing the UTF-8 encoding of the character, and return the expanded char code list.

src/server/editorServices.ts

@@ -1217,10 +1217,45 @@ namespace ts.server {
             }
             else {
                 const project = this.createProject(configFilename, projectOptions);
+                let programSizeForNonTsFiles = 0;
+
+                // As the project openning might not be complete if there are too many files,
+                // therefore to surface the diagnostics we need to make sure the given client file is opened.
+                if (clientFileName) {
+                    if (this.host.fileExists(clientFileName)) {
+                        const currentClientFileInfo = this.openFile(clientFileName, /*openedByClient*/ true);
+                        project.addRoot(currentClientFileInfo);
+                        if (!hasTypeScriptFileExtension(currentClientFileInfo.fileName) && currentClientFileInfo.content) {
+                            programSizeForNonTsFiles += currentClientFileInfo.content.length;
+                        }
+                    }
+                    else {
+                        return { errorMsg: "specified file " + clientFileName + " not found" };
+                    }
+                }
+
                 for (const rootFilename of projectOptions.files) {
+                    if (rootFilename === clientFileName) {
+                        continue;
+                    }
+
                     if (this.host.fileExists(rootFilename)) {
-                        const info = this.openFile(rootFilename, /*openedByClient*/ clientFileName == rootFilename);
-                        project.addRoot(info);
+                        if (projectOptions.compilerOptions.disableSizeLimit) {
+                            const info = this.openFile(rootFilename, /*openedByClient*/ false);
+                            project.addRoot(info);
+                        }
+                        else if (programSizeForNonTsFiles <= maxProgramSizeForNonTsFiles) {
+                            const info = this.openFile(rootFilename, /*openedByClient*/ false);
+                            project.addRoot(info);
+                            if (!hasTypeScriptFileExtension(rootFilename)) {
+                                programSizeForNonTsFiles += info.content.length;
+                            }
+                        }
+                        else {
+                            // The project size is too large. Stop loading the files on the server,
+                            // and let the compiler issue an diagnostic via `createProgram`.
+                            break;
+                        }
                     }
                     else {
                         return { errorMsg: "specified file " + rootFilename + " not found" };
@@ -1251,7 +1286,10 @@ namespace ts.server {
                     return error;
                 }
                 else {
-                    const oldFileNames = project.compilerService.host.roots.map(info => info.fileName);
+                    // if the project is too large, the root files might not have been all loaded if the total
+                    // program size reached the upper limit. In that case project.projectOptions.files should
+                    // be more precise. However this would only happen for configured project.
+                    const oldFileNames = project.projectOptions ? project.projectOptions.files : project.compilerService.host.roots.map(info => info.fileName);
                     const newFileNames = projectOptions.files;
                     const fileNamesToRemove = oldFileNames.filter(f => newFileNames.indexOf(f) < 0);
                     const fileNamesToAdd = newFileNames.filter(f => oldFileNames.indexOf(f) < 0);

src/services/services.ts

@@ -2759,7 +2759,8 @@ namespace ts {
                  oldSettings.module !== newSettings.module ||
                  oldSettings.noResolve !== newSettings.noResolve ||
                  oldSettings.jsx !== newSettings.jsx ||
-                 oldSettings.allowJs !== newSettings.allowJs);
+                 oldSettings.allowJs !== newSettings.allowJs ||
+                 oldSettings.disableSizeLimit !== oldSettings.disableSizeLimit);
 
             // Now create a new compiler
             const compilerHost: CompilerHost = {