Merge pull request #14999 from RyanCavanaugh/typesMap

Add advanced safelist for exclusions

Author: RyanCavanaugh

src/harness/unittests/tsserverProjectSystem.ts

@@ -17,6 +17,16 @@ namespace ts.projectSystem {
         })
     };
 
+    const customSafeList = {
+        path: <Path>"/typeMapList.json",
+        content: JSON.stringify({
+            "quack": {
+                "match": "/duckquack-(\\d+)\\.min\\.js",
+                "types": ["duck-types"]
+            },
+        })
+    };
+
     export interface PostExecAction {
         readonly success: boolean;
         readonly callback: TI.RequestCompletedAction;
@@ -1445,6 +1455,28 @@ namespace ts.projectSystem {
             checkProjectActualFiles(projectService.inferredProjects[1], [file3.path]);
         });
 
+        it("ignores files excluded by the safe type list", () => {
+            const file1 = {
+                path: "/a/b/f1.ts",
+                content: "export let x = 5"
+            };
+            const office = {
+                path: "/lib/duckquack-3.min.js",
+                content: "whoa do @@ not parse me ok thanks!!!"
+            };
+            const host = createServerHost([customSafeList, file1, office]);
+            const projectService = createProjectService(host);
+            projectService.loadSafeList(customSafeList.path);
+            try {
+                projectService.openExternalProject({ projectFileName: "project", options: {}, rootFiles: toExternalFiles([file1.path, office.path]) });
+                const proj = projectService.externalProjects[0];
+                assert.deepEqual(proj.getFileNames(/*excludeFilesFromExternalLibraries*/ true), [file1.path]);
+                assert.deepEqual(proj.getTypeAcquisition().include, ["duck-types"]);
+            } finally {
+                projectService.resetSafeList();
+            }
+        });
+
         it("open file become a part of configured project if it is referenced from root file", () => {
             const file1 = {
                 path: "/a/b/f1.ts",
@@ -1695,7 +1727,7 @@ namespace ts.projectSystem {
             checkProjectActualFiles(projectService.inferredProjects[1], [file2.path]);
         });
 
-        it ("loading files with correct priority", () => {
+        it("loading files with correct priority", () => {
             const f1 = {
                 path: "/a/main.ts",
                 content: "let x = 1"
@@ -1720,14 +1752,14 @@ namespace ts.projectSystem {
             });
             projectService.openClientFile(f1.path);
             projectService.checkNumberOfProjects({ configuredProjects: 1 });
-            checkProjectActualFiles(projectService.configuredProjects[0], [ f1.path ]);
+            checkProjectActualFiles(projectService.configuredProjects[0], [f1.path]);
 
             projectService.closeClientFile(f1.path);
 
             projectService.openClientFile(f2.path);
             projectService.checkNumberOfProjects({ configuredProjects: 1, inferredProjects: 1 });
-            checkProjectActualFiles(projectService.configuredProjects[0], [ f1.path ]);
-            checkProjectActualFiles(projectService.inferredProjects[0], [ f2.path ]);
+            checkProjectActualFiles(projectService.configuredProjects[0], [f1.path]);
+            checkProjectActualFiles(projectService.inferredProjects[0], [f2.path]);
         });
 
         it("tsconfig script block support", () => {
@@ -1845,7 +1877,7 @@ namespace ts.projectSystem {
             //  #3. Ensure no errors when compiler options aren't specified
             const config3 = {
                 path: "/a/b/tsconfig.json",
-                content: JSON.stringify({ })
+                content: JSON.stringify({})
             };
 
             host = createServerHost([file1, file2, config3, libFile], { executingFilePath: combinePaths(getDirectoryPath(libFile.path), "tsc.js") });
@@ -3381,13 +3413,13 @@ namespace ts.projectSystem {
             assert.equal((<protocol.CompileOnSaveAffectedFileListSingleProject[]>response)[0].projectUsesOutFile, expectedUsesOutFile, "usesOutFile");
         }
 
-        it ("projectUsesOutFile should not be returned if not set", () => {
+        it("projectUsesOutFile should not be returned if not set", () => {
             test({}, /*expectedUsesOutFile*/ false);
         });
-        it ("projectUsesOutFile should be true if outFile is set", () => {
+        it("projectUsesOutFile should be true if outFile is set", () => {
             test({ outFile: "/a/out.js" }, /*expectedUsesOutFile*/ true);
         });
-        it ("projectUsesOutFile should be true if out is set", () => {
+        it("projectUsesOutFile should be true if out is set", () => {
             test({ out: "/a/out.js" }, /*expectedUsesOutFile*/ true);
         });
     });
@@ -3468,7 +3500,7 @@ namespace ts.projectSystem {
 
             const cancellationToken = new TestServerCancellationToken();
             const host = createServerHost([f1, config]);
-            const session = createSession(host, /*typingsInstaller*/ undefined, () => {}, cancellationToken);
+            const session = createSession(host, /*typingsInstaller*/ undefined, () => { }, cancellationToken);
             {
                 session.executeCommandSeq(<protocol.OpenRequest>{
                     command: "open",
@@ -3750,4 +3782,4 @@ namespace ts.projectSystem {
             assert.isUndefined(project.getCompilerOptions().maxNodeModuleJsDepth);
         });
     });
-}
+}

src/harness/unittests/typingsInstaller.ts

@@ -381,12 +381,12 @@ namespace ts.projectSystem {
 
             const p = projectService.externalProjects[0];
             projectService.checkNumberOfProjects({ externalProjects: 1 });
-            checkProjectActualFiles(p, [file1.path, file2.path]);
+            checkProjectActualFiles(p, [file2.path]);
 
             installer.checkPendingCommands(/*expectedCount*/ 0);
 
             checkNumberOfProjects(projectService, { externalProjects: 1 });
-            checkProjectActualFiles(p, [file1.path, file2.path]);
+            checkProjectActualFiles(p, [file2.path]);
         });
 
         it("external project - with type acquisition, with only js, d.ts files", () => {

src/server/editorServices.ts

@@ -35,6 +35,10 @@ namespace ts.server {
         (event: ProjectServiceEvent): void;
     }
 
+    export interface SafeList {
+        [name: string]: { match: RegExp, exclude?: Array<Array<string | number>>, types?: string[] };
+    }
+
     function prepareConvertersForEnumLikeCompilerOptions(commandLineOptions: CommandLineOption[]): Map<Map<number>> {
         const map: Map<Map<number>> = createMap<Map<number>>();
         for (const option of commandLineOptions) {
@@ -57,6 +61,32 @@ namespace ts.server {
         "smart": IndentStyle.Smart
     });
 
+    const defaultTypeSafeList: SafeList = {
+        "jquery": {
+            // jquery files can have names like "jquery-1.10.2.min.js" (or "jquery.intellisense.js")
+            "match": /jquery(-(\.?\d+)+)?(\.intellisense)?(\.min)?\.js$/i,
+            "types": ["jquery"]
+        },
+        "WinJS": {
+            // e.g. c:/temp/UWApp1/lib/winjs-4.0.1/js/base.js
+            "match": /^(.*\/winjs-[.\d]+)\/js\/base\.js$/i,        // If the winjs/base.js file is found..
+            "exclude": [["^", 1, "/.*"]],                // ..then exclude all files under the winjs folder
+            "types": ["winjs"]                           // And fetch the @types package for WinJS
+        },
+        "Kendo": {
+            // e.g. /Kendo3/wwwroot/lib/kendo/kendo.all.min.js
+            "match": /^(.*\/kendo)\/kendo\.all\.min\.js$/i,
+            "exclude": [["^", 1, "/.*"]],
+            "types": ["kendo-ui"]
+        },
+        "Office Nuget": {
+            // e.g. /scripts/Office/1/excel-15.debug.js
+            "match": /^(.*\/office\/1)\/excel-\d+\.debug\.js$/i, // Office NuGet package is installed under a "1/office" folder
+            "exclude": [["^", 1, "/.*"]],                     // Exclude that whole folder if the file indicated above is found in it
+            "types": ["office"]                               // @types package to fetch instead
+        }
+    };
+
     export function convertFormatOptions(protocolOptions: protocol.FormatCodeSettings): FormatCodeSettings {
         if (typeof protocolOptions.indentStyle === "string") {
             protocolOptions.indentStyle = indentStyle.get(protocolOptions.indentStyle.toLowerCase());
@@ -259,6 +289,7 @@ namespace ts.server {
         private readonly throttledOperations: ThrottledOperations;
 
         private readonly hostConfiguration: HostConfiguration;
+        private static safelist: SafeList = defaultTypeSafeList;
 
         private changedFiles: ScriptInfo[];
 
@@ -284,8 +315,6 @@ namespace ts.server {
 
             this.typingsCache = new TypingsCache(this.typingsInstaller);
 
-            // ts.disableIncrementalParsing = true;
-
             this.hostConfiguration = {
                 formatCodeOptions: getDefaultFormatCodeSettings(this.host),
                 hostInfo: "Unknown host",
@@ -831,7 +860,7 @@ namespace ts.server {
                 getDirectoryPath(configFilename),
                 /*existingOptions*/ {},
                 configFilename,
-                /*resolutionStack*/ [],
+                /*resolutionStack*/[],
                 this.hostConfiguration.extraFileExtensions);
 
             if (parsedCommandLine.errors.length) {
@@ -1399,13 +1428,104 @@ namespace ts.server {
             this.refreshInferredProjects();
         }
 
+        /** Makes a filename safe to insert in a RegExp */
+        private static filenameEscapeRegexp = /[-\/\\^$*+?.()|[\]{}]/g;
+        private static escapeFilenameForRegex(filename: string) {
+            return filename.replace(this.filenameEscapeRegexp, "\\$&");
+        }
+
+        resetSafeList(): void {
+            ProjectService.safelist = defaultTypeSafeList;
+        }
+
+        loadSafeList(fileName: string): void {
+            const raw: SafeList = JSON.parse(this.host.readFile(fileName, "utf-8"));
+            // Parse the regexps
+            for (const k of Object.keys(raw)) {
+                raw[k].match = new RegExp(raw[k].match as {} as string, "i");
+            }
+            // raw is now fixed and ready
+            ProjectService.safelist = raw;
+        }
+
+        applySafeList(proj: protocol.ExternalProject): void {
+            const { rootFiles, typeAcquisition } = proj;
+            const types = (typeAcquisition && typeAcquisition.include) || [];
+
+            const excludeRules: string[] = [];
+
+            const normalizedNames = rootFiles.map(f => normalizeSlashes(f.fileName));
+
+            for (const name of Object.keys(ProjectService.safelist)) {
+                const rule = ProjectService.safelist[name];
+                for (const root of normalizedNames) {
+                    if (rule.match.test(root)) {
+                        this.logger.info(`Excluding files based on rule ${name}`);
+
+                        // If the file matches, collect its types packages and exclude rules
+                        if (rule.types) {
+                            for (const type of rule.types) {
+                                if (types.indexOf(type) < 0) {
+                                    types.push(type);
+                                }
+                            }
+                        }
+
+                        if (rule.exclude) {
+                            for (const exclude of rule.exclude) {
+                                const processedRule = root.replace(rule.match, (...groups: Array<string>) => {
+                                    return exclude.map(groupNumberOrString => {
+                                        // RegExp group numbers are 1-based, but the first element in groups
+                                        // is actually the original string, so it all works out in the end.
+                                        if (typeof groupNumberOrString === "number") {
+                                            if (typeof groups[groupNumberOrString] !== "string") {
+                                                // Specification was wrong - exclude nothing!
+                                                this.logger.info(`Incorrect RegExp specification in safelist rule ${name} - not enough groups`);
+                                                // * can't appear in a filename; escape it because it's feeding into a RegExp
+                                                return "\\*";
+                                            }
+                                            return ProjectService.escapeFilenameForRegex(groups[groupNumberOrString]);
+                                        }
+                                        return groupNumberOrString;
+                                    }).join("");
+                                });
+
+                                if (excludeRules.indexOf(processedRule) === -1) {
+                                    excludeRules.push(processedRule);
+                                }
+                            }
+                        }
+                        else {
+                            // If not rules listed, add the default rule to exclude the matched file
+                            const escaped = ProjectService.escapeFilenameForRegex(root);
+                            if (excludeRules.indexOf(escaped) < 0) {
+                                excludeRules.push(escaped);
+                            }
+                        }
+                    }
+                }
+
+                // Copy back this field into the project if needed
+                if (types.length > 0) {
+                    proj.typeAcquisition = proj.typeAcquisition || { };
+                    proj.typeAcquisition.include = types;
+                }
+            }
+
+            const excludeRegexes = excludeRules.map(e => new RegExp(e, "i"));
+            proj.rootFiles = proj.rootFiles.filter((_file, index) => !excludeRegexes.some(re => re.test(normalizedNames[index])));
+        }
+
         openExternalProject(proj: protocol.ExternalProject, suppressRefreshOfInferredProjects = false): void {
             // typingOptions has been deprecated and is only supported for backward compatibility
             // purposes. It should be removed in future releases - use typeAcquisition instead.
             if (proj.typingOptions && !proj.typeAcquisition) {
                 const typeAcquisition = convertEnableAutoDiscoveryToEnable(proj.typingOptions);
                 proj.typeAcquisition = typeAcquisition;
             }
+
+            this.applySafeList(proj);
+
             let tsConfigFiles: NormalizedPath[];
             const rootFiles: protocol.ExternalFile[] = [];
             for (const file of proj.rootFiles) {