Merge pull request #649 from blakeembrey/improve-glob-implementation

Move to using `glob` library (optimised globbing)

Author: basarat

VSProj.njsproj

@@ -39,7 +39,7 @@
     <Folder Include="lib\typings\" />
     <Folder Include="lib\typings\atom\" />
     <Folder Include="lib\typings\emissary\" />
-    <Folder Include="lib\typings\glob-expand\" />
+    <Folder Include="lib\typings\glob\" />
     <Folder Include="lib\typings\jquery\" />
     <Folder Include="lib\typings\mixto\" />
     <Folder Include="lib\typings\mkdirp\" />
@@ -93,4 +93,4 @@
       </FlavorProperties>
     </VisualStudio>
   </ProjectExtensions>
-</Project>
+</Project>

dist/main/lang/transformers/transformerRegistry.js

@@ -39,8 +39,9 @@ function transform(name, code) {
     return transformer.transform(code);
 }
 exports.transform = transform;
-var expand = require('glob-expand');
-var files = expand({ filter: 'isFile', cwd: __dirname }, [
-    "./implementations/*.js"
-]);
+var glob = require('glob');
+var files = glob.sync('./implementations/*.js', {
+    nodir: true,
+    cwd: __dirname
+});
 files = files.map(function (f) { return require(f); });

dist/main/tsconfig/tsconfig.js

@@ -62,7 +62,7 @@ function errorWithDetails(error, details) {
 }
 var fs = require('fs');
 var path = require('path');
-var expand = require('glob-expand');
+var glob = require('glob');
 var os = require('os');
 var formatting = require('./formatting');
 var projectFileName = 'tsconfig.json';
@@ -71,7 +71,7 @@ var defaultFilesGlob = [
     "**/*.tsx",
     "!node_modules/**",
 ];
-var invisibleFilesGlob = ["**/*.ts", "**/*.tsx"];
+var invisibleFilesGlob = '{**/*.ts,**/*.tsx}';
 exports.defaults = {
     target: ts.ScriptTarget.ES5,
     module: ts.ModuleKind.CommonJS,
@@ -214,18 +214,21 @@ function getProjectSync(pathOrSrcFile) {
     if (!projectSpec.compilerOptions)
         projectSpec.compilerOptions = {};
     var cwdPath = path.relative(process.cwd(), path.dirname(projectFile));
-    if (!projectSpec.files && !projectSpec.filesGlob) {
-        var toExpand = invisibleFilesGlob;
-        if (projectSpec.exclude) {
-            toExpand = toExpand.concat(projectSpec.exclude.map(function (exclude) { return ("!" + exclude + "/**"); }));
-        }
+    var filesGlob = invisibleFilesGlob;
+    var ignore = [];
+    if (Array.isArray(projectSpec.filesGlob)) {
+        filesGlob = projectSpec.filesGlob.length === 1 ? projectSpec.filesGlob[0] : "{" + projectSpec.filesGlob.join(',') + "}";
     }
-    if (projectSpec.filesGlob) {
-        var toExpand = projectSpec.filesGlob;
+    else if (projectSpec.exclude) {
+        ignore = projectSpec.exclude.map(function (path) { return (path + "/**"); });
     }
-    if (toExpand) {
+    if (filesGlob) {
         try {
-            projectSpec.files = expand({ filter: 'isFile', cwd: cwdPath }, toExpand);
+            projectSpec.files = glob.sync(filesGlob, {
+                cwd: cwdPath,
+                ignore: ignore,
+                nodir: true
+            });
         }
         catch (ex) {
             throw errorWithDetails(new Error(exports.errors.GET_PROJECT_GLOB_EXPAND_FAILED), { glob: projectSpec.filesGlob, projectFilePath: fsu.consistentPath(projectFile), errorMessage: ex.message });

lib/main/lang/transformers/transformerRegistry.ts

@@ -2,7 +2,7 @@ import utils = require("../utils");
 import {Transformer, TransformerDelimiter, FileTransformationDetails} from "./transformer";
 
 /**
- * Transformers should push themeselves into this registry. 
+ * Transformers should push themeselves into this registry.
  * This is so that the registry does not depend on the transformers
  * As that will cause a circular reference
  */
@@ -37,18 +37,18 @@ export function getInitialTransformation(code: string): FileTransformationDetail
 
     while (true) {
         let remainingCode = code.substr(processedSrcUpto);
-        // Get the next transform that exist in this file: 
+        // Get the next transform that exist in this file:
         var matches = transformFinderRegex.exec(remainingCode);
         // No more transforms:
         if (!matches || !matches.length || matches.length < 2) return { transforms };
         // Found one!
         var nextTransformName = matches.slice[1];
         // Update the processedUpto
     }
-        
+
     /**
      * TODO: for each transform we note down the src start and src end
-     * Then we transform the src code. This gives a dest start (initially same as src start) and dest end (more or less) 
+     * Then we transform the src code. This gives a dest start (initially same as src start) and dest end (more or less)
      * we use this to additionally compute a running (delta) in dest. This delta is used in the next (dest start).
      */
 
@@ -67,12 +67,13 @@ export function transform(name: string, code: string): { code: string } {
     return transformer.transform(code);
 }
 
-// Ensure that all the transformers are loaded: 
+// Ensure that all the transformers are loaded:
 // Since nothing depends on these explicitly (don't want a circular ref)
 // We need to load them manually:
 import * as path from "path";
-var expand = require('glob-expand');
-let files: string[] = expand({ filter: 'isFile', cwd: __dirname }, [
-    "./implementations/*.js"
-]);
-files = files.map(f=> require(f));
+var glob = require('glob');
+let files: string[] = glob.sync('./implementations/*.js', {
+  nodir: true,
+  cwd: __dirname
+});
+files = files.map(f=> require(f));

lib/main/tsconfig/tsconfig.ts

@@ -196,7 +196,7 @@ function errorWithDetails<T>(error: Error, details: T): Error {
 
 import fs = require('fs');
 import path = require('path');
-import expand = require('glob-expand');
+import glob = require('glob');
 import os = require('os');
 import formatting = require('./formatting');
 
@@ -212,7 +212,7 @@ var defaultFilesGlob = [
 /**
  * This is what we use when the user doens't specify a files / filesGlob
  */
-var invisibleFilesGlob = ["**/*.ts", "**/*.tsx"];
+var invisibleFilesGlob = '{**/*.ts,**/*.tsx}';
 
 export var defaults: ts.CompilerOptions = {
     target: ts.ScriptTarget.ES5,
@@ -390,18 +390,22 @@ export function getProjectSync(pathOrSrcFile: string): TypeScriptProjectFileDeta
     // Our customizations for "tsconfig.json"
     // Use grunt.file.expand type of logic
     var cwdPath = path.relative(process.cwd(), path.dirname(projectFile));
-    if (!projectSpec.files && !projectSpec.filesGlob) { // If there is no files and no filesGlob, we create an invisible one.
-        var toExpand = invisibleFilesGlob;
-        if(projectSpec.exclude){
-            toExpand = toExpand.concat(projectSpec.exclude.map((exclude) => `!${exclude}/**`));
-        }
-    }
-    if (projectSpec.filesGlob) { // If there is a files glob we will use that
-        var toExpand = projectSpec.filesGlob
+    var filesGlob = invisibleFilesGlob;
+    var ignore = [];
+
+    if (Array.isArray(projectSpec.filesGlob)) {
+      filesGlob = projectSpec.filesGlob.length === 1 ? projectSpec.filesGlob[0] : `{${projectSpec.filesGlob.join(',')}}`;
+    } else if (projectSpec.exclude) {
+      ignore = projectSpec.exclude.map(path => `${path}/**`)
     }
-    if (toExpand) { // Expand whatever needs expanding
+
+    if (filesGlob) { // Expand whatever needs expanding
         try {
-            projectSpec.files = expand({ filter: 'isFile', cwd: cwdPath }, toExpand);
+            projectSpec.files = glob.sync(filesGlob, {
+              cwd: cwdPath,
+              ignore: ignore,
+              nodir: true
+            });
         }
         catch (ex) {
             throw errorWithDetails<GET_PROJECT_GLOB_EXPAND_FAILED_Details>(

lib/tsconfig.json

@@ -105,10 +105,11 @@
         "./typings/codemirror.d.ts",
         "./typings/d3/d3.d.ts",
         "./typings/emissary/emissary.d.ts",
-        "./typings/glob-expand/glob-expand.d.ts",
+        "./typings/glob/glob.d.ts",
         "./typings/htmltojsx/htmltojsx.d.ts",
         "./typings/jquery/jquery.d.ts",
         "./typings/minimatch.d.ts",
+        "./typings/minimatch/minimatch.d.ts",
         "./typings/mixto/mixto.d.ts",
         "./typings/mkdirp/mkdirp.d.ts",
         "./typings/mustache.d.ts",

lib/tsd.json

@@ -30,7 +30,7 @@
       "commit": "69bdfb0884020e41f17a1dd80ad4c77de2636874"
     },
     "node/node.d.ts": {
-      "commit": "69bdfb0884020e41f17a1dd80ad4c77de2636874"
+      "commit": "66f88ec51858bb21bc375fa91a963c04bc09401e"
     },
     "q/Q.d.ts": {
       "commit": "69bdfb0884020e41f17a1dd80ad4c77de2636874"
@@ -52,6 +52,12 @@
     },
     "htmltojsx/htmltojsx.d.ts": {
       "commit": "1632606f82cfbaf89d4b96b8dccacfaf2a6ae2f3"
+    },
+    "glob/glob.d.ts": {
+      "commit": "66f88ec51858bb21bc375fa91a963c04bc09401e"
+    },
+    "minimatch/minimatch.d.ts": {
+      "commit": "66f88ec51858bb21bc375fa91a963c04bc09401e"
     }
   }
 }

lib/typings/glob-expand/glob-expand.d.ts

@@ -0,0 +1,112 @@
+// Type definitions for Glob 5.0.10
+// Project: https://github.com/isaacs/node-glob
+// Definitions by: vvakame <https://github.com/vvakame/>
+// Definitions: https://github.com/borisyankov/DefinitelyTyped
+
+/// <reference path="../node/node.d.ts" />
+/// <reference path="../minimatch/minimatch.d.ts" />
+
+declare module "glob" {
+
+	import events = require("events");
+	import fs = require('fs');
+	import minimatch = require("minimatch");
+
+	function G(pattern: string, cb: (err: Error, matches: string[]) => void): void;
+	function G(pattern: string, options: G.IOptions, cb: (err: Error, matches: string[]) => void): void;
+
+	module G {
+		function sync(pattern: string, options?: IOptions): string[];
+
+		function hasMagic(pattern: string, options?: IOptions): boolean;
+
+		var Glob: IGlobStatic;
+		var GlobSync: IGlobSyncStatic;
+
+		interface IOptions extends minimatch.IOptions {
+			cwd?: string;
+			root?: string;
+			dot?: boolean;
+			nomount?: boolean;
+			mark?: boolean;
+			nosort?: boolean;
+			stat?: boolean;
+			silent?: boolean;
+			strict?: boolean;
+			cache?: { [path: string]: any /* boolean | string | string[] */ };
+			statCache?: { [path: string]: fs.Stats };
+			symlinks?: any;
+			sync?: boolean;
+			nounique?: boolean;
+			nonull?: boolean;
+			debug?: boolean;
+			nobrace?: boolean;
+			noglobstar?: boolean;
+			noext?: boolean;
+			nocase?: boolean;
+			matchBase?: any;
+			nodir?: boolean;
+			ignore?: any; /* string | string[] */
+			follow?: boolean;
+			realpath?: boolean;
+			nonegate?: boolean;
+			nocomment?: boolean;
+
+			/** Deprecated. */
+			globDebug?: boolean;
+		}
+
+		interface IGlobStatic extends events.EventEmitter {
+			new (pattern: string, cb?: (err: Error, matches: string[]) => void): IGlob;
+			new (pattern: string, options: IOptions, cb?: (err: Error, matches: string[]) => void): IGlob;
+			prototype: IGlob;
+		}
+
+		interface IGlobSyncStatic {
+			new (pattern: string, options?: IOptions): IGlobBase
+			prototype: IGlobBase;
+		}
+
+		interface IGlobBase {
+			minimatch: minimatch.IMinimatch;
+			options: IOptions;
+			aborted: boolean;
+			cache: { [path: string]: any /* boolean | string | string[] */ };
+			statCache: { [path: string]: fs.Stats };
+			symlinks: { [path: string]: boolean };
+			realpathCache: { [path: string]: string };
+			found: string[];
+		}
+
+		interface IGlob extends IGlobBase, events.EventEmitter {
+			pause(): void;
+			resume(): void;
+			abort(): void;
+
+			/** Deprecated. */
+			EOF: any;
+			/** Deprecated. */
+			paused: boolean;
+			/** Deprecated. */
+			maxDepth: number;
+			/** Deprecated. */
+			maxLength: number;
+			/** Deprecated. */
+			changedCwd: boolean;
+			/** Deprecated. */
+			cwd: string;
+			/** Deprecated. */
+			root: string;
+			/** Deprecated. */
+			error: any;
+			/** Deprecated. */
+			matches: string[];
+			/** Deprecated. */
+			log(...args: any[]): void;
+			/** Deprecated. */
+			emitMatch(m: any): void;
+		}
+	}
+
+	export = G;
+}

lib/typings/glob/glob.d.ts

@@ -0,0 +1,64 @@
+// Type definitions for Minimatch 2.0.8
+// Project: https://github.com/isaacs/minimatch
+// Definitions by: vvakame <https://github.com/vvakame/>
+// Definitions: https://github.com/borisyankov/DefinitelyTyped
+
+declare module "minimatch" {
+
+	function M(target: string, pattern: string, options?: M.IOptions): boolean;
+
+	module M {
+		function match(list: string[], pattern: string, options?: IOptions): string[];
+		function filter(pattern: string, options?: IOptions): (element: string, indexed: number, array: string[]) => boolean;
+		function makeRe(pattern: string, options?: IOptions): RegExp;
+
+		var Minimatch: IMinimatchStatic;
+		
+		interface IOptions {
+			debug?: boolean;
+			nobrace?: boolean;
+			noglobstar?: boolean;
+			dot?: boolean;
+			noext?: boolean;
+			nocase?: boolean;
+			nonull?: boolean;
+			matchBase?: boolean;
+			nocomment?: boolean;
+			nonegate?: boolean;
+			flipNegate?: boolean;
+		}
+
+		interface IMinimatchStatic {
+			new (pattern: string, options?: IOptions): IMinimatch;
+			prototype: IMinimatch;
+		}
+
+		interface IMinimatch {
+			pattern: string;
+			options: IOptions;
+			/** 2-dimensional array of regexp or string expressions. */
+			set: any[][]; // (RegExp | string)[][]
+			regexp: RegExp;
+			negate: boolean;
+			comment: boolean;
+			empty: boolean;
+
+			makeRe(): RegExp; // regexp or boolean
+			match(fname: string): boolean;
+			matchOne(files: string[], pattern: string[], partial: boolean): boolean;
+
+			/** Deprecated. For internal use. */
+			debug(): void;
+			/** Deprecated. For internal use. */
+			make(): void;
+			/** Deprecated. For internal use. */
+			parseNegate(): void;
+			/** Deprecated. For internal use. */
+			braceExpand(pattern: string, options: IOptions): void;
+			/** Deprecated. For internal use. */
+			parse(pattern: string, isSub?: boolean): void;
+		}
+	}
+
+	export = M;
+}