finally fixed incremental compile

Author: benholloway

lib/build/browserify.js

@@ -19,11 +19,12 @@ var through         = require('through2'),
  */
 function init() {
 
-  // share a common cache by default
+  // share a common cache
   var caches = {
     cache       : {},
     packageCache: {}
   };
+  var internalCache = {};
 
   // complete
   return {
@@ -37,15 +38,14 @@ function init() {
    */
   function create(opt) {
 
-    // ensure options
-    //  browserify must be in debug mode
+    // user options override defaults but we always use debug mode and our own cache
     var options = merge({
       anonymous      : false,
       bowerRelative  : false,
       projectRelative: false,
       transforms     : [],
       sourceMapBase  : null
-    }, caches, opt, {
+    }, opt, caches, {
       debug: true
     });
 
@@ -127,158 +127,217 @@ function init() {
       }
     }
   }
-}
-
-module.exports = init;
 
-/**
- * Create an instance of the multiton that closes a fixed set of files
- * @param files The composition roots for the bundler
- * @param options A hash of options for both browserify and internal settings
- * @returns {function} a bundle method
- */
-function createInstance(files, options) {
+  /**
+   * Create an instance of the multiton that closes a fixed set of files
+   * @param files The composition roots for the bundler
+   * @param options A hash of options for both browserify and internal settings
+   * @returns {function} a bundle method
+   */
+  function createInstance(files, options) {
+
+    // browserify must be debug mode
+    var browserifyOpts = merge({}, options);
+
+    // create bundler
+    var bundler = browserify(browserifyOpts);
+
+    // must setup pipeline on every reset
+    bundler.on('reset', setupPipeline);
+    setupPipeline();
+
+    // transforms with possible options
+    //  transform, [opt], transform, [opt], ...
+    [].concat(options.transforms)
+      .concat(requireTransform(options.bowerRelative, options.projectRelative))
+      .forEach(function eachItem(item, i, list) {
+        if (typeof item === 'function') {
+          var opts = (typeof list[i + 1] === 'object') ? merge({global: true}, list[i + 1]) : {global: true};
+          bundler.transform(item, opts);
+        }
+      });
 
-  // browserify must be debug mode
-  var browserifyOpts = merge({}, options);
+    // require statements
+    [].concat(files)
+      .forEach(function eachItem(item) {
+        bundler.require(item, {entry: true});
+      });
 
-  // create bundler
-  var bundler = browserify(browserifyOpts);
+    // create instance
+    return {
+      bundle: bundle
+    };
 
-  // ensure anonymous module paths when we are minifying
-  if (options.anonymous) {
-    bundler.pipeline
-      .get('label')
-      .push(anonymousLabeler());
-  }
+    /**
+     * Setup the browserify pipeline
+     */
+    function setupPipeline() {
 
-  // transforms with possible options
-  //  transform, [opt], transform, [opt], ...
-  [].concat(options.transforms)
-    .concat(requireTransform(options.bowerRelative, options.projectRelative))
-    .forEach(function eachItem(item, i, list) {
-      if (typeof item === 'function') {
-        var opts = (typeof list[i + 1] === 'object') ? merge({global: true}, list[i + 1]) : {global: true};
-        bundler.transform(item, opts);
+      // ensure anonymous module paths when we are minifying
+      if (options.anonymous) {
+        bundler.pipeline
+          .get('label')
+          .push(anonymousLabeler());
       }
-    });
 
-  // require statements
-  [].concat(files)
-    .forEach(function eachItem(item) {
-      bundler.require(item, {entry: true});
-    });
+      // populate the module-deps cache to achieve incremental compile
+      var deps = bundler.pipeline.get('deps');
+      deps.push(populateCache(deps._streams[0].cache));
+      // TODO this cache is inexplicably different from options.cache
+    }
 
-  // create instance
-  return {
-    bundle: bundle
-  };
+    /**
+     * Compile any number of files into a bundle
+     * @param {stream.Through} stream A stream to push files to
+     * @param {Array.<string>|string} files Any number of files to bundle
+     * @param {string} bundleName The name for the output file
+     * @param {function} done Callback for completion
+     */
+    function bundle(stream, bundleName) {
 
-  /**
-   * Compile any number of files into a bundle
-   * @param {stream.Through} stream A stream to push files to
-   * @param {Array.<string>|string} files Any number of files to bundle
-   * @param {string} bundleName The name for the output file
-   * @param {function} done Callback for completion
-   */
-  function bundle(stream, bundleName) {
+      // create a promise
+      var deferred = q.defer();
 
-    // create a promise
-    var deferred = q.defer();
+      // setup
+      var outPath = path.resolve(bundleName);
+      var mapPath = path.basename(bundleName) + '.map';
+      var errors  = [];
+      var timeout;
 
-    // setup
-    var outPath = path.resolve(bundleName);
-    var mapPath = path.basename(bundleName) + '.map';
-    var errors  = [];
-    var timeout;
+      // compile
+      bundler.bundle()
+        .on('error', errorHandler)
+        .pipe(outputStream());
 
-    // compile
-    bundler.bundle()
-      .on('error', errorHandler)
-      .pipe(outputStream());
+      // return the promise
+      return deferred.promise;
 
-    // return the promise
-    return deferred.promise;
+      // handle an error in the context of the timeout
+      function errorHandler(error) {
 
-    // handle an error in the context of the timeout
-    function errorHandler(error) {
+        // parse the error text
+        var message = parseError(error.toString());
 
-      // parse the error text
-      var message = parseError(error.toString());
+        // add unique
+        if (errors.indexOf(message) < 0) {
+          errors.push(message);
+        }
 
-      // add unique
-      if (errors.indexOf(message) < 0) {
-        errors.push(message);
+        // complete overall only once there are no further errors
+        //  ensure idempotent in the case there are some late errors
+        clearTimeout(timeout);
+        timeout = setTimeout(onTimeout, 100);
       }
 
-      // complete overall only once there are no further errors
-      //  ensure idempotent in the case there are some late errors
-      clearTimeout(timeout);
-      timeout = setTimeout(onTimeout, 100);
-    }
+      // error has occurred and a delay has passed with no further errors
+      function onTimeout() {
+        deferred.reject(errors);  // complete overall
+      }
 
-    // error has occurred and a delay has passed with no further errors
-    function onTimeout() {
-      deferred.reject(errors);  // complete overall
-    }
+      // handle the output of the bundler (as a stream)
+      function outputStream() {
+        var code = '';
 
-    // handle the output of the bundler (as a stream)
-    function outputStream() {
-      var code = '';
+        function transform(buffer, encoding, done) {
+          code += buffer.toString();  // accumulate code
+          done();
+        }
 
-      function transform(buffer, encoding, done) {
-        code += buffer.toString();  // accumulate code
-        done();
-      }
+        function flush(done) {
+          var sourceMap = convert.fromComment(code).toObject();
+          var external = code.replace(convert.commentRegex, '//# sourceMappingURL=' + mapPath);
+          delete sourceMap.file;
+          delete sourceMap.sourceRoot;
+          delete sourceMap.sourcesContent;
+          sourceMap.sources
+            .forEach(rootRelative);
+          pushFileToStream(outPath + '.map', JSON.stringify(sourceMap, null, 2));
+          pushFileToStream(outPath, external);
+          done();
+          deferred.resolve();   // complete overall
+        }
 
-      function flush(done) {
-        var sourceMap = convert.fromComment(code).toObject();
-        var external = code.replace(convert.commentRegex, '//# sourceMappingURL=' + mapPath);
-        delete sourceMap.file;
-        delete sourceMap.sourceRoot;
-        delete sourceMap.sourcesContent;
-        sourceMap.sources
-          .forEach(rootRelative);
-        pushFileToStream(outPath + '.map', JSON.stringify(sourceMap, null, 2));
-        pushFileToStream(outPath, external);
-        done();
-        deferred.resolve();   // complete overall
+        return through.obj(transform, flush);
       }
 
-      return through.obj(transform, flush);
+      // stream output
+      function pushFileToStream(path, text) {
+        stream.push(new gutil.File({
+          path    : path,
+          contents: new Buffer(text)
+        }));
+      }
     }
 
-    // stream output
-    function pushFileToStream(path, text) {
-      stream.push(new gutil.File({
-        path    : path,
-        contents: new Buffer(text)
-      }));
+    /**
+     * Determine the root relative form of the given file path.
+     * If the file path is outside the project directory then just return its name.
+     * @param {string} filePath The input path string
+     * @param {number} An index for <code>Array.map()</code> type operations
+     * @param {object} The array for <code>Array.map()</code> type operations
+     * @return {string} The transformed file path
+     */
+    function rootRelative(filePath, i, array) {
+      var rootRelPath = slash(path.relative(process.cwd(), path.resolve(filePath))); // resolve relative references
+      var isProject = (rootRelPath.slice(0, 2) !== '..');
+      var result = [
+        options.sourceMapBase || '',
+        isProject ? rootRelPath : path.basename(rootRelPath)
+      ].join('/');
+      if ((typeof i === 'number') && (typeof array === 'object')) {
+        array[i] = result;
+      }
+      return result;
     }
-  }
 
-  /**
-   * Determine the root relative form of the given file path.
-   * If the file path is outside the project directory then just return its name.
-   * @param {string} filePath The input path string
-   * @param {number} An index for <code>Array.map()</code> type operations
-   * @param {object} The array for <code>Array.map()</code> type operations
-   * @return {string} The transformed file path
-   */
-  function rootRelative(filePath, i, array) {
-    var rootRelPath = slash(path.relative(process.cwd(), path.resolve(filePath))); // resolve relative references
-    var isProject = (rootRelPath.slice(0, 2) !== '..');
-    var result = [
-      options.sourceMapBase || '',
-      isProject ? rootRelPath : path.basename(rootRelPath)
-    ].join('/');
-    if ((typeof i === 'number') && (typeof array === 'object')) {
-      array[i] = result;
+    /**
+     * A pipeline 'deps' stage that populates cache for incremental compile.
+     * Called on fully transformed row but only when there is no cache hit.
+     * @param cache The cache used by module-deps
+     * @returns {stream.Through} a through stream
+     */
+    function populateCache(cache) {
+      function transform(row, encoding, done) {
+        /* jshint validthis:true */
+        var filename = row.file;
+
+        // set the new transformed row output
+        internalCache[filename] = {
+          input : fs.readFileSync(filename).toString(),
+          output: {
+            id    : filename,
+            source: row.source,
+            deps  : merge({}, row.deps),
+            file  : filename
+          }
+        };
+
+        // we need to use a getter as it is the only hook at which we can perform comparison
+        //  getters cannot be redefined so we create on first access and retain, hence the need
+        //  for the internal cache to store the value above
+        if (!cache.hasOwnProperty(filename)) {
+          Object.defineProperty(cache, filename, {
+            get: function () {
+              // file read and comparison is in the order of 100us
+              var cached  = internalCache[filename];
+              var input   = fs.readFileSync(filename).toString();
+              var isMatch = (cached.input === input);
+              return isMatch ? cached.output : undefined;
+            }
+          });
+        }
+
+        // complete
+        this.push(row);
+        done();
+      }
+      return through.obj(transform);
     }
-    return result;
   }
 }
 
+module.exports = init;
+
 /**
  * A pipeline labeler that ensures that final file names are anonymousd in the final output
  * @returns {stream.Through} A through stream for the labelling stage