Support for an advanced “persistentCache” to be used for persisting outside of the current memory.

Author: martinheidegger

index.js

@@ -27,6 +27,11 @@ function Deps (opts) {
     if (!opts) opts = {};
 
     this.basedir = opts.basedir || process.cwd();
+    this.persistentCache = opts.persistentCache || function (file, id, pkg, fallback, cb) {
+        setImmediate(function () {
+            fallback(null, cb);
+        });
+    };
     this.cache = opts.cache;
     this.fileCache = opts.fileCache;
     this.pkgCache = opts.packageCache || {};
@@ -376,19 +381,41 @@ Deps.prototype.walk = function (id, parent, cb) {
         var c = self.cache && self.cache[file];
         if (c) return fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps));
 
-        self.readFile(file, id, pkg)
-            .pipe(self.getTransforms(fakePath || file, pkg, {
-                builtin: builtin,
-                inNodeModules: parent.inNodeModules
-            }))
-            .pipe(concat(function (body) {
-                fromSource(file, body.toString('utf8'), pkg, fakePath);
-            }))
-        ;
+        self.persistentCache(file, id, pkg, function fallback (stream, cb) {
+            (stream || self.readFile(file, id, pkg))
+                .pipe(self.getTransforms(fakePath || file, pkg, {
+                    builtin: builtin,
+                    inNodeModules: parent.inNodeModules
+                }))
+                .pipe(concat(function (body) {
+                    var src = body.toString('utf8');
+                    var deps = getDeps(file, src);
+                    if (deps) {
+                        cb(null, {
+                            source: src,
+                            package: pkg,
+                            deps: deps.reduce(function (deps, dep) {
+                                deps[dep] = true
+                                return deps
+                            }, {})
+                        });
+                    }
+                }));
+        }, function (err, c) {
+            if (err) {
+                self.emit('error', err);
+                return;
+            }
+            fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps));
+        });
     });
 
+    function getDeps (file, src) {
+        return rec.noparse ? [] : self.parseDeps(file, src);
+    }
+
     function fromSource (file, src, pkg, fakePath) {
-        var deps = rec.noparse ? [] : self.parseDeps(file, src);
+        var deps = getDeps(file, src);
         if (deps) fromDeps(file, src, pkg, fakePath, deps);
     }
 

readme.markdown

@@ -94,6 +94,38 @@ contents for browser fields, main entries, and transforms
 * `opts.fileCache` - an object mapping filenames to raw source to avoid reading
 from disk.
 
+* `opts.persistentCache` - a complex cache handler that allows async and persistent
+    caching of data. A `persistentCache` needs to follow this interface:
+    ```
+    function peristentCache (
+        file, // the path to the file that is loaded
+        id,   // the id that is used to reference this file
+        pkg,  // the package that this file belongs to fallback
+        cb    // callback handler that receives the cache data
+    ) {
+        if (hasError()) {
+            return cb(error) // Pass any error to the callback
+        }
+        if (isInCache()) {
+            return cb(null, {
+                source: fs.readFileSync(file, 'utf8'), // String of the fully processed file
+                package: pkg, // The package for housekeeping
+                deps: {
+                    'id':  // id that is used to reference a required file
+                    'file' // file path to the required file
+                }
+            })
+        }
+        // optional (can be null) stream that provides the data from
+        // the hard disk, can be provided in case the file data is used
+        // to evaluate the cache identifier
+        stream = fs.createReadStream(file)
+
+        // fallback to the default reading
+        fallback(stream, cb)
+    }
+    ```
+
 * `opts.paths` - array of global paths to search. Defaults to splitting on `':'`
 in `process.env.NODE_PATH`
 

test/cache_persistent.js

@@ -0,0 +1,81 @@
+var parser = require('../');
+var test = require('tap').test;
+var path = require('path');
+var fs = require('fs');
+
+var files = {
+    foo: path.join(__dirname, '/files/foo.js'),
+    bar: path.join(__dirname, '/files/bar.js')
+};
+
+test('uses persistent cache', function (t) {
+    t.plan(1);
+    var p = parser({
+        persistentCache: function (file, id, pkg, fallback, cb) {
+            if (file === files.bar) {
+                return fallback(null, cb)
+            }
+            cb(null, {
+                source: 'file at ' + file + '@' + id,
+                package: pkg,
+                deps: { './bar': files.bar }
+            })
+        }
+    });
+    p.end({ id: 'foo', file: files.foo, entry: false });
+
+    var rows = [];
+    p.on('data', function (row) { rows.push(row) });
+    p.on('end', function () {
+        t.same(rows.sort(cmp), [
+            {
+                id: files.bar,
+                file: files.bar,
+                source: fs.readFileSync(files.bar, 'utf8'),
+                deps: {}
+            },
+            {
+                id: 'foo',
+                file: files.foo,
+                source: 'file at ' + files.foo + '@' + files.foo,
+                deps: { './bar': files.bar }
+            }
+        ].sort(cmp));
+    });
+});
+
+test('passes persistent cache error through', function (t) {
+    t.plan(1);
+    var p = parser({
+        persistentCache: function (file, id, pkg, fallback, cb) {
+            cb(new Error('foo'))
+        }
+    });
+    p.end({ id: 'foo', file: files.foo, entry: false });
+    p.on('error', function (err) { t.equals(err.message, 'foo') });
+});
+
+test('allow passing of a different stream', function (t) {
+    t.plan(1);
+    var p = parser({
+        persistentCache: function (file, id, pkg, fallback, cb) {
+            fallback(fs.createReadStream(files.bar), cb)
+        }
+    });
+    p.end({ id: 'foo', file: files.foo, entry: false });
+
+    var rows = [];
+    p.on('data', function (row) { rows.push(row) });
+    p.on('end', function () {
+        t.same(rows.sort(cmp), [
+            {
+                id: 'foo',
+                file: files.foo,
+                source: fs.readFileSync(files.bar, 'utf8'),
+                deps: {}
+            }
+        ].sort(cmp));
+    });
+});
+
+function cmp (a, b) { return a.id < b.id ? -1 : 1 }