v1.0.0

Author: bmacnaughton

.npmrc

@@ -0,0 +1 @@
+package-lock=false

README.md

@@ -0,0 +1,94 @@
+# action-walk
+
+Minimal utility to walk directory trees performing actions on each directory
+entry. `action-walk` has no production dependencies other than
+node core modules and has only one strong opinion - don't presume anything
+about why the directory tree is being walked.
+
+No presumptions means that this does little more than walk the tree. There
+are two options to facilitate implementing your code on top of `action-walk`.
+If the boolean option `stat` is truthy `action-walk` will execute `fs.stat`
+on the entry and pass that to you action handler. If the option `own` is
+present `action-walk` will pass that to the action functions in a context
+object.
+
+### usage
+
+`npm install action-walk`
+
+### examples
+
+```
+const walk = require('action-walk');
+
+function dirAction (path, context) {
+  const {dirent, stat, own} = context;
+  if (own.skipDirs && own.skipDirs.indexOf(dirent.name) >= 0) {
+    return 'skip';
+  }
+  own.total += stat.size;
+}
+function fileAction (path, context) {
+  const {stat, own} = context;
+  own.total += stat.size;
+}
+
+const own = {total: 0, skipDirs: ['node_modules']};
+const options = {
+  dirAction,
+  fileAction,
+  own,
+  stat: true
+};
+
+await walk('.', options);
+
+console.log('total bytes in "."', ctx.total);
+
+// executed in the await-walk package root it will print something like
+// total bytes in "." 14778
+```
+
+see `test/basics.test.js` for another example.
+
+### api
+
+`await walk(directory, options = {})`
+
+options
+- `dirAction` - called for each directory.
+- `fileAction` - called for each file.
+- `otherAction` - called for non-file, non-directory entries.
+- `stat` - call `fs.stat` on the entry and add it to the action context.
+- `own` - add this to the action context.
+
+It's possible to call `walk()` with no options but probably not useful unless
+all you're wanting to do is seed the disk cache with directory entries. The
+action functions are where task-specific work is done.
+
+Each of the action function (`dirAction`, `fileAction`, `otherAction`) is
+called with two arguments:
+- `filepath` for the entry starting with the `directory`, e.g., if
+`directory` is `test` and the entry is `basics.test.js` then `filepath`
+will be `test/basics.test.js`. (It is created using node's `path.join` so
+note that if `directory` is `.` it will *not* be present in `filepath`.)
+- `context` is an object as follows.
+```
+{
+  dirent, // the fs.Dirent object for the directory entry
+  stat,   // if `options.stat` the object returned by `fs.stat`
+  own     // `options.own` if provided.
+}
+```
+
+`dirAction` is the only function with return value that matters. If
+`dirAction` returns the string `'skip'` (either directly or via a
+Promise) then `walk()` will not walk that branch of the directory tree.
+
+All the action functions can return a promise if they need to perform
+asynchronous work but only the value of `dirAction` is meaningful.
+
+### todo
+
+- add error handling
+- let otherAction return indicator that a symbolic link should be followed.

action-walk.js

@@ -0,0 +1,47 @@
+const fsp = require('fs').promises;
+const path = require('path');
+
+async function walk (dir, options = {}) {
+  const noop = async () => undefined;
+  let fileAction = noop;
+  let dirAction = noop;
+  let otherAction = noop;
+
+  if (options.fileAction) {
+    fileAction = async (filepath, ctx) => options.fileAction(filepath, ctx);
+  }
+  if (options.dirAction) {
+    dirAction = async (filepath, ctx) => options.dirAction(filepath, ctx);
+  }
+  if (options.otherAction) {
+    otherAction = async (filepath, ctx) => options.otherAction(filepath, ctx);
+  }
+
+  //
+  // walk through a directory tree calling user functions for each entry.
+  //
+  async function walker (dir) {
+    for await (const dirent of await fsp.opendir(dir)) {
+      const entry = path.join(dir, dirent.name);
+      const ctx = {dirent};
+      if (options.own) {
+        ctx.own = options.own;
+      }
+      if (options.stat) {
+        ctx.stat = await fsp.stat(entry);
+      }
+      if (dirent.isDirectory() && await dirAction(entry, ctx) !== 'skip') {
+        await walker(entry);
+      } else if (dirent.isFile()) {
+        await fileAction(entry, ctx);
+      } else {
+        await otherAction(entry, ctx);
+      }
+    }
+    return undefined;
+  }
+
+  return walker(dir);
+}
+
+module.exports = walk;

package.json

@@ -0,0 +1,31 @@
+{
+  "name": "action-walk",
+  "version": "1.0.0",
+  "description": "walk a directory tree performing actions",
+  "main": "action-walk.js",
+  "directories": {
+    "test": "test"
+  },
+  "scripts": {
+    "test": "mocha test/*.test.js"
+  },
+  "keywords": [
+    "directory",
+    "tree",
+    "action",
+    "iterate",
+    "general",
+    "utility",
+    "recursive",
+    "walk",
+    "flexible"
+  ],
+  "author": "Bruce A. MacNaughton",
+  "license": "ISC",
+  "dependencies": {
+  },
+  "devDependencies": {
+    "mocha": "^8.1.3",
+    "chai": "^4.2.0"
+  }
+}

test/basics.test.js

@@ -0,0 +1,126 @@
+const fsp = require('fs').promises;
+const {execCommandLine} = require('./utilities/exec');
+const walk = require('../action-walk');
+const {expect} = require('chai');
+
+const testdir = '.';
+let testdirStat;
+const duOutput = {
+  wo_node: {},
+  w_node: {},
+}
+
+describe('verify that action-walk works as expected', function () {
+  // tests need to account for du counting the target directory itself
+  // while walk treats that as a starting point and only counts the
+  // contents of the directory.
+  before(function getTestDirSize () {
+    return fsp.stat(testdir)
+      .then(s => {
+        testdirStat = s;
+      });
+  })
+  before(function getDuOutput () {
+    // output is size-in-bytes <tab> path-starting-with-dir-name
+    const p = [
+      execCommandLine(`du -ab --exclude=node_modules ${testdir}`),
+      execCommandLine(`du -ab ${testdir}`),
+    ];
+    return Promise.all(p)
+      .then(r => {
+        expect(r[0].stderr).equal('');
+        expect(r[1].stderr).equal('');
+        duOutput.wo_node = parseDuOutput(r[0].stdout);
+        duOutput.w_node = parseDuOutput(r[1].stdout);
+      });
+  });
+
+  it('should match du -ab output', function () {
+    const own = {total: 0};
+    const options = {dirAction, fileAction, own, stat: true};
+    return walk(testdir, options)
+      .then(() => {
+        expect(own.total + testdirStat.size).equal(duOutput.w_node[testdir]);
+      })
+  });
+
+  it('should match du -ab --exclude=node_modules', function () {
+    const own = {total: 0, skipDirs: ['node_modules']};
+    const options = {dirAction, fileAction, own, stat: true};
+    return walk(testdir, options)
+      .then(() => {
+        expect(own.total + testdirStat.size).equal(duOutput.wo_node[testdir]);
+      })
+  });
+
+  it('should execute recursively matching du -b --exclude=node_modules', function () {
+    const own = {total: 0, dirTotals: {}, skipDirs: ['node_modules']};
+    const options = {dirAction: daDirOnly, fileAction, own, stat: true};
+    return walk(testdir, options)
+      .then(() => {
+        expect(own.total + testdirStat.size).equal(duOutput.wo_node[testdir]);
+        for (const dir in own.dirTotals) {
+          expect(own.dirTotals[dir]).equal(duOutput.w_node[`./${dir}`]);
+        }
+      });
+  });
+
+  it('should execute recursively matching du -b', function () {
+    const own = {total: 0, dirTotals: {}, skipDirs: []};
+    const options = {dirAction: daDirOnly, fileAction, own, stat: true};
+    return walk(testdir, options)
+      .then(() => {
+        expect(own.total + testdirStat.size).equal(duOutput.w_node[testdir]);
+        for (const dir in own.dirTotals) {
+          expect(own.dirTotals[dir]).equal(duOutput.w_node[`./${dir}`]);
+        }
+      });
+  });
+
+
+});
+
+
+//
+// utilities
+//
+function dirAction (path, ctx) {
+  const {dirent, stat, own} = ctx;
+  if (own.skipDirs && own.skipDirs.indexOf(dirent.name) >= 0) {
+    return 'skip';
+  }
+  own.total += stat.size;
+}
+function fileAction (path, ctx) {
+  const {stat, own} = ctx;
+  own.total += stat.size;
+}
+
+async function daDirOnly (path, ctx) {
+  const {dirent, stat, own} = ctx;
+  if (own.skipDirs && own.skipDirs.indexOf(dirent.name) >= 0) {
+    return 'skip';
+  }
+  own.dirTotals[path] = 0;
+  const newown = {total: 0, dirTotals: own.dirTotals};
+  const options = {
+    dirAction: daDirOnly,
+    fileAction,
+    own: newown,
+    stat: true,
+  };
+  await walk(path, options);
+  own.dirTotals[path] = newown.total + stat.size;
+  own.total += newown.total + stat.size;
+
+  // skip it because the recursive call counted the subtree.
+  return 'skip';
+}
+
+function parseDuOutput (text) {
+  const o = {};
+  for (const m of text.matchAll(/(?<size>\d+)\s+(?<path>.+)/g)) {
+    o[m.groups.path] = +m.groups.size;
+  }
+  return o;
+}

test/utilities/exec.js

@@ -0,0 +1,18 @@
+const exec = require('child_process').exec;
+
+async function execCommandLine (cmdline, options = {}) {
+  return new Promise((resolve, reject) => {
+    // eslint-disable-next-line no-unused-vars
+    const cp = exec(cmdline, options, function (error, stdout, stderr) {
+      if (error) {
+        reject({error, stdout, stderr});
+      } else {
+        resolve({stdout, stderr});
+      }
+    });
+  });
+}
+
+module.exports = {
+  execCommandLine,
+}