Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 94 additions & 66 deletions lib/cli/lookup-files.js
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

performance

@Uzlopak this looks like a lot of good optimizations, but I'm not aware of any known performance woes with file lookups. Can you demonstrate that there are issues for users? A reproduction case + measurement script, etc.?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is my main concern as well. Reviewing a PR like this takes a lot of time. I don't want this work to languish but I don't want to ignore potentially bigger issues with Mocha (pipeline failures, known bugs, easy requested features, etc.) I think we should follow the perf issue template and discuss before taking up significant work like this so we're on the same page with review timelines and priority

Original file line number Diff line number Diff line change
@@ -1,18 +1,22 @@
'use strict';

/**
* Contains `lookupFiles`, which takes some globs/dirs/options and returns a list of files.
* @module
* @private
*/

var fs = require('node:fs');
var path = require('node:path');
var glob = require('glob');
var errors = require('../errors');
var createNoFilesMatchPatternError = errors.createNoFilesMatchPatternError;
var createMissingArgumentError = errors.createMissingArgumentError;
const fs = require('node:fs');
const path = require('node:path');
const glob = require('glob');
const {
createNoFilesMatchPatternError,
createMissingArgumentError
} = require('../errors');
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you separate out these unrelated changes, like var -> const? I'm in favor of them in general but the diff is already pretty big.

If you want to holistically change most/all vars to consts separately I'd be very in favor.

const debug = require('debug')('mocha:cli:lookup-files');

/** @typedef {`.${string}`} HiddenUnixPathname */

/**
* Determines if pathname would be a "hidden" file (or directory) on UN*X.
*
Expand All @@ -24,30 +28,54 @@ const debug = require('debug')('mocha:cli:lookup-files');
*
* @private
* @param {string} pathname - Pathname to check for match.
* @return {boolean} whether pathname would be considered a hidden file.
* @returns {value is HiddenUnixPathname} whether pathname would be considered a hidden file.
* @example
* isHiddenOnUnix('.profile'); // => true
*/
const isHiddenOnUnix = pathname => path.basename(pathname).startsWith('.');
const isHiddenOnUnix = pathname => path.basename(pathname)[0] === '.';

/** @typedef {`.${string}`} FileExtension */

/**
* Determines if pathname has a matching file extension.
* Normalize file extensions to ensure they have a leading period character.
*
* Supports multi-part extensions.
* @private
* @param {FileExtension[]|string[]|undefined|null} exts
* @returns {FileExtension[]}
*/
const normalizeFileExtensions = (exts) => {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is the expected behavior for normalizeFileExtensions(['hello.world']) really to return [.hello.world']? extensions should just be the stuff after the last . . Maybe we are careful about this in the callers but it's something to consider.

That said, I haven't read the rest of the PR yet!

if (!exts) {
return [];
}

for (var i = 0; i < exts.length; i++) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
for (var i = 0; i < exts.length; i++) {
for (let i = 0; i < exts.length; i++) {

if (exts[i][0] !== '.') {
exts[i] = `.${exts[i]}`;
}
}
return /** @type {FileExtension[]} */ (exts);
}

/**
* Determines if pathname has a matching file extension.
*
* @private
* @param {string} pathname - Pathname to check for match.
* @param {string[]} exts - List of file extensions, w/-or-w/o leading period
* @return {boolean} `true` if file extension matches.
* @param {FileExtension[]} fileExtensions - List of file extensions, w/-or-w/o leading period
* @returns {boolean} `true` if file extension matches.
* @example
* hasMatchingExtname('foo.html', ['js', 'css']); // false
* hasMatchingExtname('foo.js', ['.js']); // true
* hasMatchingExtname('foo.js', ['js']); // ture
* hasMatchingFileExtension('foo.html', ['js', 'css']); // false
* hasMatchingFileExtension('foo.js', ['.js']); // true
* hasMatchingFileExtension('foo.js', ['js']); // ture
*/
const hasMatchingExtname = (pathname, exts = []) =>
exts
.map(ext => (ext.startsWith('.') ? ext : `.${ext}`))
.some(ext => pathname.endsWith(ext));
const hasMatchingFileExtension = (pathname, fileExtensions) => {
for (var i = 0; i < fileExtensions.length; i++) {
if (pathname.endsWith(fileExtensions[i])) {
return true;
}
}
return false;
}

/**
* Lookup file names at the given `path`.
Expand All @@ -59,31 +87,30 @@ const hasMatchingExtname = (pathname, exts = []) =>
* @public
* @alias module:lib/cli.lookupFiles
* @param {string} filepath - Base path to start searching from.
* @param {string[]} [extensions=[]] - File extensions to look for.
* @param {string[]} [fileExtensions=[]] - File extensions to look for.
* @param {boolean} [recursive=false] - Whether to recurse into subdirectories.
* @return {string[]} An array of paths.
* @returns {string[]} An array of paths.
* @throws {Error} if no files match pattern.
* @throws {TypeError} if `filepath` is directory and `extensions` not provided.
* @throws {TypeError} if `filepath` is directory and `fileExtensions` not
* provided or an empty array.
*/
module.exports = function lookupFiles(
filepath,
extensions = [],
fileExtensions,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's make the code match the comment. I know normalizeFileExtensions does this for us but explicit is a bit more readable

Suggested change
fileExtensions,
fileExtensions = [],

recursive = false
) {
const files = [];
let stat;
fileExtensions = normalizeFileExtensions(fileExtensions);

// Detect glob patterns by checking if the path does not exist as-is
if (!fs.existsSync(filepath)) {
let pattern;
if (glob.hasMagic(filepath, {windowsPathsNoEscape: true})) {
if (glob.hasMagic(filepath, { windowsPathsNoEscape: true })) {
// Handle glob as is without extensions
pattern = filepath;
} else {
// glob pattern e.g. 'filepath+(.js|.ts)'
const strExtensions = extensions
.map(ext => (ext.startsWith('.') ? ext : `.${ext}`))
.join('|');
pattern = `${filepath}+(${strExtensions})`;
pattern = `${filepath}+(${fileExtensions.join('|')})`;
debug('looking for files using glob pattern: %s', pattern);
}
files.push(
Expand All @@ -106,50 +133,51 @@ module.exports = function lookupFiles(
return files;
}

// Handle file
try {
stat = fs.statSync(filepath);
if (stat.isFile()) {
return filepath;
}
} catch (err) {
// ignore error
return;
}

// Handle directory
fs.readdirSync(filepath).forEach(dirent => {
const pathname = path.join(filepath, dirent);
let stat;
const stat = fs.statSync(filepath, {
throwIfNoEntry: false
});

try {
stat = fs.statSync(pathname);
if (stat.isDirectory()) {
if (recursive) {
files.push(...lookupFiles(pathname, extensions, recursive));
}
return;
}
} catch (ignored) {
return;
}
if (!extensions.length) {
if (stat === undefined) {
// Unreachable because glob check already checks if path exists, but for
// completeness...
} else if (stat.isFile()) {
files.push(filepath);
} else if (stat.isDirectory()) {
if (fileExtensions.length === 0) {
throw createMissingArgumentError(
`Argument '${extensions}' required when argument '${filepath}' is a directory`,
`Argument '${fileExtensions}' required when argument '${filepath}' is a directory`,
'extensions',
'array'
);
)
}

if (
!stat.isFile() ||
!hasMatchingExtname(pathname, extensions) ||
isHiddenOnUnix(pathname)
) {
return;
// Handle directory
const dirEnts = fs.readdirSync(filepath, { recursive, withFileTypes: true });

for (var i = 0; i < dirEnts.length; i++) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
for (var i = 0; i < dirEnts.length; i++) {
for (const dirEnt of dirEnts) {

const dirEnt = dirEnts[i];

const pathname = dirEnt.parentPath
? path.join(dirEnt.parentPath, dirEnt.name)
: path.join(filepath, dirEnt.name);
Comment on lines +160 to +162
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
const pathname = dirEnt.parentPath
? path.join(dirEnt.parentPath, dirEnt.name)
: path.join(filepath, dirEnt.name);
const pathToJoin = dirEnt.parentPath ?? filepath
const pathname = path.join(pathToJoin, dirEnt.name)


if (dirEnt.isFile() || dirEnt.isSymbolicLink()) {
if (dirEnt.isSymbolicLink()) {
const stat = fs.statSync(pathname, { throwIfNoEntry: false });
if (!stat || !stat.isFile()) {
continue; // Skip broken symlinks or symlinks to directories
}
}

if (
hasMatchingFileExtension(pathname, /** @type {FileExtension[]} */(fileExtensions)) &&
!isHiddenOnUnix(pathname)
) {
files.push(pathname);
}
}
}
files.push(pathname);
});
}

return files;
};
27 changes: 26 additions & 1 deletion test/integration/file-utils.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ describe('file utils', function () {
tmpDir = result.dirpath;
removeTempDir = result.removeTempDir;

tmpFile = filepath => path.join(tmpDir, filepath);
tmpFile = (...args) => path.join(...[tmpDir, ...args]);

touchFile(tmpFile('mocha-utils.js'));
if (SYMLINK_SUPPORT) {
Expand Down Expand Up @@ -64,6 +64,22 @@ describe('file utils', function () {
ex.and('to have length', expectedLength);
});

describe('when containing subdirectories', function () {
beforeEach(function () {
touchFile(tmpFile('types','mocha-utils.d.ts'));
});

it('should find files in subdirectories', function () {
expect(
lookupFiles(tmpDir, ['.d.ts'], true).map(foundFilepath =>
path.normalize(foundFilepath)
),
'to contain',
tmpFile('types', 'mocha-utils.d.ts')
).and('to have length', 1);
});
})

describe('when given `extension` option', function () {
describe('when provided a directory for the filepath', function () {
let filepath;
Expand Down Expand Up @@ -253,6 +269,15 @@ describe('file utils', function () {
});
});

describe('when provided path to non existing file', function () {
it('should throw an exception', function () {
expect(() => lookupFiles(tmpFile('mocha-utils-faux')), 'to throw', {
name: 'Error',
code: 'ERR_MOCHA_NO_FILES_MATCH_PATTERN'
});
});
});

describe('when no files match', function () {
it('should throw an exception', function () {
expect(() => lookupFiles(tmpFile('mocha-utils')), 'to throw', {
Expand Down
Loading