-
-
Notifications
You must be signed in to change notification settings - Fork 3.1k
perf: cli/lookup-files.js, refactor, add jsdoc, reduce intermediary variables #5489
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -1,18 +1,22 @@ | ||||||||||||
| 'use strict'; | ||||||||||||
|
|
||||||||||||
| /** | ||||||||||||
| * Contains `lookupFiles`, which takes some globs/dirs/options and returns a list of files. | ||||||||||||
| * @module | ||||||||||||
| * @private | ||||||||||||
| */ | ||||||||||||
|
|
||||||||||||
| var fs = require('node:fs'); | ||||||||||||
| var path = require('node:path'); | ||||||||||||
| var glob = require('glob'); | ||||||||||||
| var errors = require('../errors'); | ||||||||||||
| var createNoFilesMatchPatternError = errors.createNoFilesMatchPatternError; | ||||||||||||
| var createMissingArgumentError = errors.createMissingArgumentError; | ||||||||||||
| const fs = require('node:fs'); | ||||||||||||
| const path = require('node:path'); | ||||||||||||
| const glob = require('glob'); | ||||||||||||
| const { | ||||||||||||
| createNoFilesMatchPatternError, | ||||||||||||
| createMissingArgumentError | ||||||||||||
| } = require('../errors'); | ||||||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you separate out these unrelated changes, like If you want to holistically change most/all |
||||||||||||
| const debug = require('debug')('mocha:cli:lookup-files'); | ||||||||||||
|
|
||||||||||||
| /** @typedef {`.${string}`} HiddenUnixPathname */ | ||||||||||||
|
|
||||||||||||
| /** | ||||||||||||
| * Determines if pathname would be a "hidden" file (or directory) on UN*X. | ||||||||||||
| * | ||||||||||||
|
|
@@ -24,30 +28,54 @@ const debug = require('debug')('mocha:cli:lookup-files'); | |||||||||||
| * | ||||||||||||
| * @private | ||||||||||||
| * @param {string} pathname - Pathname to check for match. | ||||||||||||
| * @return {boolean} whether pathname would be considered a hidden file. | ||||||||||||
| * @returns {value is HiddenUnixPathname} whether pathname would be considered a hidden file. | ||||||||||||
Uzlopak marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||||||||
| * @example | ||||||||||||
| * isHiddenOnUnix('.profile'); // => true | ||||||||||||
| */ | ||||||||||||
| const isHiddenOnUnix = pathname => path.basename(pathname).startsWith('.'); | ||||||||||||
| const isHiddenOnUnix = pathname => path.basename(pathname)[0] === '.'; | ||||||||||||
|
|
||||||||||||
| /** @typedef {`.${string}`} FileExtension */ | ||||||||||||
|
|
||||||||||||
| /** | ||||||||||||
| * Determines if pathname has a matching file extension. | ||||||||||||
| * Normalize file extensions to ensure they have a leading period character. | ||||||||||||
| * | ||||||||||||
| * Supports multi-part extensions. | ||||||||||||
| * @private | ||||||||||||
| * @param {FileExtension[]|string[]|undefined|null} exts | ||||||||||||
| * @returns {FileExtension[]} | ||||||||||||
| */ | ||||||||||||
| const normalizeFileExtensions = (exts) => { | ||||||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is the expected behavior for That said, I haven't read the rest of the PR yet! |
||||||||||||
| if (!exts) { | ||||||||||||
| return []; | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| for (var i = 0; i < exts.length; i++) { | ||||||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||||
| if (exts[i][0] !== '.') { | ||||||||||||
| exts[i] = `.${exts[i]}`; | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
| return /** @type {FileExtension[]} */ (exts); | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| /** | ||||||||||||
| * Determines if pathname has a matching file extension. | ||||||||||||
| * | ||||||||||||
| * @private | ||||||||||||
| * @param {string} pathname - Pathname to check for match. | ||||||||||||
| * @param {string[]} exts - List of file extensions, w/-or-w/o leading period | ||||||||||||
| * @return {boolean} `true` if file extension matches. | ||||||||||||
| * @param {FileExtension[]} fileExtensions - List of file extensions, w/-or-w/o leading period | ||||||||||||
| * @returns {boolean} `true` if file extension matches. | ||||||||||||
| * @example | ||||||||||||
| * hasMatchingExtname('foo.html', ['js', 'css']); // false | ||||||||||||
| * hasMatchingExtname('foo.js', ['.js']); // true | ||||||||||||
| * hasMatchingExtname('foo.js', ['js']); // ture | ||||||||||||
| * hasMatchingFileExtension('foo.html', ['js', 'css']); // false | ||||||||||||
| * hasMatchingFileExtension('foo.js', ['.js']); // true | ||||||||||||
| * hasMatchingFileExtension('foo.js', ['js']); // ture | ||||||||||||
| */ | ||||||||||||
| const hasMatchingExtname = (pathname, exts = []) => | ||||||||||||
| exts | ||||||||||||
| .map(ext => (ext.startsWith('.') ? ext : `.${ext}`)) | ||||||||||||
| .some(ext => pathname.endsWith(ext)); | ||||||||||||
| const hasMatchingFileExtension = (pathname, fileExtensions) => { | ||||||||||||
| for (var i = 0; i < fileExtensions.length; i++) { | ||||||||||||
| if (pathname.endsWith(fileExtensions[i])) { | ||||||||||||
| return true; | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
| return false; | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| /** | ||||||||||||
| * Lookup file names at the given `path`. | ||||||||||||
|
|
@@ -59,31 +87,30 @@ const hasMatchingExtname = (pathname, exts = []) => | |||||||||||
| * @public | ||||||||||||
| * @alias module:lib/cli.lookupFiles | ||||||||||||
| * @param {string} filepath - Base path to start searching from. | ||||||||||||
| * @param {string[]} [extensions=[]] - File extensions to look for. | ||||||||||||
| * @param {string[]} [fileExtensions=[]] - File extensions to look for. | ||||||||||||
| * @param {boolean} [recursive=false] - Whether to recurse into subdirectories. | ||||||||||||
| * @return {string[]} An array of paths. | ||||||||||||
| * @returns {string[]} An array of paths. | ||||||||||||
| * @throws {Error} if no files match pattern. | ||||||||||||
| * @throws {TypeError} if `filepath` is directory and `extensions` not provided. | ||||||||||||
| * @throws {TypeError} if `filepath` is directory and `fileExtensions` not | ||||||||||||
| * provided or an empty array. | ||||||||||||
| */ | ||||||||||||
| module.exports = function lookupFiles( | ||||||||||||
| filepath, | ||||||||||||
| extensions = [], | ||||||||||||
| fileExtensions, | ||||||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. let's make the code match the comment. I know
Suggested change
|
||||||||||||
| recursive = false | ||||||||||||
| ) { | ||||||||||||
| const files = []; | ||||||||||||
| let stat; | ||||||||||||
| fileExtensions = normalizeFileExtensions(fileExtensions); | ||||||||||||
|
|
||||||||||||
| // Detect glob patterns by checking if the path does not exist as-is | ||||||||||||
| if (!fs.existsSync(filepath)) { | ||||||||||||
| let pattern; | ||||||||||||
| if (glob.hasMagic(filepath, {windowsPathsNoEscape: true})) { | ||||||||||||
| if (glob.hasMagic(filepath, { windowsPathsNoEscape: true })) { | ||||||||||||
| // Handle glob as is without extensions | ||||||||||||
| pattern = filepath; | ||||||||||||
| } else { | ||||||||||||
| // glob pattern e.g. 'filepath+(.js|.ts)' | ||||||||||||
| const strExtensions = extensions | ||||||||||||
| .map(ext => (ext.startsWith('.') ? ext : `.${ext}`)) | ||||||||||||
| .join('|'); | ||||||||||||
| pattern = `${filepath}+(${strExtensions})`; | ||||||||||||
| pattern = `${filepath}+(${fileExtensions.join('|')})`; | ||||||||||||
| debug('looking for files using glob pattern: %s', pattern); | ||||||||||||
| } | ||||||||||||
| files.push( | ||||||||||||
|
|
@@ -106,50 +133,51 @@ module.exports = function lookupFiles( | |||||||||||
| return files; | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| // Handle file | ||||||||||||
| try { | ||||||||||||
| stat = fs.statSync(filepath); | ||||||||||||
| if (stat.isFile()) { | ||||||||||||
| return filepath; | ||||||||||||
| } | ||||||||||||
| } catch (err) { | ||||||||||||
| // ignore error | ||||||||||||
| return; | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| // Handle directory | ||||||||||||
| fs.readdirSync(filepath).forEach(dirent => { | ||||||||||||
| const pathname = path.join(filepath, dirent); | ||||||||||||
| let stat; | ||||||||||||
| const stat = fs.statSync(filepath, { | ||||||||||||
| throwIfNoEntry: false | ||||||||||||
| }); | ||||||||||||
|
|
||||||||||||
| try { | ||||||||||||
| stat = fs.statSync(pathname); | ||||||||||||
| if (stat.isDirectory()) { | ||||||||||||
| if (recursive) { | ||||||||||||
| files.push(...lookupFiles(pathname, extensions, recursive)); | ||||||||||||
| } | ||||||||||||
| return; | ||||||||||||
| } | ||||||||||||
| } catch (ignored) { | ||||||||||||
| return; | ||||||||||||
| } | ||||||||||||
| if (!extensions.length) { | ||||||||||||
| if (stat === undefined) { | ||||||||||||
| // Unreachable because glob check already checks if path exists, but for | ||||||||||||
| // completeness... | ||||||||||||
| } else if (stat.isFile()) { | ||||||||||||
| files.push(filepath); | ||||||||||||
| } else if (stat.isDirectory()) { | ||||||||||||
| if (fileExtensions.length === 0) { | ||||||||||||
| throw createMissingArgumentError( | ||||||||||||
| `Argument '${extensions}' required when argument '${filepath}' is a directory`, | ||||||||||||
| `Argument '${fileExtensions}' required when argument '${filepath}' is a directory`, | ||||||||||||
| 'extensions', | ||||||||||||
| 'array' | ||||||||||||
| ); | ||||||||||||
| ) | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| if ( | ||||||||||||
| !stat.isFile() || | ||||||||||||
| !hasMatchingExtname(pathname, extensions) || | ||||||||||||
| isHiddenOnUnix(pathname) | ||||||||||||
| ) { | ||||||||||||
| return; | ||||||||||||
| // Handle directory | ||||||||||||
| const dirEnts = fs.readdirSync(filepath, { recursive, withFileTypes: true }); | ||||||||||||
|
|
||||||||||||
| for (var i = 0; i < dirEnts.length; i++) { | ||||||||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||||
| const dirEnt = dirEnts[i]; | ||||||||||||
|
|
||||||||||||
| const pathname = dirEnt.parentPath | ||||||||||||
| ? path.join(dirEnt.parentPath, dirEnt.name) | ||||||||||||
| : path.join(filepath, dirEnt.name); | ||||||||||||
|
Comment on lines
+160
to
+162
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||||
|
|
||||||||||||
| if (dirEnt.isFile() || dirEnt.isSymbolicLink()) { | ||||||||||||
| if (dirEnt.isSymbolicLink()) { | ||||||||||||
| const stat = fs.statSync(pathname, { throwIfNoEntry: false }); | ||||||||||||
| if (!stat || !stat.isFile()) { | ||||||||||||
| continue; // Skip broken symlinks or symlinks to directories | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| if ( | ||||||||||||
| hasMatchingFileExtension(pathname, /** @type {FileExtension[]} */(fileExtensions)) && | ||||||||||||
| !isHiddenOnUnix(pathname) | ||||||||||||
| ) { | ||||||||||||
| files.push(pathname); | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
| files.push(pathname); | ||||||||||||
| }); | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| return files; | ||||||||||||
| }; | ||||||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@Uzlopak this looks like a lot of good optimizations, but I'm not aware of any known performance woes with file lookups. Can you demonstrate that there are issues for users? A reproduction case + measurement script, etc.?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is my main concern as well. Reviewing a PR like this takes a lot of time. I don't want this work to languish but I don't want to ignore potentially bigger issues with Mocha (pipeline failures, known bugs, easy requested features, etc.) I think we should follow the perf issue template and discuss before taking up significant work like this so we're on the same page with review timelines and priority