diff --git a/gyp/pylib/gyp/easy_xml.py b/gyp/pylib/gyp/easy_xml.py index e475b5530c..399d872b4f 100644 --- a/gyp/pylib/gyp/easy_xml.py +++ b/gyp/pylib/gyp/easy_xml.py @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import sys import re import os import locale @@ -121,7 +122,10 @@ def WriteXmlIfChanged(content, path, encoding="utf-8", pretty=False, win32=False default_encoding = locale.getdefaultlocale()[1] if default_encoding and default_encoding.upper() != encoding.upper(): - xml_string = xml_string.encode(encoding) + if sys.platform == "win32" and sys.version_info < (3, 7): + xml_string = xml_string.decode("cp1251").encode(encoding) + else: + xml_string = xml_string.encode(encoding) # Get the old content try: diff --git a/gyp/pylib/gyp/input.py b/gyp/pylib/gyp/input.py index ca7ce44eab..354958bfb2 100644 --- a/gyp/pylib/gyp/input.py +++ b/gyp/pylib/gyp/input.py @@ -225,7 +225,7 @@ def LoadOneBuildFile(build_file_path, data, aux_data, includes, is_target, check return data[build_file_path] if os.path.exists(build_file_path): - build_file_contents = open(build_file_path).read() + build_file_contents = open(build_file_path, encoding='utf-8').read() else: raise GypError(f"{build_file_path} not found (cwd: {os.getcwd()})") diff --git a/lib/find-python-script.py b/lib/find-python-script.py new file mode 100644 index 0000000000..dae48e0e95 --- /dev/null +++ b/lib/find-python-script.py @@ -0,0 +1,4 @@ +import sys +if sys.stdout.encoding != "utf-8" and sys.platform == "win32": + sys.stdout.reconfigure(encoding='utf-8') +print(sys.executable) diff --git a/lib/find-python.js b/lib/find-python.js index e6464d12c6..16885bb4c8 100644 --- a/lib/find-python.js +++ b/lib/find-python.js @@ -1,6 +1,8 @@ +// @ts-check 'use strict' -const log = require('npmlog') +const path = require('path') +const npmlog = require('npmlog') const semver = require('semver') const cp = require('child_process') const extend = require('util')._extend // eslint-disable-line @@ -39,280 +41,606 @@ function getOsUserInfo () { } catch {} } -function PythonFinder (configPython, callback) { - this.callback = callback - this.configPython = configPython - this.errorLog = [] -} +//! after editing file dont forget run "npm test" and +//! change tests for this file if needed + +// ? may be some addition info in silly and verbose levels +// ? add safety to colorizeOutput function. E.g. when terminal doesn't +// ? support colorizing, just disable it (return given string) +// i hope i made not bad error handling but may be some improvements would be nice +// TODO: better error handler on linux/macOS + +class Logger { + constructor () { + /** @private */ + this._npmlog = npmlog + this.log = logWithPrefix(this._npmlog, 'find Python') -PythonFinder.prototype = { - log: logWithPrefix(log, 'find Python'), - argsExecutable: ['-c', 'import sys; print(sys.executable);'], - argsVersion: ['-c', 'import sys; print("%s.%s.%s" % sys.version_info[:3]);'], - semverRange: '>=3.6.0', - - // These can be overridden for testing: - execFile: cp.execFile, - env: process.env, - win: win, - pyLauncher: 'py.exe', - winDefaultLocations: winDefaultLocationsArray, - - // Logs a message at verbose level, but also saves it to be displayed later - // at error level if an error occurs. This should help diagnose the problem. - addLog: function addLog (message) { + /** @private */ + this._errorLog = [] + } + + /** + * Logs a message at verbose level, but also saves it to internal buffer + * from which they can be accessed later by calling `this.dumpErrorLog()`. + * This should help diagnose the problem. + * + * ?message is array or one string + */ + addLog (message) { + // write also to verbose for consistent output this.log.verbose(message) - this.errorLog.push(message) - }, - - // Find Python by trying a sequence of possibilities. - // Ignore errors, keep trying until Python is found. - findPython: function findPython () { - const SKIP = 0; const FAIL = 1 - var toCheck = getChecks.apply(this) - - function getChecks () { - if (this.env.NODE_GYP_FORCE_PYTHON) { - return [{ - before: () => { - this.addLog( - 'checking Python explicitly set from NODE_GYP_FORCE_PYTHON') - this.addLog('- process.env.NODE_GYP_FORCE_PYTHON is ' + - `"${this.env.NODE_GYP_FORCE_PYTHON}"`) - }, - check: this.checkCommand, - arg: this.env.NODE_GYP_FORCE_PYTHON - }] - } + this._errorLog.push(message) + } - var checks = [ - { - before: () => { - if (!this.configPython) { - this.addLog( - 'Python is not set from command line or npm configuration') - return SKIP - } - this.addLog('checking Python explicitly set from command line or ' + - 'npm configuration') - this.addLog('- "--python=" or "npm config get python" is ' + - `"${this.configPython}"`) - }, - check: this.checkCommand, - arg: this.configPython - }, + /** + * Clear buffer with error messages and return colleted messages about error as a string + * To add messages to error buffer use `this.addLog()` + * + * @returns string contain all collected error messages + */ + dumpErrorLog () { + const errString = this._errorLog.map((str) => str.trim()).join('\n') + this._errorLog = [] + return errString + } + + /** + * Paint (not print, just colorize) string with selected color + * + * @param color color to set: colorHighlight.RED or colorHighlight.GREEN + * @param {string} string string to colorize + */ + colorizeOutput (color, string) { + return color + string + Logger.colors.RESET + } +} + +// `standard` package version we using doesn't work +// with class property declaration syntax. +// As soon as we upgrade standard, this should be +// declared as static property of `Logger` class +Logger.colors = { + RED: '\x1b[31m', + GREEN: '\x1b[32m', + RESET: '\x1b[0m' +} + +//! on windows debug running with locale cmd encoding (e. g. chcp 866) +// to avoid that uncomment next lines +// locale encodings cause issues. See run func for more info +// this lines only for testing +// win ? cp.execSync("chcp 65001") : null +// npmlog.level = "silly"; + +/** + * @class + */ +class PythonFinder { + /** + * + * @param {string} configPython force setted from terminal or npm config python path + * @param {(err:Error, found:string) => void} callback succeed/error callback from where result + * is available + */ + constructor (configPython, callback) { + this.callback = callback + this.configPython = configPython + + this.catchErrors = this.catchErrors.bind(this) + this.checkExecPath = this.checkExecPath.bind(this) + this.succeed = this.succeed.bind(this) + + this.SKIP = 0 + this.FAIL = 1 + + this.logger = new Logger() + + this.argsExecutable = [path.resolve(__dirname, 'find-python-script.py')] + this.argsVersion = [ + '-c', + 'import sys; print("%s.%s.%s" % sys.version_info[:3]);' + // for testing + // 'print("2.1.1")' + ] + this.semverRange = '>=3.6.0' + // These can be overridden for testing: + this.execFile = cp.execFile + this.env = process.env + this.win = win + this.pyLauncher = 'py.exe' + this.winDefaultLocations = winDefaultLocationsArray + } + + /** + * Find Python by trying a sequence of possibilities. + * Ignore errors, keep trying until Python is found. + * + * @public + */ + findPython () { + this.toCheck = this.getChecks() + + this.runChecks(this.toCheck) + } + + /** + * Getting list of checks which should be checked + * + * @private + * @returns {check[]} + */ + getChecks () { + if (this.env.NODE_GYP_FORCE_PYTHON) { + /** + * @type {check[]} + */ + return [ { before: () => { - if (!this.env.PYTHON) { - this.addLog('Python is not set from environment variable ' + - 'PYTHON') - return SKIP - } - this.addLog('checking Python explicitly set from environment ' + - 'variable PYTHON') - this.addLog(`- process.env.PYTHON is "${this.env.PYTHON}"`) + this.logger.addLog( + 'checking Python explicitly set from NODE_GYP_FORCE_PYTHON' + ) + this.logger.addLog( + '- process.env.NODE_GYP_FORCE_PYTHON is ' + + `"${this.env.NODE_GYP_FORCE_PYTHON}"` + ) }, - check: this.checkCommand, - arg: this.env.PYTHON - }, - { - before: () => { this.addLog('checking if "python3" can be used') }, - check: this.checkCommand, - arg: 'python3' - }, - { - before: () => { this.addLog('checking if "python" can be used') }, - check: this.checkCommand, - arg: 'python' + checkFunc: this.checkCommand, + arg: this.env.NODE_GYP_FORCE_PYTHON } ] + } - if (this.win) { - for (var i = 0; i < this.winDefaultLocations.length; ++i) { - const location = this.winDefaultLocations[i] - checks.push({ - before: () => { - this.addLog('checking if Python is ' + - `${location}`) - }, - check: this.checkExecPath, - arg: location - }) - } + /** + * @type {check[]} + */ + const checks = [ + { + before: (name) => { + if (!this.configPython) { + this.logger.addLog( + `${this.logger.colorizeOutput( + Logger.colors.GREEN, + 'Python is not set from command line or npm configuration' + )}` + ) + this.logger.addLog('') + return this.SKIP + } + this.logger.addLog( + 'checking Python explicitly set from command line or ' + + 'npm configuration' + ) + this.logger.addLog( + '- "--python=" or "npm config get python" is ' + + `"${this.logger.colorizeOutput(Logger.colors.GREEN, this.configPython)}"` + ) + }, + checkFunc: this.checkCommand, + arg: this.configPython + }, + { + before: (name) => { + if (!this.env.PYTHON) { + this.logger.addLog( + `Python is not set from environment variable ${this.logger.colorizeOutput( + Logger.colors.GREEN, + 'PYTHON' + )}` + ) + return this.SKIP + } + this.logger.addLog( + 'checking Python explicitly set from environment ' + + 'variable PYTHON' + ) + this.logger.addLog( + `${this.logger.colorizeOutput( + Logger.colors.GREEN, + 'process.env.PYTHON' + )} is "${this.logger.colorizeOutput(Logger.colors.GREEN, this.env.PYTHON)}"` + ) + }, + checkFunc: this.checkCommand, + arg: this.env.PYTHON, + // name used as very short description + name: 'process.env.PYTHON' + }, + { + checkFunc: this.checkCommand, + name: 'python3', + arg: 'python3' + }, + { + checkFunc: this.checkCommand, + name: 'python', + arg: 'python' + } + ] + + if (this.win) { + for (let i = 0; i < this.winDefaultLocations.length; ++i) { + const location = this.winDefaultLocations[i] checks.push({ before: () => { - this.addLog( - 'checking if the py launcher can be used to find Python 3') + this.logger.addLog( + `checking if Python is "${this.logger.colorizeOutput(Logger.colors.GREEN, location)}"` + ) }, - check: this.checkPyLauncher + checkFunc: this.checkExecPath, + arg: location }) } - - return checks + checks.push({ + before: () => { + this.logger.addLog( + `checking if the ${this.logger.colorizeOutput( + Logger.colors.GREEN, + 'py launcher' + )} can be used to find Python` + ) + }, + checkFunc: this.checkPyLauncher, + name: 'py Launcher' + }) } - function runChecks (err) { - this.log.silly('runChecks: err = %j', (err && err.stack) || err) + return checks + } - const check = toCheck.shift() - if (!check) { - return this.fail() + /** + * Type for possible place where python is + * + * @typedef check + * @type {object} + * @property {(name: string) => number|void} [before] what to execute before running check itself + * @property {function} checkFunc function which will perform check + * @property {string} [arg] what will be executed + * @property {string} [name] how check is named. this name is displayed to user + * @property {{shell: boolean}} [options] additional data, may be extended later, if shell true, exec command as in shell + */ + + /** + * + * + * @private + * @argument {check[]} checks + */ + async runChecks (checks) { + // using this flag because Fail is happen when ALL checks fail + let fail = true + + for (const check of checks) { + if (check.before) { + const beforeResult = check.before.apply(this, [check.name]) + + // if pretask fail - skip + if (beforeResult === this.SKIP || beforeResult === this.FAIL) { + // ?optional + // TODO: write to result arr which tests are SKIPPED + continue + } } - const before = check.before.apply(this) - if (before === SKIP) { - return runChecks.apply(this) - } - if (before === FAIL) { - return this.fail() - } + try { + if (!check.before) { + this.logger.addLog( + `checking if "${this.logger.colorizeOutput( + Logger.colors.GREEN, + check.name || check.arg + )}" can be used` + ) + } - const args = [runChecks.bind(this)] - if (check.arg) { - args.unshift(check.arg) + this.logger.log.verbose( + `executing "${this.logger.colorizeOutput( + Logger.colors.GREEN, + // DONE: swap in favor of arg (user want to see what we actually will run not how it is named) + check.arg || check.name + )}" to get Python executable path` + ) + + const result = await check.checkFunc.apply(this, [ + check ? check.arg : null + ]) + fail = false + this.succeed(result.path, result.version) + + break + } catch (err) { + this.catchErrors(err, check) } - check.check.apply(this, args) } - runChecks.apply(this) - }, + if (fail) { + this.fail() + } + } - // Check if command is a valid Python to use. - // Will exit the Python finder on success. - // If on Windows, run in a CMD shell to support BAT/CMD launchers. - checkCommand: function checkCommand (command, errorCallback) { - var exec = command - var args = this.argsExecutable - var shell = false + /** + * Check if command is a valid Python to use. + * Will exit the Python finder on success. + * If on Windows, run in a CMD shell to support BAT/CMD launchers. + * + * @private + * @argument {string} command command which will be executed in shell + * @returns {Promise} + */ + checkCommand (command) { + let exec = command + let args = this.argsExecutable + let shell = false + + // TODO: add explanation why shell is needed if (this.win) { - // Arguments have to be manually quoted - exec = `"${exec}"` - args = args.map(a => `"${a}"`) + // Arguments have to be manually quoted to avoid bugs with spaces in paths shell = true + exec = `"${exec}"` + args = args.map((a) => `"${a}"`) } - this.log.verbose(`- executing "${command}" to get executable path`) - this.run(exec, args, shell, function (err, execPath) { - // Possible outcomes: - // - Error: not in PATH, not executable or execution fails - // - Gibberish: the next command to check version will fail - // - Absolute path to executable - if (err) { - this.addLog(`- "${command}" is not in PATH or produced an error`) - return errorCallback(err) - } - this.addLog(`- executable path is "${execPath}"`) - this.checkExecPath(execPath, errorCallback) - }.bind(this)) - }, - - // Check if the py launcher can find a valid Python to use. - // Will exit the Python finder on success. - // Distributions of Python on Windows by default install with the "py.exe" - // Python launcher which is more likely to exist than the Python executable - // being in the $PATH. - // Because the Python launcher supports Python 2 and Python 3, we should - // explicitly request a Python 3 version. This is done by supplying "-3" as - // the first command line argument. Since "py.exe -3" would be an invalid - // executable for "execFile", we have to use the launcher to figure out - // where the actual "python.exe" executable is located. - checkPyLauncher: function checkPyLauncher (errorCallback) { - this.log.verbose( - `- executing "${this.pyLauncher}" to get Python 3 executable path`) - this.run(this.pyLauncher, ['-3', ...this.argsExecutable], false, - function (err, execPath) { - // Possible outcomes: same as checkCommand - if (err) { - this.addLog( - `- "${this.pyLauncher}" is not in PATH or produced an error`) - return errorCallback(err) - } - this.addLog(`- executable path is "${execPath}"`) - this.checkExecPath(execPath, errorCallback) - }.bind(this)) - }, - - // Check if a Python executable is the correct version to use. - // Will exit the Python finder on success. - checkExecPath: function checkExecPath (execPath, errorCallback) { - this.log.verbose(`- executing "${execPath}" to get version`) - this.run(execPath, this.argsVersion, false, function (err, version) { - // Possible outcomes: - // - Error: executable can not be run (likely meaning the command wasn't - // a Python executable and the previous command produced gibberish) - // - Gibberish: somehow the last command produced an executable path, - // this will fail when verifying the version - // - Version of the Python executable - if (err) { - this.addLog(`- "${execPath}" could not be run`) - return errorCallback(err) - } - this.addLog(`- version is "${version}"`) + return new Promise((resolve, reject) => { + this.run(exec, args, shell) + .then(this.checkExecPath) + .then(resolve) + .catch(reject) + }) + } - const range = new semver.Range(this.semverRange) - var valid = false - try { - valid = range.test(version) - } catch (err) { - this.log.silly('range.test() threw:\n%s', err.stack) - this.addLog(`- "${execPath}" does not have a valid version`) - this.addLog('- is it a Python executable?') - return errorCallback(err) - } + /** + * Check if the py launcher can find a valid Python to use. + * Will exit the Python finder on success. + * Distributions of Python on Windows by default install with the "py.exe" + * Python launcher which is more likely to exist than the Python executable + * being in the $PATH. + * + * @private + * @returns {Promise} + */ + // theoretically this method can be removed in favor of checkCommand and getChecks. + // the only difference between checkCommand and checkPyLauncher is the shell arg for run function + // BUT! if we will use declarative style (would be cool i think) + // then we should somehow instruct checkCommand esp. on windows, that + // we do not want to execute command in the shell mode. + // Have tried to do this via "optional.shell" property of check object + // but have failed, because to support high modularity of file + // consistent interface across functions should be supported. + // Thus we have to pass check object not only in checkCommand but in + // every other function in conveyor. + // Passing check to every function from previous in promise chain would lead to + // hard to fix errors and overcomplicate structure of module + + checkPyLauncher () { + return new Promise((resolve, reject) => { + this.run(this.pyLauncher, this.argsExecutable, false) + .then(this.checkExecPath) + .then(resolve) + .catch(reject) + }) + } - if (!valid) { - this.addLog(`- version is ${version} - should be ${this.semverRange}`) - this.addLog('- THIS VERSION OF PYTHON IS NOT SUPPORTED') - return errorCallback(new Error( - `Found unsupported Python version ${version}`)) - } - this.succeed(execPath, version) - }.bind(this)) - }, - - // Run an executable or shell command, trimming the output. - run: function run (exec, args, shell, callback) { - var env = extend({}, this.env) - env.TERM = 'dumb' - const opts = { env: env, shell: shell } - - this.log.silly('execFile: exec = %j', exec) - this.log.silly('execFile: args = %j', args) - this.log.silly('execFile: opts = %j', opts) - try { - this.execFile(exec, args, opts, execFileCallback.bind(this)) - } catch (err) { - this.log.silly('execFile: threw:\n%s', err.stack) - return callback(err) - } + /** + * + * Check if a gotten path is correct and + * Python executable has the correct version to use. + * + * @private + * @argument {string} execPath path to check + * @returns {Promise} + */ + checkExecPath (execPath) { + // Returning new Promise instead of forwarding existing + // to pass both path and version + return new Promise((resolve, reject) => { + this.logger.log.verbose(`executing "${this.logger.colorizeOutput(Logger.colors.GREEN, execPath)}" to get version`) + + // do not wrap with quotes because executing without shell + this.run(execPath, this.argsVersion, false) + .then((ver) => { + this.logger.log.silly(this.logger.colorizeOutput(Logger.colors.GREEN, 'got version:'), ver) + // ? may be better code for version check + // ? may be move log messages to catchError func + const range = new semver.Range(this.semverRange) + let valid = false + + try { + valid = range.test(ver) + // throw new Error("test error") + } catch (err) { + this.logger.log.silly(`semver.satisfies(${ver}, ${this.semverRange}) threw:\n${err.stack}`) + this.logger.addLog(`"${this.logger.colorizeOutput(Logger.colors.RED, execPath)}" does not have a valid version`) + this.logger.addLog('Is it a Python executable?') + + // if you need to pass additional data, use ErrorWithData class + // you can also use any Error capable object + return reject(err) + } + + if (!valid) { + reject(new PythonVersionError({ received: ver, wanted: this.semverRange }, `Found unsupported Python version ${ver}`)) + } + + resolve({ path: execPath, version: ver }) + }) + .catch(reject) + }) + } - function execFileCallback (err, stdout, stderr) { - this.log.silly('execFile result: err = %j', (err && err.stack) || err) - this.log.silly('execFile result: stdout = %j', stdout) - this.log.silly('execFile result: stderr = %j', stderr) - if (err) { - return callback(err) + /** + * Run an executable or shell command, trimming the output. + * + * @private + * @argument {string} exec command or path without arguments to execute + * @argument {string[]} args command args + * @argument {boolean} shell need be documented + * @returns {Promise} + */ + run (exec, args, shell) { + return new Promise( + /** + * @this {PythonFinder} + * @argument {function} resolve + * @argument {function} reject + */ + function (resolve, reject) { + const env = extend({}, this.env) + env.TERM = 'dumb' + /** @type {cp.ExecFileOptions} */ + const opts = { env: env, shell: shell } + + this.logger.log.verbose(`${this.logger.colorizeOutput(Logger.colors.GREEN, 'execFile')}: exec = `, exec) + this.logger.log.verbose(`${this.logger.colorizeOutput(Logger.colors.GREEN, 'execFile')}: args = `, args) + // TODO: make beauty print of PATH property (new line by semicolon) + this.logger.log.silly(`${this.logger.colorizeOutput(Logger.colors.GREEN, 'execFile')}: opts = `, JSON.stringify(opts, null, 2), '\n\n') + + //* possible outcomes with error messages on Windows (err.message, error.stack?, stderr) + // issue of encoding (garbage in terminal ) when 866 or any other locale code + // page is setted + // possible solutions: + // 1. leave it as is and just warn the user that it should use utf8 + // (already done in this.catchError's info statement) + // 2. somehow determine the user's terminal encoding and use utils.TextDecoder + // with the raw buffer from execFile. + // Requires to correct error.message because garbage persists there + // 3. Force the user's terminal to use utf8 encoding via e.g. run "chcp 65001". May break user's programs + // 4. use "cmd" command with flag "/U" and "/C" (for more information run "help cmd") + // which "Causes the output of + // internal commands ... to be Unicode" (utf16le) + //* note: find-python-script.py already send output in utf8 then may become necessary + //* to reencode string with Buffer.from(stderr).toString() or something + //* similar (if needed) + // for this solution all execFile call should look like execFile("cmd", ["/U", "/C", command to run, arg1, arg2, ...]) + //* all paths/commands and each argument must be in quotes if they contain spaces + + // ! potential bug + // if "shell" is true and is users default shell on windows is powershell then executables in PATH which name contain spaces will not work. + // it is feature of powershell which handle first arg in quotes as string + // thus if exec name has spaces, we can shield them (every space) with ` (backtick) + // or & (ampersand) can be placed before string in quotes, to tell to shell that + // it is executable, not string + + //* assume we have a utf8 compatible terminal + this.execFile(exec, args, opts, execFileCallback.bind(this)) + + // ? may be better to use utils.promisify + /** + * + * @param {Error} err + * @param {string} stdout + * @param {string} stderr + * @this {PythonFinder} + */ + function execFileCallback (err, stdout, stderr) { + // Done: add silly logs as in previous version + this.logger.log.silly(`${this.logger.colorizeOutput(Logger.colors.RED, 'execFile result')}: err =`, (err && err.stack) || err) + this.logger.log.verbose(`${this.logger.colorizeOutput(Logger.colors.RED, 'execFile result')}: stdout =`, stdout) + this.logger.log.silly(`${this.logger.colorizeOutput(Logger.colors.RED, 'execFile result')}: stderr =`, stderr) + + // executed script shouldn't pass anything to stderr if successful + if (err || stderr) { + reject(new ErrorWithData({ data: { stderr: stderr || null }, messageOrError: err || null })) + } else { + // trim() function removes string endings that would break string comparison + const stdoutTrimmed = stdout.trim() + resolve(stdoutTrimmed) + } + } + }.bind(this) + ) + } + + /** + * Main error handling function in module + * Promises should throw errors up to this function + * Also used for logging + * + * @private + * TODO: figure out err type + * @param {ErrorWithData} err + * @param {check} check + */ + catchErrors (err, check) { + this.logger.addLog(this.logger.colorizeOutput(Logger.colors.RED, `FAIL: ${check.name || check.arg}`)) + + // array of error codes (type of errors) that we handling + const catchedErrorsCods = ['ENOENT', 9009] + + // don't know type of terminal errors + // @ts-ignore + if (catchedErrorsCods.includes(err.error && err.error.code)) { + const { error } = err + // @ts-ignore + switch (error && error.code) { + case 'ENOENT': + this.logger.addLog( + `${this.logger.colorizeOutput( + Logger.colors.RED, + 'ERROR:' + // @ts-ignore + )} No such file or directory: "${this.logger.colorizeOutput(Logger.colors.RED, error.path)}"` + ) + break + + case 9009: + this.logger.addLog( + `${this.logger.colorizeOutput( + Logger.colors.RED, + 'ERROR:' + )} Command failed: file not found or not in PATH` + ) + break + } + } else if (err instanceof PythonVersionError) { + err.log(this.logger) + } else { + this.logger.addLog( + `${this.logger.colorizeOutput(Logger.colors.RED, 'ERROR:')} ${ + err ? (err.message || err) : '' + }` + ) + this.logger.log.silly(this.logger.colorizeOutput(Logger.colors.RED, 'FULL ERROR:'), err ? (err.stack || err) : '') + + if (err.log) { + err.log(this.logger) } - const execPath = stdout.trim() - callback(null, execPath) } - }, + this.logger.addLog('--------------------------------------------') + } - succeed: function succeed (execPath, version) { - this.log.info(`using Python version ${version} found at "${execPath}"`) + /** + * Function which is called if python path founded + * + * @private + * @param {string} execPath founded path + * @param {string} version python version + */ + succeed (execPath, version) { + this.logger.log.info( + `using Python version ${this.logger.colorizeOutput( + Logger.colors.GREEN, + version + )} found at "${this.logger.colorizeOutput(Logger.colors.GREEN, execPath)}"` + ) process.nextTick(this.callback.bind(null, null, execPath)) - }, + } - fail: function fail () { - const errorLog = this.errorLog.join('\n') + /** + * @private + */ + fail () { + const errorLog = this.logger.dumpErrorLog() - const pathExample = this.win ? 'C:\\Path\\To\\python.exe' + const pathExample = this.win + ? 'C:\\Path\\To\\python.exe' : '/path/to/pythonexecutable' // For Windows 80 col console, use up to the column before the one marked // with X (total 79 chars including logger prefix, 58 chars usable here): // X const info = [ '**********************************************************', + 'If you have non-displayed characters, please set "UTF-8"', + 'encoding.', 'You need to install the latest version of Python.', 'Node-gyp should be able to find and use Python. If not,', 'you can try one of the following options:', @@ -326,17 +654,126 @@ PythonFinder.prototype = { '**********************************************************' ].join('\n') - this.log.error(`\n${errorLog}\n\n${info}\n`) - process.nextTick(this.callback.bind(null, new Error( - 'Could not find any Python installation to use'))) + this.logger.log.error(`\n${errorLog}\n\n${info}\n`) + process.nextTick( + this.callback.bind( + null, + // if changing error message don't forget also change it test file too + new Error('Could not find any Python installation to use') + ) + ) + } +} + +/** + * Error with additional data. + * If you do not want to pass any additional data use regular Error + * + * !ALL MEMBERS EXCEPT "DATA" ARE OPTIONAL! + * @see Error + * + * @class + * @extends Error +*/ +class ErrorWithData extends Error { + // DONE: give to user possibility to pass existing error for which provide additional data + /** + * + * @typedef ErrorConstructor + * @property {{[key:string]: any}} data additional data to pass in data property of error object + * @property {string|Error} [messageOrError] + * @private + */ + /** + * @constructor + * @param {ErrorConstructor} [options] + * @throws {TypeError} + */ + constructor (options) { + if (typeof options.messageOrError === 'string') { + const message = options.messageOrError + super(message) + } else if (options.messageOrError instanceof Error) { + const error = options.messageOrError + super(error.message) + this.error = error + } else { + super() + } + + if (!options.data) { + throw new TypeError('"data" property is required. If you do not want pass any additional data use regular Error instead this one') + } + + this.data = options.data + } + + /** + * + * @param {Logger} logger + */ + log (logger) { + // map through data object to print it as KEY: value + for (const prop in this.data) { + if (this.data[prop]) { + logger.addLog(`${logger.colorizeOutput(Logger.colors.RED, `${prop.toUpperCase()}:`)} ${this.data[prop].trim()}`) + } + } } } +class PythonVersionError extends ErrorWithData { + /** + * + * @param {{wanted: string, received: string}} ver + * @param {string} message + */ + constructor (ver, message) { + super({ messageOrError: message, data: { version: ver.received } }) + + /** @private */ + this._ver = ver + } + + /** + * Log error + * @param {Logger} logger + */ + log (logger) { + if (!this.data.version) { + logger.addLog(this.message) + } else { + logger.addLog( + `version is ${logger.colorizeOutput( + Logger.colors.RED, + this.data.version + )} - should be ${logger.colorizeOutput(Logger.colors.GREEN, this._ver.wanted)}` + ) + + logger.addLog('') + logger.addLog(logger.colorizeOutput(Logger.colors.RED, 'THIS VERSION OF PYTHON IS NOT SUPPORTED')) + logger.addLog('') + } + } +} + +/** + * + * @param {string} configPython force setted from terminal or npm config python path + * @param {(err:Error, found:string)=> void} callback succeed/error callback from where result + * is available + */ function findPython (configPython, callback) { - var finder = new PythonFinder(configPython, callback) + const finder = new PythonFinder(configPython, callback) finder.findPython() } +// function for tests +/* findPython(null, (err, found) => { + console.log('found:', (new Logger()).colorizeOutput(Logger.colors.GREEN, found)) + console.log('err:', err) +}) */ + module.exports = findPython module.exports.test = { PythonFinder: PythonFinder, diff --git a/test/rm.js b/test/rm.js new file mode 100644 index 0000000000..c32bfe0553 --- /dev/null +++ b/test/rm.js @@ -0,0 +1,20 @@ +const fs = require('fs') +const path = require('path') + +/** recursively delete files, symlinks (without following them) and dirs */ +module.exports = function rmRecSync (pth) { + pth = path.normalize(pth) + + rm(pth) + + function rm (pth) { + const pathStat = fs.statSync(pth) + // trick with lstat is used to avoid following symlinks (especially junctions on windows) + if (pathStat.isDirectory() && !fs.lstatSync(pth).isSymbolicLink()) { + fs.readdirSync(pth).forEach((nextPath) => rm(path.join(pth, nextPath))) + fs.rmdirSync(pth) + } else { + fs.unlinkSync(pth) + } + } +} diff --git a/test/test-configure-python.js b/test/test-configure-python.js index ac25f7972e..6a52668502 100644 --- a/test/test-configure-python.js +++ b/test/test-configure-python.js @@ -5,7 +5,7 @@ const path = require('path') const devDir = require('./common').devDir() const gyp = require('../lib/node-gyp') const requireInject = require('require-inject') -const configure = requireInject('../lib/configure', { +const configure = requireInject('../lib/configure.js', { 'graceful-fs': { openSync: function () { return 0 }, closeSync: function () { }, diff --git a/test/test-find-accessible-sync.js b/test/test-find-accessible-sync.js index 0a2e584c4f..9bffd338ce 100644 --- a/test/test-find-accessible-sync.js +++ b/test/test-find-accessible-sync.js @@ -3,7 +3,7 @@ const test = require('tap').test const path = require('path') const requireInject = require('require-inject') -const configure = requireInject('../lib/configure', { +const configure = requireInject('../lib/configure.js', { 'graceful-fs': { closeSync: function () { return undefined }, openSync: function (path) { diff --git a/test/test-find-python-script.js b/test/test-find-python-script.js new file mode 100644 index 0000000000..2b4376c36e --- /dev/null +++ b/test/test-find-python-script.js @@ -0,0 +1,84 @@ +// @ts-check +'use strict' +/** @typedef {import("tap")} Tap */ + +const test = require('tap').test +const execFile = require('child_process').execFile +const path = require('path') + +require('npmlog').level = 'warn' + +//* name can be used as short descriptions + +/** + * @typedef Check + * @property {string} path - path to executable or command + * @property {string} name - very little description + */ + +// TODO: add symlinks to python which will contain utf-8 chars +/** + * @type {Check[]} + */ +const checks = [ + { path: process.env.PYTHON, name: 'env var PYTHON' }, + { path: 'python3', name: 'python3 in PATH' }, + { path: 'python', name: 'python in PATH' } +] +const args = [path.resolve('./lib/find-python-script.py')] +const options = { + windowsHide: true +} + +/** + Getting output from find-python-script.py, + compare it to path provided to terminal. + If equals - test pass + + runs for all checks + + @private + @argument {Error} err - exec error + @argument {string} stdout - stdout buffer of child process + @argument {string} stderr + @this {{t: Tap, exec: Check}} + */ +function check (err, stdout, stderr) { + const { t, exec } = this + if (!err && !stderr) { + t.ok( + stdout.trim(), + `${exec.name}: check path ${exec.path} equals ${stdout.trim()}` + ) + } else { + // @ts-ignore + if (err.code === 9009 || err.code === 'ENOENT') { + t.skip(`skipped: ${exec.name} file not found`) + } else { + t.skip(`error: ${err}\n\nstderr: ${stderr}`) + } + } +} + +test('find-python-script', { buffered: false }, (t) => { + t.plan(checks.length) + + // ? may be more elegant way to pass context + // context for check functions + const ctx = { + t: t, + exec: {} + } + + for (const exec of checks) { + // checking if env var exist + if (!(exec.path === undefined || exec.path === null)) { + ctx.exec = exec + // passing ctx as copied object to make properties immutable from here + const boundedCheck = check.bind(Object.assign({}, ctx)) + execFile(exec.path, args, options, boundedCheck) + } else { + t.skip(`skipped: ${exec.name} doesn't exist or unavailable`) + } + } +}) diff --git a/test/test-find-python.js b/test/test-find-python.js index 67d0b2664f..02f5985751 100644 --- a/test/test-find-python.js +++ b/test/test-find-python.js @@ -1,226 +1,351 @@ -'use strict' - -delete process.env.PYTHON - -const test = require('tap').test -const findPython = require('../lib/find-python') -const execFile = require('child_process').execFile -const PythonFinder = findPython.test.PythonFinder - -require('npmlog').level = 'warn' - -test('find python', function (t) { - t.plan(4) - - findPython.test.findPython(null, function (err, found) { - t.strictEqual(err, null) - var proc = execFile(found, ['-V'], function (err, stdout, stderr) { - t.strictEqual(err, null) - t.ok(/Python 3/.test(stdout)) - t.strictEqual(stderr, '') - }) - proc.stdout.setEncoding('utf-8') - proc.stderr.setEncoding('utf-8') - }) -}) - -function poison (object, property) { - function fail () { - console.error(Error(`Property ${property} should not have been accessed.`)) - process.abort() - } - var descriptor = { - configurable: false, - enumerable: false, - get: fail, - set: fail - } - Object.defineProperty(object, property, descriptor) -} - -function TestPythonFinder () { - PythonFinder.apply(this, arguments) -} -TestPythonFinder.prototype = Object.create(PythonFinder.prototype) -// Silence npmlog - remove for debugging -TestPythonFinder.prototype.log = { - silly: () => {}, - verbose: () => {}, - info: () => {}, - warn: () => {}, - error: () => {} -} -delete TestPythonFinder.prototype.env.NODE_GYP_FORCE_PYTHON - -test('find python - python', function (t) { - t.plan(6) - - var f = new TestPythonFinder('python', done) - f.execFile = function (program, args, opts, cb) { - f.execFile = function (program, args, opts, cb) { - poison(f, 'execFile') - t.strictEqual(program, '/path/python') - t.ok(/sys\.version_info/.test(args[1])) - cb(null, '3.9.1') - } - t.strictEqual(program, - process.platform === 'win32' ? '"python"' : 'python') - t.ok(/sys\.executable/.test(args[1])) - cb(null, '/path/python') - } - f.findPython() - - function done (err, python) { - t.strictEqual(err, null) - t.strictEqual(python, '/path/python') - } -}) - -test('find python - python too old', function (t) { - t.plan(2) - - var f = new TestPythonFinder(null, done) - f.execFile = function (program, args, opts, cb) { - if (/sys\.executable/.test(args[args.length - 1])) { - cb(null, '/path/python') - } else if (/sys\.version_info/.test(args[args.length - 1])) { - cb(null, '2.3.4') - } else { - t.fail() - } - } - f.findPython() - - function done (err) { - t.ok(/Could not find any Python/.test(err)) - t.ok(/not supported/i.test(f.errorLog)) - } -}) - -test('find python - no python', function (t) { - t.plan(2) - - var f = new TestPythonFinder(null, done) - f.execFile = function (program, args, opts, cb) { - if (/sys\.executable/.test(args[args.length - 1])) { - cb(new Error('not found')) - } else if (/sys\.version_info/.test(args[args.length - 1])) { - cb(new Error('not a Python executable')) - } else { - t.fail() - } - } - f.findPython() - - function done (err) { - t.ok(/Could not find any Python/.test(err)) - t.ok(/not in PATH/.test(f.errorLog)) - } -}) - -test('find python - no python2, no python, unix', function (t) { - t.plan(2) - - var f = new TestPythonFinder(null, done) - f.checkPyLauncher = t.fail - f.win = false - - f.execFile = function (program, args, opts, cb) { - if (/sys\.executable/.test(args[args.length - 1])) { - cb(new Error('not found')) - } else { - t.fail() - } - } - f.findPython() - - function done (err) { - t.ok(/Could not find any Python/.test(err)) - t.ok(/not in PATH/.test(f.errorLog)) - } -}) - -test('find python - no python, use python launcher', function (t) { - t.plan(4) - - var f = new TestPythonFinder(null, done) - f.win = true - - f.execFile = function (program, args, opts, cb) { - if (program === 'py.exe') { - t.notEqual(args.indexOf('-3'), -1) - t.notEqual(args.indexOf('-c'), -1) - return cb(null, 'Z:\\snake.exe') - } - if (/sys\.executable/.test(args[args.length - 1])) { - cb(new Error('not found')) - } else if (f.winDefaultLocations.includes(program)) { - cb(new Error('not found')) - } else if (/sys\.version_info/.test(args[args.length - 1])) { - if (program === 'Z:\\snake.exe') { - cb(null, '3.9.0') - } else { - t.fail() - } - } else { - t.fail() - } - } - f.findPython() - - function done (err, python) { - t.strictEqual(err, null) - t.strictEqual(python, 'Z:\\snake.exe') - } -}) - -test('find python - no python, no python launcher, good guess', function (t) { - t.plan(2) - - var f = new TestPythonFinder(null, done) - f.win = true - const expectedProgram = f.winDefaultLocations[0] - - f.execFile = function (program, args, opts, cb) { - if (program === 'py.exe') { - return cb(new Error('not found')) - } - if (/sys\.executable/.test(args[args.length - 1])) { - cb(new Error('not found')) - } else if (program === expectedProgram && - /sys\.version_info/.test(args[args.length - 1])) { - cb(null, '3.7.3') - } else { - t.fail() - } - } - f.findPython() - - function done (err, python) { - t.strictEqual(err, null) - t.ok(python === expectedProgram) - } -}) - -test('find python - no python, no python launcher, bad guess', function (t) { - t.plan(2) - - var f = new TestPythonFinder(null, done) - f.win = true - - f.execFile = function (program, args, opts, cb) { - if (/sys\.executable/.test(args[args.length - 1])) { - cb(new Error('not found')) - } else if (/sys\.version_info/.test(args[args.length - 1])) { - cb(new Error('not a Python executable')) - } else { - t.fail() - } - } - f.findPython() - - function done (err) { - t.ok(/Could not find any Python/.test(err)) - t.ok(/not in PATH/.test(f.errorLog)) - } -}) +'use strict' + +const tap = require('tap') +const { test } = tap +const findPython = require('../lib/find-python') +const cp = require('child_process') +const PythonFinder = findPython.test.PythonFinder +const util = require('util') +const path = require('path') +const npmlog = require('npmlog') +const fs = require('fs') +// just comment this line to see log output +// useful to not to test output by hand +// ! keep uncommented when committing +npmlog.level = 'silent' + +// what final error message displayed in terminal should contain +const finalErrorMessage = 'Could not find any Python' + +//! don't forget manually call pythonFinderInstance.findPython() + +// String emulating path command or anything else with spaces +// and UTF-8 characters. +// Is returned by execFile +//! USE FOR ALL STRINGS +const testString = 'python one love♥' +const testVersions = { + outdated: '2.0.0', + normal: '3.9.0', + testError: new Error('test error') +} + +function strictDeepEqual (received, wanted) { + let result = false + + for (let i = 0; i < received.length; i++) { + if (Array.isArray(received[i]) && Array.isArray(wanted[i])) { + result = strictDeepEqual(received[i], wanted[i]) + } else { + result = received[i] === wanted[i] + } + + if (!result) { + return result + } + } + + return result +} + +/** + * @typedef OptionsObj + * @property {boolean} [shouldProduceError] pass test error to callback + * @property {boolean} [checkingPyLauncher] + * @property {boolean} [isPythonOutdated] return outdated version + * @property {boolean} [checkingWinDefaultPathes] + * + */ + +/** + * implement custom childProcess.execFile for testing proposes + * + * ! ***DO NOT FORGET TO OVERRIDE DEFAULT `PythonFinder.execFile` AFTER INSTANCING `PythonFinder`*** + * + * TODO: do overriding if automotive way + * + * @param {OptionsObj} [optionsObj] + */ +function TestExecFile (optionsObj) { + /** + * + * @this {PythonFinder} + */ + return function testExecFile (exec, args, options, callback) { + if (!(optionsObj && optionsObj.shouldProduceError)) { + // when checking version in checkExecPath, thus need to use PythonFinder.argsVersion + if (args === this.argsVersion) { + if (optionsObj && optionsObj.checkingWinDefaultPathes) { + if (this.winDefaultLocations.includes(exec)) { + callback(null, testVersions.normal) + } else { + callback(new Error('not found')) + } + } else if (optionsObj && optionsObj.isPythonOutdated) { + callback(null, testVersions.outdated, null) + } else { + callback(null, testVersions.normal, null) + } + } else if ( + // DONE: map through argsExecutable to check that all args are equals + strictDeepEqual(args, this.win ? this.argsExecutable.map((arg) => `"${arg}"`) : this.argsExecutable) + ) { + if (optionsObj && optionsObj.checkingPyLauncher) { + if ( + exec === 'py.exe' || + exec === (this.win ? '"python"' : 'python') + ) { + callback(null, testString, null) + } else { + callback(new Error('not found')) + } + } else if (optionsObj && optionsObj.checkingWinDefaultPathes) { + // return "not found" for regular checks (env-vars etc.) + // which are running twice: + // first to get path, second to check it + callback(new Error('not found')) + } else { + // returned string should be trimmed + callback(null, testString + '\n', null) + } + } else { + throw new Error( + `invalid arguments are provided! provided args +are: ${args};\n\nValid are: \n${this.argsExecutable}\n${this.argsVersion}` + ) + } + } else { + const testError = new Error( + `test error ${testString}; optionsObj: ${JSON.stringify(optionsObj)}` + ) + callback(testError) + } + } +} + +/** + * + * @param {boolean} isPythonOutdated if true will return outdated version of python + * @param {OptionsObj} optionsObj + */ + +test('find-python', { buffered: true }, (t) => { + t.test('whole module tests', (t) => { + t.test('python found', (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + if (err) { + t.fail( + `mustn't produce any errors if execFile doesn't produced error. ${err}` + ) + } else { + t.equal(path, testString) + t.end() + } + }) + pythonFinderInstance.execFile = TestExecFile() + + pythonFinderInstance.findPython() + }) + + t.test('outdated version of python found', (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + if (!err) { + t.fail("mustn't return path for outdated version") + } else { + t.end() + } + }) + + pythonFinderInstance.execFile = TestExecFile({ isPythonOutdated: true }) + + pythonFinderInstance.findPython() + }) + + t.test('no python on computer', (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + t.ok(err.message.includes(finalErrorMessage)) + t.end() + }) + + pythonFinderInstance.execFile = TestExecFile({ + shouldProduceError: true + }) + + pythonFinderInstance.findPython() + }) + + t.test('no python, unix', (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + t.notOk(path) + + t.ok(err) + t.ok(err.message.includes(finalErrorMessage)) + t.end() + }) + + pythonFinderInstance.win = false + pythonFinderInstance.checkPyLauncher = t.fail + + pythonFinderInstance.execFile = TestExecFile({ + shouldProduceError: true + }) + + pythonFinderInstance.findPython() + }) + + t.test('no python, use python launcher', (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + t.equal(err, null) + + t.equal(path, testString) + + t.end() + }) + + pythonFinderInstance.win = true + + pythonFinderInstance.execFile = TestExecFile({ + checkingPyLauncher: true + }) + + pythonFinderInstance.findPython() + }) + + t.test( + 'no python, no python launcher, checking win default locations', + (t) => { + const pythonFinderInstance = new PythonFinder(null, (err, path) => { + t.equal(err, null) + t.ok(pythonFinderInstance.winDefaultLocations.includes(path)) + t.end() + }) + + pythonFinderInstance.win = true + + pythonFinderInstance.execFile = TestExecFile({ + checkingWinDefaultPathes: true + }) + pythonFinderInstance.findPython() + } + ) + + t.test('python is setted from config', (t) => { + const pythonFinderInstance = new PythonFinder(testString, (err, path) => { + t.equal(err, null) + + t.equal(path, testString) + + t.end() + }) + + pythonFinderInstance.win = true + + pythonFinderInstance.execFile = TestExecFile() + pythonFinderInstance.findPython() + }) + + t.end() + }) + + // DONE: make symlink to python with utf-8 chars + t.test('real testing', async (t) => { + const paths = { + python: '', + pythonDir: '', + testDir: '', + baseDir: __dirname + } + + const execFile = util.promisify(cp.execFile) + + // a bit tricky way to make PythonFinder promisified + function promisifyPythonFinder (config) { + let pythonFinderInstance + + const result = new Promise((resolve, reject) => { + pythonFinderInstance = new PythonFinder(config, (err, path) => { + if (err) { + reject(err) + } else { + resolve(path) + } + }) + }) + + return { pythonFinderInstance, result } + } + + async function testPythonPath (t, pythonPath) { + try { + const { stderr, stdout } = await execFile(pythonPath, ['-V']) + + console.log('stdout:', stdout) + console.log('stderr:', stderr) + + if (t.ok(stdout.includes('Python 3'), 'is it python with major version 3') && + t.equal(stderr, '', 'is stderr empty')) { + return true + } + + return false + } catch (err) { + t.equal(err, null, 'is error null') + return false + } + } + + // await is needed because test func is async + await t.test('trying to find real python exec', async (t) => { + const { pythonFinderInstance, result } = promisifyPythonFinder(null) + + try { + pythonFinderInstance.findPython() + + const pythonPath = await result + + if (t.ok(await testPythonPath(t, pythonPath), 'is path valid')) { + // stdout contain output of "python -V" command, not python path + // using found path as trusted + paths.python = pythonPath + paths.pythonDir = path.join(paths.python, '../') + } + } catch (err) { + t.notOk(err, 'are we having error') + } + + t.end() + }) + + await t.test(`test with path containing "${testString}"`, async (t) => { + // making fixture + paths.testDir = fs.mkdtempSync(path.resolve(paths.baseDir, 'node_modules', 'pythonFindTestFolder-')) + + // using "junction" to avoid permission error on windows (ignored on other platforms) + fs.symlinkSync(paths.pythonDir, path.resolve(paths.testDir, testString), 'junction') + console.log('🚀 ~ file: test-find-python.js ~ line 312 ~ await.test ~ path.resolve(paths.testDir, testString)', path.resolve(paths.testDir, testString)) + console.log('🚀 ~ file: test-find-python.js ~ line 312 ~ await.test ~ paths.pythonDir', paths.pythonDir) + + const { pythonFinderInstance, result } = promisifyPythonFinder(path.resolve(paths.testDir, 'python')) + + pythonFinderInstance.findPython() + + const pythonPath = await result + + t.ok(await testPythonPath(t, pythonPath), 'is path valid') + + t.end() + }) + + // remove fixture + if (fs.rmSync) { + fs.rmSync(paths.testDir, { recursive: true }) + } else { + // + require('./rm.js')(paths.testDir) + } + + t.end() + }) + + t.end() +})