From 1942cc889ff73a8fbcc2542521664555d4051789 Mon Sep 17 00:00:00 2001 From: Bradley Farias Date: Mon, 5 Jun 2017 19:44:56 -0500 Subject: [PATCH 1/8] module: Allow runMain to be ESM This follows the EPS an allows the node CLI to have ESM as an entry point. `node ./example.mjs`. A newer V8 is needed for `import()` so that is not included. `import.meta` is still in specification stage so that also is not included. Author: Bradley Farias Author: Guy Bedford Author: Jan Krems fix eslint errors add missing include restore module.exports remove TryCatch Use anonymous namespace, fix compilation warnings Clean up string usage Fix c++ linting Allow unlinked cwd Fixes make test. fix indentation eslint fix more common whitespace usage, fix missing header include try to address Windows build failure throw no-base error in JS No need to go to C++ for this. Fix resolve module dir URL Fix module.load usage Makes CJS module loading work process.exit(1) when there is an error in ESM loading Add support for .mjs in test harness tools: support ESM in ESLint required-modules rule Start porting ESM tests Disable required-modules for basic mjs tests Avoid using auto for easy-to-mix up cases fix linter errors for new linter rules on `master` More linting use null as [[Prototype]] directly move src/loader/* out of its own directory fix more linting properly decode url encoding in pathnames Cleanup URL->file path conversion Error cases, add tests. lint fix module._cache regression move test out of fixtures nits on test properly realpath after getting path from URL reject module instantiation when binding throws call e.stack getter preserve-symlinks support remove unnecessary index file flag as harmony-modules lint ordering of lists add .mjs to coverage, ESM support appears buggy though simple ESM namespace keys test move logic for making URL from file path f test and fix snapshot timing of CJS use internal/errors fix bug with moduleProvider in CJS snapshot nits from @Fishrock123 linting gate --- Makefile | 4 +- lib/internal/errors.js | 7 +- lib/internal/loader/Loader.js | 75 +++ lib/internal/loader/ModuleJob.js | 116 ++++ lib/internal/loader/ModuleMap.js | 33 ++ lib/internal/loader/ModuleWrap.js | 61 ++ lib/internal/loader/resolveRequestUrl.js | 104 ++++ lib/internal/loader/search.js | 33 ++ lib/internal/safe_globals.js | 26 + lib/internal/url.js | 7 + lib/module.js | 58 +- node.gyp | 9 + src/module_wrap.cc | 531 ++++++++++++++++++ src/module_wrap.h | 58 ++ src/node.cc | 8 + src/node_config.cc | 3 + src/node_internals.h | 4 + src/node_url.cc | 63 +++ src/node_url.h | 4 + test/cctest/test_url.cc | 25 + test/es-module/es-module.status | 7 + test/es-module/esm-snapshot-mutator.js | 5 + test/es-module/esm-snapshot.js | 3 + test/es-module/test-esm-basic-imports.mjs | 8 + .../es-module/test-esm-encoded-path-native.js | 10 + test/es-module/test-esm-encoded-path.mjs | 7 + test/es-module/test-esm-forbidden-globals.mjs | 24 + test/es-module/test-esm-namespace.mjs | 7 + test/es-module/test-esm-ok.mjs | 5 + test/es-module/test-esm-pkg-over-ext.mjs | 8 + test/es-module/test-esm-preserve-symlinks.js | 38 ++ test/es-module/test-esm-require-cache.mjs | 7 + test/es-module/test-esm-shebang.mjs | 6 + test/es-module/test-esm-snapshot.mjs | 7 + test/es-module/test-esm-symlink.js | 48 ++ test/es-module/testcfg.py | 6 + .../es-module-require-cache/counter.js | 2 + .../es-module-require-cache/preload.js | 1 + test/fixtures/es-module-url/empty.js | 0 test/fixtures/es-module-url/native.mjs | 2 + test/testpy/__init__.py | 13 +- tools/eslint-rules/required-modules.js | 60 +- tools/test.py | 18 +- 43 files changed, 1481 insertions(+), 40 deletions(-) create mode 100644 lib/internal/loader/Loader.js create mode 100644 lib/internal/loader/ModuleJob.js create mode 100644 lib/internal/loader/ModuleMap.js create mode 100644 lib/internal/loader/ModuleWrap.js create mode 100644 lib/internal/loader/resolveRequestUrl.js create mode 100644 lib/internal/loader/search.js create mode 100644 lib/internal/safe_globals.js create mode 100644 src/module_wrap.cc create mode 100644 src/module_wrap.h create mode 100644 test/es-module/es-module.status create mode 100644 test/es-module/esm-snapshot-mutator.js create mode 100644 test/es-module/esm-snapshot.js create mode 100644 test/es-module/test-esm-basic-imports.mjs create mode 100644 test/es-module/test-esm-encoded-path-native.js create mode 100644 test/es-module/test-esm-encoded-path.mjs create mode 100644 test/es-module/test-esm-forbidden-globals.mjs create mode 100644 test/es-module/test-esm-namespace.mjs create mode 100644 test/es-module/test-esm-ok.mjs create mode 100644 test/es-module/test-esm-pkg-over-ext.mjs create mode 100644 test/es-module/test-esm-preserve-symlinks.js create mode 100644 test/es-module/test-esm-require-cache.mjs create mode 100644 test/es-module/test-esm-shebang.mjs create mode 100644 test/es-module/test-esm-snapshot.mjs create mode 100644 test/es-module/test-esm-symlink.js create mode 100644 test/es-module/testcfg.py create mode 100644 test/fixtures/es-module-require-cache/counter.js create mode 100644 test/fixtures/es-module-require-cache/preload.js create mode 100644 test/fixtures/es-module-url/empty.js create mode 100644 test/fixtures/es-module-url/native.mjs diff --git a/Makefile b/Makefile index 67a568cfa54efe..5cc1a8dae7c9a3 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ coverage-build: all "$(CURDIR)/testing/coverage/gcovr-patches.diff"); fi if [ -d lib_ ]; then $(RM) -r lib; mv lib_ lib; fi mv lib lib_ - $(NODE) ./node_modules/.bin/nyc instrument lib_/ lib/ + $(NODE) ./node_modules/.bin/nyc instrument --extension .js --extension .mjs lib_/ lib/ $(MAKE) coverage-test: coverage-build @@ -888,6 +888,8 @@ jslint: @echo "Running JS linter..." $(NODE) tools/eslint/bin/eslint.js --cache --rulesdir=tools/eslint-rules --ext=.js,.md \ $(JSLINT_TARGETS) + $(NODE) tools/eslint/bin/eslint.js --cache --rulesdir=tools/eslint-rules --parser-options=sourceType:module --ext=.mjs \ + $(JSLINT_TARGETS) jslint-ci: @echo "Running JS linter..." diff --git a/lib/internal/errors.js b/lib/internal/errors.js index c90cd82e305d3c..97fe042a58b7e9 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -209,6 +209,7 @@ E('ERR_INVALID_OPT_VALUE', }); E('ERR_INVALID_OPT_VALUE_ENCODING', (value) => `The value "${String(value)}" is invalid for option "encoding"`); +E('ERR_INVALID_PERFORMANCE_MARK', 'The "%s" performance mark has not been set'); E('ERR_INVALID_PROTOCOL', (protocol, expectedProtocol) => `Protocol "${protocol}" not supported. Expected "${expectedProtocol}"`); E('ERR_INVALID_REPL_EVAL_CONFIG', @@ -226,6 +227,9 @@ E('ERR_IPC_ONE_PIPE', 'Child process can have only one IPC pipe'); E('ERR_IPC_SYNC_FORK', 'IPC cannot be used with synchronous forks'); E('ERR_METHOD_NOT_IMPLEMENTED', 'The %s method is not implemented'); E('ERR_MISSING_ARGS', missingArgs); +E('ERR_MISSING_MODULE', 'Cannot find module %s'); +E('ERR_MODULE_RESOLUTION_LEGACY', '%s not found by import in %s.' + + 'Legacy behavior in require would have found it at %s'); E('ERR_MULTIPLE_CALLBACK', 'Callback called multiple times'); E('ERR_NAPI_CONS_FUNCTION', 'Constructor must be a function'); E('ERR_NAPI_CONS_PROTOTYPE_OBJECT', 'Constructor.prototype must be an object'); @@ -233,7 +237,7 @@ E('ERR_NO_CRYPTO', 'Node.js is not compiled with OpenSSL crypto support'); E('ERR_NO_ICU', '%s is not supported on Node.js compiled without ICU'); E('ERR_NO_LONGER_SUPPORTED', '%s is no longer supported'); E('ERR_PARSE_HISTORY_DATA', 'Could not parse history data in %s'); -E('ERR_INVALID_PERFORMANCE_MARK', 'The "%s" performance mark has not been set'); +E('ERR_REQUIRE_ESM', 'Must use import to load ES Module: %s'); E('ERR_SOCKET_ALREADY_BOUND', 'Socket is already bound'); E('ERR_SOCKET_BAD_PORT', 'Port should be > 0 and < 65536'); E('ERR_SOCKET_BAD_TYPE', @@ -270,6 +274,7 @@ E('ERR_VALID_PERFORMANCE_ENTRY_TYPE', 'At least one valid performance entry type is required'); E('ERR_VALUE_OUT_OF_RANGE', 'The value of "%s" must be %s. Received "%s"'); + function invalidArgType(name, expected, actual) { assert(name, 'name is required'); diff --git a/lib/internal/loader/Loader.js b/lib/internal/loader/Loader.js new file mode 100644 index 00000000000000..a409d397f85dd6 --- /dev/null +++ b/lib/internal/loader/Loader.js @@ -0,0 +1,75 @@ +'use strict'; + +const { URL } = require('url'); +const { getURLFromFilePath } = require('internal/url'); + +const { + getNamespaceOfModuleWrap +} = require('internal/loader/ModuleWrap'); + +const ModuleMap = require('internal/loader/ModuleMap'); +const ModuleJob = require('internal/loader/ModuleJob'); +const resolveRequestUrl = require('internal/loader/resolveRequestUrl'); +const errors = require('internal/errors'); + +function getBase() { + try { + return getURLFromFilePath(`${process.cwd()}/`); + } catch (e) { + e.stack; + // If the current working directory no longer exists. + if (e.code === 'ENOENT') { + return undefined; + } + throw e; + } +} + +class Loader { + constructor(base = getBase()) { + this.moduleMap = new ModuleMap(); + if (typeof base !== 'undefined' && base instanceof URL !== true) { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'URL'); + } + this.base = base; + } + + async resolve(specifier) { + const request = resolveRequestUrl(this.base, specifier); + if (request.url.protocol !== 'file:') { + throw new errors.Error('ERR_INVALID_PROTOCOL', + request.url.protocol, 'file:'); + } + return request.url; + } + + async getModuleJob(dependentJob, specifier) { + if (!this.moduleMap.has(dependentJob.url)) { + throw new errors.Error('ERR_MISSING_MODULE', dependentJob.url); + } + const request = await resolveRequestUrl(dependentJob.url, specifier); + const url = `${request.url}`; + if (this.moduleMap.has(url)) { + return this.moduleMap.get(url); + } + const dependencyJob = new ModuleJob(this, request); + this.moduleMap.set(url, dependencyJob); + return dependencyJob; + } + + async import(specifier) { + const request = await resolveRequestUrl(this.base, specifier); + const url = `${request.url}`; + let job; + if (this.moduleMap.has(url)) { + job = this.moduleMap.get(url); + } else { + job = new ModuleJob(this, request); + this.moduleMap.set(url, job); + } + const module = await job.run(); + return getNamespaceOfModuleWrap(module); + } +} +Object.setPrototypeOf(Loader.prototype, null); +module.exports = Loader; diff --git a/lib/internal/loader/ModuleJob.js b/lib/internal/loader/ModuleJob.js new file mode 100644 index 00000000000000..db4cb6ae5c5031 --- /dev/null +++ b/lib/internal/loader/ModuleJob.js @@ -0,0 +1,116 @@ +'use strict'; + +const { SafeSet, SafePromise } = require('internal/safe_globals'); +const resolvedPromise = SafePromise.resolve(); +const resolvedArrayPromise = SafePromise.resolve([]); +const { ModuleWrap } = require('internal/loader/ModuleWrap'); + +const NOOP = () => { /* No-op */ }; +class ModuleJob { + /** + * @param {module: ModuleWrap?, compiled: Promise} moduleProvider + */ + constructor(loader, moduleProvider, url) { + this.url = `${moduleProvider.url}`; + this.moduleProvider = moduleProvider; + this.loader = loader; + this.error = null; + this.hadError = false; + + if (moduleProvider instanceof ModuleWrap !== true) { + // linked == promise for dependency jobs, with module populated, + // module wrapper linked + this.modulePromise = this.moduleProvider.createModule(); + this.module = undefined; + const linked = async () => { + const dependencyJobs = []; + this.module = await this.modulePromise; + this.module.link(async (dependencySpecifier) => { + const dependencyJobPromise = + this.loader.getModuleJob(this, dependencySpecifier); + dependencyJobs.push(dependencyJobPromise); + const dependencyJob = await dependencyJobPromise; + return dependencyJob.modulePromise; + }); + return SafePromise.all(dependencyJobs); + }; + this.linked = linked(); + + // instantiated == deep dependency jobs wrappers instantiated, + //module wrapper instantiated + this.instantiated = undefined; + } else { + const getModuleProvider = async () => moduleProvider; + this.modulePromise = getModuleProvider(); + this.moduleProvider = { finish: NOOP }; + this.module = moduleProvider; + this.linked = resolvedArrayPromise; + this.instantiated = this.modulePromise; + } + } + + instantiate() { + if (this.instantiated) { + return this.instantiated; + } + return this.instantiated = new Promise(async (resolve, reject) => { + const jobsInGraph = new SafeSet(); + let jobsReadyToInstantiate = 0; + // (this must be sync for counter to work) + const queueJob = (moduleJob) => { + if (jobsInGraph.has(moduleJob)) { + return; + } + jobsInGraph.add(moduleJob); + moduleJob.linked.then((dependencyJobs) => { + for (const dependencyJob of dependencyJobs) { + queueJob(dependencyJob); + } + checkComplete(); + }, (e) => { + if (!this.hadError) { + this.error = e; + this.hadError = true; + } + checkComplete(); + }); + }; + const checkComplete = () => { + if (++jobsReadyToInstantiate === jobsInGraph.size) { + // I believe we only throw once the whole tree is finished loading? + // or should the error bail early, leaving entire tree to still load? + if (this.hadError) { + reject(this.error); + } else { + try { + this.module.instantiate(); + for (const dependencyJob of jobsInGraph) { + dependencyJob.instantiated = resolvedPromise; + } + resolve(this.module); + } catch (e) { + e.stack; + reject(e); + } + } + } + }; + queueJob(this); + }); + } + + async run() { + const module = await this.instantiate(); + try { + module.evaluate(); + } catch (e) { + e.stack; + this.hadError = true; + this.error = e; + throw e; + } + return module; + } +} +Object.setPrototypeOf(ModuleJob.prototype, null); +module.exports = ModuleJob; diff --git a/lib/internal/loader/ModuleMap.js b/lib/internal/loader/ModuleMap.js new file mode 100644 index 00000000000000..aa238afbaedc05 --- /dev/null +++ b/lib/internal/loader/ModuleMap.js @@ -0,0 +1,33 @@ +'use strict'; + +const ModuleJob = require('internal/loader/ModuleJob'); +const { SafeMap } = require('internal/safe_globals'); +const debug = require('util').debuglog('esm'); +const errors = require('internal/errors'); + +// Tracks the state of the loader-level module cache +class ModuleMap extends SafeMap { + get(url) { + if (typeof url !== 'string') { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string'); + } + return super.get(url); + } + set(url, job) { + if (typeof url !== 'string') { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string'); + } + if (job instanceof ModuleJob !== true) { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'job', 'ModuleJob'); + } + debug(`Storing ${url} in ModuleMap`); + return super.set(url, job); + } + has(url) { + if (typeof url !== 'string') { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string'); + } + return super.has(url); + } +} +module.exports = ModuleMap; diff --git a/lib/internal/loader/ModuleWrap.js b/lib/internal/loader/ModuleWrap.js new file mode 100644 index 00000000000000..4d35356ec2433e --- /dev/null +++ b/lib/internal/loader/ModuleWrap.js @@ -0,0 +1,61 @@ +'use strict'; + +const { ModuleWrap } = process.binding('module_wrap'); +const debug = require('util').debuglog('esm'); +const ArrayJoin = Function.call.bind(Array.prototype.join); +const ArrayMap = Function.call.bind(Array.prototype.map); + +const getNamespaceOfModuleWrap = (m) => { + const tmp = new ModuleWrap('import * as _ from "";_;', ''); + tmp.link(async () => m); + tmp.instantiate(); + return tmp.evaluate(); +}; + +const createDynamicModule = (exports, url = '', evaluate) => { + debug( + `creating ESM facade for ${url} with exports: ${ArrayJoin(exports, ', ')}` + ); + const names = ArrayMap(exports, (name) => `${name}`); + // sanitized ESM for reflection purposes + const src = `export let executor; + ${ArrayJoin(ArrayMap(names, (name) => `export let $${name}`), ';\n')} + ;(() => [ + fn => executor = fn, + { exports: { ${ + ArrayJoin(ArrayMap(names, (name) => `${name}: { + get: () => $${name}, + set: v => $${name} = v + }`), ',\n') +} } } + ]); + `; + const reflectiveModule = new ModuleWrap(src, `cjs-facade:${url}`); + reflectiveModule.instantiate(); + const [setExecutor, reflect] = reflectiveModule.evaluate()(); + // public exposed ESM + const reexports = `import { executor, + ${ArrayMap(names, (name) => `$${name}`)} + } from ""; + export { + ${ArrayJoin(ArrayMap(names, (name) => `$${name} as ${name}`), ', ')} + } + // add await to this later if top level await comes along + typeof executor === "function" ? executor() : void 0;`; + if (typeof evaluate === 'function') { + setExecutor(() => evaluate(reflect)); + } + const runner = new ModuleWrap(reexports, `${url}`); + runner.link(async () => reflectiveModule); + runner.instantiate(); + return { + module: runner, + reflect + }; +}; + +module.exports = { + createDynamicModule, + getNamespaceOfModuleWrap, + ModuleWrap +}; diff --git a/lib/internal/loader/resolveRequestUrl.js b/lib/internal/loader/resolveRequestUrl.js new file mode 100644 index 00000000000000..2245064bfe4ba8 --- /dev/null +++ b/lib/internal/loader/resolveRequestUrl.js @@ -0,0 +1,104 @@ +'use strict'; + +const { URL } = require('url'); +const internalCJSModule = require('internal/module'); +const internalURLModule = require('internal/url'); +const internalFS = require('internal/fs'); +const NativeModule = require('native_module'); +const { extname } = require('path'); +const { realpathSync } = require('fs'); +const preserveSymlinks = !!process.binding('config').preserveSymlinks; +const { + ModuleWrap, + createDynamicModule +} = require('internal/loader/ModuleWrap'); +const errors = require('internal/errors'); + +const search = require('internal/loader/search'); +const asyncReadFile = require('util').promisify(require('fs').readFile); +const debug = require('util').debuglog('esm'); + +const realpathCache = new Map(); + +class ModuleRequest { + constructor(url) { + this.url = url; + } +} +Object.setPrototypeOf(ModuleRequest.prototype, null); + +// Strategy for loading a standard JavaScript module +class StandardModuleRequest extends ModuleRequest { + async createModule() { + const source = `${await asyncReadFile(this.url)}`; + debug(`Loading StandardModule ${this.url}`); + return new ModuleWrap(internalCJSModule.stripShebang(source), + `${this.url}`); + } +} + +// Strategy for loading a node-style CommonJS module +class CJSModuleRequest extends ModuleRequest { + async createModule() { + const ctx = createDynamicModule(['default'], this.url, (reflect) => { + debug(`Loading CJSModule ${this.url.pathname}`); + const CJSModule = require('module'); + const pathname = internalURLModule.getPathFromURL(this.url); + CJSModule._load(pathname); + }); + this.finish = (module) => { + ctx.reflect.exports.default.set(module.exports); + }; + return ctx.module; + } +} + +// Strategy for loading a node builtin CommonJS module that isn't +// through normal resolution +class NativeModuleRequest extends CJSModuleRequest { + async createModule() { + const ctx = createDynamicModule(['default'], this.url, (reflect) => { + debug(`Loading NativeModule ${this.url.pathname}`); + const exports = require(this.url.pathname); + reflect.exports.default.set(exports); + }); + return ctx.module; + } +} + +const normalizeBaseURL = (baseURLOrString) => { + if (baseURLOrString instanceof URL) return baseURLOrString; + if (typeof baseURLOrString === 'string') return new URL(baseURLOrString); + return undefined; +}; + +const resolveRequestUrl = (baseURLOrString, specifier) => { + if (NativeModule.nonInternalExists(specifier)) { + return new NativeModuleRequest(new URL(`node:${specifier}`)); + } + + const baseURL = normalizeBaseURL(baseURLOrString); + let url = search(specifier, baseURL); + + if (url.protocol !== 'file:') { + throw new errors.Error('ERR_INVALID_PROTOCOL', url.protocol, 'file:'); + } + + if (!preserveSymlinks) { + const real = realpathSync(internalURLModule.getPathFromURL(url), { + [internalFS.realpathCacheKey]: realpathCache + }); + const old = url; + url = internalURLModule.getURLFromFilePath(real); + url.search = old.search; + url.hash = old.hash; + } + + const ext = extname(url.pathname); + if (ext === '.mjs') { + return new StandardModuleRequest(url); + } + + return new CJSModuleRequest(url); +}; +module.exports = resolveRequestUrl; diff --git a/lib/internal/loader/search.js b/lib/internal/loader/search.js new file mode 100644 index 00000000000000..f0ec34ae4e77c2 --- /dev/null +++ b/lib/internal/loader/search.js @@ -0,0 +1,33 @@ +'use strict'; + +const { URL } = require('url'); +const CJSmodule = require('module'); +const errors = require('internal/errors'); +const { resolve } = process.binding('module_wrap'); + +module.exports = (target, base) => { + target = `${target}`; + if (base === undefined) { + // We cannot search without a base. + throw new errors.Error('ERR_MISSING_MODULE', target); + } + base = `${base}`; + try { + return resolve(target, base); + } catch (e) { + e.stack; // cause V8 to generate stack before rethrow + let error = e; + try { + const questionedBase = new URL(base); + const tmpMod = new CJSmodule(questionedBase.pathname, null); + tmpMod.paths = CJSmodule._nodeModulePaths( + new URL('./', questionedBase).pathname); + const found = CJSmodule._resolveFilename(target, tmpMod); + error = new errors.Error('ERR_MODULE_RESOLUTION_LEGACY', target, + base, found); + } catch (problemChecking) { + // ignore + } + throw error; + } +}; diff --git a/lib/internal/safe_globals.js b/lib/internal/safe_globals.js new file mode 100644 index 00000000000000..ad58fa662b53ef --- /dev/null +++ b/lib/internal/safe_globals.js @@ -0,0 +1,26 @@ +'use strict'; + +const copyProps = (unsafe, safe) => { + for (const key of [...Object.getOwnPropertyNames(unsafe), + ...Object.getOwnPropertySymbols(unsafe) + ]) { + if (!Object.getOwnPropertyDescriptor(safe, key)) { + Object.defineProperty( + safe, + key, + Object.getOwnPropertyDescriptor(unsafe, key)); + } + } +}; +const makeSafe = (unsafe, safe) => { + copyProps(unsafe.prototype, safe.prototype); + copyProps(unsafe, safe); + Object.setPrototypeOf(safe.prototype, null); + Object.freeze(safe.prototype); + Object.freeze(safe); + return safe; +}; + +exports.SafeMap = makeSafe(Map, class SafeMap extends Map {}); +exports.SafeSet = makeSafe(Set, class SafeSet extends Set {}); +exports.SafePromise = makeSafe(Promise, class SafePromise extends Promise {}); diff --git a/lib/internal/url.js b/lib/internal/url.js index 95b5e9c5fd4240..f9d22f877db048 100644 --- a/lib/internal/url.js +++ b/lib/internal/url.js @@ -1377,6 +1377,12 @@ function getPathFromURL(path) { return isWindows ? getPathFromURLWin32(path) : getPathFromURLPosix(path); } +function getURLFromFilePath(filepath) { + return new URL(`file://${ + encodeURIComponent(filepath).replace(/%2F/g, '/') + }`); +} + function NativeURL(ctx) { this[context] = ctx; } @@ -1405,6 +1411,7 @@ setURLConstructor(constructUrl); module.exports = { toUSVString, getPathFromURL, + getURLFromFilePath, URL, URLSearchParams, domainToASCII, diff --git a/lib/module.js b/lib/module.js index 339a228da91bdf..d02a676c90270f 100644 --- a/lib/module.js +++ b/lib/module.js @@ -24,6 +24,7 @@ const NativeModule = require('native_module'); const util = require('util'); const internalModule = require('internal/module'); +const { getURLFromFilePath } = require('internal/url'); const vm = require('vm'); const assert = require('assert').ok; const fs = require('fs'); @@ -32,6 +33,14 @@ const path = require('path'); const internalModuleReadFile = process.binding('fs').internalModuleReadFile; const internalModuleStat = process.binding('fs').internalModuleStat; const preserveSymlinks = !!process.binding('config').preserveSymlinks; +const experimentalModules = !!process.binding('config').experimentalModules; + +const errors = require('internal/errors'); + +const Loader = require('internal/loader/Loader'); +const ModuleJob = require('internal/loader/ModuleJob'); +const { createDynamicModule } = require('internal/loader/ModuleWrap'); +const ESMLoader = new Loader(); function stat(filename) { filename = path._makeLong(filename); @@ -436,7 +445,36 @@ Module._load = function(request, parent, isMain) { debug('Module._load REQUEST %s parent: %s', request, parent.id); } - var filename = Module._resolveFilename(request, parent, isMain); + var filename = null; + + if (isMain) { + let err; + try { + filename = Module._resolveFilename(request, parent, isMain); + } catch (e) { + // try to keep stack + e.stack; + err = e; + } + if (experimentalModules) { + if (filename === null || /\.mjs$/.test(filename)) { + try { + ESMLoader.import(request).catch((e) => { + console.error(e); + process.exit(1); + }); + return; + } catch (e) { + // well, it isn't ESM + } + } + } + if (err) { + throw err; + } + } else { + filename = Module._resolveFilename(request, parent, isMain); + } var cachedModule = Module._cache[filename]; if (cachedModule) { @@ -506,6 +544,19 @@ Module.prototype.load = function(filename) { if (!Module._extensions[extension]) extension = '.js'; Module._extensions[extension](this, filename); this.loaded = true; + + if (experimentalModules) { + const url = getURLFromFilePath(filename); + if (ESMLoader.moduleMap.has(`${url}`) !== true) { + const ctx = createDynamicModule(['default'], url); + ctx.reflect.exports.default.set(this.exports); + ESMLoader.moduleMap.set(`${url}`, + new ModuleJob(ESMLoader, ctx.module)); + } else { + ESMLoader.moduleMap.get(`${url}`).moduleProvider.finish( + Module._cache[filename]); + } + } }; @@ -602,6 +653,11 @@ Module._extensions['.node'] = function(module, filename) { return process.dlopen(module, path._makeLong(filename)); }; +if (experimentalModules) { + Module._extensions['.mjs'] = function(module, filename) { + throw new errors.Error('ERR_REQUIRE_ESM', filename); + }; +} // bootstrap main module. Module.runMain = function() { diff --git a/node.gyp b/node.gyp index 14acf375e128a4..79d9e0a68dcedf 100644 --- a/node.gyp +++ b/node.gyp @@ -91,6 +91,13 @@ 'lib/internal/http.js', 'lib/internal/inspector_async_hook.js', 'lib/internal/linkedlist.js', + 'lib/internal/loader/Loader.js', + 'lib/internal/loader/ModuleMap.js', + 'lib/internal/loader/ModuleJob.js', + 'lib/internal/loader/ModuleWrap.js', + 'lib/internal/loader/resolveRequestUrl.js', + 'lib/internal/loader/search.js', + 'lib/internal/safe_globals.js', 'lib/internal/net.js', 'lib/internal/module.js', 'lib/internal/os.js', @@ -177,6 +184,7 @@ 'src/fs_event_wrap.cc', 'src/handle_wrap.cc', 'src/js_stream.cc', + 'src/module_wrap.cc', 'src/node.cc', 'src/node_api.cc', 'src/node_api.h', @@ -230,6 +238,7 @@ 'src/env-inl.h', 'src/handle_wrap.h', 'src/js_stream.h', + 'src/module_wrap.h', 'src/node.h', 'src/node_http2_core.h', 'src/node_http2_core-inl.h', diff --git a/src/module_wrap.cc b/src/module_wrap.cc new file mode 100644 index 00000000000000..05bbe04ef2e605 --- /dev/null +++ b/src/module_wrap.cc @@ -0,0 +1,531 @@ +#include +#include // PATH_MAX +#include // S_IFDIR +#include "module_wrap.h" + +#include "env.h" +#include "node_url.h" +#include "util.h" +#include "util-inl.h" + +namespace node { +namespace loader { + +using node::url::URL; +using node::url::URL_FLAGS_FAILED; +using v8::Context; +using v8::EscapableHandleScope; +using v8::Function; +using v8::FunctionCallbackInfo; +using v8::FunctionTemplate; +using v8::Integer; +using v8::IntegrityLevel; +using v8::Isolate; +using v8::JSON; +using v8::Local; +using v8::MaybeLocal; +using v8::Module; +using v8::Object; +using v8::Persistent; +using v8::Promise; +using v8::ScriptCompiler; +using v8::ScriptOrigin; +using v8::String; +using v8::Value; + +static const char* EXTENSIONS[] = {".mjs", ".js", ".json", ".node"}; +std::map*> ModuleWrap::module_map_; + +ModuleWrap::ModuleWrap(Environment* env, + Local object, + Local module, + Local url) : BaseObject(env, object) { + Isolate* iso = Isolate::GetCurrent(); + module_.Reset(iso, module); + url_.Reset(iso, url); +} + +ModuleWrap::~ModuleWrap() { + Local module = module_.Get(Isolate::GetCurrent()); + std::vector* same_hash = module_map_[module->GetIdentityHash()]; + auto it = std::find(same_hash->begin(), same_hash->end(), this); + + if (it != same_hash->end()) { + same_hash->erase(it); + } + + module_.Reset(); +} + +void ModuleWrap::New(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + + Isolate* iso = args.GetIsolate(); + + if (!args.IsConstructCall()) { + env->ThrowError("constructor must be called using new"); + return; + } + + if (args.Length() != 2) { + env->ThrowError("constructor must have exactly 2 arguments " + "(string, string)"); + return; + } + + if (!args[0]->IsString()) { + env->ThrowError("first argument is not a string"); + return; + } + + auto source_text = args[0].As(); + + if (!args[1]->IsString()) { + env->ThrowError("second argument is not a string"); + return; + } + + Local url = args[1].As(); + + Local mod; + + // compile + { + ScriptOrigin origin(url, + Integer::New(iso, 0), + Integer::New(iso, 0), + False(iso), + Integer::New(iso, 0), + FIXED_ONE_BYTE_STRING(iso, ""), + False(iso), + False(iso), + True(iso)); + ScriptCompiler::Source source(source_text, origin); + auto maybe_mod = ScriptCompiler::CompileModule(iso, &source); + if (maybe_mod.IsEmpty()) { + return; + } + mod = maybe_mod.ToLocalChecked(); + } + + auto that = args.This(); + auto ctx = that->CreationContext(); + auto url_str = FIXED_ONE_BYTE_STRING(iso, "url"); + + if (!that->Set(ctx, url_str, url).FromMaybe(false)) { + return; + } + + ModuleWrap* obj = + new ModuleWrap(Environment::GetCurrent(ctx), that, mod, url); + + if (ModuleWrap::module_map_.count(mod->GetIdentityHash()) == 0) { + ModuleWrap::module_map_[mod->GetIdentityHash()] = + new std::vector(); + } + + ModuleWrap::module_map_[mod->GetIdentityHash()]->push_back(obj); + Wrap(that, obj); + + that->SetIntegrityLevel(ctx, IntegrityLevel::kFrozen); + args.GetReturnValue().Set(that); +} + +void ModuleWrap::Link(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + Isolate* iso = args.GetIsolate(); + EscapableHandleScope handle_scope(iso); + if (!args[0]->IsFunction()) { + env->ThrowError("first argument is not a function"); + return; + } + + Local resolver_arg = args[0].As(); + + auto that = args.This(); + ModuleWrap* obj = Unwrap(that); + auto mod_context = that->CreationContext(); + if (obj->linked_) return; + obj->linked_ = true; + Local mod(obj->module_.Get(iso)); + + // call the dependency resolve callbacks + for (int i = 0; i < mod->GetModuleRequestsLength(); i++) { + Local specifier = mod->GetModuleRequest(i); + Utf8Value specifier_utf(env->isolate(), specifier); + std::string specifier_std(*specifier_utf, specifier_utf.length()); + + Local argv[] = { + specifier + }; + + MaybeLocal maybe_resolve_return_value = + resolver_arg->Call(mod_context, that, 1, argv); + if (maybe_resolve_return_value.IsEmpty()) { + return; + } + Local resolve_return_value = + maybe_resolve_return_value.ToLocalChecked(); + if (!resolve_return_value->IsPromise()) { + env->ThrowError("linking error, expected resolver to return a promise"); + } + Local resolve_promise = resolve_return_value.As(); + obj->resolve_cache_[specifier_std] = new Persistent(); + obj->resolve_cache_[specifier_std]->Reset(iso, resolve_promise); + } + + args.GetReturnValue().Set(handle_scope.Escape(that)); +} + +void ModuleWrap::Instantiate(const FunctionCallbackInfo& args) { + auto iso = args.GetIsolate(); + auto that = args.This(); + auto ctx = that->CreationContext(); + + ModuleWrap* obj = Unwrap(that); + Local mod = obj->module_.Get(iso); + bool ok = mod->Instantiate(ctx, ModuleWrap::ResolveCallback); + + // clear resolve cache on instantiate + obj->resolve_cache_.clear(); + + if (!ok) { + return; + } +} + +void ModuleWrap::Evaluate(const FunctionCallbackInfo& args) { + auto iso = args.GetIsolate(); + auto that = args.This(); + auto ctx = that->CreationContext(); + ModuleWrap* obj = Unwrap(that); + auto result = obj->module_.Get(iso)->Evaluate(ctx); + + if (result.IsEmpty()) { + return; + } + + auto ret = result.ToLocalChecked(); + args.GetReturnValue().Set(ret); +} + +MaybeLocal ModuleWrap::ResolveCallback(Local context, + Local specifier, + Local referrer) { + Environment* env = Environment::GetCurrent(context); + Isolate* iso = Isolate::GetCurrent(); + if (ModuleWrap::module_map_.count(referrer->GetIdentityHash()) == 0) { + env->ThrowError("linking error, unknown module"); + return MaybeLocal(); + } + + std::vector* possible_deps = + ModuleWrap::module_map_[referrer->GetIdentityHash()]; + ModuleWrap* dependent = nullptr; + + for (auto possible_dep : *possible_deps) { + if (possible_dep->module_ == referrer) { + dependent = possible_dep; + } + } + + if (dependent == nullptr) { + env->ThrowError("linking error, null dep"); + return MaybeLocal(); + } + + Utf8Value specifier_utf(env->isolate(), specifier); + std::string specifier_std(*specifier_utf, specifier_utf.length()); + + if (dependent->resolve_cache_.count(specifier_std) != 1) { + env->ThrowError("linking error, not in local cache"); + return MaybeLocal(); + } + + Local resolve_promise = + dependent->resolve_cache_[specifier_std]->Get(iso); + + if (resolve_promise->State() != Promise::kFulfilled) { + env->ThrowError("linking error, dependency promises must be resolved on " + "instantiate"); + return MaybeLocal(); + } + + auto module_object = resolve_promise->Result().As(); + if (module_object.IsEmpty() || !module_object->IsObject()) { + env->ThrowError("linking error, expected a valid module object from " + "resolver"); + return MaybeLocal(); + } + + ModuleWrap* mod; + ASSIGN_OR_RETURN_UNWRAP(&mod, module_object, MaybeLocal()); + return mod->module_.Get(env->isolate()); +} + +namespace { + +URL __init_cwd() { + std::string specifier = "file://"; +#ifdef _WIN32 + // MAX_PATH is in characters, not bytes. Make sure we have enough headroom. + char buf[MAX_PATH * 4]; +#else + char buf[PATH_MAX]; +#endif + + size_t cwd_len = sizeof(buf); + int err = uv_cwd(buf, &cwd_len); + if (err) { + return URL(""); + } + specifier += buf; + specifier += "/"; + return URL(specifier); +} +static URL INITIAL_CWD(__init_cwd()); +inline bool is_relative_or_absolute_path(std::string specifier) { + auto len = specifier.length(); + if (len <= 0) { + return false; + } else if (specifier[0] == '/') { + return true; + } else if (specifier[0] == '.') { + if (len == 1 || specifier[1] == '/') { + return true; + } else if (specifier[1] == '.') { + if (len == 2 || specifier[2] == '/') { + return true; + } + } + } + return false; +} +struct read_result { + bool had_error = false; + std::string source; +} read_result; +inline const struct read_result read_file(uv_file file) { + struct read_result ret; + std::string src; + uv_fs_t req; + void* base = malloc(4096); + if (base == nullptr) { + ret.had_error = true; + return ret; + } + uv_buf_t buf = uv_buf_init(static_cast(base), 4096); + uv_fs_read(uv_default_loop(), &req, file, &buf, 1, 0, nullptr); + while (req.result > 0) { + src += std::string(static_cast(buf.base), req.result); + uv_fs_read(uv_default_loop(), &req, file, &buf, 1, src.length(), nullptr); + } + ret.source = src; + return ret; +} +struct file_check { + bool failed = true; + uv_file file; +} file_check; +inline const struct file_check check_file(URL search, + bool close = false, + bool allow_dir = false) { + struct file_check ret; + uv_fs_t fs_req; + std::string path = search.ToFilePath(); + if (path.empty()) { + return ret; + } + uv_fs_open(nullptr, &fs_req, path.c_str(), O_RDONLY, 0, nullptr); + auto fd = fs_req.result; + if (fd < 0) { + return ret; + } + if (!allow_dir) { + uv_fs_fstat(nullptr, &fs_req, fd, nullptr); + if (fs_req.statbuf.st_mode & S_IFDIR) { + uv_fs_close(nullptr, &fs_req, fd, nullptr); + return ret; + } + } + ret.failed = false; + ret.file = fd; + if (close) uv_fs_close(nullptr, &fs_req, fd, nullptr); + return ret; +} +URL resolve_extensions(URL search, bool check_exact = true) { + if (check_exact) { + auto check = check_file(search, true); + if (!check.failed) { + return search; + } + } + for (auto extension : EXTENSIONS) { + URL guess(search.path() + extension, &search); + auto check = check_file(guess, true); + if (!check.failed) { + return guess; + } + } + return URL(""); +} +inline URL resolve_index(URL search) { + return resolve_extensions(URL("index", &search), false); +} +URL resolve_main(URL search) { + URL pkg("package.json", &search); + auto check = check_file(pkg); + if (!check.failed) { + auto iso = Isolate::GetCurrent(); + auto ctx = iso->GetCurrentContext(); + auto read = read_file(check.file); + uv_fs_t fs_req; + // if we fail to close :-/ + uv_fs_close(nullptr, &fs_req, check.file, nullptr); + if (read.had_error) return URL(""); + std::string pkg_src = read.source; + Local src = + String::NewFromUtf8(iso, pkg_src.c_str(), + String::kNormalString, pkg_src.length()); + if (src.IsEmpty()) return URL(""); + auto maybe_pkg_json = JSON::Parse(ctx, src); + if (maybe_pkg_json.IsEmpty()) return URL(""); + auto pkg_json_obj = maybe_pkg_json.ToLocalChecked().As(); + if (!pkg_json_obj->IsObject()) return URL(""); + auto maybe_pkg_main = pkg_json_obj->Get( + ctx, FIXED_ONE_BYTE_STRING(iso, "main")); + if (maybe_pkg_main.IsEmpty()) return URL(""); + auto pkg_main_str = maybe_pkg_main.ToLocalChecked().As(); + if (!pkg_main_str->IsString()) return URL(""); + Utf8Value main_utf8(iso, pkg_main_str); + std::string main_std(*main_utf8, main_utf8.length()); + if (!is_relative_or_absolute_path(main_std)) { + main_std.insert(0, "./"); + } + return Resolve(main_std, &search); + } + return URL(""); +} +URL resolve_module(std::string specifier, URL* base) { + URL parent(".", base); + URL dir(""); + do { + dir = parent; + auto check = Resolve("./node_modules/" + specifier, &dir, true); + if (!(check.flags() & URL_FLAGS_FAILED)) { + const auto limit = specifier.find('/'); + const auto spec_len = limit == std::string::npos ? + specifier.length() : + limit + 1; + std::string chroot = + dir.path() + "node_modules/" + specifier.substr(0, spec_len); + if (check.path().substr(0, chroot.length()) != chroot) { + return URL(""); + } + return check; + } else { + // TODO(bmeck) PREVENT FALLTHROUGH + } + parent = URL("..", &dir); + } while (parent.path() != dir.path()); + return URL(""); +} + +URL resolve_directory(URL search, bool read_pkg_json) { + if (read_pkg_json) { + auto main = resolve_main(search); + if (!(main.flags() & URL_FLAGS_FAILED)) return main; + } + return resolve_index(search); +} + +} // anonymous namespace + + +URL Resolve(std::string specifier, URL* base, bool read_pkg_json) { + URL pure_url(specifier); + if (!(pure_url.flags() & URL_FLAGS_FAILED)) { + return pure_url; + } + if (specifier.length() == 0) { + return URL(""); + } + if (is_relative_or_absolute_path(specifier)) { + URL resolved(specifier, base); + auto file = resolve_extensions(resolved); + if (!(file.flags() & URL_FLAGS_FAILED)) return file; + if (specifier.back() != '/') { + resolved = URL(specifier + "/", base); + } + return resolve_directory(resolved, read_pkg_json); + } else { + return resolve_module(specifier, base); + } + return URL(""); +} + +void ModuleWrap::Resolve(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + + if (args.IsConstructCall()) { + env->ThrowError("resolve() must not be called as a constructor"); + return; + } + if (args.Length() != 2) { + env->ThrowError("resolve must have exactly 2 arguments (string, string)"); + return; + } + + if (!args[0]->IsString()) { + env->ThrowError("first argument is not a string"); + return; + } + Utf8Value specifier_utf(env->isolate(), args[0]); + + if (!args[1]->IsString()) { + env->ThrowError("second argument is not a string"); + return; + } + Utf8Value url_utf(env->isolate(), args[1]); + URL url(*url_utf, url_utf.length()); + + if (url.flags() & URL_FLAGS_FAILED) { + env->ThrowError("second argument is not a URL string"); + return; + } + + URL result = node::loader::Resolve(*specifier_utf, &url, true); + if (result.flags() & URL_FLAGS_FAILED) { + std::string msg = "module "; + msg += *specifier_utf; + msg += " not found"; + env->ThrowError(msg.c_str()); + return; + } + + args.GetReturnValue().Set(result.ToObject(env)); +} + +void ModuleWrap::Initialize(Local target, + Local unused, + Local context) { + Environment* env = Environment::GetCurrent(context); + Isolate* isolate = env->isolate(); + + Local tpl = env->NewFunctionTemplate(New); + tpl->SetClassName(FIXED_ONE_BYTE_STRING(isolate, "ModuleWrap")); + tpl->InstanceTemplate()->SetInternalFieldCount(1); + + env->SetProtoMethod(tpl, "link", Link); + env->SetProtoMethod(tpl, "instantiate", Instantiate); + env->SetProtoMethod(tpl, "evaluate", Evaluate); + + target->Set(FIXED_ONE_BYTE_STRING(isolate, "ModuleWrap"), tpl->GetFunction()); + env->SetMethod(target, "resolve", node::loader::ModuleWrap::Resolve); +} + +} // namespace loader +} // namespace node + +NODE_MODULE_CONTEXT_AWARE_BUILTIN(module_wrap, + node::loader::ModuleWrap::Initialize) diff --git a/src/module_wrap.h b/src/module_wrap.h new file mode 100644 index 00000000000000..c669834c6f3ce5 --- /dev/null +++ b/src/module_wrap.h @@ -0,0 +1,58 @@ +#ifndef SRC_MODULE_WRAP_H_ +#define SRC_MODULE_WRAP_H_ + +#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#include +#include +#include +#include "node_url.h" +#include "base-object.h" +#include "base-object-inl.h" + +namespace node { +namespace loader { + +node::url::URL Resolve(std::string specifier, node::url::URL* base, + bool read_pkg_json = false); + +class ModuleWrap : public BaseObject { + public: + static const std::string EXTENSIONS[]; + static void Initialize(v8::Local target, + v8::Local unused, + v8::Local context); + + private: + ModuleWrap(node::Environment* env, + v8::Local object, + v8::Local module, + v8::Local url); + ~ModuleWrap(); + + static void New(const v8::FunctionCallbackInfo& args); + static void Link(const v8::FunctionCallbackInfo& args); + static void Instantiate(const v8::FunctionCallbackInfo& args); + static void Evaluate(const v8::FunctionCallbackInfo& args); + static void GetUrl(v8::Local property, + const v8::PropertyCallbackInfo& info); + static void Resolve(const v8::FunctionCallbackInfo& args); + static v8::MaybeLocal ResolveCallback( + v8::Local context, + v8::Local specifier, + v8::Local referrer); + + v8::Persistent module_; + v8::Persistent url_; + bool linked_ = false; + std::map*> resolve_cache_; + + static std::map*> module_map_; +}; + +} // namespace loader +} // namespace node + +#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#endif // SRC_MODULE_WRAP_H_ diff --git a/src/node.cc b/src/node.cc index bdff4527c67857..5cd0ffc29db1e9 100644 --- a/src/node.cc +++ b/src/node.cc @@ -225,6 +225,11 @@ bool trace_warnings = false; // that is used by lib/module.js bool config_preserve_symlinks = false; +// Set in node.cc by ParseArgs when --experimental-modules is used. +// Used in node_config.cc to set a constant on process.binding('config') +// that is used by lib/module.js +bool config_experimental_modules = false; + // Set by ParseArgs when --pending-deprecation or NODE_PENDING_DEPRECATION // is used. bool config_pending_deprecation = false; @@ -3711,6 +3716,7 @@ static void PrintHelp() { " note: linked-in ICU data is present\n" #endif " --preserve-symlinks preserve symbolic links when resolving\n" + " --experimental-modules experimental ES Module support\n" " and caching modules\n" #endif "\n" @@ -3947,6 +3953,8 @@ static void ParseArgs(int* argc, Revert(cve); } else if (strcmp(arg, "--preserve-symlinks") == 0) { config_preserve_symlinks = true; + } else if (strcmp(arg, "--experimental-modules") == 0) { + config_experimental_modules = true; } else if (strcmp(arg, "--prof-process") == 0) { prof_process = true; short_circuit = true; diff --git a/src/node_config.cc b/src/node_config.cc index 87110dd8c644f7..2f45a5e9712ce7 100644 --- a/src/node_config.cc +++ b/src/node_config.cc @@ -65,6 +65,9 @@ static void InitConfig(Local target, if (config_preserve_symlinks) READONLY_BOOLEAN_PROPERTY("preserveSymlinks"); + if (config_experimental_modules) + READONLY_BOOLEAN_PROPERTY("experimentalModules"); + if (config_pending_deprecation) READONLY_BOOLEAN_PROPERTY("pendingDeprecation"); diff --git a/src/node_internals.h b/src/node_internals.h index 1e099325a35a90..a241e671edda48 100644 --- a/src/node_internals.h +++ b/src/node_internals.h @@ -86,6 +86,10 @@ extern bool config_preserve_symlinks; // Set in node.cc by ParseArgs when --expose-http2 is used. extern bool config_expose_http2; +// Set in node.cc by ParseArgs when --experimental-modules is used. +// Used in node_config.cc to set a constant on process.binding('config') +// that is used by lib/module.js +extern bool config_experimental_modules; // Set in node.cc by ParseArgs when --expose-internals or --expose_internals is // used. diff --git a/src/node_url.cc b/src/node_url.cc index dd3da1133ebf2a..f8adc7d7af5509 100644 --- a/src/node_url.cc +++ b/src/node_url.cc @@ -2080,6 +2080,69 @@ static void DomainToUnicode(const FunctionCallbackInfo& args) { v8::NewStringType::kNormal).ToLocalChecked()); } +std::string URL::ToFilePath() { + if (context_.scheme != "file:") { + return ""; + } + +#ifdef _WIN32 + const char* slash = "\\"; + auto is_slash = [] (char ch) { + return ch == '/' || ch == '\\'; + }; +#else + const char* slash = "/"; + auto is_slash = [] (char ch) { + return ch == '/'; + }; + if ((context_.flags & URL_FLAGS_HAS_HOST) && + context_.host.length() > 0) { + return ""; + } +#endif + std::string decoded_path; + for (std::string& part : context_.path) { + std::string decoded; + PercentDecode(part.c_str(), part.length(), &decoded); + for (char& ch : decoded) { + if (is_slash(ch)) { + return ""; + } + } + decoded_path += slash + decoded; + } + +#ifdef _WIN32 + // TODO(TimothyGu): Use "\\?\" long paths on Windows. + + // If hostname is set, then we have a UNC path. Pass the hostname through + // ToUnicode just in case it is an IDN using punycode encoding. We do not + // need to worry about percent encoding because the URL parser will have + // already taken care of that for us. Note that this only causes IDNs with an + // appropriate `xn--` prefix to be decoded. + if ((context_.flags & URL_FLAGS_HAS_HOST) && + context_.host.length() > 0) { + std::string unicode_host; + if (!ToUnicode(&context_.host, &unicode_host)) { + return ""; + } + return "\\\\" + unicode_host + decoded_path; + } + // Otherwise, it's a local path that requires a drive letter. + if (decoded_path.length() < 3) { + return ""; + } + if (decoded_path[2] != ':' || + !IsASCIIAlpha(decoded_path[1])) { + return ""; + } + // Strip out the leading '\'. + return decoded_path.substr(1); +#else + return decoded_path; +#endif +} + // This function works by calling out to a JS function that creates and // returns the JS URL object. Be mindful of the JS<->Native boundary // crossing that is required. diff --git a/src/node_url.h b/src/node_url.h index 72ac366ec1386b..cb7bdca7f2cd6a 100644 --- a/src/node_url.h +++ b/src/node_url.h @@ -163,6 +163,10 @@ class URL { return ret; } + // Get the path of the file: URL in a format consumable by native file system + // APIs. Returns an empty string if something went wrong. + std::string ToFilePath(); + const Local ToObject(Environment* env) const; private: diff --git a/test/cctest/test_url.cc b/test/cctest/test_url.cc index 2cede1a8a3deb3..0b80d44caad807 100644 --- a/test/cctest/test_url.cc +++ b/test/cctest/test_url.cc @@ -79,3 +79,28 @@ TEST_F(URLTest, Base3) { EXPECT_EQ(simple.host(), "example.org"); EXPECT_EQ(simple.path(), "/baz"); } + +TEST_F(URLTest, ToFilePath) { +#define T(url, path) EXPECT_EQ(path, URL(url).ToFilePath()) + T("http://example.org/foo/bar", ""); + +#ifdef _WIN32 + T("file:///C:/Program%20Files/", "C:\\Program Files\\"); + T("file:///C:/a/b/c?query#fragment", "C:\\a\\b\\c"); + T("file://host/path/a/b/c?query#fragment", "\\\\host\\path\\a\\b\\c"); + T("file://xn--weird-prdj8vva.com/host/a", "\\\\wͪ͊eiͬ͋rd.com\\host\\a"); + T("file:///C:/a%2Fb", ""); + T("file:///", ""); + T("file:///home", ""); +#else + T("file:///", "/"); + T("file:///home/user?query#fragment", "/home/user"); + T("file:///home/user/?query#fragment", "/home/user/"); + T("file:///home/user/%20space", "/home/user/ space"); + T("file:///home/us%5Cer", "/home/us\\er"); + T("file:///home/us%2Fer", ""); + T("file://host/path", ""); +#endif + +#undef T +} diff --git a/test/es-module/es-module.status b/test/es-module/es-module.status new file mode 100644 index 00000000000000..343e622ca02ac4 --- /dev/null +++ b/test/es-module/es-module.status @@ -0,0 +1,7 @@ +prefix parallel + +# To mark a test as flaky, list the test name in the appropriate section +# below, without ".js", followed by ": PASS,FLAKY". Example: +# sample-test : PASS,FLAKY + +[true] # This section applies to all platforms diff --git a/test/es-module/esm-snapshot-mutator.js b/test/es-module/esm-snapshot-mutator.js new file mode 100644 index 00000000000000..a0dfa0c28a92bd --- /dev/null +++ b/test/es-module/esm-snapshot-mutator.js @@ -0,0 +1,5 @@ +/* eslint-disable required-modules */ +'use strict'; +const shouldSnapshotFilePath = require.resolve('./esm-snapshot.js'); +require('./esm-snapshot.js'); +require.cache[shouldSnapshotFilePath].exports++; diff --git a/test/es-module/esm-snapshot.js b/test/es-module/esm-snapshot.js new file mode 100644 index 00000000000000..2c3c3a459a738b --- /dev/null +++ b/test/es-module/esm-snapshot.js @@ -0,0 +1,3 @@ +/* eslint-disable required-modules */ +'use strict'; +module.exports = 1; diff --git a/test/es-module/test-esm-basic-imports.mjs b/test/es-module/test-esm-basic-imports.mjs new file mode 100644 index 00000000000000..23989bddd5b0bd --- /dev/null +++ b/test/es-module/test-esm-basic-imports.mjs @@ -0,0 +1,8 @@ +// Flags: --experimental-modules +import '../common'; +import assert from 'assert'; +import ok from './test-esm-ok.mjs'; +import okShebang from './test-esm-shebang.mjs'; + +assert(ok); +assert(okShebang); diff --git a/test/es-module/test-esm-encoded-path-native.js b/test/es-module/test-esm-encoded-path-native.js new file mode 100644 index 00000000000000..f32297efdb9d7b --- /dev/null +++ b/test/es-module/test-esm-encoded-path-native.js @@ -0,0 +1,10 @@ +'use strict'; +const common = require('../common'); +const assert = require('assert'); +const { spawn } = require('child_process'); + +const native = `${common.fixturesDir}/es-module-url/native.mjs`; +const child = spawn(process.execPath, ['--experimental-modules', native]); +child.on('exit', (code) => { + assert.strictEqual(code, 1); +}); diff --git a/test/es-module/test-esm-encoded-path.mjs b/test/es-module/test-esm-encoded-path.mjs new file mode 100644 index 00000000000000..2c6e145927a1de --- /dev/null +++ b/test/es-module/test-esm-encoded-path.mjs @@ -0,0 +1,7 @@ +// Flags: --experimental-modules +import '../common'; +import assert from 'assert'; +// ./test-esm-ok.mjs +import ok from './test-%65%73%6d-ok.mjs'; + +assert(ok); diff --git a/test/es-module/test-esm-forbidden-globals.mjs b/test/es-module/test-esm-forbidden-globals.mjs new file mode 100644 index 00000000000000..d3e92b9238adba --- /dev/null +++ b/test/es-module/test-esm-forbidden-globals.mjs @@ -0,0 +1,24 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ + +if (typeof arguments !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof this !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof exports !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof require !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof module !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof __filename !== 'undefined') { + throw new Error('not an ESM'); +} +if (typeof __dirname !== 'undefined') { + throw new Error('not an ESM'); +} diff --git a/test/es-module/test-esm-namespace.mjs b/test/es-module/test-esm-namespace.mjs new file mode 100644 index 00000000000000..72b7fed4b33dfa --- /dev/null +++ b/test/es-module/test-esm-namespace.mjs @@ -0,0 +1,7 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ + +import * as fs from 'fs'; +import assert from 'assert'; + +assert.deepStrictEqual(Object.keys(fs), ['default']); diff --git a/test/es-module/test-esm-ok.mjs b/test/es-module/test-esm-ok.mjs new file mode 100644 index 00000000000000..6712e1ab7dfca1 --- /dev/null +++ b/test/es-module/test-esm-ok.mjs @@ -0,0 +1,5 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ + +const isJs = true; +export default isJs; diff --git a/test/es-module/test-esm-pkg-over-ext.mjs b/test/es-module/test-esm-pkg-over-ext.mjs new file mode 100644 index 00000000000000..7e47c4c326942f --- /dev/null +++ b/test/es-module/test-esm-pkg-over-ext.mjs @@ -0,0 +1,8 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ + +import resolved from '../fixtures/module-pkg-over-ext/inner'; +import expected from '../fixtures/module-pkg-over-ext/inner/package.json'; +import assert from 'assert'; + +assert.strictEqual(resolved, expected); diff --git a/test/es-module/test-esm-preserve-symlinks.js b/test/es-module/test-esm-preserve-symlinks.js new file mode 100644 index 00000000000000..eea5bf061b2fa3 --- /dev/null +++ b/test/es-module/test-esm-preserve-symlinks.js @@ -0,0 +1,38 @@ +// Flags: --experimental-modules +'use strict'; + +const common = require('../common'); +const { spawn } = require('child_process'); +const assert = require('assert'); +const path = require('path'); +const fs = require('fs'); + +common.refreshTmpDir(); +const tmpDir = common.tmpDir; + +const entry = path.join(tmpDir, 'entry.js'); +const real = path.join(tmpDir, 'real.js'); +const link_absolute_path = path.join(tmpDir, 'link.js'); + +fs.writeFileSync(entry, ` +const assert = require('assert'); +global.x = 0; +require('./real.js'); +assert.strictEqual(x, 1); +require('./link.js'); +assert.strictEqual(x, 2); +`); +fs.writeFileSync(real, 'x++;'); + +try { + fs.symlinkSync(real, link_absolute_path); +} catch (err) { + if (err.code !== 'EPERM') throw err; + common.skip('insufficient privileges for symlinks'); +} + +spawn(process.execPath, + ['--experimental-modules', '--preserve-symlinks', entry], + { stdio: 'inherit' }).on('exit', (code) => { + assert.strictEqual(code, 0); +}); diff --git a/test/es-module/test-esm-require-cache.mjs b/test/es-module/test-esm-require-cache.mjs new file mode 100644 index 00000000000000..ff32cde36ff2a8 --- /dev/null +++ b/test/es-module/test-esm-require-cache.mjs @@ -0,0 +1,7 @@ +// Flags: --experimental-modules +import '../common'; +import '../fixtures/es-module-require-cache/preload.js'; +import '../fixtures/es-module-require-cache/counter.js'; +import assert from 'assert'; +assert.strictEqual(global.counter, 1); +delete global.counter; diff --git a/test/es-module/test-esm-shebang.mjs b/test/es-module/test-esm-shebang.mjs new file mode 100644 index 00000000000000..43cc0f8367d8a2 --- /dev/null +++ b/test/es-module/test-esm-shebang.mjs @@ -0,0 +1,6 @@ +#! }]) // isn't js +// Flags: --experimental-modules +/* eslint-disable required-modules */ + +const isJs = true; +export default isJs; diff --git a/test/es-module/test-esm-snapshot.mjs b/test/es-module/test-esm-snapshot.mjs new file mode 100644 index 00000000000000..89034f56681238 --- /dev/null +++ b/test/es-module/test-esm-snapshot.mjs @@ -0,0 +1,7 @@ +// Flags: --experimental-modules +/* eslint-disable required-modules */ +import './esm-snapshot-mutator'; +import one from './esm-snapshot'; +import assert from 'assert'; + +assert.strictEqual(one, 1); diff --git a/test/es-module/test-esm-symlink.js b/test/es-module/test-esm-symlink.js new file mode 100644 index 00000000000000..3b7d689bf8f5f2 --- /dev/null +++ b/test/es-module/test-esm-symlink.js @@ -0,0 +1,48 @@ +'use strict'; + +const common = require('../common'); +const { spawn } = require('child_process'); +const assert = require('assert'); +const path = require('path'); +const fs = require('fs'); + +common.refreshTmpDir(); +const tmpDir = common.tmpDir; + +const entry = path.join(tmpDir, 'entry.mjs'); +const real = path.join(tmpDir, 'index.mjs'); +const link_absolute_path = path.join(tmpDir, 'absolute'); +const link_relative_path = path.join(tmpDir, 'relative'); +const link_ignore_extension = path.join(tmpDir, + 'ignore_extension.json'); +const link_directory = path.join(tmpDir, 'directory'); + +fs.writeFileSync(real, 'export default [];'); +fs.writeFileSync(entry, ` +import assert from 'assert'; +import real from './index.mjs'; +import absolute from './absolute'; +import relative from './relative'; +import ignoreExtension from './ignore_extension.json'; +import directory from './directory'; + +assert.strictEqual(absolute, real); +assert.strictEqual(relative, real); +assert.strictEqual(ignoreExtension, real); +assert.strictEqual(directory, real); +`); + +try { + fs.symlinkSync(real, link_absolute_path); + fs.symlinkSync(path.basename(real), link_relative_path); + fs.symlinkSync(real, link_ignore_extension); + fs.symlinkSync(path.dirname(real), link_directory); +} catch (err) { + if (err.code !== 'EPERM') throw err; + common.skip('insufficient privileges for symlinks'); +} + +spawn(process.execPath, ['--experimental-modules', entry], + { stdio: 'inherit' }).on('exit', (code) => { + assert.strictEqual(code, 0); +}); diff --git a/test/es-module/testcfg.py b/test/es-module/testcfg.py new file mode 100644 index 00000000000000..0d8dfeed463ec8 --- /dev/null +++ b/test/es-module/testcfg.py @@ -0,0 +1,6 @@ +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import testpy + +def GetConfiguration(context, root): + return testpy.SimpleTestConfiguration(context, root, 'es-module') diff --git a/test/fixtures/es-module-require-cache/counter.js b/test/fixtures/es-module-require-cache/counter.js new file mode 100644 index 00000000000000..2640d3e372dcd2 --- /dev/null +++ b/test/fixtures/es-module-require-cache/counter.js @@ -0,0 +1,2 @@ +global.counter = global.counter || 0; +global.counter++; \ No newline at end of file diff --git a/test/fixtures/es-module-require-cache/preload.js b/test/fixtures/es-module-require-cache/preload.js new file mode 100644 index 00000000000000..6090dc0d5821eb --- /dev/null +++ b/test/fixtures/es-module-require-cache/preload.js @@ -0,0 +1 @@ +require('./counter'); \ No newline at end of file diff --git a/test/fixtures/es-module-url/empty.js b/test/fixtures/es-module-url/empty.js new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/test/fixtures/es-module-url/native.mjs b/test/fixtures/es-module-url/native.mjs new file mode 100644 index 00000000000000..c8831f9bfee1be --- /dev/null +++ b/test/fixtures/es-module-url/native.mjs @@ -0,0 +1,2 @@ +// path +import 'p%61th'; diff --git a/test/testpy/__init__.py b/test/testpy/__init__.py index 37e5ac710bcdf8..f113c1253a477f 100644 --- a/test/testpy/__init__.py +++ b/test/testpy/__init__.py @@ -27,7 +27,7 @@ import test import os -from os.path import join, dirname, exists +from os.path import join, dirname, exists, splitext import re import ast @@ -109,18 +109,17 @@ def __init__(self, context, root, section, additional=None): self.additional_flags = [] def Ls(self, path): - def SelectTest(name): - return name.startswith('test-') and name.endswith('.js') - return [f[:-3] for f in os.listdir(path) if SelectTest(f)] + return [f for f in os.listdir(path) if re.match('^test-.*\.m?js$', f)] def ListTests(self, current_path, path, arch, mode): all_tests = [current_path + [t] for t in self.Ls(join(self.root))] result = [] for test in all_tests: if self.Contains(path, test): - file_path = join(self.root, reduce(join, test[1:], "") + ".js") - result.append(SimpleTestCase(test, file_path, arch, mode, self.context, - self, self.additional_flags)) + file_path = join(self.root, reduce(join, test[1:], "")) + test_name = test[:-1] + [splitext(test[-1])[0]] + result.append(SimpleTestCase(test_name, file_path, arch, mode, + self.context, self, self.additional_flags)) return result def GetBuildRequirements(self): diff --git a/tools/eslint-rules/required-modules.js b/tools/eslint-rules/required-modules.js index 47ade5cd9f9b42..948c46c036d99d 100644 --- a/tools/eslint-rules/required-modules.js +++ b/tools/eslint-rules/required-modules.js @@ -13,6 +13,7 @@ const path = require('path'); module.exports = function(context) { // trim required module names var requiredModules = context.options; + const isESM = context.parserOptions.sourceType === 'module'; const foundModules = []; @@ -39,39 +40,35 @@ module.exports = function(context) { return node.callee.type === 'Identifier' && node.callee.name === 'require'; } + /** + * Function to check if the path is a required module and return its name. + * @param {String} str The path to check + * @returns {undefined|String} required module name or undefined + */ + function getRequiredModuleName(str) { + var value = path.basename(str); + + // check if value is in required modules array + return requiredModules.indexOf(value) !== -1 ? value : undefined; + } + /** * Function to check if a node has an argument that is a required module and * return its name. * @param {ASTNode} node The node to check * @returns {undefined|String} required module name or undefined */ - function getRequiredModuleName(node) { - var moduleName; - + function getRequiredModuleNameFromCall(node) { // node has arguments and first argument is string if (node.arguments.length && isString(node.arguments[0])) { - var argValue = path.basename(node.arguments[0].value.trim()); - - // check if value is in required modules array - if (requiredModules.indexOf(argValue) !== -1) { - moduleName = argValue; - } + return getRequiredModuleName(node.arguments[0].value.trim()); } - return moduleName; + return undefined; } - return { - 'CallExpression': function(node) { - if (isRequireCall(node)) { - var requiredModuleName = getRequiredModuleName(node); - - if (requiredModuleName) { - foundModules.push(requiredModuleName); - } - } - }, - 'Program:exit': function(node) { + const rules = { + 'Program:exit'(node) { if (foundModules.length < requiredModules.length) { var missingModules = requiredModules.filter( function(module) { @@ -88,6 +85,27 @@ module.exports = function(context) { } } }; + + if (isESM) { + rules.ImportDeclaration = (node) => { + var requiredModuleName = getRequiredModuleName(node.source.value); + if (requiredModuleName) { + foundModules.push(requiredModuleName); + } + }; + } else { + rules.CallExpression = (node) => { + if (isRequireCall(node)) { + var requiredModuleName = getRequiredModuleNameFromCall(node); + + if (requiredModuleName) { + foundModules.push(requiredModuleName); + } + } + }; + } + + return rules; }; module.exports.schema = { diff --git a/tools/test.py b/tools/test.py index 5a50c7f2e6ccc6..6839f4e1b2acb1 100755 --- a/tools/test.py +++ b/tools/test.py @@ -279,9 +279,7 @@ def HasRun(self, output): # hard to decipher what test is running when only the filename is printed. prefix = abspath(join(dirname(__file__), '../test')) + os.sep command = output.command[-1] - if command.endswith('.js'): command = command[:-3] - if command.startswith(prefix): command = command[len(prefix):] - command = command.replace('\\', '/') + command = NormalizePath(command, prefix) if output.UnexpectedOutput(): status_line = 'not ok %i %s' % (self._done, command) @@ -352,9 +350,7 @@ def HasRun(self, output): # hard to decipher what test is running when only the filename is printed. prefix = abspath(join(dirname(__file__), '../test')) + os.sep command = output.command[-1] - if command.endswith('.js'): command = command[:-3] - if command.startswith(prefix): command = command[len(prefix):] - command = command.replace('\\', '/') + command = NormalizePath(command, prefix) stdout = output.output.stdout.strip() printed_file = False @@ -1509,12 +1505,16 @@ def SplitPath(s): stripped = [ c.strip() for c in s.split('/') ] return [ Pattern(s) for s in stripped if len(s) > 0 ] -def NormalizePath(path): +def NormalizePath(path, prefix='test/'): # strip the extra path information of the specified test - if path.startswith('test/'): - path = path[5:] + prefix = prefix.replace('\\', '/') + path = path.replace('\\', '/') + if path.startswith(prefix): + path = path[len(prefix):] if path.endswith('.js'): path = path[:-3] + elif path.endswith('.mjs'): + path = path[:-4] return path def GetSpecialCommandProcessor(value): From 918e936d30afa8bf1bf6cc9ab8ef8787678845ef Mon Sep 17 00:00:00 2001 From: Bradley Farias Date: Thu, 31 Aug 2017 14:16:09 -0500 Subject: [PATCH 2/8] warn on using ESM loader --- lib/internal/bootstrap_node.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/internal/bootstrap_node.js b/lib/internal/bootstrap_node.js index 9d776674d2edab..ebe0ff525189a1 100644 --- a/lib/internal/bootstrap_node.js +++ b/lib/internal/bootstrap_node.js @@ -109,6 +109,13 @@ 'DeprecationWarning', 'DEP0062', startup, true); } + if (!!process.binding('config').experimentalModules) { + process.emitWarning( + 'The ESM module loader is experimental.', + 'ExperimentalWarning', undefined); + } + + // There are various modes that Node can run in. The most common two // are running from a script and running the REPL - but there are a few // others like the debugger or running --eval arguments. Here we decide From 118d18a518a1a046f3f93184cbc5d68c51ec6435 Mon Sep 17 00:00:00 2001 From: Bradley Farias Date: Thu, 31 Aug 2017 15:54:23 -0500 Subject: [PATCH 3/8] basic docs --- doc/api/esm.md | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 doc/api/esm.md diff --git a/doc/api/esm.md b/doc/api/esm.md new file mode 100644 index 00000000000000..304f9bcd65e7bb --- /dev/null +++ b/doc/api/esm.md @@ -0,0 +1,85 @@ +# Modules + + + +> Stability: 1 - Experimental + + + +Node contains support for ES Modules based upon [the Node EP for ES Modules](https://github.com/nodejs/node-eps/blob/master/002-es-modules.md). + +Not all features of the EP are complete and will be landing as both VM support and implementation is ready. Error messages are still being polished. + +## Enabling + + + +The `--experimental-modules` flag can be used to enable features for loading ESM modules. + +Once this has been set, you can use files ending with `.mjs` as ES Modules. + +```sh +node --experimental-modules my-app.mjs +``` + +## Features + + + +### Supported + +Only the CLI argument for the main entry point to your program can be an entry point into an ESM graph. In the future you can use `import()` to create entry points into ESM graphs at run time. + +### Unsupported + +| Feature | Reason | +| `require('./foo.mjs')` | ES Modules have differing resolution and timing, use language standard `import()` | +| `import()` | pending newer V8 release used in Node.js | +| `import.meta` | pending V8 implementation | +| Loader Hooks | pending Node.js EP creation/consensus | + +## Notable differences between `import` and `require` + +### No NODE_PATH + +`NODE_PATH` is not part of resolving `import` specifiers. Please use symlinks if you need behavior like this. + +### No `require.extensions` + +`require.extensions` is not used by `import`. The expectation is that loader hooks can provide this workflow in the future. + +### No `require.cache` + +`require.cache` is not used by `import`. It has a separate cache. + +### URL based paths + +ESM are resolved and cached based upon [URL](url.spec.whatwg.org) semantics. This means that files containing special characters such as `#` and `?` need to be escaped. + +Modules will be loaded multiple times if the `import` specifier used to resolve them have a different query or fragment. + +```js +import "./foo?query=1"; // loads ./foo with query of "?query=1" +import "./foo?query=2"; // loads ./foo with query of "?query=2" +``` + +For now, only modules using the `file:` protocol can be loaded. + +## Interop with existing modules + +All CommonJS, JSON, and C++ modules can be used with `import`. + +Modules loaded this way will only be loaded once, even if their query or fragment string differs between `import` statements. + +When loaded via `import` these modules will provide a single `default` export representing the value of `module.exports` at the time they finished evaluating. + +```js +import fs from 'fs'; +fs.readFile('./foo.txt', (err, body) { + if (err) { + console.error(err); + } else { + console.log(body); + } +}); +``` From 762cf03865643b96eb71516aa885bedb455d0c00 Mon Sep 17 00:00:00 2001 From: Bradley Farias Date: Fri, 1 Sep 2017 09:37:19 -0500 Subject: [PATCH 4/8] doc nits --- doc/api/esm.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/api/esm.md b/doc/api/esm.md index 304f9bcd65e7bb..905e4d45932e95 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -16,7 +16,7 @@ Not all features of the EP are complete and will be landing as both VM support a The `--experimental-modules` flag can be used to enable features for loading ESM modules. -Once this has been set, you can use files ending with `.mjs` as ES Modules. +Once this has been set, files ending with `.mjs` will be able to be loaded as ES Modules. ```sh node --experimental-modules my-app.mjs @@ -28,11 +28,12 @@ node --experimental-modules my-app.mjs ### Supported -Only the CLI argument for the main entry point to your program can be an entry point into an ESM graph. In the future you can use `import()` to create entry points into ESM graphs at run time. +Only the CLI argument for the main entry point to the program can be an entry point into an ESM graph. In the future `import()` can be used to create entry points into ESM graphs at run time. ### Unsupported | Feature | Reason | +| --- | --- | | `require('./foo.mjs')` | ES Modules have differing resolution and timing, use language standard `import()` | | `import()` | pending newer V8 release used in Node.js | | `import.meta` | pending V8 implementation | @@ -42,7 +43,7 @@ Only the CLI argument for the main entry point to your program can be an entry p ### No NODE_PATH -`NODE_PATH` is not part of resolving `import` specifiers. Please use symlinks if you need behavior like this. +`NODE_PATH` is not part of resolving `import` specifiers. Please use symlinks if this behavior is desired. ### No `require.extensions` From 1adf7cb58e339f9e8cda10f974a1046f03644f3f Mon Sep 17 00:00:00 2001 From: Bradley Farias Date: Fri, 1 Sep 2017 09:57:02 -0500 Subject: [PATCH 5/8] doc nits --- doc/api/esm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api/esm.md b/doc/api/esm.md index 905e4d45932e95..e916add8a176be 100644 --- a/doc/api/esm.md +++ b/doc/api/esm.md @@ -1,4 +1,4 @@ -# Modules +# ECMAScript Modules From 6e328d048065fe6b393b5408dcccfb9bea48941f Mon Sep 17 00:00:00 2001 From: guybedford Date: Sun, 3 Sep 2017 13:20:06 +0200 Subject: [PATCH 6/8] functional providers, format separation, resolve hook prototype --- lib/internal/loader/Loader.js | 54 +++++----- lib/internal/loader/ModuleJob.js | 16 ++- lib/internal/loader/ModuleRequest.js | 121 +++++++++++++++++++++++ lib/internal/loader/resolveRequestUrl.js | 104 ------------------- lib/module.js | 12 ++- node.gyp | 2 +- 6 files changed, 164 insertions(+), 145 deletions(-) create mode 100644 lib/internal/loader/ModuleRequest.js delete mode 100644 lib/internal/loader/resolveRequestUrl.js diff --git a/lib/internal/loader/Loader.js b/lib/internal/loader/Loader.js index a409d397f85dd6..4eccbc7e4fbeb7 100644 --- a/lib/internal/loader/Loader.js +++ b/lib/internal/loader/Loader.js @@ -9,7 +9,7 @@ const { const ModuleMap = require('internal/loader/ModuleMap'); const ModuleJob = require('internal/loader/ModuleJob'); -const resolveRequestUrl = require('internal/loader/resolveRequestUrl'); +const { formatProviders, resolve } = require('internal/loader/ModuleRequest'); const errors = require('internal/errors'); function getBase() { @@ -34,39 +34,41 @@ class Loader { this.base = base; } - async resolve(specifier) { - const request = resolveRequestUrl(this.base, specifier); - if (request.url.protocol !== 'file:') { + setModuleResolver(resolver) { + resolve = resolver; + } + + async resolve(specifier, parentUrlOrString = this.base) { + const { url, format } = await resolve(specifier, parentUrlOrString); + if (typeof url === 'string') { + url = new URL(url); + } + else if (!(url instanceof URL)) { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'URL'); + } + if (url.protocol !== 'file:') { throw new errors.Error('ERR_INVALID_PROTOCOL', request.url.protocol, 'file:'); } - return request.url; + if (!formatProviders.has(format)) { + throw new errors.Error('ERR_INVALID_FORMAT', format); + } + return { url, format }; } - async getModuleJob(dependentJob, specifier) { - if (!this.moduleMap.has(dependentJob.url)) { - throw new errors.Error('ERR_MISSING_MODULE', dependentJob.url); - } - const request = await resolveRequestUrl(dependentJob.url, specifier); - const url = `${request.url}`; - if (this.moduleMap.has(url)) { - return this.moduleMap.get(url); + async getModuleJob(specifier, parentUrlOrString = this.base) { + const { url, format } = await this.resolve(specifier, parentUrlOrString); + const urlString = `${url}`; + let job = this.moduleMap.get(urlString); + if (job === undefined) { + job = new ModuleJob(this, url, urlString, formatProviders.get(format)); + this.moduleMap.set(urlString, job); } - const dependencyJob = new ModuleJob(this, request); - this.moduleMap.set(url, dependencyJob); - return dependencyJob; + return job; } - async import(specifier) { - const request = await resolveRequestUrl(this.base, specifier); - const url = `${request.url}`; - let job; - if (this.moduleMap.has(url)) { - job = this.moduleMap.get(url); - } else { - job = new ModuleJob(this, request); - this.moduleMap.set(url, job); - } + async import(specifier, parentUrlOrString = this.base) { + const job = await this.getModuleJob(specifier, parentUrlOrString); const module = await job.run(); return getNamespaceOfModuleWrap(module); } diff --git a/lib/internal/loader/ModuleJob.js b/lib/internal/loader/ModuleJob.js index db4cb6ae5c5031..f4c2b62ca93f6b 100644 --- a/lib/internal/loader/ModuleJob.js +++ b/lib/internal/loader/ModuleJob.js @@ -5,14 +5,12 @@ const resolvedPromise = SafePromise.resolve(); const resolvedArrayPromise = SafePromise.resolve([]); const { ModuleWrap } = require('internal/loader/ModuleWrap'); -const NOOP = () => { /* No-op */ }; class ModuleJob { /** * @param {module: ModuleWrap?, compiled: Promise} moduleProvider */ - constructor(loader, moduleProvider, url) { - this.url = `${moduleProvider.url}`; - this.moduleProvider = moduleProvider; + constructor(loader, url, urlString, moduleProvider) { + this.url = url; this.loader = loader; this.error = null; this.hadError = false; @@ -20,14 +18,15 @@ class ModuleJob { if (moduleProvider instanceof ModuleWrap !== true) { // linked == promise for dependency jobs, with module populated, // module wrapper linked - this.modulePromise = this.moduleProvider.createModule(); + this.moduleProvider = moduleProvider; + this.modulePromise = this.moduleProvider(url); this.module = undefined; const linked = async () => { const dependencyJobs = []; this.module = await this.modulePromise; this.module.link(async (dependencySpecifier) => { const dependencyJobPromise = - this.loader.getModuleJob(this, dependencySpecifier); + this.loader.getModuleJob(dependencySpecifier, urlString); dependencyJobs.push(dependencyJobPromise); const dependencyJob = await dependencyJobPromise; return dependencyJob.modulePromise; @@ -40,9 +39,8 @@ class ModuleJob { //module wrapper instantiated this.instantiated = undefined; } else { - const getModuleProvider = async () => moduleProvider; - this.modulePromise = getModuleProvider(); - this.moduleProvider = { finish: NOOP }; + this.moduleProvider = async () => moduleProvider; + this.modulePromise = this.moduleProvider(); this.module = moduleProvider; this.linked = resolvedArrayPromise; this.instantiated = this.modulePromise; diff --git a/lib/internal/loader/ModuleRequest.js b/lib/internal/loader/ModuleRequest.js new file mode 100644 index 00000000000000..12c2a4ef290145 --- /dev/null +++ b/lib/internal/loader/ModuleRequest.js @@ -0,0 +1,121 @@ +'use strict'; + +const { URL } = require('url'); +const internalCJSModule = require('internal/module'); +const internalURLModule = require('internal/url'); +const internalFS = require('internal/fs'); +const NativeModule = require('native_module'); +const { extname } = require('path'); +const { realpathSync } = require('fs'); +const preserveSymlinks = !!process.binding('config').preserveSymlinks; +const { + ModuleWrap, + createDynamicModule +} = require('internal/loader/ModuleWrap'); +const errors = require('internal/errors'); + +const search = require('internal/loader/search'); +const asyncReadFile = require('util').promisify(require('fs').readFile); +const debug = require('util').debuglog('esm'); + +const realpathCache = new Map(); + +const formatProviders = new Map(); +exports.formatProviders = formatProviders; + +// Strategy for loading a standard JavaScript module +formatProviders.set('esm', async (url) => { + const source = `${await asyncReadFile(url)}`; + debug(`Loading StandardModule ${url}`); + return new ModuleWrap(internalCJSModule.stripShebang(source), + `${url}`); +}); + +// Strategy for loading a node-style CommonJS module +formatProviders.set('cjs', async (url) => { + const ctx = createDynamicModule(['default'], url, (reflect) => { + debug(`Loading CJSModule ${url.pathname}`); + const CJSModule = require('module'); + const pathname = internalURLModule.getPathFromURL(url); + ctx.reflect.exports.default.set(CJSModule._load(pathname)); + }); + return ctx.module; +}); + +// Strategy for loading a node builtin CommonJS module that isn't +// through normal resolution +formatProviders.set('native', async (url) => { + const ctx = createDynamicModule(['default'], url, (reflect) => { + debug(`Loading CJSModule ${url.pathname}`); + const CJSModule = require('module'); + const pathname = internalURLModule.getPathFromURL(url); + ctx.reflect.exports.default.set(CJSModule._load(pathname)); + }); + return ctx.module; +}); + +formatProviders.set('json', async (url) => { + const source = `${await asyncReadFile(url)}`; + const ctx = createDynamicModule(['default'], url, (reflect) => { + debug(`Loading JSONModule ${url.pathname}`); + const json = JSON.parse(source); + ctx.reflect.exports.default.set(json); + }); + return ctx.module; +}); + +// TODO: make this native binary handling only +formatProviders.set('binary', async (url) => { + const source = `${await asyncReadFile(url)}`; + const ctx = createDynamicModule(['default'], url, (reflect) => { + debug(`Loading JSONModule ${url.pathname}`); + const json = JSON.parse(source); + ctx.reflect.exports.default.set(json); + }); + return ctx.module; +}); + +function normalizeBaseURL(baseURLOrString) { + if (baseURLOrString instanceof URL) return baseURLOrString; + if (typeof baseURLOrString === 'string') return new URL(baseURLOrString); + return undefined; +} + +exports.resolve = resolve; +function resolve(specifier, parentURLOrString) { + if (NativeModule.nonInternalExists(specifier)) { + return { + url: new URL(`node:${specifier}`), + format: 'native' + }; + } + + const parentURL = normalizeBaseURL(parentURLOrString); + let url = search(specifier, parentURL); + + if (url.protocol !== 'file:') { + throw new errors.Error('ERR_INVALID_PROTOCOL', url.protocol, 'file:'); + } + + if (!preserveSymlinks) { + const real = realpathSync(internalURLModule.getPathFromURL(url), { + [internalFS.realpathCacheKey]: realpathCache + }); + const old = url; + url = internalURLModule.getURLFromFilePath(real); + url.search = old.search; + url.hash = old.hash; + } + + const ext = extname(url.pathname); + switch (ext) { + case '.mjs': + return { url, format: 'esm' }; + case '.json': + return { url, format: 'json' }; + case '.node': + return { url, format: 'binary' }; + default: + return { url, format: 'cjs' }; + } +}; diff --git a/lib/internal/loader/resolveRequestUrl.js b/lib/internal/loader/resolveRequestUrl.js deleted file mode 100644 index 2245064bfe4ba8..00000000000000 --- a/lib/internal/loader/resolveRequestUrl.js +++ /dev/null @@ -1,104 +0,0 @@ -'use strict'; - -const { URL } = require('url'); -const internalCJSModule = require('internal/module'); -const internalURLModule = require('internal/url'); -const internalFS = require('internal/fs'); -const NativeModule = require('native_module'); -const { extname } = require('path'); -const { realpathSync } = require('fs'); -const preserveSymlinks = !!process.binding('config').preserveSymlinks; -const { - ModuleWrap, - createDynamicModule -} = require('internal/loader/ModuleWrap'); -const errors = require('internal/errors'); - -const search = require('internal/loader/search'); -const asyncReadFile = require('util').promisify(require('fs').readFile); -const debug = require('util').debuglog('esm'); - -const realpathCache = new Map(); - -class ModuleRequest { - constructor(url) { - this.url = url; - } -} -Object.setPrototypeOf(ModuleRequest.prototype, null); - -// Strategy for loading a standard JavaScript module -class StandardModuleRequest extends ModuleRequest { - async createModule() { - const source = `${await asyncReadFile(this.url)}`; - debug(`Loading StandardModule ${this.url}`); - return new ModuleWrap(internalCJSModule.stripShebang(source), - `${this.url}`); - } -} - -// Strategy for loading a node-style CommonJS module -class CJSModuleRequest extends ModuleRequest { - async createModule() { - const ctx = createDynamicModule(['default'], this.url, (reflect) => { - debug(`Loading CJSModule ${this.url.pathname}`); - const CJSModule = require('module'); - const pathname = internalURLModule.getPathFromURL(this.url); - CJSModule._load(pathname); - }); - this.finish = (module) => { - ctx.reflect.exports.default.set(module.exports); - }; - return ctx.module; - } -} - -// Strategy for loading a node builtin CommonJS module that isn't -// through normal resolution -class NativeModuleRequest extends CJSModuleRequest { - async createModule() { - const ctx = createDynamicModule(['default'], this.url, (reflect) => { - debug(`Loading NativeModule ${this.url.pathname}`); - const exports = require(this.url.pathname); - reflect.exports.default.set(exports); - }); - return ctx.module; - } -} - -const normalizeBaseURL = (baseURLOrString) => { - if (baseURLOrString instanceof URL) return baseURLOrString; - if (typeof baseURLOrString === 'string') return new URL(baseURLOrString); - return undefined; -}; - -const resolveRequestUrl = (baseURLOrString, specifier) => { - if (NativeModule.nonInternalExists(specifier)) { - return new NativeModuleRequest(new URL(`node:${specifier}`)); - } - - const baseURL = normalizeBaseURL(baseURLOrString); - let url = search(specifier, baseURL); - - if (url.protocol !== 'file:') { - throw new errors.Error('ERR_INVALID_PROTOCOL', url.protocol, 'file:'); - } - - if (!preserveSymlinks) { - const real = realpathSync(internalURLModule.getPathFromURL(url), { - [internalFS.realpathCacheKey]: realpathCache - }); - const old = url; - url = internalURLModule.getURLFromFilePath(real); - url.search = old.search; - url.hash = old.hash; - } - - const ext = extname(url.pathname); - if (ext === '.mjs') { - return new StandardModuleRequest(url); - } - - return new CJSModuleRequest(url); -}; -module.exports = resolveRequestUrl; diff --git a/lib/module.js b/lib/module.js index d02a676c90270f..53aaeea09ca943 100644 --- a/lib/module.js +++ b/lib/module.js @@ -40,6 +40,7 @@ const errors = require('internal/errors'); const Loader = require('internal/loader/Loader'); const ModuleJob = require('internal/loader/ModuleJob'); const { createDynamicModule } = require('internal/loader/ModuleWrap'); +const { resolve: defaultResolver } = require('internal/loader/ModuleRequest'); const ESMLoader = new Loader(); function stat(filename) { @@ -78,6 +79,9 @@ Module._extensions = Object.create(null); var modulePaths = []; Module.globalPaths = []; +Module.resolve = defaultResolver; +Module.setModuleResolver = ESMLoader.setModuleResolver; + Module.wrapper = NativeModule.wrapper; Module.wrap = NativeModule.wrap; Module._debug = util.debuglog('module'); @@ -547,14 +551,12 @@ Module.prototype.load = function(filename) { if (experimentalModules) { const url = getURLFromFilePath(filename); - if (ESMLoader.moduleMap.has(`${url}`) !== true) { + const urlString = `${url}`; + if (ESMLoader.moduleMap.has(urlString) !== true) { const ctx = createDynamicModule(['default'], url); ctx.reflect.exports.default.set(this.exports); - ESMLoader.moduleMap.set(`${url}`, + ESMLoader.moduleMap.set(urlString, new ModuleJob(ESMLoader, ctx.module)); - } else { - ESMLoader.moduleMap.get(`${url}`).moduleProvider.finish( - Module._cache[filename]); } } }; diff --git a/node.gyp b/node.gyp index 79d9e0a68dcedf..a45fcf0257dc71 100644 --- a/node.gyp +++ b/node.gyp @@ -95,7 +95,7 @@ 'lib/internal/loader/ModuleMap.js', 'lib/internal/loader/ModuleJob.js', 'lib/internal/loader/ModuleWrap.js', - 'lib/internal/loader/resolveRequestUrl.js', + 'lib/internal/loader/ModuleRequest.js', 'lib/internal/loader/search.js', 'lib/internal/safe_globals.js', 'lib/internal/net.js', From 5496f771d3e6eb9ca26181c68e518f8f9bc4f43b Mon Sep 17 00:00:00 2001 From: guybedford Date: Sun, 3 Sep 2017 18:57:21 +0200 Subject: [PATCH 7/8] stricter signature, parent url tweaks --- lib/internal/loader/Loader.js | 20 +++++++++++++------- lib/internal/loader/ModuleJob.js | 4 ++-- lib/internal/loader/ModuleRequest.js | 24 ++++++++---------------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/lib/internal/loader/Loader.js b/lib/internal/loader/Loader.js index 4eccbc7e4fbeb7..ca606bb4cc3d42 100644 --- a/lib/internal/loader/Loader.js +++ b/lib/internal/loader/Loader.js @@ -38,8 +38,14 @@ class Loader { resolve = resolver; } - async resolve(specifier, parentUrlOrString = this.base) { - const { url, format } = await resolve(specifier, parentUrlOrString); + async resolve(specifier, parentURLOrString = this.base) { + if (typeof parentURLOrString === 'string') { + parentURLOrString = new URL(parentURLOrString); + } + else if (parentURLOrString instanceof URL === false) { + throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'parentURLOrString', 'URL'); + } + const { url, format } = await resolve(specifier, parentURLOrString); if (typeof url === 'string') { url = new URL(url); } @@ -56,19 +62,19 @@ class Loader { return { url, format }; } - async getModuleJob(specifier, parentUrlOrString = this.base) { - const { url, format } = await this.resolve(specifier, parentUrlOrString); + async getModuleJob(specifier, parentURLOrString = this.base) { + const { url, format } = await this.resolve(specifier, parentURLOrString); const urlString = `${url}`; let job = this.moduleMap.get(urlString); if (job === undefined) { - job = new ModuleJob(this, url, urlString, formatProviders.get(format)); + job = new ModuleJob(this, url, formatProviders.get(format)); this.moduleMap.set(urlString, job); } return job; } - async import(specifier, parentUrlOrString = this.base) { - const job = await this.getModuleJob(specifier, parentUrlOrString); + async import(specifier, parentURLOrString = this.base) { + const job = await this.getModuleJob(specifier, parentURLOrString); const module = await job.run(); return getNamespaceOfModuleWrap(module); } diff --git a/lib/internal/loader/ModuleJob.js b/lib/internal/loader/ModuleJob.js index f4c2b62ca93f6b..0d1f2947434b5d 100644 --- a/lib/internal/loader/ModuleJob.js +++ b/lib/internal/loader/ModuleJob.js @@ -9,7 +9,7 @@ class ModuleJob { /** * @param {module: ModuleWrap?, compiled: Promise} moduleProvider */ - constructor(loader, url, urlString, moduleProvider) { + constructor(loader, url, moduleProvider) { this.url = url; this.loader = loader; this.error = null; @@ -26,7 +26,7 @@ class ModuleJob { this.module = await this.modulePromise; this.module.link(async (dependencySpecifier) => { const dependencyJobPromise = - this.loader.getModuleJob(dependencySpecifier, urlString); + this.loader.getModuleJob(dependencySpecifier, url); dependencyJobs.push(dependencyJobPromise); const dependencyJob = await dependencyJobPromise; return dependencyJob.modulePromise; diff --git a/lib/internal/loader/ModuleRequest.js b/lib/internal/loader/ModuleRequest.js index 12c2a4ef290145..533df597396878 100644 --- a/lib/internal/loader/ModuleRequest.js +++ b/lib/internal/loader/ModuleRequest.js @@ -46,10 +46,9 @@ formatProviders.set('cjs', async (url) => { // through normal resolution formatProviders.set('native', async (url) => { const ctx = createDynamicModule(['default'], url, (reflect) => { - debug(`Loading CJSModule ${url.pathname}`); - const CJSModule = require('module'); - const pathname = internalURLModule.getPathFromURL(url); - ctx.reflect.exports.default.set(CJSModule._load(pathname)); + debug(`Loading NativeModule ${url.pathname}`); + const exports = NativeModule.require(url.pathname); + reflect.exports.default.set(exports); }); return ctx.module; }); @@ -66,23 +65,17 @@ formatProviders.set('json', async (url) => { // TODO: make this native binary handling only formatProviders.set('binary', async (url) => { - const source = `${await asyncReadFile(url)}`; const ctx = createDynamicModule(['default'], url, (reflect) => { - debug(`Loading JSONModule ${url.pathname}`); - const json = JSON.parse(source); - ctx.reflect.exports.default.set(json); + debug(`Loading CJSModule ${url.pathname}`); + const CJSModule = require('module'); + const pathname = internalURLModule.getPathFromURL(url); + ctx.reflect.exports.default.set(CJSModule._load(pathname)); }); return ctx.module; }); -function normalizeBaseURL(baseURLOrString) { - if (baseURLOrString instanceof URL) return baseURLOrString; - if (typeof baseURLOrString === 'string') return new URL(baseURLOrString); - return undefined; -} - exports.resolve = resolve; -function resolve(specifier, parentURLOrString) { +function resolve(specifier, parentURL) { if (NativeModule.nonInternalExists(specifier)) { return { url: new URL(`node:${specifier}`), @@ -90,7 +83,6 @@ function resolve(specifier, parentURLOrString) { }; } - const parentURL = normalizeBaseURL(parentURLOrString); let url = search(specifier, parentURL); if (url.protocol !== 'file:') { From 31aa66f19fca97b2d2357f68fff48cebb1556ab4 Mon Sep 17 00:00:00 2001 From: guybedford Date: Tue, 5 Sep 2017 13:24:34 +0200 Subject: [PATCH 8/8] const fix --- lib/internal/loader/Loader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/internal/loader/Loader.js b/lib/internal/loader/Loader.js index ca606bb4cc3d42..65f6b09379bb28 100644 --- a/lib/internal/loader/Loader.js +++ b/lib/internal/loader/Loader.js @@ -9,7 +9,7 @@ const { const ModuleMap = require('internal/loader/ModuleMap'); const ModuleJob = require('internal/loader/ModuleJob'); -const { formatProviders, resolve } = require('internal/loader/ModuleRequest'); +let { formatProviders, resolve } = require('internal/loader/ModuleRequest'); const errors = require('internal/errors'); function getBase() {