node imports (#1156)

* checkpoint import from node_modules

* it works!

* minify

* only resolve bare imports

* coalesce imports; ignore errors

* preload transitive dependencies

* DRY parseImports

* parseImports tests

* fix windows?

* add logging

* test only windows

* don’t resolve input

* restore tests

* build _node

* fix import order

* 30s timeout

* adopt @rollup/plugin-node-resolve

* more tests

* fix file path resolution

* restore tests

* add missing dependency

* adopt pkg-dir

---------

Co-authored-by: Philippe Rivière <[email protected]>

Author: mbostock

package.json

@@ -27,8 +27,8 @@
     "test": "concurrently npm:test:mocha npm:test:tsc npm:test:lint npm:test:prettier",
     "test:coverage": "c8 --check-coverage --lines 80 --per-file yarn test:mocha",
     "test:build": "rimraf test/build && node build.js --sourcemap --outdir=test/build \"{src,test}/**/*.{ts,js,css}\" --ignore \"test/input/**\" --ignore \"test/output/**\" --ignore \"test/preview/dashboard/**\" --ignore \"**/*.d.ts\" && cp -r templates test/build",
-    "test:mocha": "yarn test:build && rimraf --glob test/.observablehq/cache test/input/build/*/.observablehq/cache && cross-env OBSERVABLE_TELEMETRY_DISABLE=1 TZ=America/Los_Angeles mocha --timeout 5000 -p \"test/build/test/**/*-test.js\"",
-    "test:mocha:serial": "yarn test:build && rimraf --glob test/.observablehq/cache test/input/build/*/.observablehq/cache && cross-env OBSERVABLE_TELEMETRY_DISABLE=1 TZ=America/Los_Angeles mocha --timeout 5000 \"test/build/test/**/*-test.js\"",
+    "test:mocha": "yarn test:build && rimraf --glob test/.observablehq/cache test/input/build/*/.observablehq/cache && cross-env OBSERVABLE_TELEMETRY_DISABLE=1 TZ=America/Los_Angeles mocha --timeout 30000 -p \"test/build/test/**/*-test.js\"",
+    "test:mocha:serial": "yarn test:build && rimraf --glob test/.observablehq/cache test/input/build/*/.observablehq/cache && cross-env OBSERVABLE_TELEMETRY_DISABLE=1 TZ=America/Los_Angeles mocha --timeout 30000 \"test/build/test/**/*-test.js\"",
     "test:lint": "eslint src test --max-warnings=0",
     "test:prettier": "prettier --check src test",
     "test:tsc": "tsc --noEmit",
@@ -77,6 +77,7 @@
     "mime": "^4.0.0",
     "minisearch": "^6.3.0",
     "open": "^10.1.0",
+    "pkg-dir": "^8.0.0",
     "rollup": "^4.6.0",
     "rollup-plugin-esbuild": "^6.1.0",
     "semver": "^7.5.4",

src/build.ts

@@ -10,6 +10,7 @@ import {transpileModule} from "./javascript/transpile.js";
 import type {Logger, Writer} from "./logger.js";
 import type {MarkdownPage} from "./markdown.js";
 import {parseMarkdown} from "./markdown.js";
+import {extractNodeSpecifier} from "./node.js";
 import {extractNpmSpecifier, populateNpmCache, resolveNpmImport} from "./npm.js";
 import {isPathImport, relativePath, resolvePath} from "./path.js";
 import {renderPage} from "./render.js";
@@ -175,10 +176,14 @@ export async function build(
   // these, too, but it would involve rewriting the files since populateNpmCache
   // doesn’t let you pass in a resolver.
   for (const path of globalImports) {
-    if (!path.startsWith("/_npm/")) continue; // skip _observablehq
-    effects.output.write(`${faint("copy")} npm:${extractNpmSpecifier(path)} ${faint("→")} `);
-    const sourcePath = await populateNpmCache(root, path); // TODO effects
-    await effects.copyFile(sourcePath, path);
+    if (path.startsWith("/_npm/")) {
+      effects.output.write(`${faint("copy")} npm:${extractNpmSpecifier(path)} ${faint("→")} `);
+      const sourcePath = await populateNpmCache(root, path); // TODO effects
+      await effects.copyFile(sourcePath, path);
+    } else if (path.startsWith("/_node/")) {
+      effects.output.write(`${faint("copy")} ${extractNodeSpecifier(path)} ${faint("→")} `);
+      await effects.copyFile(join(root, ".observablehq", "cache", path), path);
+    }
   }
 
   // Copy over imported local modules, overriding import resolution so that

src/javascript/imports.ts

@@ -1,8 +1,11 @@
+import {readFile} from "node:fs/promises";
+import {join} from "node:path/posix";
 import type {Node} from "acorn";
 import type {CallExpression} from "acorn";
 import type {ExportAllDeclaration, ExportNamedDeclaration, ImportDeclaration, ImportExpression} from "acorn";
 import {simple} from "acorn-walk";
 import {isPathImport, relativePath, resolveLocalPath} from "../path.js";
+import {parseProgram} from "./parse.js";
 import {getStringLiteralValue, isStringLiteral} from "./source.js";
 import {syntaxError} from "./syntaxError.js";
 
@@ -59,6 +62,7 @@ export function hasImportDeclaration(body: Node): boolean {
  */
 export function findImports(body: Node, path: string, input: string): ImportReference[] {
   const imports: ImportReference[] = [];
+  const keys = new Set<string>();
 
   simple(body, {
     ImportDeclaration: findImport,
@@ -68,6 +72,11 @@ export function findImports(body: Node, path: string, input: string): ImportRefe
     CallExpression: findImportMetaResolve
   });
 
+  function addImport(ref: ImportReference) {
+    const key = `${ref.type}:${ref.method}:${ref.name}`;
+    if (!keys.has(key)) keys.add(key), imports.push(ref);
+  }
+
   function findImport(node: ImportNode | ExportNode) {
     const source = node.source;
     if (!source || !isStringLiteral(source)) return;
@@ -76,9 +85,9 @@ export function findImports(body: Node, path: string, input: string): ImportRefe
     if (isPathImport(name)) {
       const localPath = resolveLocalPath(path, name);
       if (!localPath) throw syntaxError(`non-local import: ${name}`, node, input); // prettier-ignore
-      imports.push({name: relativePath(path, localPath), type: "local", method});
+      addImport({name: relativePath(path, localPath), type: "local", method});
     } else {
-      imports.push({name, type: "global", method});
+      addImport({name, type: "global", method});
     }
   }
 
@@ -89,9 +98,9 @@ export function findImports(body: Node, path: string, input: string): ImportRefe
     if (isPathImport(name)) {
       const localPath = resolveLocalPath(path, name);
       if (!localPath) throw syntaxError(`non-local import: ${name}`, node, input); // prettier-ignore
-      imports.push({name: relativePath(path, localPath), type: "local", method: "dynamic"});
+      addImport({name: relativePath(path, localPath), type: "local", method: "dynamic"});
     } else {
-      imports.push({name, type: "global", method: "dynamic"});
+      addImport({name, type: "global", method: "dynamic"});
     }
   }
 
@@ -109,3 +118,28 @@ export function isImportMetaResolve(node: CallExpression): boolean {
     node.arguments.length > 0
   );
 }
+
+export function isJavaScript(path: string): boolean {
+  return /\.(m|c)?js$/i.test(path);
+}
+
+const parseImportsCache = new Map<string, Promise<ImportReference[]>>();
+
+export async function parseImports(root: string, path: string): Promise<ImportReference[]> {
+  if (!isJavaScript(path)) return []; // TODO traverse CSS, too
+  const filePath = join(root, path);
+  let promise = parseImportsCache.get(filePath);
+  if (promise) return promise;
+  promise = (async function () {
+    try {
+      const source = await readFile(filePath, "utf-8");
+      const body = parseProgram(source);
+      return findImports(body, path, source);
+    } catch (error: any) {
+      console.warn(`unable to fetch or parse ${path}: ${error.message}`);
+      return [];
+    }
+  })();
+  parseImportsCache.set(filePath, promise);
+  return promise;
+}

src/node.ts

@@ -0,0 +1,112 @@
+import {existsSync} from "node:fs";
+import {copyFile, readFile, writeFile} from "node:fs/promises";
+import {createRequire} from "node:module";
+import op from "node:path";
+import {extname, join} from "node:path/posix";
+import {pathToFileURL} from "node:url";
+import {nodeResolve} from "@rollup/plugin-node-resolve";
+import {packageDirectory} from "pkg-dir";
+import type {AstNode, OutputChunk, Plugin, ResolveIdResult} from "rollup";
+import {rollup} from "rollup";
+import esbuild from "rollup-plugin-esbuild";
+import {prepareOutput, toOsPath} from "./files.js";
+import type {ImportReference} from "./javascript/imports.js";
+import {isJavaScript, parseImports} from "./javascript/imports.js";
+import {parseNpmSpecifier} from "./npm.js";
+import {isPathImport} from "./path.js";
+import {faint} from "./tty.js";
+
+export async function resolveNodeImport(root: string, spec: string): Promise<string> {
+  return resolveNodeImportInternal(op.join(root, ".observablehq", "cache", "_node"), root, spec);
+}
+
+const bundlePromises = new Map<string, Promise<void>>();
+
+async function resolveNodeImportInternal(cacheRoot: string, packageRoot: string, spec: string): Promise<string> {
+  const {name, path = "."} = parseNpmSpecifier(spec);
+  const require = createRequire(pathToFileURL(op.join(packageRoot, "/")));
+  const pathResolution = require.resolve(spec);
+  const packageResolution = await packageDirectory({cwd: op.dirname(pathResolution)});
+  if (!packageResolution) throw new Error(`unable to resolve package.json: ${spec}`);
+  const {version} = JSON.parse(await readFile(op.join(packageResolution, "package.json"), "utf-8"));
+  const resolution = `${name}@${version}/${extname(path) ? path : path === "." ? "index.js" : `${path}.js`}`;
+  const outputPath = op.join(cacheRoot, toOsPath(resolution));
+  if (!existsSync(outputPath)) {
+    let promise = bundlePromises.get(outputPath);
+    if (!promise) {
+      promise = (async () => {
+        process.stdout.write(`${spec} ${faint("→")} ${resolution}\n`);
+        await prepareOutput(outputPath);
+        if (isJavaScript(pathResolution)) {
+          await writeFile(outputPath, await bundle(spec, cacheRoot, packageResolution));
+        } else {
+          await copyFile(pathResolution, outputPath);
+        }
+      })();
+      bundlePromises.set(outputPath, promise);
+      promise.catch(() => {}).then(() => bundlePromises.delete(outputPath));
+    }
+    await promise;
+  }
+  return `/_node/${resolution}`;
+}
+
+/**
+ * Resolves the direct dependencies of the specified node import path, such as
+ * "/_node/[email protected]/src/index.js", returning a set of node import paths.
+ */
+export async function resolveNodeImports(root: string, path: string): Promise<ImportReference[]> {
+  if (!path.startsWith("/_node/")) throw new Error(`invalid node path: ${path}`);
+  return parseImports(join(root, ".observablehq", "cache"), path);
+}
+
+/**
+ * Given a local npm path such as "/_node/[email protected]/src/index.js", returns
+ * the corresponding npm specifier such as "[email protected]/src/index.js".
+ */
+export function extractNodeSpecifier(path: string): string {
+  if (!path.startsWith("/_node/")) throw new Error(`invalid node path: ${path}`);
+  return path.replace(/^\/_node\//, "");
+}
+
+async function bundle(input: string, cacheRoot: string, packageRoot: string): Promise<string> {
+  const bundle = await rollup({
+    input,
+    plugins: [
+      nodeResolve({browser: true, rootDir: packageRoot}),
+      importResolve(input, cacheRoot, packageRoot),
+      esbuild({
+        target: ["es2022", "chrome96", "firefox96", "safari16", "node18"],
+        exclude: [], // don’t exclude node_modules
+        minify: true
+      })
+    ],
+    onwarn(message, warn) {
+      if (message.code === "CIRCULAR_DEPENDENCY") return;
+      warn(message);
+    }
+  });
+  try {
+    const output = await bundle.generate({format: "es"});
+    const code = output.output.find((o): o is OutputChunk => o.type === "chunk")!.code; // TODO don’t assume one chunk?
+    return code;
+  } finally {
+    await bundle.close();
+  }
+}
+
+function importResolve(input: string, cacheRoot: string, packageRoot: string): Plugin {
+  async function resolve(specifier: string | AstNode): Promise<ResolveIdResult> {
+    return typeof specifier !== "string" || // AST node?
+      isPathImport(specifier) || // relative path, e.g., ./foo.js
+      /^\w+:/.test(specifier) || // windows file path, https: URL, etc.
+      specifier === input // entry point
+      ? null // don’t do any additional resolution
+      : {id: await resolveNodeImportInternal(cacheRoot, packageRoot, specifier), external: true}; // resolve bare import
+  }
+  return {
+    name: "resolve-import",
+    resolveId: resolve,
+    resolveDynamicImport: resolve
+  };
+}

src/npm.ts

@@ -6,7 +6,7 @@ import {simple} from "acorn-walk";
 import {rsort, satisfies} from "semver";
 import {isEnoent} from "./error.js";
 import type {ExportNode, ImportNode, ImportReference} from "./javascript/imports.js";
-import {findImports, isImportMetaResolve} from "./javascript/imports.js";
+import {isImportMetaResolve, parseImports} from "./javascript/imports.js";
 import {parseProgram} from "./javascript/parse.js";
 import type {StringLiteral} from "./javascript/source.js";
 import {getStringLiteralValue, isStringLiteral} from "./javascript/source.js";
@@ -252,30 +252,14 @@ export async function resolveNpmImport(root: string, specifier: string): Promise
   return `/_npm/${name}@${await resolveNpmVersion(root, {name, range})}/${path.replace(/\+esm$/, "_esm.js")}`;
 }
 
-const npmImportsCache = new Map<string, Promise<ImportReference[]>>();
-
 /**
  * Resolves the direct dependencies of the specified npm path, such as
  * "/_npm/[email protected]/_esm.js", returning the corresponding set of npm paths.
  */
 export async function resolveNpmImports(root: string, path: string): Promise<ImportReference[]> {
   if (!path.startsWith("/_npm/")) throw new Error(`invalid npm path: ${path}`);
-  let promise = npmImportsCache.get(path);
-  if (promise) return promise;
-  promise = (async function () {
-    try {
-      const filePath = await populateNpmCache(root, path);
-      if (!/\.(m|c)?js$/i.test(path)) return []; // not JavaScript; TODO traverse CSS, too
-      const source = await readFile(filePath, "utf-8");
-      const body = parseProgram(source);
-      return findImports(body, path, source);
-    } catch (error: any) {
-      console.warn(`unable to fetch or parse ${path}: ${error.message}`);
-      return [];
-    }
-  })();
-  npmImportsCache.set(path, promise);
-  return promise;
+  await populateNpmCache(root, path);
+  return parseImports(join(root, ".observablehq", "cache"), path);
 }
 
 /**

src/preview.ts

@@ -124,6 +124,8 @@ export class PreviewServer {
       } else if (pathname.startsWith("/_observablehq/") && pathname.endsWith(".css")) {
         const path = getClientPath(pathname.slice("/_observablehq/".length));
         end(req, res, await bundleStyles({path}), "text/css");
+      } else if (pathname.startsWith("/_node/")) {
+        send(req, pathname, {root: join(root, ".observablehq", "cache")}).pipe(res);
       } else if (pathname.startsWith("/_npm/")) {
         await populateNpmCache(root, pathname);
         send(req, pathname, {root: join(root, ".observablehq", "cache")}).pipe(res);

src/resolvers.ts

@@ -8,6 +8,7 @@ import {getImplicitDependencies, getImplicitDownloads} from "./libraries.js";
 import {getImplicitFileImports, getImplicitInputImports} from "./libraries.js";
 import {getImplicitStylesheets} from "./libraries.js";
 import type {MarkdownPage} from "./markdown.js";
+import {extractNodeSpecifier, resolveNodeImport, resolveNodeImports} from "./node.js";
 import {extractNpmSpecifier, populateNpmCache, resolveNpmImport, resolveNpmImports} from "./npm.js";
 import {isAssetPath, isPathImport, relativePath, resolveLocalPath, resolvePath} from "./path.js";
 
@@ -169,39 +170,70 @@ export async function getResolvers(
     globalImports.add(i);
   }
 
-  // Resolve npm: imports.
+  // Resolve npm: and bare imports.
   for (const i of globalImports) {
     if (i.startsWith("npm:") && !builtins.has(i)) {
       resolutions.set(i, await resolveNpmImport(root, i.slice("npm:".length)));
+    } else if (!/^\w+:/.test(i)) {
+      try {
+        resolutions.set(i, await resolveNodeImport(root, i));
+      } catch {
+        // ignore error; allow the import to be resolved at runtime
+      }
     }
   }
 
-  // Follow transitive imports of npm imports. This has the side-effect of
-  // populating the npm cache.
-  for (const value of resolutions.values()) {
-    for (const i of await resolveNpmImports(root, value)) {
-      if (i.type === "local") {
-        const path = resolvePath(value, i.name);
-        const specifier = `npm:${extractNpmSpecifier(path)}`;
-        globalImports.add(specifier);
-        resolutions.set(specifier, path);
+  // Follow transitive imports of npm and bare imports. This has the side-effect
+  // of populating the npm cache; the node import cache is already transitively
+  // populated above.
+  for (const [key, value] of resolutions) {
+    if (key.startsWith("npm:")) {
+      for (const i of await resolveNpmImports(root, value)) {
+        if (i.type === "local") {
+          const path = resolvePath(value, i.name);
+          const specifier = `npm:${extractNpmSpecifier(path)}`;
+          globalImports.add(specifier);
+          resolutions.set(specifier, path);
+        }
+      }
+    } else if (!/^\w+:/.test(key)) {
+      for (const i of await resolveNodeImports(root, value)) {
+        if (i.type === "local") {
+          const path = resolvePath(value, i.name);
+          const specifier = extractNodeSpecifier(path);
+          globalImports.add(specifier);
+          resolutions.set(specifier, path);
+        }
       }
     }
   }
 
-  // Resolve transitive static npm: imports.
-  const npmStaticResolutions = new Set<string>();
+  // Resolve transitive static npm: and bare imports.
+  const staticResolutions = new Map<string, string>();
   for (const i of staticImports) {
-    const r = resolutions.get(i);
-    if (r) npmStaticResolutions.add(r);
+    if (i.startsWith("npm:") || !/^\w+:/.test(i)) {
+      const r = resolutions.get(i);
+      if (r) staticResolutions.set(i, r);
+    }
   }
-  for (const value of npmStaticResolutions) {
-    for (const i of await resolveNpmImports(root, value)) {
-      if (i.type === "local" && i.method === "static") {
-        const path = resolvePath(value, i.name);
-        const specifier = `npm:${extractNpmSpecifier(path)}`;
-        staticImports.add(specifier);
-        npmStaticResolutions.add(path);
+  for (const [key, value] of staticResolutions) {
+    if (key.startsWith("npm:")) {
+      for (const i of await resolveNpmImports(root, value)) {
+        if (i.type === "local" && i.method === "static") {
+          const path = resolvePath(value, i.name);
+          const specifier = `npm:${extractNpmSpecifier(path)}`;
+          staticImports.add(specifier);
+          staticResolutions.set(specifier, path);
+        }
+      }
+    } else if (!/^\w+:/.test(key)) {
+      for (const i of await resolveNodeImports(root, value)) {
+        if (i.type === "local" && i.method === "static") {
+          const path = resolvePath(value, i.name);
+          const specifier = extractNodeSpecifier(path);
+          staticImports.add(specifier);
+          staticResolutions.set(specifier, path);
+        }
       }
     }
   }