diff --git a/package.json b/package.json index a715a4cb62..28abadf173 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,7 @@ }, "pnpm": { "onlyBuiltDependencies": [ + "isolated-vm", "sqlite3" ], "overrides": { diff --git a/packages/@n8n/expression-runtime/README.md b/packages/@n8n/expression-runtime/README.md index d19a6de604..3394fc69b0 100644 --- a/packages/@n8n/expression-runtime/README.md +++ b/packages/@n8n/expression-runtime/README.md @@ -7,12 +7,12 @@ Secure, isolated expression evaluation runtime for n8n workflows. **In progress — landing as a series of incremental PRs.** Implemented so far: -- ✅ TypeScript interfaces and architecture design -- ✅ Core architecture documentation +- ✅ TypeScript interfaces and architecture design (PR 1) +- ✅ Core architecture documentation (PR 1) +- ✅ Runtime bundle: extension functions, deep lazy proxy system (PR 2) +- ✅ `IsolatedVmBridge`: V8 isolate management via `isolated-vm` (PR 3) Coming in later PRs: -- 🚧 Runtime bundle: extension functions, deep lazy proxy system (PR 2) -- 🚧 `IsolatedVmBridge`: V8 isolate management via `isolated-vm` (PR 3) - 🚧 `ExpressionEvaluator`: tournament integration, expression code caching (PR 4) - 🚧 Integration tests (PR 4) - 🚧 Workflow integration behind `N8N_EXPRESSION_ENGINE=vm` flag (PR 5) @@ -165,7 +165,7 @@ interface RuntimeBridge { ### Bridge Implementations -- **IsolatedVmBridge**: 🚧 For Node.js backend (isolated-vm with V8 isolates) - coming in PR 3 +- **IsolatedVmBridge**: ✅ For Node.js backend (isolated-vm with V8 isolates) - Memory isolation with hard 128MB limit - Timeout enforcement (5s default) - Deep lazy proxy system for workflow data @@ -185,9 +185,9 @@ interface EvaluatorConfig { } interface BridgeConfig { - memoryLimit?: number; // Default: 128 MB (PR 3) - timeout?: number; // Default: 5000 ms (PR 3) - debug?: boolean; // Default: false (PR 3) + memoryLimit?: number; // Default: 128 MB + timeout?: number; // Default: 5000 ms + debug?: boolean; // Default: false } ``` diff --git a/packages/@n8n/expression-runtime/docs/architecture-diagram.mmd b/packages/@n8n/expression-runtime/docs/architecture-diagram.mmd index d5fa16552a..f87b8a16c1 100644 --- a/packages/@n8n/expression-runtime/docs/architecture-diagram.mmd +++ b/packages/@n8n/expression-runtime/docs/architecture-diagram.mmd @@ -69,24 +69,23 @@ graph TB sequenceDiagram participant WF as Workflow participant Eval as ExpressionEvaluator - participant Bridge as RuntimeBridge + participant Bridge as IsolatedVmBridge participant Runtime as Runtime (Isolated) - participant Store as Data Store WF->>Eval: evaluate(expr, data) Eval->>Eval: Transform with Tournament - Eval->>Eval: Check cache - Eval->>Store: Store data with ID - Eval->>Bridge: execute(code, dataId) - Bridge->>Runtime: Run code in isolation + Eval->>Eval: Check code cache + Eval->>Bridge: execute(code, data) + Bridge->>Bridge: Register ivm.Reference callbacks with data + Bridge->>Runtime: evalSync("resetDataProxies()") + Runtime->>Runtime: Create lazy proxies for $json, $input, etc. + Bridge->>Runtime: Run compiled script Runtime->>Runtime: Access $json.email - Runtime->>Bridge: getData(dataId, 'email') - Bridge->>Store: Lookup 'email' - Store-->>Bridge: Value + Runtime->>Bridge: __getValueAtPath(['$json','email']) [ivm.Reference] + Bridge->>Bridge: Navigate path in data Bridge-->>Runtime: Value Runtime-->>Bridge: Expression result Bridge-->>Eval: Result - Eval->>Eval: Cache result Eval-->>WF: Result diff --git a/packages/@n8n/expression-runtime/docs/deep-lazy-proxy.md b/packages/@n8n/expression-runtime/docs/deep-lazy-proxy.md index b13b0d6bde..d947380f3f 100644 --- a/packages/@n8n/expression-runtime/docs/deep-lazy-proxy.md +++ b/packages/@n8n/expression-runtime/docs/deep-lazy-proxy.md @@ -14,8 +14,9 @@ The Deep Lazy Proxy is a memory-efficient mechanism for providing workflow data ## Architecture -The deep lazy proxy is implemented entirely within `src/runtime/index.ts`, which is -bundled into `dist/bundle/runtime.iife.js` and injected into the V8 isolate at startup. +The deep lazy proxy is implemented in `src/runtime/lazy-proxy.ts`, which is bundled +together with the other runtime modules into `dist/bundle/runtime.iife.js` and injected +into the V8 isolate at startup. Key functions exposed on `globalThis` inside the isolate: @@ -223,6 +224,9 @@ When modifying the proxy implementation: ## Related Files -- Implementation: `packages/@n8n/expression-runtime/src/runtime/index.ts` — proxy system, `resetDataProxies`, `__sanitize`, `SafeObject`, `SafeError` +- Proxy implementation: `packages/@n8n/expression-runtime/src/runtime/lazy-proxy.ts` — `createDeepLazyProxy` +- Reset: `packages/@n8n/expression-runtime/src/runtime/reset.ts` — `resetDataProxies` +- Security globals: `packages/@n8n/expression-runtime/src/runtime/safe-globals.ts` — `SafeObject`, `SafeError`, `__sanitize` +- Runtime entry: `packages/@n8n/expression-runtime/src/runtime/index.ts` — wires all modules to `globalThis` - Bridge: `packages/@n8n/expression-runtime/src/bridge/isolated-vm-bridge.ts` — registers `ivm.Reference` callbacks, loads bundle, calls `resetDataProxies` - Build: `packages/@n8n/expression-runtime/esbuild.config.js` — bundles runtime to `dist/bundle/runtime.iife.js` diff --git a/packages/@n8n/expression-runtime/package.json b/packages/@n8n/expression-runtime/package.json index f90235472a..6e6f2e6c42 100644 --- a/packages/@n8n/expression-runtime/package.json +++ b/packages/@n8n/expression-runtime/package.json @@ -21,6 +21,7 @@ ], "license": "SEE LICENSE IN LICENSE.md", "dependencies": { + "isolated-vm": "^6.0.2", "js-base64": "catalog:", "jssha": "3.3.1", "lodash": "catalog:", diff --git a/packages/@n8n/expression-runtime/src/bridge/isolated-vm-bridge.ts b/packages/@n8n/expression-runtime/src/bridge/isolated-vm-bridge.ts new file mode 100644 index 0000000000..7b9dfaa2a5 --- /dev/null +++ b/packages/@n8n/expression-runtime/src/bridge/isolated-vm-bridge.ts @@ -0,0 +1,465 @@ +import ivm from 'isolated-vm'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { RuntimeBridge, BridgeConfig } from '../types'; + +// Get __dirname equivalent for ES modules +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +/** + * IsolatedVmBridge - Runtime bridge using isolated-vm for secure expression evaluation. + * + * This bridge creates a V8 isolate with: + * - Hard memory limit (128MB default) + * - No access to Node.js APIs + * - Timeout enforcement + * - Complete isolation from host process + * + * Context reuse pattern: Create isolate/context once, reset state between evaluations. + */ +export class IsolatedVmBridge implements RuntimeBridge { + private isolate: ivm.Isolate; + private context?: ivm.Context; + private initialized = false; + private disposed = false; + private config: Required; + + // Script compilation cache for performance + // Maps expression code -> compiled ivm.Script + private scriptCache = new Map(); + + constructor(config: BridgeConfig = {}) { + this.config = { + memoryLimit: config.memoryLimit ?? 128, + timeout: config.timeout ?? 5000, + debug: config.debug ?? false, + }; + + // Create isolate with memory limit + // Note: memoryLimit is in MB + this.isolate = new ivm.Isolate({ memoryLimit: this.config.memoryLimit }); + } + + /** + * Initialize the isolate and create execution context. + * + * Steps: + * 1. Create context + * 2. Set up basic globals (global reference) + * 3. Load runtime bundle (DateTime, extend, proxy system) + * 4. Verify proxy system + * + * Must be called before execute(). + */ + async initialize(): Promise { + if (this.initialized) { + return; + } + + // Create context in the isolate + this.context = await this.isolate.createContext(); + + // Set up basic globals + // jail is a reference to the context's global object + const jail = this.context.global; + + // Set 'global' to reference itself (pattern from POC) + // This allows code in isolate to access 'global.something' + await jail.set('global', jail.derefInto()); + + // Load runtime bundle (DateTime, extend, SafeObject, proxy system) + await this.loadVendorLibraries(); + + // Verify proxy system loaded correctly + await this.verifyProxySystem(); + + // Inject E() error handler needed by tournament-generated try-catch code + await this.injectErrorHandler(); + + this.initialized = true; + + if (this.config.debug) { + console.log('[IsolatedVmBridge] Initialized successfully'); + } + } + + /** + * Load runtime bundle into the isolate. + * + * The runtime bundle includes: + * - DateTime, extend, extendOptional (expression engine globals) + * - SafeObject and SafeError wrappers + * - createDeepLazyProxy function + * - __data object initialization + * + * @private + * @throws {Error} If context not initialized or bundle loading fails + */ + private async loadVendorLibraries(): Promise { + if (!this.context) { + throw new Error('Context not initialized'); + } + + try { + // Load runtime bundle (includes vendor libraries + proxy system) + // Path: dist/bundle/runtime.iife.js + const runtimeBundlePath = path.join(__dirname, '../../dist/bundle/runtime.iife.js'); + const runtimeBundle = fs.readFileSync(runtimeBundlePath, 'utf-8'); + + // Evaluate bundle in isolate context + // This makes all exported globals available (DateTime, extend, extendOptional, SafeObject, SafeError, createDeepLazyProxy, resetDataProxies, __data) + await this.context.eval(runtimeBundle); + + if (this.config.debug) { + console.log('[IsolatedVmBridge] Runtime bundle loaded from:', runtimeBundlePath); + } + + // Verify vendor libraries loaded correctly + const hasDateTime = await this.context.eval('typeof DateTime !== "undefined"'); + const hasExtend = await this.context.eval('typeof extend !== "undefined"'); + + if (!hasDateTime || !hasExtend) { + throw new Error( + `Library verification failed: DateTime=${hasDateTime}, extend=${hasExtend}`, + ); + } + + if (this.config.debug) { + console.log('[IsolatedVmBridge] Vendor libraries verified successfully'); + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to load runtime bundle: ${errorMessage}`); + } + } + + /** + * Verify the proxy system loaded correctly. + * + * The proxy system is loaded as part of the runtime bundle in loadVendorLibraries(). + * This method verifies all required components are available. + * + * @private + * @throws {Error} If context not initialized or proxy system verification fails + */ + private async verifyProxySystem(): Promise { + if (!this.context) { + throw new Error('Context not initialized'); + } + + try { + // Verify proxy system components loaded correctly + const hasProxyCreator = await this.context.eval('typeof createDeepLazyProxy !== "undefined"'); + const hasData = await this.context.eval('typeof __data !== "undefined"'); + const hasSafeObject = await this.context.eval('typeof SafeObject !== "undefined"'); + const hasSafeError = await this.context.eval('typeof SafeError !== "undefined"'); + const hasResetFunction = await this.context.eval('typeof resetDataProxies !== "undefined"'); + + if (!hasProxyCreator || !hasData || !hasSafeObject || !hasSafeError || !hasResetFunction) { + throw new Error( + `Proxy system verification failed: ` + + `createDeepLazyProxy=${hasProxyCreator}, __data=${hasData}, ` + + `SafeObject=${hasSafeObject}, SafeError=${hasSafeError}, ` + + `resetDataProxies=${hasResetFunction}`, + ); + } + + if (this.config.debug) { + console.log('[IsolatedVmBridge] Proxy system verified successfully'); + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to verify proxy system: ${errorMessage}`); + } + } + + /** + * Inject the E() error handler into the isolate context. + * + * Tournament wraps expressions with try-catch that calls E(error, this). + * This handler: + * - Re-throws security violations from __sanitize + * - Swallows TypeErrors (failed attack attempts return undefined) + * - Re-throws all other errors + * + * @private + * @throws {Error} If context not initialized + */ + private async injectErrorHandler(): Promise { + if (!this.context) { + throw new Error('Context not initialized'); + } + + await this.context.eval(` + if (typeof E === 'undefined') { + globalThis.E = function(error, _context) { + // Re-throw security violations from __sanitize + if (error && error.message && error.message.includes('due to security concerns')) { + throw error; + } + // Swallow TypeErrors (failed attack attempts return undefined) + if (error instanceof TypeError) { + return undefined; + } + throw error; + }; + } + `); + + if (this.config.debug) { + console.log('[IsolatedVmBridge] Error handler injected successfully'); + } + } + + /** + * Reset data proxies in the isolate context. + * + * This method should be called before each execute() to: + * 1. Clear proxy caches from previous evaluations + * 2. Initialize fresh workflow data references + * 3. Expose workflow properties to globalThis + * + * The reset function runs in the isolate and calls back to the host + * via ivm.Reference callbacks to fetch workflow data. + * + * @private + * @throws {Error} If context not initialized or reset fails + */ + private resetDataProxies(): void { + if (!this.context) { + throw new Error('Context not initialized'); + } + + try { + // Call the resetDataProxies function in the isolate + // This function is loaded as part of the runtime bundle + this.context.evalSync('resetDataProxies()'); + + if (this.config.debug) { + console.log('[IsolatedVmBridge] Data proxies reset successfully'); + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to reset data proxies: ${errorMessage}`); + } + } + + /** + * Register callback functions for cross-isolate communication. + * + * Creates three ivm.Reference callbacks that the runtime bundle uses + * to fetch data from the host process: + * + * - __getValueAtPath: Returns metadata or primitive for a property path + * - __getArrayElement: Returns individual array elements + * - __callFunctionAtPath: Executes functions in host context + * + * These callbacks are called synchronously from isolate proxy traps. + * + * @param data - Current workflow data to use for callback responses + * @private + */ + private registerCallbacks(data: Record): void { + if (!this.context) { + throw new Error('Context not initialized'); + } + + // Callback 1: Get value/metadata at path + // Used by createDeepLazyProxy when accessing properties + const getValueAtPath = new ivm.Reference((path: string[]) => { + // Navigate to value + let value: unknown = data; + for (const key of path) { + value = (value as Record)?.[key]; + if (value === undefined || value === null) { + return value; + } + } + + // Handle functions - return metadata marker + if (typeof value === 'function') { + const fnString = value.toString(); + // Block native functions for security + if (fnString.includes('[native code]')) { + return undefined; + } + return { __isFunction: true, __name: path[path.length - 1] }; + } + + // Handle arrays - always lazy, only transfer length + if (Array.isArray(value)) { + return { + __isArray: true, + __length: value.length, + __data: null, + }; + } + + // Handle objects - return metadata with keys + if (value !== null && typeof value === 'object') { + return { + __isObject: true, + __keys: Object.keys(value), + }; + } + + // Primitive value + return value; + }); + + // Callback 2: Get array element at index + // Used by array proxy when accessing numeric indices + const getArrayElement = new ivm.Reference((path: string[], index: number) => { + // Navigate to array + let arr: unknown = data; + for (const key of path) { + arr = (arr as Record)?.[key]; + } + + if (!Array.isArray(arr)) { + return undefined; + } + + const element = arr[index]; + + // If element is object/array, return metadata + if (element !== null && typeof element === 'object') { + if (Array.isArray(element)) { + return { + __isArray: true, + __length: element.length, + __data: null, + }; + } + return { + __isObject: true, + __keys: Object.keys(element), + }; + } + + // Primitive element + return element; + }); + + // Callback 3: Call function at path with arguments + // Used when expressions invoke functions from workflow data + const callFunctionAtPath = new ivm.Reference((path: string[], ...args: unknown[]) => { + // Navigate to function + let fn: unknown = data; + for (const key of path) { + fn = (fn as Record)?.[key]; + } + + if (typeof fn !== 'function') { + throw new Error(`${path.join('.')} is not a function`); + } + + // Execute function in host context + return (fn as Function)(...args); + }); + + // Register all callbacks in isolate global context + this.context.global.setSync('__getValueAtPath', getValueAtPath); + this.context.global.setSync('__getArrayElement', getArrayElement); + this.context.global.setSync('__callFunctionAtPath', callFunctionAtPath); + + if (this.config.debug) { + console.log('[IsolatedVmBridge] Callbacks registered successfully'); + } + } + + /** + * Execute JavaScript code in the isolated context. + * + * Flow: + * 1. Register callbacks as ivm.Reference for cross-isolate communication + * 2. Call resetDataProxies() to initialize workflow data proxies + * 3. Compile script (with caching for performance) + * 4. Execute with timeout enforcement + * 5. Return result (copied from isolate) + * + * @param code - JavaScript expression to evaluate + * @param data - Workflow data (e.g., { $json: {...}, $runIndex: 0 }) + * @returns Result of the expression + * @throws {Error} If bridge not initialized or execution fails + */ + execute(code: string, data: Record): unknown { + if (!this.initialized || !this.context) { + throw new Error('Bridge not initialized. Call initialize() first.'); + } + + try { + // Step 1: Register callbacks with current data context + this.registerCallbacks(data); + + // Step 2: Reset proxies for this evaluation + // This initializes $json, $binary, etc. as lazy proxies + this.resetDataProxies(); + + // Step 3: Wrap transformed code so 'this' === __data in the isolate. + // Tournament generates: this.$json.email, this.$items(), etc. + // __data has $json, $items, etc. as lazy proxies (set in resetDataProxies). + const wrappedCode = `(function() {\n${code}\n}).call(__data)`; + + let script = this.scriptCache.get(code); + if (!script) { + script = this.isolate.compileScriptSync(wrappedCode); + this.scriptCache.set(code, script); + + if (this.config.debug) { + console.log('[IsolatedVmBridge] Script compiled and cached'); + } + } + + // Step 4: Execute with timeout and copy result back + const result = script.runSync(this.context, { + timeout: this.config.timeout, + copy: true, + }); + + if (this.config.debug) { + console.log('[IsolatedVmBridge] Expression executed successfully'); + } + + return result; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + throw new Error(`Expression evaluation failed: ${errorMessage}`); + } + } + + /** + * Dispose of the isolate and free resources. + * + * After disposal, the bridge cannot be used again. + */ + async dispose(): Promise { + if (this.disposed) { + return; + } + + // Dispose isolate (this also disposes all contexts, references, etc.) + if (!this.isolate.isDisposed) { + this.isolate.dispose(); + } + + this.disposed = true; + this.initialized = false; + this.scriptCache.clear(); + + if (this.config.debug) { + console.log('[IsolatedVmBridge] Disposed'); + } + } + + /** + * Check if the bridge has been disposed. + * + * @returns true if disposed, false otherwise + */ + isDisposed(): boolean { + return this.disposed; + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index efb519f56d..946046f01f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1213,6 +1213,9 @@ importers: packages/@n8n/expression-runtime: dependencies: + isolated-vm: + specifier: ^6.0.2 + version: 6.0.2 js-base64: specifier: 'catalog:' version: 3.7.2(patch_hash=bb02fdf69495c7b0768791b60ab6e1a002053b8decd19a174f5755691e5c9500) @@ -13687,6 +13690,10 @@ packages: resolution: {integrity: sha512-7c7mBznZu2ktfvyT582E2msM+Udc1EjOyhVRE/0ZsjD9LBtWSm23h3PtiRh2a35XoUsTQQjJXaJzuLjXsOdFDg==} engines: {node: '>=6.0'} + isolated-vm@6.0.2: + resolution: {integrity: sha512-Qw6AJuagG/VJuh2AIcSWmQPsAArti/L+lKhjXU+lyhYkbt3J57XZr+ZjgfTnOr4NJcY1r3f8f0eePS7MRGp+pg==} + engines: {node: '>=22.0.0'} + isomorphic-timers-promises@1.0.1: resolution: {integrity: sha512-u4sej9B1LPSxTGKB/HiuzvEQnXH0ECYkSVQU39koSwmFAxhlEAFl9RdTvLv4TOTQUgBS5O3O5fwUxk6byBZ+IQ==} engines: {node: '>=10'} @@ -15353,10 +15360,6 @@ packages: resolution: {integrity: sha512-IJN4O9pturuRdn60NjQ7YkFt6Rwei7ZKaOwb1tvUIIqTgeD0SDDAX3vrqZD4wcXczeEy/AsUXxpGpP/yHqV7xg==} engines: {node: '>=18.20.0 <20 || >=20.12.1'} - node-abi@3.54.0: - resolution: {integrity: sha512-p7eGEiQil0YUV3ItH4/tBb781L5impVmmx2E9FRKF7d18XXzp4PGT2tdYMFY6wQqgxD0IwNZOiSJ0/K0fSi/OA==} - engines: {node: '>=10'} - node-abi@3.75.0: resolution: {integrity: sha512-OhYaY5sDsIka7H7AtijtI9jwGYLyl29eQn/W623DiN/MIv5sUqc4g7BIDThX+gb7di9f6xK02nkp8sdfFWZLTg==} engines: {node: '>=10'} @@ -32127,6 +32130,10 @@ snapshots: iso-639-1@2.1.15: {} + isolated-vm@6.0.2: + dependencies: + prebuild-install: 7.1.3 + isomorphic-timers-promises@1.0.1: {} isomorphic.js@0.2.5: {} @@ -34509,10 +34516,6 @@ snapshots: json-stringify-safe: 5.0.1 propagate: 2.0.1 - node-abi@3.54.0: - dependencies: - semver: 7.7.3 - node-abi@3.75.0: dependencies: semver: 7.7.3 @@ -35506,7 +35509,7 @@ snapshots: minimist: 1.2.8 mkdirp-classic: 0.5.3 napi-build-utils: 2.0.0 - node-abi: 3.54.0 + node-abi: 3.75.0 pump: 3.0.0 rc: 1.2.8 simple-get: 4.0.1