Merge pull request #209 from HarperFast/vm-load-plugins

Use the VM loader for loading plugin modules as well resource modules

Author: kriszyp

components/ApplicationScope.ts

@@ -0,0 +1,49 @@
+import type { Resources } from '../resources/Resources.ts';
+import { type Server } from '../server/Server.ts';
+import { loggerWithTag } from '../utility/logging/harper_logger.js';
+import { scopedImport } from '../security/jsLoader.ts';
+import * as env from '../utility/environment/environmentManager.js';
+import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
+
+export class MissingDefaultFilesOptionError extends Error {
+	constructor() {
+		super('No default files option exists. Ensure `files` is specified in config.yaml');
+		this.name = 'MissingDefaultFilesOptionError';
+	}
+}
+
+/**
+ * This class is used to represent the application scope for the VM context used for loading modules within an application
+ */
+export class ApplicationScope {
+	logger: any;
+	resources: Resources;
+	server: Server;
+	mode?: 'none' | 'vm' | 'compartment'; // option to set this from the scope
+	dependencyContainment?: boolean; // option to set this from the scope
+	verifyPath?: string;
+	config: any;
+	constructor(name: string, resources: Resources, server: Server, verifyPath?: string) {
+		this.logger = loggerWithTag(name);
+
+		this.resources = resources;
+		this.server = server;
+
+		this.mode = env.get(CONFIG_PARAMS.APPLICATIONS_CONTAINMENT) ?? 'vm';
+		this.dependencyContainment = Boolean(env.get(CONFIG_PARAMS.APPLICATIONS_DEPENDENCYCONTAINMENT));
+		this.verifyPath = verifyPath;
+	}
+
+	/**
+	 * The compartment that is used for this scope and any imports that it makes
+	 */
+	compartment?: Promise<any>;
+	/**
+	 * Import a file into the scope's sandbox.
+	 * @param filePath - The path of the file to import.
+	 * @returns A promise that resolves with the imported module or value.
+	 */
+	async import(filePath: string): Promise<unknown> {
+		return scopedImport(filePath, this);
+	}
+}

components/Scope.ts

@@ -1,16 +1,14 @@
 import { type Logger } from '../utility/logging/logger.ts';
 import { loggerWithTag } from '../utility/logging/harper_logger.js';
 import { EventEmitter, once } from 'node:events';
-import { type Server } from '../server/Server.ts';
+import { server, type Server } from '../server/Server.ts';
 import { EntryHandler, type EntryHandlerEventMap, type onEntryEventHandler } from './EntryHandler.ts';
 import { OptionsWatcher, OptionsWatcherEventMap } from './OptionsWatcher.ts';
-import type { Resources } from '../resources/Resources.ts';
+import { resources, type Resources } from '../resources/Resources.ts';
 import type { FileAndURLPathConfig } from './Component.ts';
 import { FilesOption } from './deriveGlobOptions.ts';
 import { requestRestart } from './requestRestart.ts';
-import { scopedImport } from '../security/jsLoader.ts';
-import * as env from '../utility/environment/environmentManager.js';
-import { CONFIG_PARAMS } from '../utility/hdbTerms';
+import { ApplicationScope } from './ApplicationScope.ts';
 
 export class MissingDefaultFilesOptionError extends Error {
 	constructor() {
@@ -19,12 +17,6 @@ export class MissingDefaultFilesOptionError extends Error {
 	}
 }
 
-export interface ApplicationContainment {
-	mode?: 'none' | 'vm' | 'compartment'; // option to set this from the scope
-	dependencyContainment?: boolean; // option to set this from the scope
-	verifyPath?: string;
-}
-
 /**
  * This class is what is passed to the `handleApplication` function of an extension.
  *
@@ -40,22 +32,23 @@ export class Scope extends EventEmitter {
 	#entryHandlers: EntryHandler[];
 	#logger: Logger;
 	#pendingInitialLoads: Set<Promise<void>>;
+	applicationScope?: ApplicationScope;
 
 	options: OptionsWatcher;
-	resources: Resources;
-	server: Server;
+	resources?: Resources;
+	server?: Server;
 	ready: Promise<any[]>;
-	applicationContainment?: ApplicationContainment;
-	constructor(name: string, directory: string, configFilePath: string, resources: Resources, server: Server) {
+	constructor(name: string, directory: string, configFilePath: string, applicationScope: ApplicationScope) {
 		super();
 
 		this.#name = name;
 		this.#directory = directory;
 		this.#configFilePath = configFilePath;
 		this.#logger = loggerWithTag(this.#name);
 
-		this.resources = resources;
-		this.server = server;
+		this.applicationScope = applicationScope;
+		this.resources = applicationScope?.resources ?? resources;
+		this.server = applicationScope?.server ?? server;
 
 		this.#entryHandlers = [];
 		this.#pendingInitialLoads = new Set();
@@ -67,12 +60,6 @@ export class Scope extends EventEmitter {
 			.on('error', this.#handleError.bind(this))
 			.on('change', this.#optionsWatcherChangeListener.bind(this)())
 			.on('ready', this.#handleOptionsWatcherReady.bind(this));
-
-		this.applicationContainment = {
-			mode: env.get(CONFIG_PARAMS.APPLICATIONS_CONTAINMENT) ?? 'vm',
-			dependencyContainment: Boolean(env.get(CONFIG_PARAMS.APPLICATIONS_DEPENDENCYCONTAINMENT)),
-			verifyPath: directory,
-		};
 	}
 
 	get logger(): Logger {
@@ -307,16 +294,12 @@ export class Scope extends EventEmitter {
 		}
 	}
 
-	/**
-	 * The compartment that is used for this scope and any imports that it makes
-	 */
-	compartment?: Promise<any>;
 	/**
 	 * Import a file into the scope's sandbox.
 	 * @param filePath - The path of the file to import.
 	 * @returns A promise that resolves with the imported module or value.
 	 */
 	async import(filePath: string): Promise<unknown> {
-		return scopedImport(filePath, this);
+		return this.applicationScope ? this.applicationScope.import(filePath) : import(filePath);
 	}
 }

components/componentLoader.ts

@@ -29,7 +29,8 @@ import * as mqtt from '../server/mqtt.ts';
 import { getConfigObj, resolvePath } from '../config/configUtils.js';
 import { createReuseportFd } from '../server/serverHelpers/Request.ts';
 import { ErrorResource } from '../resources/ErrorResource.ts';
-import { Scope, type ApplicationContainment } from './Scope.ts';
+import { Scope } from './Scope.ts';
+import { ApplicationScope } from './ApplicationScope.ts';
 import { ComponentV1, processResourceExtensionComponent } from './ComponentV1.ts';
 import * as httpComponent from '../server/http.ts';
 import { Status } from '../server/status/index.ts';
@@ -219,6 +220,7 @@ function sequentiallyHandleApplication(scope: Scope, plugin: PluginModule) {
 
 export interface LoadComponentOptions {
 	isRoot?: boolean;
+	applicationScope?: ApplicationScope;
 	autoReload?: boolean;
 	applicationContainment?: ApplicationContainment;
 	providedLoadedComponents?: Map<any, any>;
@@ -241,7 +243,14 @@ export async function loadComponent(
 	const resolvedFolder = realpathSync(componentDirectory);
 	if (loadedPaths.has(resolvedFolder)) return loadedPaths.get(resolvedFolder);
 	loadedPaths.set(resolvedFolder, true);
-	const { providedLoadedComponents, isRoot, autoReload } = options;
+
+	const {
+		providedLoadedComponents,
+		applicationScope = new ApplicationScope(basename(componentDirectory), resources, server),
+		isRoot,
+		autoReload,
+	} = options;
+	applicationScope.verifyPath ??= componentDirectory;
 	if (providedLoadedComponents) loadedComponents = providedLoadedComponents;
 	try {
 		let config;
@@ -257,6 +266,7 @@ export async function loadComponent(
 		} else {
 			config = DEFAULT_CONFIG;
 		}
+		applicationScope.config ??= config;
 
 		if (!isRoot) {
 			try {
@@ -286,6 +296,8 @@ export async function loadComponent(
 			// Initialize loading status for all components (applications and extensions)
 			componentLifecycle.loading(componentStatusName);
 
+			const subApplicationScope = isRoot ? new ApplicationScope(componentName, resources, server) : applicationScope;
+
 			let extensionModule: any;
 			const pkg = componentConfig.package;
 			try {
@@ -306,8 +318,11 @@ export async function loadComponent(
 						}
 					}
 					if (componentPath) {
+						subApplicationScope.verifyPath = componentPath;
 						if (!process.env.HARPER_SAFE_MODE) {
-							extensionModule = await loadComponent(componentPath, resources, origin);
+							extensionModule = await loadComponent(componentPath, resources, origin, {
+								applicationScope: subApplicationScope,
+							});
 							componentFunctionality[componentName] = true;
 						}
 					} else {
@@ -317,7 +332,7 @@ export async function loadComponent(
 					const plugin = TRUSTED_RESOURCE_PLUGINS[componentName];
 					extensionModule =
 						typeof plugin === 'string'
-							? await scopedImport(plugin.startsWith('@/') ? join(PACKAGE_ROOT, plugin.slice(1)) : plugin)
+							? await import(plugin.startsWith('@/') ? join(PACKAGE_ROOT, plugin.slice(1)) : plugin)
 							: plugin;
 				}
 
@@ -358,8 +373,7 @@ export async function loadComponent(
 
 				// New Plugin API (`handleApplication`)
 				if (resources.isWorker && extensionModule.handleApplication) {
-					const scope = new Scope(componentName, componentDirectory, configPath, resources, server);
-					if (options.applicationContainment) scope.applicationContainment = options.applicationContainment;
+					const scope = new Scope(componentName, componentDirectory, configPath, applicationScope);
 
 					await sequentiallyHandleApplication(scope, extensionModule);
 
@@ -464,7 +478,10 @@ export async function loadComponent(
 			});
 		}
 		if (config.extensionModule || config.pluginModule) {
-			const extensionModule = await import(join(componentDirectory, config.extensionModule || config.pluginModule));
+			const extensionModule = await scopedImport(
+				join(componentDirectory, config.extensionModule || config.pluginModule),
+				applicationScope
+			);
 			loadedPaths.set(resolvedFolder, extensionModule);
 			return extensionModule;
 		}

security/jsLoader.ts

@@ -7,7 +7,7 @@ import { readFileSync } from 'node:fs';
 import { dirname, isAbsolute } from 'node:path';
 import { pathToFileURL, fileURLToPath } from 'node:url';
 import { SourceTextModule, SyntheticModule, createContext, runInContext } from 'node:vm';
-import { Scope } from '../components/Scope.ts';
+import { ApplicationScope } from '../components/ApplicationScope.ts';
 import logger from '../utility/logging/harper_logger.js';
 import { createRequire } from 'node:module';
 import * as env from '../utility/environment/environmentManager';
@@ -25,7 +25,7 @@ let lockedDown = false;
  * @param moduleUrl
  * @param scope
  */
-export async function scopedImport(filePath: string | URL, scope?: Scope) {
+export async function scopedImport(filePath: string | URL, scope?: ApplicationScope) {
 	if (!lockedDown && APPLICATIONS_LOCKDOWN && APPLICATIONS_LOCKDOWN !== 'none') {
 		lockedDown = true;
 		if (APPLICATIONS_LOCKDOWN === 'ses') {
@@ -76,7 +76,7 @@ export async function scopedImport(filePath: string | URL, scope?: Scope) {
 	}
 	const moduleUrl = (filePath instanceof URL ? filePath : pathToFileURL(filePath)).toString();
 	try {
-		const containmentMode = scope?.applicationContainment.mode;
+		const containmentMode = scope?.mode;
 		if (scope && containmentMode !== 'none') {
 			if (containmentMode === 'compartment') {
 				// use SES Compartments
@@ -110,7 +110,7 @@ export async function scopedImport(filePath: string | URL, scope?: Scope) {
 /**
  * Load a module using Node's vm.Module API with (not really secure) sandboxing
  */
-async function loadModuleWithVM(moduleUrl: string, scope: Scope) {
+async function loadModuleWithVM(moduleUrl: string, scope: ApplicationScope) {
 	const moduleCache = new Map<string, Promise<SourceTextModule | SyntheticModule>>();
 
 	// Create a secure context with limited globals
@@ -167,7 +167,7 @@ async function loadModuleWithVM(moduleUrl: string, scope: Scope) {
 			filename: url,
 			async importModuleDynamically(specifier: string, script) {
 				const resolvedUrl = resolveModule(specifier, script.sourceURL);
-				const useContainment = specifier.startsWith('.') || scope.applicationContainment.dependencyContainment;
+				const useContainment = specifier.startsWith('.') || scope.dependencyContainment;
 				const dynamicModule = await loadModuleWithCache(resolvedUrl, useContainment);
 				return dynamicModule;
 			},
@@ -213,7 +213,7 @@ async function loadModuleWithVM(moduleUrl: string, scope: Scope) {
 			return moduleCache.get(resolvedUrl)!;
 		}
 
-		const useContainment = specifier.startsWith('.') || scope.applicationContainment.dependencyContainment;
+		const useContainment = specifier.startsWith('.') || scope.dependencyContainment;
 		// Load the module
 		return await loadModuleWithCache(resolvedUrl, useContainment);
 	}
@@ -246,7 +246,7 @@ async function loadModuleWithVM(moduleUrl: string, scope: Scope) {
 				{ identifier: url, context }
 			);
 		} else if (usePrivateGlobal && url.startsWith('file://')) {
-			checkAllowedModulePath(url, scope.applicationContainment.verifyPath);
+			checkAllowedModulePath(url, scope.verifyPath);
 			// Load source text from file
 			const source = await readFile(new URL(url), { encoding: 'utf-8' });
 
@@ -286,7 +286,7 @@ async function loadModuleWithVM(moduleUrl: string, scope: Scope) {
 				}
 			}
 		} else {
-			checkAllowedModulePath(url, scope.applicationContainment.verifyPath);
+			checkAllowedModulePath(url, scope.verifyPath);
 			// For Node.js built-in modules (node:) and npm packages, use dynamic import
 			const importedModule = await import(url);
 			const exportNames = Object.keys(importedModule);
@@ -318,7 +318,7 @@ async function loadModuleWithVM(moduleUrl: string, scope: Scope) {
 	return entryModule.namespace;
 }
 
-async function getCompartment(scope: Scope, globals) {
+async function getCompartment(scope: ApplicationScope, globals) {
 	const { StaticModuleRecord } = await import('@endo/static-module-record');
 	require('ses');
 	const compartment: CompartmentOptions = new (Compartment as typeof CompartmentOptions)(
@@ -351,7 +351,7 @@ async function getCompartment(scope: Scope, globals) {
 					const moduleText = await readFile(new URL(moduleSpecifier), { encoding: 'utf-8' });
 					return new StaticModuleRecord(moduleText, moduleSpecifier);
 				} else {
-					checkAllowedModulePath(moduleSpecifier, scope.applicationContainment.verifyPath);
+					checkAllowedModulePath(moduleSpecifier, scope.verifyPath);
 					const moduleExports = await import(moduleSpecifier);
 					return {
 						imports: [],
@@ -398,7 +398,7 @@ function getDefaultJSGlobalNames() {
 /**
  * Get the set of global variables that should be available to modules that run in scoped compartments/contexts.
  */
-function getGlobalObject(scope: Scope) {
+function getGlobalObject(scope: ApplicationScope) {
 	const appGlobal = {};
 	// create the new global object, assigning all the global variables from this global
 	// except those that will be natural intrinsics of the new VM
@@ -411,20 +411,20 @@ function getGlobalObject(scope: Scope) {
 		server: scope.server ?? server,
 		logger: scope.logger ?? logger,
 		resources: scope.resources,
-		config: scope.options.getRoot() ?? {},
+		config: scope.config ?? {},
 		fetch: APPLICATIONS_LOCKDOWN === 'ses' ? secureOnlyFetch : fetch,
 		console,
 		global: appGlobal,
 		harper: getHarperExports(scope),
 	});
 	return appGlobal;
 }
-function getHarperExports(scope: Scope) {
+function getHarperExports(scope: ApplicationScope) {
 	return {
 		server: scope.server ?? server,
 		logger: scope.logger ?? logger,
 		resources: scope.resources,
-		config: scope.options.getRoot() ?? {},
+		config: scope.config ?? {},
 		Resource,
 		tables,
 		databases,
@@ -465,10 +465,10 @@ const ALLOWED_NODE_BUILTIN_MODULES = new Set([
 function checkAllowedModulePath(moduleUrl: string, containingFolder: string): boolean {
 	if (moduleUrl.startsWith('file:')) {
 		const path = moduleUrl.slice(7);
-		if (path.startsWith(containingFolder)) {
+		if (!containingFolder || path.startsWith(containingFolder)) {
 			return true;
 		}
-		throw new Error(`Can not load module outside of application folder`);
+		throw new Error(`Can not load module outside of application folder ${containingFolder}`);
 	}
 	let simpleName = moduleUrl.startsWith('node:') ? moduleUrl.slice(5) : moduleUrl;
 	simpleName = simpleName.split('/')[0];