WIP: Share /wordpress and /internal FS between PHP processes

Author: brandonpayton

packages/php-wasm/compile/php/php_wasm.c

@@ -241,7 +241,7 @@ EM_JS(int, wasm_poll_socket, (php_socket_t socketd, int events, int timeout), {
 						if (stream.stream_ops?.poll) {
 							mask = stream.stream_ops.poll(stream, -1);
 						}
-						
+
 						mask &= events | POLLERR | POLLHUP;
 						if (mask) {
 							return mask;
@@ -418,7 +418,7 @@ EM_JS(__wasi_errno_t, js_fd_read, (__wasi_fd_t fd, const __wasi_iovec_t *iov, si
                 HEAPU32[pnum >> 2] = num;
                 return wakeUp(returnCode);
             }
-            
+
             // It's a blocking stream and we Blocking stream with no data available yet.
             // Let's poll up to a timeout.
             await new Promise(resolve => setTimeout(resolve, interval));
@@ -1026,8 +1026,8 @@ int main(int argc, char *argv[]);
 int run_cli()
 {
 	// See wasm_sapi_request_init() for details on why we need to redirect stdout and stderr.
-	stdout_replacement = redirect_stream_to_file(stdout, "/internal/stdout");
-	stderr_replacement = redirect_stream_to_file(stderr, "/internal/stderr");
+	stdout_replacement = redirect_stream_to_file(stdout, "/request/stdout");
+	stderr_replacement = redirect_stream_to_file(stderr, "/request/stderr");
 	if (stdout_replacement == -1 || stderr_replacement == -1)
 	{
 		return -1;
@@ -1337,7 +1337,7 @@ void wasm_set_request_port(int port)
  *
  *   stream: The stream to redirect, e.g. stdout or stderr.
  *
- *   path: The path to the file to redirect to, e.g. "/internal/stdout".
+ *   path: The path to the file to redirect to, e.g. "/request/stdout".
  *
  *   returns: The exit code: 0 on success, -1 on failure.
  */
@@ -1559,11 +1559,11 @@ int wasm_sapi_request_init()
 	// Write to files instead of stdout and stderr because Emscripten truncates null
 	// bytes from stdout and stderr, and null bytes are a valid output when streaming
 	// binary data.
-	// We use our custom Emscripten-defined /internal/std* devices and handle the output in JavaScript.
+	// We use our custom Emscripten-defined /request/std* devices and handle the output in JavaScript.
 	// These /internal devices are not thread-safe and should always stay in per-process MEMFS space.
 	// Sharing them between PHP instances may cause intertwined output.
-	stdout_replacement = redirect_stream_to_file(stdout, "/internal/stdout");
-	stderr_replacement = redirect_stream_to_file(stderr, "/internal/stderr");
+	stdout_replacement = redirect_stream_to_file(stdout, "/request/stdout");
+	stderr_replacement = redirect_stream_to_file(stderr, "/request/stderr");
 	if (stdout_replacement == -1 || stderr_replacement == -1)
 	{
 		return -1;
@@ -1821,7 +1821,7 @@ FILE *headers_file;
  */
 static int wasm_sapi_send_headers(sapi_headers_struct *sapi_headers TSRMLS_DC)
 {
-	headers_file = fopen("/internal/headers", "w");
+	headers_file = fopen("/request/headers", "w");
 	if (headers_file == NULL)
 	{
 		return FAILURE;

packages/php-wasm/compile/php/phpwasm-emscripten-library.js

@@ -38,22 +38,27 @@ const LibraryExample = {
 				.filter(Boolean)
 				.join(':');
 
-			// The /internal directory is required by the C module. It's where the
+			// The /request directory is required by the C module. It's where the
 			// stdout, stderr, and headers information are written for the JavaScript
-			// code to read later on.
+			// code to read later on. This is per-request state that is isolated to a
+			// single PHP process.
+			FS.mkdir('/request');
+			// The /internal directory is shared amongst all PHP processes
+			// and contains wp-config.php, constants, etc.
 			FS.mkdir('/internal');
-			// The files from the shared directory are shared between all th
-			// PHP processes managed by PHPProcessManager.
-			FS.mkdir('/internal/shared');
 
 			if (phpWasmInitOptions?.nativeInternalDirPath) {
 				FS.mount(
 					FS.filesystems.NODEFS,
 					{ root: phpWasmInitOptions.nativeInternalDirPath },
-					'/internal/shared'
+					'/internal'
 				);
 			}
 
+			// The files from the shared directory are shared between all the
+			// PHP processes managed by PHPProcessManager.
+			FS.mkdirTree('/internal/shared');
+
 			// The files from the preload directory are preloaded using the
 			// auto_prepend_file php.ini directive.
 			FS.mkdirTree('/internal/shared/preload');
@@ -92,7 +97,7 @@ const LibraryExample = {
 					return length;
 				},
 			});
-			FS.mkdev('/internal/stdout', FS.makedev(64, 0));
+			FS.mkdev('/request/stdout', FS.makedev(64, 0));
 
 			FS.registerDevice(FS.makedev(63, 0), {
 				open: () => {},
@@ -104,7 +109,7 @@ const LibraryExample = {
 					return length;
 				},
 			});
-			FS.mkdev('/internal/stderr', FS.makedev(63, 0));
+			FS.mkdev('/request/stderr', FS.makedev(63, 0));
 
 			FS.registerDevice(FS.makedev(62, 0), {
 				open: () => {},
@@ -116,7 +121,7 @@ const LibraryExample = {
 					return length;
 				},
 			});
-			FS.mkdev('/internal/headers', FS.makedev(62, 0));
+			FS.mkdev('/request/headers', FS.makedev(62, 0));
 
 			// Handle events.
 			PHPWASM.EventEmitter = ENVIRONMENT_IS_NODE

packages/php-wasm/universal/src/lib/index.ts

@@ -79,7 +79,10 @@ export {
 } from './urls';
 
 export { isExitCode } from './is-exit-code';
-export { proxyFileSystem } from './proxy-file-system';
+export {
+	proxyFileSystem,
+	proxyNonSharedMemoryFileSystems,
+} from './proxy-file-system';
 export { sandboxedSpawnHandlerFactory } from './sandboxed-spawn-handler-factory';
 
 export * from './api';

packages/playground/cli/src/blueprints-v1/blueprints-v1-handler.ts

@@ -136,9 +136,9 @@ export class BlueprintsV1Handler {
 
 		await playground.useFileLockManager(fileLockManagerPort);
 		await playground.bootAsPrimaryWorker({
-			phpVersion: this.phpVersion,
+			php: this.phpVersion,
 			wpVersion: compiledBlueprint.versions.wp,
-			absoluteUrl: this.siteUrl,
+			siteUrl: this.siteUrl,
 			mountsBeforeWpInstall,
 			mountsAfterWpInstall,
 			wordPressZip: wordPressZip && (await wordPressZip!.arrayBuffer()),
@@ -187,17 +187,10 @@ export class BlueprintsV1Handler {
 		await additionalPlayground.isConnected();
 		await additionalPlayground.useFileLockManager(fileLockManagerPort);
 		await additionalPlayground.bootAsSecondaryWorker({
-			phpVersion: this.phpVersion,
-			absoluteUrl: this.siteUrl,
+			php: this.phpVersion!,
+			siteUrl: this.siteUrl,
 			mountsBeforeWpInstall: this.args['mount-before-install'] || [],
 			mountsAfterWpInstall: this.args['mount'] || [],
-			// Skip WordPress zip because we share the /wordpress directory
-			// populated by the initial worker.
-			wordPressZip: undefined,
-			// Skip SQLite integration plugin for now because we
-			// will copy it from primary's `/internal` directory.
-			sqliteIntegrationPluginZip: undefined,
-			dataSqlPath: '/wordpress/wp-content/database/.ht.sqlite',
 			firstProcessId,
 			processIdSpaceLength: this.processIdSpaceLength,
 			followSymlinks: this.args.followSymlinks === true,

packages/playground/cli/src/blueprints-v1/worker-thread-v1.ts

@@ -11,7 +11,7 @@ import {
 } from '@php-wasm/universal';
 import { sprintf } from '@php-wasm/util';
 import { RecommendedPHPVersion } from '@wp-playground/common';
-import { bootWordPress } from '@wp-playground/wordpress';
+import { bootWordPress, bootRequestHandler } from '@wp-playground/wordpress';
 import { rootCertificates } from 'tls';
 import { jspi } from 'wasm-feature-detect';
 import { MessageChannel, type MessagePort, parentPort } from 'worker_threads';
@@ -23,16 +23,12 @@ export interface Mount {
 }
 
 export type WorkerBootOptions = {
-	wpVersion?: string;
-	phpVersion?: SupportedPHPVersion;
-	absoluteUrl: string;
+	php: SupportedPHPVersion;
+	siteUrl: string;
 	mountsBeforeWpInstall: Array<Mount>;
 	mountsAfterWpInstall: Array<Mount>;
-	wordPressZip?: ArrayBuffer;
-	sqliteIntegrationPluginZip?: ArrayBuffer;
 	firstProcessId: number;
 	processIdSpaceLength: number;
-	dataSqlPath?: string;
 	followSymlinks: boolean;
 	trace: boolean;
 	/**
@@ -44,9 +40,26 @@ export type WorkerBootOptions = {
 	 */
 	internalCookieStore?: boolean;
 	withXdebug?: boolean;
-	nativeInternalDirPath?: string;
+	nativeInternalDirPath: string;
+};
+
+export type PrimaryWorkerBootOptions = WorkerBootOptions & {
+	wpVersion?: string;
+	wordPressZip?: ArrayBuffer;
+	sqliteIntegrationPluginZip?: ArrayBuffer;
+	dataSqlPath?: string;
 };
 
+interface WorkerBootRequestHandlerOptions {
+	siteUrl: string;
+	allow?: string;
+	php: SupportedPHPVersion;
+	firstProcessId: number;
+	processIdSpaceLength: number;
+	trace: boolean;
+	nativeInternalDirPath: string;
+}
+
 /**
  * Print trace messages from PHP-WASM.
  *
@@ -105,10 +118,10 @@ export class PlaygroundCliBlueprintV1Worker extends PHPWorker {
 	}
 
 	async bootAsPrimaryWorker({
-		absoluteUrl,
+		siteUrl,
 		mountsBeforeWpInstall,
 		mountsAfterWpInstall,
-		phpVersion = RecommendedPHPVersion,
+		php = RecommendedPHPVersion,
 		wordPressZip,
 		sqliteIntegrationPluginZip,
 		firstProcessId,
@@ -119,7 +132,7 @@ export class PlaygroundCliBlueprintV1Worker extends PHPWorker {
 		internalCookieStore,
 		withXdebug,
 		nativeInternalDirPath,
-	}: WorkerBootOptions) {
+	}: PrimaryWorkerBootOptions) {
 		if (this.booted) {
 			throw new Error('Playground already booted');
 		}
@@ -137,7 +150,7 @@ export class PlaygroundCliBlueprintV1Worker extends PHPWorker {
 				};
 
 			const requestHandler = await bootWordPress({
-				siteUrl: absoluteUrl,
+				siteUrl,
 				createPhpRuntime: async () => {
 					const processId = nextProcessId;
 
@@ -148,7 +161,7 @@ export class PlaygroundCliBlueprintV1Worker extends PHPWorker {
 						nextProcessId = firstProcessId;
 					}
 
-					return await loadNodeRuntime(phpVersion, {
+					return await loadNodeRuntime(php, {
 						emscriptenOptions: {
 							fileLockManager: this.fileLockManager!,
 							processId,
@@ -207,7 +220,72 @@ export class PlaygroundCliBlueprintV1Worker extends PHPWorker {
 	}
 
 	async bootAsSecondaryWorker(args: WorkerBootOptions) {
-		return this.bootAsPrimaryWorker(args);
+		await this.bootRequestHandler(args);
+		const primaryPhp = this.__internal_getPHP()!;
+		// When secondary workers are spawned, WordPress is already installed.
+		await mountResources(primaryPhp, args.mountsBeforeWpInstall || []);
+		await mountResources(primaryPhp, args.mountsAfterWpInstall || []);
+	}
+
+	async bootRequestHandler({
+		siteUrl,
+		allow,
+		php,
+		firstProcessId,
+		processIdSpaceLength,
+		trace,
+		nativeInternalDirPath,
+	}: WorkerBootRequestHandlerOptions) {
+		if (this.booted) {
+			throw new Error('Playground already booted');
+		}
+		this.booted = true;
+
+		let nextProcessId = firstProcessId;
+		const lastProcessId = firstProcessId + processIdSpaceLength - 1;
+
+		try {
+			const requestHandler = await bootRequestHandler({
+				siteUrl,
+				createPhpRuntime: async () => {
+					const processId = nextProcessId;
+
+					if (nextProcessId < lastProcessId) {
+						nextProcessId++;
+					} else {
+						// We've reached the end of the process ID space. Start over.
+						nextProcessId = firstProcessId;
+					}
+
+					return await loadNodeRuntime(php!, {
+						emscriptenOptions: {
+							fileLockManager: this.fileLockManager!,
+							processId,
+							trace: trace ? tracePhpWasm : undefined,
+							ENV: {
+								DOCROOT: '/wordpress',
+							},
+							phpWasmInitOptions: {
+								nativeInternalDirPath,
+							},
+						},
+						followSymlinks: allow?.includes('follow-symlinks'),
+					});
+				},
+				sapiName: 'cli',
+				cookieStore: false,
+				spawnHandler: sandboxedSpawnHandlerFactory,
+			});
+			this.__internal_setRequestHandler(requestHandler);
+
+			const primaryPhp = await requestHandler.getPrimaryPhp();
+			await this.setPrimaryPHP(primaryPhp);
+
+			setApiReady();
+		} catch (e) {
+			setAPIError(e as Error);
+			throw e;
+		}
 	}
 
 	// Provide a named disposal method that can be invoked via comlink.