feat: add VirtualModulesPlugin (#11021)

Author: nilptr

Cargo.toml

@@ -254,7 +254,7 @@ strip         = "debuginfo"
 [profile.release]
 codegen-units = 1
 debug         = false
-# Performs “fat” LTO which attempts to perform optimizations across all crates within the dependency graph.
+# Performs "fat" LTO which attempts to perform optimizations across all crates within the dependency graph.
 lto       = "fat"
 opt-level = 3
 panic     = "abort"

crates/node_binding/napi-binding.d.ts

@@ -328,6 +328,7 @@ export declare class JsCompiler {
   /** Rebuild with the given option passed to the constructor */
   rebuild(changed_files: string[], removed_files: string[], callback: (err: null | Error) => void): void
   close(): Promise<void>
+  getVirtualFileStore(): VirtualFileStore | null
 }
 
 export declare class JsContextModuleFactoryAfterResolveData {
@@ -443,6 +444,7 @@ export declare class ModuleGraphConnection {
 export declare class NativeWatcher {
   constructor(options: NativeWatcherOptions)
   watch(files: [Array<string>, Array<string>], directories: [Array<string>, Array<string>], missing: [Array<string>, Array<string>], callback: (err: Error | null, result: NativeWatchResult) => void, callbackUndelayed: (path: string) => void): void
+  triggerEvent(kind: 'change' | 'remove' | 'create', path: string): void
   /**
    * # Safety
    *
@@ -487,6 +489,12 @@ export declare class Sources {
   _get(sourceType: string): JsCompatSourceOwned | null
 }
 
+export declare class VirtualFileStore {
+  writeVirtualFileSync(path: string, content: string): void
+  batchWriteVirtualFilesSync(files: Array<JsVirtualFile>): void
+}
+export type JsVirtualFileStore = VirtualFileStore
+
 export declare function async(path: string, request: string): Promise<ResolveResult>
 
 export interface BuiltinPlugin {
@@ -1479,6 +1487,11 @@ export interface JsTap {
   stage: number
 }
 
+export interface JsVirtualFile {
+  path: string
+  content: string
+}
+
 export interface KnownAssetInfo {
   /** if the asset can be long term cached forever (contains a hash) */
   immutable?: boolean
@@ -2535,6 +2548,7 @@ export interface RawOptions {
   amd?: string
   bail: boolean
   __references: Record<string, any>
+  __virtual_files?: Array<JsVirtualFile>
 }
 
 export interface RawOutputOptions {

crates/node_binding/rspack.wasi-browser.js

@@ -102,6 +102,8 @@ export const RawExternalItemFnCtx = __napiModule.exports.RawExternalItemFnCtx
 export const ReadonlyResourceData = __napiModule.exports.ReadonlyResourceData
 export const ResolverFactory = __napiModule.exports.ResolverFactory
 export const Sources = __napiModule.exports.Sources
+export const VirtualFileStore = __napiModule.exports.VirtualFileStore
+export const JsVirtualFileStore = __napiModule.exports.JsVirtualFileStore
 export const async = __napiModule.exports.async
 export const BuiltinPluginName = __napiModule.exports.BuiltinPluginName
 export const cleanupGlobalTrace = __napiModule.exports.cleanupGlobalTrace

crates/node_binding/rspack.wasi.cjs

@@ -147,6 +147,8 @@ module.exports.RawExternalItemFnCtx = __napiModule.exports.RawExternalItemFnCtx
 module.exports.ReadonlyResourceData = __napiModule.exports.ReadonlyResourceData
 module.exports.ResolverFactory = __napiModule.exports.ResolverFactory
 module.exports.Sources = __napiModule.exports.Sources
+module.exports.VirtualFileStore = __napiModule.exports.VirtualFileStore
+module.exports.JsVirtualFileStore = __napiModule.exports.JsVirtualFileStore
 module.exports.async = __napiModule.exports.async
 module.exports.BuiltinPluginName = __napiModule.exports.BuiltinPluginName
 module.exports.cleanupGlobalTrace = __napiModule.exports.cleanupGlobalTrace

crates/rspack_binding_api/src/lib.rs

@@ -97,8 +97,12 @@ mod stats;
 mod swc;
 mod trace_event;
 mod utils;
+mod virtual_modules;
 
-use std::{cell::RefCell, sync::Arc};
+use std::{
+  cell::RefCell,
+  sync::{Arc, RwLock},
+};
 
 use napi::{CallContext, bindgen_prelude::*};
 pub use raw_options::{CustomPluginBuilder, register_custom_plugin};
@@ -137,6 +141,9 @@ use crate::{
   resolver_factory::JsResolverFactory,
   trace_event::RawTraceEvent,
   utils::callbackify,
+  virtual_modules::{
+    JsVirtualFileStore, TrieVirtualFileStore, VirtualFileStore, VirtualFileSystem,
+  },
 };
 
 // Export expected @rspack/core version
@@ -166,6 +173,7 @@ struct JsCompiler {
   include_dependencies_map: FxHashMap<String, FxHashMap<EntryOptions, BoxDependency>>,
   entry_dependencies_map: FxHashMap<String, FxHashMap<EntryOptions, BoxDependency>>,
   compiler_context: Arc<CompilerContext>,
+  virtual_file_store: Option<Arc<RwLock<dyn VirtualFileStore>>>,
 }
 
 #[napi]
@@ -218,12 +226,14 @@ impl JsCompiler {
         bp.append_to(env, &mut this, &mut plugins)?;
       }
 
+      let pnp = options.resolve.pnp.unwrap_or(false);
+      let virtual_files = options.__virtual_files.take();
       let use_input_fs = options.experiments.use_input_file_system.take();
       let compiler_options: rspack_core::CompilerOptions = options.try_into().to_napi_result()?;
 
       tracing::debug!(name:"normalized_options", options=?&compiler_options);
 
-      let input_file_system: Option<Arc<dyn ReadableFileSystem>> =
+      let mut input_file_system: Option<Arc<dyn ReadableFileSystem>> =
         input_filesystem.and_then(|fs| {
           use_input_fs.and_then(|use_input_file_system| {
             let node_fs = NodeFileSystem::new(fs).expect("Failed to create readable filesystem");
@@ -245,6 +255,31 @@ impl JsCompiler {
           })
         });
 
+      let mut virtual_file_store: Option<Arc<RwLock<dyn VirtualFileStore>>> = None;
+      if let Some(list) = virtual_files {
+        let store = {
+          let mut store = TrieVirtualFileStore::new();
+          for f in list {
+            store.write_file(f.path.as_str().into(), f.content.into());
+          }
+          Arc::new(RwLock::new(store))
+        };
+        input_file_system = input_file_system
+          .map(|real_fs| {
+            let binding: Arc<dyn ReadableFileSystem> =
+              Arc::new(VirtualFileSystem::new(real_fs, store.clone()));
+            binding
+          })
+          .or_else(|| {
+            let binding: Arc<dyn ReadableFileSystem> = Arc::new(VirtualFileSystem::new(
+              Arc::new(NativeFileSystem::new(pnp)),
+              store.clone(),
+            ));
+            Some(binding)
+          });
+        virtual_file_store = Some(store);
+      }
+
       resolver_factory_reference.update_options(
         input_file_system.clone(),
         compiler_options.resolve.clone(),
@@ -288,6 +323,7 @@ impl JsCompiler {
         include_dependencies_map: Default::default(),
         entry_dependencies_map: Default::default(),
         compiler_context,
+        virtual_file_store,
       })
     })
   }
@@ -368,6 +404,14 @@ impl JsCompiler {
       })?;
     Ok(())
   }
+
+  #[napi]
+  pub fn get_virtual_file_store(&self) -> Option<JsVirtualFileStore> {
+    self
+      .virtual_file_store
+      .as_ref()
+      .map(|store| JsVirtualFileStore::new(store.clone()))
+  }
 }
 
 struct RunGuard {

crates/rspack_binding_api/src/native_watcher.rs

@@ -1,8 +1,11 @@
-use std::{boxed::Box, path::PathBuf};
+use std::{
+  boxed::Box,
+  path::{Path, PathBuf},
+};
 
 use napi::bindgen_prelude::*;
 use napi_derive::*;
-use rspack_fs::{FsWatcher, FsWatcherIgnored, FsWatcherOptions};
+use rspack_fs::{FsEventKind, FsWatcher, FsWatcherIgnored, FsWatcherOptions};
 use rspack_paths::ArcPath;
 use rspack_regex::RspackRegex;
 
@@ -106,6 +109,20 @@ impl NativeWatcher {
     Ok(())
   }
 
+  #[napi(ts_type = "(kind: 'change' | 'remove' | 'create', path: string): void")]
+  pub fn trigger_event(&self, kind: String, path: String) {
+    if let Some(kind) = match kind.as_str() {
+      "change" => Some(FsEventKind::Change),
+      "remove" => Some(FsEventKind::Remove),
+      "create" => Some(FsEventKind::Create),
+      _ => None,
+    } {
+      self
+        .watcher
+        .trigger_event(&ArcPath::from(AsRef::<Path>::as_ref(&path)), kind);
+    }
+  }
+
   #[napi]
   /// # Safety
   ///

crates/rspack_binding_api/src/raw_options/mod.rs

@@ -39,6 +39,7 @@ pub use raw_split_chunks::*;
 pub use raw_stats::*;
 
 pub use crate::options::raw_resolve::*;
+use crate::virtual_modules::JsVirtualFile;
 
 #[derive(Debug)]
 #[napi(object, object_to_js = false)]
@@ -61,6 +62,8 @@ pub struct RawOptions {
   pub bail: bool,
   #[napi(js_name = "__references", ts_type = "Record<string, any>")]
   pub __references: References,
+  #[napi(js_name = "__virtual_files")]
+  pub __virtual_files: Option<Vec<JsVirtualFile>>,
 }
 
 impl TryFrom<RawOptions> for CompilerOptions {

crates/rspack_binding_api/src/virtual_modules/fs.rs

@@ -0,0 +1,138 @@
+use std::{
+  fmt::Debug,
+  sync::{Arc, RwLock},
+};
+
+use rspack_fs::{FileMetadata, FilePermissions, ReadableFileSystem, Result};
+use rspack_paths::{Utf8Path, Utf8PathBuf};
+
+use crate::virtual_modules::VirtualFileStore;
+
+pub struct VirtualFileSystem {
+  real_fs: Arc<dyn ReadableFileSystem>,
+  virtual_file_store: Arc<RwLock<dyn VirtualFileStore>>,
+}
+
+impl VirtualFileSystem {
+  pub fn new(
+    real_fs: Arc<dyn ReadableFileSystem>,
+    virtual_file_store: Arc<RwLock<dyn VirtualFileStore>>,
+  ) -> Self {
+    Self {
+      real_fs,
+      virtual_file_store,
+    }
+  }
+}
+
+impl Debug for VirtualFileSystem {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    f.debug_struct("VirtualFileSystem")
+      .field("virtual_file_store", &"<VirtualFileStore>")
+      .field("real_fs", &"<ReadableFileSystem>")
+      .finish()
+  }
+}
+
+#[async_trait::async_trait]
+impl ReadableFileSystem for VirtualFileSystem {
+  async fn read(&self, path: &Utf8Path) -> Result<Vec<u8>> {
+    if let Ok(store) = self.virtual_file_store.read()
+      && let Some(content) = store.get_file_content(path)
+    {
+      return Ok(content.clone());
+    }
+
+    self.real_fs.read(path).await
+  }
+
+  fn read_sync(&self, path: &Utf8Path) -> Result<Vec<u8>> {
+    if let Ok(store) = self.virtual_file_store.read()
+      && let Some(content) = store.get_file_content(path)
+    {
+      return Ok(content.clone());
+    }
+
+    self.real_fs.read_sync(path)
+  }
+
+  async fn metadata(&self, path: &Utf8Path) -> Result<FileMetadata> {
+    if let Ok(store) = self.virtual_file_store.read()
+      && let Some(metadata) = store.get_file_metadata(path)
+    {
+      return Ok(metadata.clone());
+    }
+
+    self.real_fs.metadata(path).await
+  }
+
+  fn metadata_sync(&self, path: &Utf8Path) -> Result<FileMetadata> {
+    if let Ok(store) = self.virtual_file_store.read()
+      && let Some(metadata) = store.get_file_metadata(path)
+    {
+      return Ok(metadata.clone());
+    }
+
+    self.real_fs.metadata_sync(path)
+  }
+
+  async fn symlink_metadata(&self, path: &Utf8Path) -> Result<FileMetadata> {
+    if let Ok(store) = self.virtual_file_store.read()
+      && let Some(metadata) = store.get_file_metadata(path)
+    {
+      return Ok(metadata.clone());
+    }
+
+    self.real_fs.symlink_metadata(path).await
+  }
+
+  async fn canonicalize(&self, path: &Utf8Path) -> Result<Utf8PathBuf> {
+    if let Ok(store) = self.virtual_file_store.read()
+      && store.contains(path)
+    {
+      return Ok(path.to_path_buf());
+    }
+
+    self.real_fs.canonicalize(path).await
+  }
+
+  async fn read_dir(&self, dir: &Utf8Path) -> Result<Vec<String>> {
+    if let Some(mut vlist) = self
+      .virtual_file_store
+      .read()
+      .ok()
+      .and_then(|store| store.read_dir(dir))
+    {
+      let mut list = self.real_fs.read_dir(dir).await.unwrap_or_default();
+      list.append(&mut vlist);
+      Ok(list)
+    } else {
+      self.real_fs.read_dir(dir).await
+    }
+  }
+
+  fn read_dir_sync(&self, dir: &Utf8Path) -> Result<Vec<String>> {
+    if let Some(mut vlist) = self
+      .virtual_file_store
+      .read()
+      .ok()
+      .and_then(|store| store.read_dir(dir))
+    {
+      let mut list = self.real_fs.read_dir_sync(dir).unwrap_or_default();
+      list.append(&mut vlist);
+      Ok(list)
+    } else {
+      self.real_fs.read_dir_sync(dir)
+    }
+  }
+
+  async fn permissions(&self, path: &Utf8Path) -> Result<Option<FilePermissions>> {
+    if let Ok(store) = self.virtual_file_store.read()
+      && store.contains(path)
+    {
+      return Ok(Some(FilePermissions::from_mode(0o700)));
+    }
+
+    self.real_fs.permissions(path).await
+  }
+}