feat(turbo-tasks-fs): use tokio-fs-ext::offload on wasm

Author: xusd320

Cargo.lock

@@ -55,7 +55,10 @@ urlencoding = { workspace = true }
 notify = { workspace = true }
 
 [target.'cfg(all(target_family = "wasm", target_os = "unknown"))'.dependencies]
-tokio-fs-ext = "0.3.11"
+tokio-fs-ext = "0.5.1"
+wasm_thread = { git = "https://github.com/utooland/wasm_thread.git", default-features = false, features = [
+  "spawn_from_worker",
+] }
 
 [dev-dependencies]
 criterion = { workspace = true, features = ["async_tokio"] }

turbopack/crates/turbo-tasks-fs/Cargo.toml

@@ -37,6 +37,9 @@ use std::{
     time::Duration,
 };
 
+#[cfg(all(target_family = "wasm", target_os = "unknown"))]
+pub mod wasm_fs_offload;
+
 use anyhow::{Context, Result, anyhow, bail};
 use auto_hash_map::{AutoMap, AutoSet};
 use bitflags::bitflags;
@@ -584,13 +587,7 @@ impl FileSystem for DiskFileSystem {
         };
 
         #[cfg(all(target_family = "wasm", target_os = "unknown"))]
-        let content = match tokio_fs_ext::read(full_path.clone())
-            .instrument(tracing::info_span!(
-                "read file",
-                name = display(full_path.display())
-            ))
-            .await
-        {
+        let content = match wasm_fs_offload::CLIENT.read(full_path.clone()).await {
             Ok(buf) => Ok(FileContent::from(File::from_bytes(buf))),
             Err(e) if e.kind() == ErrorKind::NotFound || e.kind() == ErrorKind::InvalidFilename => {
                 Ok(FileContent::NotFound)
@@ -630,7 +627,21 @@ impl FileSystem for DiskFileSystem {
             }
         };
 
-        #[cfg(not(all(target_family = "wasm", target_os = "unknown")))]
+        #[cfg(all(target_family = "wasm", target_os = "unknown"))]
+        let read_dir = match wasm_fs_offload::CLIENT.read_dir(full_path.clone()).await {
+            Ok(dir) => dir,
+            Err(e)
+                if e.kind() == ErrorKind::NotFound
+                    || e.kind() == ErrorKind::NotADirectory
+                    || e.kind() == ErrorKind::InvalidFilename =>
+            {
+                return Ok(RawDirectoryContent::not_found());
+            }
+            Err(e) => {
+                bail!(anyhow!(e).context(format!("reading dir {}", full_path.display())))
+            }
+        };
+
         let entries = read_dir
             .filter_map(|r| {
                 let e = match r {
@@ -654,47 +665,6 @@ impl FileSystem for DiskFileSystem {
             .collect::<Result<_>>()
             .with_context(|| format!("reading directory item in {}", full_path.display()))?;
 
-        #[cfg(all(target_family = "wasm", target_os = "unknown"))]
-        let mut read_dir = match tokio_fs_ext::read_dir(full_path.clone())
-            .instrument(tracing::info_span!(
-                "read dir",
-                name = display(full_path.display())
-            ))
-            .await
-        {
-            Ok(dir) => dir,
-            Err(e)
-                if e.kind() == ErrorKind::NotFound
-                    || e.kind() == ErrorKind::NotADirectory
-                    || e.kind() == ErrorKind::InvalidFilename =>
-            {
-                return Ok(RawDirectoryContent::not_found());
-            }
-            Err(e) => {
-                bail!(anyhow!(e).context(format!("reading dir {}", full_path.display())))
-            }
-        };
-        #[cfg(all(target_family = "wasm", target_os = "unknown"))]
-        let entries = {
-            let mut entries = AutoMap::<RcStr, RawDirectoryEntry>::new();
-            while let Some(e) = read_dir
-                .next_entry()
-                .await
-                .with_context(|| format!("reading directory item in {}", full_path.display()))?
-            {
-                let file_name: RcStr = e.file_name().to_string_lossy().to_string().into();
-                let entry = match e.file_type().await {
-                    Ok(t) if t.is_file() => RawDirectoryEntry::File,
-                    Ok(t) if t.is_dir() => RawDirectoryEntry::Directory,
-                    Ok(t) if t.is_symlink() => RawDirectoryEntry::Symlink,
-                    Ok(_) => RawDirectoryEntry::Other,
-                    Err(err) => return Err(err.into()),
-                };
-                entries.insert(file_name, entry);
-            }
-            entries
-        };
-
         Ok(RawDirectoryContent::new(entries))
     }
 
@@ -720,15 +690,9 @@ impl FileSystem for DiskFileSystem {
             };
 
         #[cfg(all(target_family = "wasm", target_os = "unknown"))]
-        let link_path = match tokio_fs_ext::read_link(full_path.clone())
-            .instrument(tracing::info_span!(
-                "read symlink",
-                name = display(full_path.display())
-            ))
-            .await
-        {
-            Ok(res) => res,
-            Err(_) => return Ok(LinkContent::NotFound.cell()),
+        let link_path: PathBuf = {
+            // TODO: not supported now
+            todo!()
         };
 
         let is_link_absolute = link_path.is_absolute();
@@ -810,6 +774,7 @@ impl FileSystem for DiskFileSystem {
         let inner = self.inner.clone();
         let invalidator = turbo_tasks::get_invalidator();
 
+        #[cfg(not(all(target_family = "wasm", target_os = "unknown")))]
         effect(async move {
             let full_path = validate_path_length(&full_path)?;
 
@@ -862,7 +827,6 @@ impl FileSystem for DiskFileSystem {
 
                     let full_path_to_write = full_path.clone();
                     let content = content.clone();
-                    #[cfg(not(all(target_family = "wasm", target_os = "unknown")))]
                     retry_blocking(full_path_to_write.into_owned(), move |full_path| {
                         use std::io::Write;
 
@@ -904,26 +868,8 @@ impl FileSystem for DiskFileSystem {
                     ))
                     .await
                     .with_context(|| format!("failed to write to {}", full_path.display()))?;
-
-                    #[cfg(all(target_family = "wasm", target_os = "unknown"))]
-                    if let FileContent::Content(file) = &*content {
-                        let mut content_buf = vec![];
-                        std::io::copy(&mut file.read(), &mut content_buf)?;
-                        tokio_fs_ext::write(full_path_to_write, content_buf)
-                            .instrument(tracing::info_span!(
-                                "write file",
-                                name = display(full_path.display())
-                            ))
-                            .await
-                            .with_context(|| {
-                                format!("failed to write to {}", full_path.display())
-                            })?;
-                    } else {
-                        unreachable!()
-                    };
                 }
                 FileContent::NotFound => {
-                    #[cfg(not(all(target_family = "wasm", target_os = "unknown")))]
                     retry_blocking(full_path.clone().into_owned(), |path| {
                         std::fs::remove_file(path)
                     })
@@ -941,22 +887,6 @@ impl FileSystem for DiskFileSystem {
                         }
                     })
                     .with_context(|| anyhow!("removing {} failed", full_path.display()))?;
-
-                    #[cfg(all(target_family = "wasm", target_os = "unknown"))]
-                    tokio_fs_ext::remove_file(full_path.clone())
-                        .instrument(tracing::info_span!(
-                            "remove file",
-                            name = display(full_path.display())
-                        ))
-                        .await
-                        .or_else(|err| {
-                            if err.kind() == ErrorKind::NotFound {
-                                Ok(())
-                            } else {
-                                Err(err)
-                            }
-                        })
-                        .with_context(|| anyhow!("removing {} failed", full_path.display()))?;
                 }
             }
 
@@ -965,6 +895,30 @@ impl FileSystem for DiskFileSystem {
             Ok(())
         });
 
+        #[cfg(all(target_family = "wasm", target_os = "unknown"))]
+        effect(async move {
+            let full_path = validate_path_length(&full_path)?;
+
+            let _lock = inner.lock_path(&full_path).await;
+
+            if let FileContent::Content(file) = &*content {
+                if let Some(create_directory) = full_path.parent() {
+                    wasm_fs_offload::CLIENT
+                        .create_dir_all(create_directory)
+                        .await?;
+                }
+
+                let mut content_buf = vec![];
+                std::io::copy(&mut file.read(), &mut content_buf)?;
+                wasm_fs_offload::CLIENT
+                    .write(full_path, &content_buf)
+                    .await?;
+                Ok(())
+            } else {
+                unreachable!()
+            }
+        });
+
         Ok(())
     }
 
@@ -1121,11 +1075,8 @@ impl FileSystem for DiskFileSystem {
             .with_context(|| format!("reading metadata for {}", full_path.display()))?;
 
         #[cfg(all(target_family = "wasm", target_os = "unknown"))]
-        let meta = tokio_fs_ext::metadata(full_path.clone())
-            .instrument(tracing::info_span!(
-                "read metadata",
-                name = display(full_path.display())
-            ))
+        let meta = wasm_fs_offload::CLIENT
+            .metadata(&full_path)
             .await
             .with_context(|| format!("reading metadata for {}", full_path.display()))?;
         Ok(FileMeta::cell(meta.into()))

turbopack/crates/turbo-tasks-fs/src/lib.rs

@@ -0,0 +1,17 @@
+use std::sync::LazyLock;
+
+use anyhow::{Context, Result};
+use parking_lot::Mutex;
+use tokio::sync::oneshot;
+use tokio_fs_ext::offload::{self, FsOffload};
+
+static WASM_FS_OFFLOAD: LazyLock<(Mutex<offload::Server>, offload::Client)> = LazyLock::new(|| {
+    let (server, client) = offload::split();
+    (Mutex::new(server), client)
+});
+
+pub static CLIENT: LazyLock<offload::Client> = LazyLock::new(|| WASM_FS_OFFLOAD.1.clone());
+
+pub async fn server(offload: impl FsOffload) {
+    WASM_FS_OFFLOAD.0.lock().serve(offload).await
+}