feat: Include eszip extractor, refactor references for payload, fs crate

Author: andreespirela

Cargo.toml

@@ -27,7 +27,7 @@ deno_core = { version = "0.222.0" }
 deno_console = { version = "0.121.0" }
 deno_crypto =  { version = "0.135.0" }
 deno_fetch = { version = "0.145.0" }
-deno_fs = "0.31.0"
+deno_fs = { version = "0.31.0", features = ["sync_fs"] }
 deno_config = "=0.3.1"
 deno_io = "0.31.0"
 deno_graph = "=0.55.0"

crates/base/Cargo.toml

@@ -17,7 +17,7 @@ anyhow = { workspace = true }
 bytes = { version = "1.2.1" }
 cityhash = { version = "0.1.1" }
 deno_ast = { workspace = true }
-deno_fs = { workspace = true, features = ["sync_fs"] }
+deno_fs.workspace = true
 deno_io = { workspace = true }
 deno_core = { workspace = true }
 deno_console = { workspace = true }
@@ -69,7 +69,7 @@ sb_node = { version = "0.1.0", path = "../node" }
 anyhow = { workspace = true }
 bytes = { version = "1.2.1" }
 deno_ast = { workspace = true }
-deno_fs = { workspace = true, features = ["sync_fs"] }
+deno_fs.workspace = true
 deno_io = { workspace = true }
 deno_core = { workspace = true }
 deno_console = { workspace = true }

crates/base/test_cases/npm/folder1/folder2/numbers.ts

@@ -0,0 +1,4 @@
+export const numbers = {
+    "Uno": 1,
+    "Dos": 2
+}

crates/base/test_cases/npm/hello.js

@@ -0,0 +1 @@
+export const hello = "";

crates/base/test_cases/npm/index.ts

@@ -1,9 +1,11 @@
 import isEven from "npm:is-even";
 import { serve } from "https://deno.land/[email protected]/http/server.ts"
+import { hello } from "./hello.js";
+import { numbers } from "./folder1/folder2/numbers.ts"
 
 serve(async (req: Request) => {
     return new Response(
-        JSON.stringify({ is_even: isEven(10) }),
+        JSON.stringify({ is_even: isEven(10), hello, numbers }),
         { status: 200, headers: { "Content-Type": "application/json" } },
     )
 })

crates/cli/src/main.rs

@@ -7,8 +7,8 @@ use clap::builder::FalseyValueParser;
 use clap::{arg, crate_version, value_parser, ArgAction, Command};
 use deno_core::url::Url;
 use sb_graph::emitter::EmitterFactory;
-use sb_graph::generate_binary_eszip;
 use sb_graph::import_map::load_import_map;
+use sb_graph::{extract_from_file, generate_binary_eszip};
 use std::fs::File;
 use std::io::Write;
 use std::path::PathBuf;
@@ -58,7 +58,12 @@ fn cli() -> Command {
                 .arg(arg!(--"output" <DIR> "Path to output eszip file").default_value("bin.eszip"))
                 .arg(arg!(--"entrypoint" <Path> "Path to entrypoint to bundle as an eszip").required(true))
                 .arg(arg!(--"import-map" <Path> "Path to import map file"))
-        )
+        ).subcommand(
+        Command::new("unbundle")
+            .about("Unbundles an .eszip file into the specified directory")
+            .arg(arg!(--"output" <DIR> "Path to extract the ESZIP content").default_value("./"))
+            .arg(arg!(--"eszip" <DIR> "Path of eszip to extract").required(true))
+    )
 }
 
 //async fn exit_with_code(result: Result<(), Error>) {
@@ -162,6 +167,20 @@ fn main() -> Result<(), anyhow::Error> {
                 let mut file = File::create(output_path.as_str()).unwrap();
                 file.write_all(&bin).unwrap();
             }
+            Some(("unbundle", sub_matches)) => {
+                let output_path = sub_matches.get_one::<String>("output").cloned().unwrap();
+                let eszip_path = sub_matches.get_one::<String>("eszip").cloned().unwrap();
+
+                let output_path = PathBuf::from(output_path.as_str());
+                let eszip_path = PathBuf::from(eszip_path.as_str());
+
+                extract_from_file(eszip_path, output_path.clone()).await;
+
+                println!(
+                    "Eszip extracted successfully inside path {}",
+                    output_path.to_str().unwrap()
+                );
+            }
             _ => {
                 // unrecognized command
             }

crates/sb_core/util/path.rs

@@ -1,5 +1,5 @@
 use deno_ast::ModuleSpecifier;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 /// Gets if the provided character is not supported on all
 /// kinds of file systems.
@@ -41,3 +41,22 @@ pub fn root_url_to_safe_local_dirname(root: &ModuleSpecifier) -> PathBuf {
 
     result
 }
+
+pub fn find_lowest_path(paths: &Vec<String>) -> Option<String> {
+    let mut lowest_path: Option<(&str, usize)> = None;
+
+    for path_str in paths {
+        // Extract the path part from the URL
+        let path = Path::new(path_str);
+
+        // Count the components
+        let component_count = path.components().count();
+
+        // Update the lowest path if this one has fewer components
+        if lowest_path.is_none() || component_count < lowest_path.unwrap().1 {
+            lowest_path = Some((path_str, component_count));
+        }
+    }
+
+    lowest_path.map(|(path, _)| path.to_string())
+}

crates/sb_graph/lib.rs

@@ -2,12 +2,16 @@ use crate::emitter::EmitterFactory;
 use crate::graph_util::{create_eszip_from_graph_raw, create_graph};
 use deno_ast::MediaType;
 use deno_core::error::AnyError;
+use deno_core::futures::io::{AllowStdIo, BufReader};
 use deno_core::{serde_json, FastString, JsBuffer, ModuleSpecifier};
 use deno_fs::{FileSystem, RealFs};
 use deno_npm::NpmSystemInfo;
 use eszip::{EszipV2, ModuleKind};
 use sb_fs::{build_vfs, VfsOpts};
-use std::path::PathBuf;
+use std::fs;
+use std::fs::{create_dir_all, File};
+use std::io::Write;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
 pub mod emitter;
@@ -25,6 +29,26 @@ pub enum EszipPayloadKind {
     Eszip(EszipV2),
 }
 
+pub async fn payload_to_eszip(eszip_payload_kind: EszipPayloadKind) -> EszipV2 {
+    match eszip_payload_kind {
+        EszipPayloadKind::Eszip(data) => data,
+        _ => {
+            let bytes = match eszip_payload_kind {
+                EszipPayloadKind::JsBufferKind(js_buffer) => Vec::from(&*js_buffer),
+                EszipPayloadKind::VecKind(vec) => vec,
+                _ => panic!("It should not get here"),
+            };
+
+            let bufreader = BufReader::new(AllowStdIo::new(bytes.as_slice()));
+            let (eszip, loader) = eszip::EszipV2::parse(bufreader).await.unwrap();
+
+            loader.await.unwrap();
+
+            eszip
+        }
+    }
+}
+
 pub async fn generate_binary_eszip(
     file: PathBuf,
     emitter_factory: Arc<EmitterFactory>,
@@ -95,3 +119,116 @@ pub async fn generate_binary_eszip(
         eszip
     }
 }
+
+fn extract_file_specifiers(eszip: &EszipV2) -> Vec<String> {
+    eszip
+        .specifiers()
+        .iter()
+        .filter(|specifier| specifier.starts_with("file:"))
+        .cloned()
+        .collect()
+}
+
+pub struct ExtractEszipPayload {
+    pub data: EszipPayloadKind,
+    pub folder: PathBuf,
+}
+
+fn create_module_path(
+    global_specifier: &str,
+    entry_path: &Path,
+    output_folder: &Path,
+) -> PathBuf {
+    let cleaned_specifier = global_specifier.replace(entry_path.to_str().unwrap(), "");
+    let module_path = PathBuf::from(cleaned_specifier);
+
+    if let Some(parent) = module_path.parent() {
+        if parent.parent().is_some() {
+            let output_folder_and_mod_folder =
+                output_folder.join(parent.strip_prefix("/").unwrap());
+            if !output_folder_and_mod_folder.exists() {
+                create_dir_all(&output_folder_and_mod_folder).unwrap();
+            }
+        }
+    }
+
+    output_folder.join(module_path.strip_prefix("/").unwrap())
+}
+
+async fn extract_modules(
+    eszip: &EszipV2,
+    specifiers: &[String],
+    lowest_path: &str,
+    output_folder: &Path,
+) {
+    let main_path = PathBuf::from(lowest_path);
+    let entry_path = main_path.parent().unwrap();
+    for global_specifier in specifiers {
+        let module_path = create_module_path(global_specifier, entry_path, output_folder);
+        let module_content = eszip
+            .get_module(global_specifier)
+            .unwrap()
+            .take_source()
+            .await
+            .unwrap();
+
+        let mut file = File::create(&module_path).unwrap();
+        file.write_all(module_content.as_ref()).unwrap();
+    }
+}
+
+pub async fn extract_eszip(payload: ExtractEszipPayload) {
+    let eszip = payload_to_eszip(payload.data).await;
+    let output_folder = payload.folder;
+
+    if !output_folder.exists() {
+        create_dir_all(&output_folder).unwrap();
+    }
+
+    let file_specifiers = extract_file_specifiers(&eszip);
+    if let Some(lowest_path) = sb_core::util::path::find_lowest_path(&file_specifiers) {
+        extract_modules(&eszip, &file_specifiers, &lowest_path, &output_folder).await;
+    } else {
+        panic!("Path seems to be invalid");
+    }
+}
+
+pub async fn extract_from_file(eszip_file: PathBuf, output_path: PathBuf) {
+    let eszip_content = fs::read(eszip_file).expect("File does not exist");
+    extract_eszip(ExtractEszipPayload {
+        data: EszipPayloadKind::VecKind(eszip_content),
+        folder: output_path,
+    })
+    .await;
+}
+
+#[cfg(test)]
+mod test {
+    use crate::{
+        extract_eszip, generate_binary_eszip, EmitterFactory, EszipPayloadKind, ExtractEszipPayload,
+    };
+    use std::fs::remove_dir_all;
+    use std::path::PathBuf;
+    use std::sync::Arc;
+
+    #[tokio::test]
+    #[allow(clippy::arc_with_non_send_sync)]
+    async fn test_module_code_no_eszip() {
+        let eszip = generate_binary_eszip(
+            PathBuf::from("../base/test_cases/npm/index.ts"),
+            Arc::new(EmitterFactory::new()),
+            None,
+            None,
+        )
+        .await;
+        let eszip = eszip.unwrap();
+        extract_eszip(ExtractEszipPayload {
+            data: EszipPayloadKind::Eszip(eszip),
+            folder: PathBuf::from("../base/test_cases/extracted-npm/"),
+        })
+        .await;
+
+        assert!(PathBuf::from("../base/test_cases/extracted-npm/hello.js").exists());
+        remove_dir_all(PathBuf::from("../base/test_cases/extracted-npm/")).unwrap();
+    }
+}

crates/sb_module_loader/standalone/mod.rs

@@ -20,7 +20,7 @@ use sb_core::util::http_util::HttpClient;
 use sb_fs::file_system::DenoCompileFileSystem;
 use sb_fs::load_npm_vfs;
 use sb_graph::graph_resolver::MappedSpecifierResolver;
-use sb_graph::{EszipPayloadKind, SOURCE_CODE_ESZIP_KEY, VFS_ESZIP_KEY};
+use sb_graph::{payload_to_eszip, EszipPayloadKind, SOURCE_CODE_ESZIP_KEY, VFS_ESZIP_KEY};
 use sb_node::analyze::NodeCodeTranslator;
 use sb_node::NodeResolver;
 use sb_npm::package_json::PackageJsonDepsProvider;
@@ -198,25 +198,7 @@ pub async fn create_module_loader_for_standalone_from_eszip_kind(
     maybe_import_map_arc: Option<Arc<ImportMap>>,
     maybe_import_map_path: Option<String>,
 ) -> Result<RuntimeProviders, AnyError> {
-    use deno_core::futures::io::{AllowStdIo, BufReader};
-
-    let eszip = match eszip_payload_kind {
-        EszipPayloadKind::Eszip(data) => data,
-        _ => {
-            let bytes = match eszip_payload_kind {
-                EszipPayloadKind::JsBufferKind(js_buffer) => Vec::from(&*js_buffer),
-                EszipPayloadKind::VecKind(vec) => vec,
-                _ => panic!("It should not get here"),
-            };
-
-            let bufreader = BufReader::new(AllowStdIo::new(bytes.as_slice()));
-            let (eszip, loader) = eszip::EszipV2::parse(bufreader).await.unwrap();
-
-            loader.await.unwrap();
-
-            eszip
-        }
-    };
+    let eszip = payload_to_eszip(eszip_payload_kind).await;
 
     let mut maybe_import_map: Option<ImportMap> = None;