chore: add old eszip migration test part 2 (#517)

* chore: add `get_eszip_testdata.sh`

* stamp: polishing

* chore(base): add a dev dependency

* chore: update `Cargo.lock`

* chore: add old eszip migration test

* fix: introduce eszip migrate options

* fix: allow specifying context regardless worker mode

* chore: update `scripts/get_eszip_testdata.sh`

* chore(ci): add a step that brings test eszip binaries

* fix(cli): reintroduce `--import-map` flag of the bundle command

* chore(ci): using checkout action instead of s3 to get test binaries

* stamp: token

* stamp: test

* stamp: polishing

* stamp: debug

* stamp: decomp

* stamp: test

* stamp: polishing

* chore: add old eszip migration test (#512)

* chore(base): add a dependency

* chore(base): add `.gitignore`

* chore: add scripts

* chore(ci): add a step that brings test eszip binaries

* chore(base): update `.gitignore`

* chore(base): add old eszip migration test

* chore(base): add snapshots for old eszip migration tests

* stamp(ci): change bucket name

* stamp: update snapshots

* stamp: polishing

* stamp: update snapshots

Author: nyannyacha

.github/workflows/ci.yml

@@ -15,6 +15,7 @@ env:
   RUSTUP_MAX_RETRIES: 10
   ORT_DYLIB_PATH: /tmp/onnxruntime/lib/libonnxruntime.so
   RUST_LOG: ext_event_worker=trace
+  ESZIP_TESTDATA_REPO: supabase/edge-runtime-test-eszip
 
 jobs:
   cargo-fmt:
@@ -75,7 +76,31 @@ jobs:
       - name: Install ONNX Runtime Library
         run: ./scripts/install_onnx.sh ${{ env.ONNXRUNTIME_VERSION }} linux x64 /tmp/onnxruntime
 
-      - name: Install S3 credentials for testing
+      - name: Prepare test eszip binaries
+        if: |
+          github.actor != 'dependabot[bot]' &&
+          (
+            github.event_name != 'pull_request'
+            || github.event.pull_request.head.repo.full_name == github.repository
+          )
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.ESZIP_TESTDATA_REPO }}
+          path: ./edge-runtime-test-eszip
+          token: ${{ secrets.EDGE_RUNTIME_ORG_TOKEN }}
+      - name: Decompress test eszip binaries
+        if: |
+          github.actor != 'dependabot[bot]' &&
+          (
+            github.event_name != 'pull_request'
+            || github.event.pull_request.head.repo.full_name == github.repository
+          )
+        run: |
+          mkdir -p crates/base/tests/fixture/testdata
+          ./scripts/esbr.cjs decompress ./edge-runtime-test-eszip/testdata
+          mv ./edge-runtime-test-eszip/testdata/*.out crates/base/tests/fixture/testdata
+
+      - name: Install Supabase Storage S3 credentials for testing
         if: |
           github.actor != 'dependabot[bot]' &&
           (

Cargo.lock

@@ -301,6 +301,9 @@ fn get_bundle_command() -> Command {
         .help("Glob pattern for static files to be included")
         .action(ArgAction::Append),
     )
+    .arg(
+      arg!(--"import-map" <Path>).help("(DEPRECATED) Path to import map file"),
+    )
     .arg(
       arg!(--"checksum" <KIND>)
         .env("EDGE_RUNTIME_BUNDLE_CHECKSUM")

cli/src/flags.rs

@@ -18,6 +18,7 @@ use base::worker::pool::WorkerPoolPolicy;
 use base::CacheSetting;
 use base::InspectorOption;
 use base::WorkerKind;
+use deno::ConfigMode;
 use deno::DenoOptionsBuilder;
 use deno_facade::extract_from_file;
 use deno_facade::generate_binary_eszip;
@@ -269,6 +270,8 @@ fn main() -> Result<ExitCode, anyhow::Error> {
       Some(("bundle", sub_matches)) => {
         let output_path =
           sub_matches.get_one::<String>("output").cloned().unwrap();
+        let import_map_path =
+          sub_matches.get_one::<String>("import-map").cloned();
         let static_patterns =
           if let Some(val_ref) = sub_matches.get_many::<String>("static") {
             val_ref.map(|s| s.as_str()).collect::<Vec<&str>>()
@@ -311,11 +314,18 @@ fn main() -> Result<ExitCode, anyhow::Error> {
         emitter_factory.set_permissions_options(Some(
           base::get_default_permissions(WorkerKind::MainWorker),
         ));
-        emitter_factory.set_deno_options(
-          DenoOptionsBuilder::new()
-            .entrypoint(entrypoint_script_path.clone())
-            .build()?,
-        );
+
+        let mut builder =
+          DenoOptionsBuilder::new().entrypoint(entrypoint_script_path.clone());
+
+        if let Some(path) = import_map_path {
+          let path = PathBuf::from(path);
+          if !path.exists() || !path.is_file() {
+            bail!("import map path is incorrect");
+          }
+          builder.set_config(Some(ConfigMode::Path(path)));
+        }
+        emitter_factory.set_deno_options(builder.build()?);
 
         let mut metadata = Metadata::default();
         let eszip = generate_binary_eszip(

cli/src/main.rs

@@ -0,0 +1,3 @@
+tests/fixture/testdata/*
+!tests/fixture/testdata/**/*.snapshot
+!tests/fixture/testdata/**/*.metadata

crates/base/.gitignore

@@ -80,6 +80,7 @@ tokio-rustls = "0.25.0"
 winapi = { workspace = true, features = ["knownfolders", "mswsock", "objbase", "shlobj", "tlhelp32", "winbase", "winerror", "winsock2"] }
 
 [dev-dependencies]
+eszip_trait.workspace = true
 serial_test.workspace = true
 tokio-util = { workspace = true, features = ["rt", "compat"] }
 tracing-subscriber = { workspace = true, features = ["env-filter", "tracing-log"] }

crates/base/Cargo.toml

@@ -68,6 +68,7 @@ use deno_core::ResolutionKind;
 use deno_core::RuntimeOptions;
 use deno_facade::generate_binary_eszip;
 use deno_facade::metadata::Entrypoint;
+use deno_facade::migrate::MigrateOptions;
 use deno_facade::module_loader::standalone::create_module_loader_for_standalone_from_eszip_kind;
 use deno_facade::module_loader::RuntimeProviders;
 use deno_facade::EmitterFactory;
@@ -491,9 +492,7 @@ where
     let is_some_entry_point = maybe_entrypoint.is_some();
 
     let maybe_user_conf = conf.as_user_worker();
-    let user_context = maybe_user_conf
-      .and_then(|it| it.context.clone())
-      .unwrap_or_default();
+    let context = conf.context().cloned().unwrap_or_default();
 
     let permissions_options = maybe_user_conf
       .and_then(|it| it.permissions.clone())
@@ -608,15 +607,22 @@ where
     };
 
     let has_inspector = worker.inspector.is_some();
-    let need_source_map = user_context
+    let need_source_map = context
       .get("sourceMap")
       .and_then(serde_json::Value::as_bool)
       .unwrap_or_default();
+    let maybe_import_map_path = context
+      .get("importMapPath")
+      .and_then(|it| it.as_str())
+      .map(str::to_string);
 
     let rt_provider = create_module_loader_for_standalone_from_eszip_kind(
       eszip,
       permissions_options,
       has_inspector || need_source_map,
+      Some(MigrateOptions {
+        maybe_import_map_path,
+      }),
     )
     .await?;
 
@@ -907,16 +913,16 @@ where
       };
 
       let extra_context = {
-        let mut context =
+        let mut extra_context =
           serde_json::json!(RuntimeContext::get_extra_context());
 
         json::merge_object(
-          &mut context,
-          &serde_json::Value::Object(user_context),
+          &mut extra_context,
+          &serde_json::Value::Object(context),
         );
-        json::merge_object(&mut context, &tokens);
+        json::merge_object(&mut extra_context, &tokens);
 
-        context
+        extra_context
       };
 
       let context = js_runtime.main_context();
@@ -2077,6 +2083,7 @@ mod test {
                   worker_pool_tx,
                   shared_metric_src: None,
                   event_worker_metric_src: None,
+                  context: None,
                 })
               }
             },
@@ -2189,6 +2196,7 @@ mod test {
               worker_pool_tx,
               shared_metric_src: None,
               event_worker_metric_src: None,
+              context: None,
             })
           },
           static_patterns: vec![],
@@ -2253,6 +2261,7 @@ mod test {
               worker_pool_tx,
               shared_metric_src: None,
               event_worker_metric_src: None,
+              context: None,
             })
           },
           static_patterns: vec![],
@@ -2332,6 +2341,7 @@ mod test {
               worker_pool_tx,
               shared_metric_src: None,
               event_worker_metric_src: None,
+              context: None,
             })
           },
           static_patterns: vec![],

crates/base/src/runtime/mod.rs

@@ -724,17 +724,13 @@ pub struct Builder {
   tls: Option<Tls>,
   main_service_path: String,
   event_worker_path: Option<String>,
-  // decorator: Option<DecoratorType>,
   user_worker_policy: Option<WorkerPoolPolicy>,
-  // import_map_path: Option<String>,
   flags: ServerFlags,
   callback_tx: Option<Sender<ServerHealth>>,
   entrypoints: WorkerEntrypoints,
   termination_token: Option<TerminationToken>,
   static_patterns: Vec<String>,
   inspector: Option<Inspector>,
-  // jsx_specifier: Option<String>,
-  // jsx_module: Option<String>,
 }
 
 impl Builder {
@@ -768,21 +764,11 @@ impl Builder {
     self
   }
 
-  // pub fn decorator(&mut self, val: DecoratorType) -> &mut Self {
-  //     self.decorator = Some(val);
-  //     self
-  // }
-
   pub fn user_worker_policy(&mut self, val: WorkerPoolPolicy) -> &mut Self {
     self.user_worker_policy = Some(val);
     self
   }
 
-  // pub fn import_map_path(&mut self, val: &str) -> &mut Self {
-  //     self.import_map_path = Some(val.to_owned());
-  //     self
-  // }
-
   pub fn event_callback(
     &mut self,
     val: mpsc::Sender<ServerHealth>,
@@ -818,16 +804,6 @@ impl Builder {
     self
   }
 
-  // pub fn jsx_specifier(&mut self, val: &str) -> &mut Self {
-  //     self.jsx_specifier = Some(val.to_owned());
-  //     self
-  // }
-
-  // pub fn jsx_module(&mut self, val: &str) -> &mut Self {
-  //     self.jsx_module = Some(val.to_owned());
-  //     self
-  // }
-
   pub fn flags_mut(&mut self) -> &mut ServerFlags {
     &mut self.flags
   }

crates/base/src/server.rs

@@ -251,6 +251,7 @@ impl TestBedBuilder {
           worker_pool_tx,
           shared_metric_src: None,
           event_worker_metric_src: None,
+          context: None,
         }),
         static_patterns: vec![],
 

crates/base/src/utils/test_utils.rs

@@ -581,10 +581,7 @@ impl MainWorkerSurfaceBuilder {
       mut inner,
       main_worker_path,
       no_module_cache,
-      // import_map_path,
       entrypoint,
-      // decorator,
-      // jsx,
       worker_pool_tx,
       shared_metric_src,
       event_worker_metric_src,
@@ -611,13 +608,13 @@ impl MainWorkerSurfaceBuilder {
       timing: None,
       maybe_eszip,
       maybe_entrypoint: entrypoint,
-      // maybe_decorator: decorator,
       maybe_module_code: None,
       conf: WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts {
         worker_pool_tx: worker_pool_tx
           .context("worker_pool_sender must be specified")?,
         shared_metric_src,
         event_worker_metric_src,
+        context: None,
       }),
       env_vars: std::env::vars().collect(),
       static_patterns: vec![],
@@ -696,9 +693,7 @@ impl EventWorkerSurfaceBuilder {
 
       event_worker_path: event_worker_path.as_ref().to_path_buf(),
       no_module_cache: None,
-      // import_map_path: None,
       entrypoint: None,
-      // decorator: None,
     }
   }
 
@@ -707,49 +702,27 @@ impl EventWorkerSurfaceBuilder {
     self
   }
 
-  // pub fn import_map_path(mut self, value: &str) -> Self {
-  //   self.import_map_path = Some(value.to_string());
-  //   self
-  // }
-
   pub fn entrypoint(mut self, value: &str) -> Self {
     self.entrypoint = Some(value.to_string());
     self
   }
 
-  // pub fn decorator(mut self, value: DecoratorType) -> Self {
-  //   self.decorator = Some(value);
-  //   self
-  // }
-
   pub fn set_no_module_cache(&mut self, value: Option<bool>) -> &mut Self {
     self.no_module_cache = value;
     self
   }
 
-  // pub fn set_import_map_path(&mut self, value: Option<&str>) -> &mut Self {
-  //   self.import_map_path = value.map(str::to_string);
-  //   self
-  // }
-
   pub fn set_entrypoint(&mut self, value: Option<&str>) -> &mut Self {
     self.entrypoint = value.map(str::to_string);
     self
   }
 
-  // pub fn set_decorator(&mut self, value: Option<DecoratorType>) -> &mut Self {
-  //   self.decorator = value;
-  //   self
-  // }
-
   pub async fn build(self) -> Result<EventWorkerSurface, anyhow::Error> {
     let Self {
       mut inner,
       event_worker_path,
       no_module_cache,
-      // import_map_path,
       entrypoint,
-      // decorator,
     } = self;
 
     let (event_msg_tx, event_msg_rx) =
@@ -773,16 +746,15 @@ impl EventWorkerSurfaceBuilder {
       service_path,
       no_module_cache: no_module_cache.unwrap_or(flags.no_module_cache),
 
-      // import_map_path,
       env_vars: std::env::vars().collect(),
       timing: None,
       maybe_eszip,
       maybe_entrypoint: entrypoint,
-      // maybe_decorator: decorator,
       maybe_module_code: None,
       conf: WorkerRuntimeOpts::EventsWorker(EventWorkerRuntimeOpts {
         events_msg_rx: Some(event_msg_rx),
         event_worker_exit_deadline_sec: Some(event_worker_exit_deadline_sec),
+        context: None,
       }),
       static_patterns: vec![],