[WIP] feat: add technical docs

.github/workflows/release.yml

@@ -18,6 +18,9 @@ jobs:
       version: ${{ steps.semantic.outputs.new_release_version }}
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          fetch-tags: true
 
       - name: Semantic Release
         id: semantic

.github/workflows/stale.yml

@@ -6,6 +6,7 @@ on:
   workflow_dispatch:
 
 permissions:
+  actions: write
   issues: write
   pull-requests: write
 

README.md

@@ -15,7 +15,13 @@ Options**
 
 ## Architecture
 
-![Sequence diagram of Edge Runtime request flow](assets/edge-runtime-diagram.svg?raw=true)
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="/assets/edge-runtime-diagram-dark.svg">
+    <source media="(prefers-color-scheme: light)" srcset="/assets/edge-runtime-diagram.svg">
+    <img alt="Sequence diagram of Edge Runtime request flow" src="/assets/edge-runtime-diagram.svg" style="max-width: 100%;">
+  </picture>
+</p>
 
 The edge runtime can be divided into two runtimes with different purposes.
 
@@ -30,7 +36,22 @@ The edge runtime can be divided into two runtimes with different purposes.
 - User runtime:
   - An instance for the _user runtime_ is responsible for executing users' code.
   - Limits are required to be set such as: Memory and Timeouts.
-  - Has access to environment variables explictly allowed by the main runtime.
+  - Has access to environment variables explicitly allowed by the main runtime.
+
+### Usage & Self-Hosting
+
+For completely usage and self-host details, visit [usage.md](/docs/usage.md)
+
+### Edge Runtime in Deep
+
+#### Conceptual
+
+- [EdgeRuntime Base](/crates/base/README.md): Overalls about how EdgeRuntime is based on Deno.
+
+#### Extension Modules
+
+- [AI](/ext/ai/README.md): Implements AI related features.
+- [NodeJs](/ext/node/README.md) & [NodeJs Polyfills](/ext/node/polyfills/README.md): Implements the NodeJs compatibility layer.
 
 ## Developers
 

cli/src/flags.rs

@@ -364,6 +364,11 @@ fn get_bundle_command() -> Command {
         .default_value("false")
         .value_parser(FalseyValueParser::new()),
     )
+    .arg(
+      arg!(--"timeout" <SECONDS>)
+        .help("Maximum time in seconds that can be waited for the bundle to complete.")
+        .value_parser(value_parser!(u64).range(..u64::MAX))
+    )
 }
 
 fn get_unbundle_command() -> Command {

cli/src/main.rs

@@ -6,6 +6,7 @@ use std::path::PathBuf;
 use std::process::ExitCode;
 use std::str::FromStr;
 use std::sync::Arc;
+use std::time::Duration;
 
 use anyhow::bail;
 use anyhow::Context;
@@ -34,6 +35,7 @@ use flags::EszipV2ChecksumKind;
 use flags::OtelConsoleConfig;
 use flags::OtelKind;
 use log::warn;
+use tokio::time::timeout;
 
 mod env;
 mod flags;
@@ -319,6 +321,10 @@ fn main() -> Result<ExitCode, anyhow::Error> {
           } else {
             vec![]
           };
+        let timeout_dur = sub_matches
+          .get_one::<u64>("timeout")
+          .cloned()
+          .map(Duration::from_secs);
 
         if import_map_path.is_some() {
           warn!(concat!(
@@ -382,14 +388,24 @@ fn main() -> Result<ExitCode, anyhow::Error> {
         emitter_factory.set_deno_options(builder.build()?);
 
         let mut metadata = Metadata::default();
-        let eszip = generate_binary_eszip(
+        let eszip_fut = generate_binary_eszip(
           &mut metadata,
           Arc::new(emitter_factory),
           None,
           maybe_checksum_kind,
           Some(static_patterns),
-        )
-        .await?;
+        );
+
+        let eszip = if let Some(dur) = timeout_dur {
+          match timeout(dur, eszip_fut).await {
+            Ok(eszip) => eszip,
+            Err(_) => {
+              bail!("Failed to complete the bundle within the given time.")
+            }
+          }
+        } else {
+          eszip_fut.await
+        }?;
 
         let bin = eszip.into_bytes();
 

crates/base/README.md

@@ -0,0 +1,14 @@
+# Supabase EdgeRuntime base
+
+This crate is part of the Supabase Edge Runtime stack and implements the runtime
+core features.
+
+## Architecture
+
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="/assets/edge-runtime-diagram-dark.svg">
+    <source media="(prefers-color-scheme: light)" srcset="/assets/edge-runtime-diagram.svg">
+    <img alt="Sequence diagram of Edge Runtime request flow" src="/assets/edge-runtime-diagram.svg" style="max-width: 100%;">
+  </picture>
+</p>

crates/base/src/runtime/mod.rs

@@ -55,7 +55,6 @@ use deno_cache::SqliteBackedCache;
 use deno_core::error::AnyError;
 use deno_core::error::JsError;
 use deno_core::serde_json;
-use deno_core::serde_json::Value;
 use deno_core::url::Url;
 use deno_core::v8;
 use deno_core::v8::GCCallbackFlags;
@@ -80,7 +79,6 @@ use deno_facade::EmitterFactory;
 use deno_facade::EszipPayloadKind;
 use deno_facade::Metadata;
 use either::Either;
-use ext_event_worker::events::EventMetadata;
 use ext_event_worker::events::WorkerEventWithMetadata;
 use ext_runtime::cert::ValueRootCertStoreProvider;
 use ext_runtime::external_memory::CustomAllocator;
@@ -462,6 +460,7 @@ where
   pub(crate) async fn new(mut worker: Worker) -> Result<Self, Error> {
     let init_opts = worker.init_opts.take();
     let flags = worker.flags.clone();
+    let event_metadata = worker.event_metadata.clone();
 
     debug_assert!(init_opts.is_some(), "init_opts must not be None");
 
@@ -658,6 +657,10 @@ where
           base_url,
         } = rt_provider;
 
+        let node_modules = metadata
+          .node_modules()
+          .ok()
+          .flatten();
         let entrypoint = metadata.entrypoint.clone();
         let main_module_url = match entrypoint.as_ref() {
           Some(Entrypoint::Key(key)) => base_url.join(key)?,
@@ -717,6 +720,7 @@ where
 
         let (fs, s3_fs) = build_file_system_fn(if is_user_worker {
           Arc::new(StaticFs::new(
+            node_modules,
             static_files,
             if matches!(entrypoint, Some(Entrypoint::ModuleCode(_)) | None)
               && is_some_entry_point
@@ -1106,7 +1110,6 @@ where
       s3_fs,
       beforeunload_cpu_threshold,
       beforeunload_mem_threshold,
-      context,
       ..
     } = match bootstrap_ret {
       Ok(Ok(v)) => v,
@@ -1118,7 +1121,7 @@ where
       }
     };
 
-    let context = context.unwrap_or_default();
+    let otel_attributes = event_metadata.otel_attributes.clone();
     let span = Span::current();
     let post_task_ret = unsafe {
       spawn_blocking_non_send(|| {
@@ -1146,46 +1149,30 @@ where
             op_state.put::<HashMap<usize, CancellationToken>>(HashMap::new());
           }
 
-          let mut otel_attributes = HashMap::new();
-
-          otel_attributes.insert(
-            "edge_runtime.worker.kind".into(),
-            conf.to_worker_kind().to_string().into(),
-          );
-
           if conf.is_user_worker() {
             let conf = conf.as_user_worker().unwrap();
             let key = conf.key.map_or("".to_string(), |k| k.to_string());
 
             // set execution id for user workers
             env_vars.insert("SB_EXECUTION_ID".to_string(), key.clone());
 
-            if let Some(Value::Object(attributes)) = context.get("otel") {
-              for (k, v) in attributes {
-                otel_attributes.insert(
-                  k.to_string().into(),
-                  match v {
-                    Value::String(str) => str.to_string().into(),
-                    others => others.to_string().into(),
-                  },
-                );
-              }
-            }
-
             if let Some(events_msg_tx) = conf.events_msg_tx.clone() {
               op_state.put::<mpsc::UnboundedSender<WorkerEventWithMetadata>>(
                 events_msg_tx,
               );
-              op_state.put::<EventMetadata>(EventMetadata {
-                service_path: conf.service_path.clone(),
-                execution_id: conf.key,
-              });
+              op_state.put(event_metadata);
             }
           }
 
           op_state.put(ext_env::EnvVars(env_vars));
           op_state.put(DenoRuntimeDropToken(DropToken(drop_token.clone())));
-          op_state.put(RuntimeOtelExtraAttributes(otel_attributes));
+          op_state.put(RuntimeOtelExtraAttributes(
+            otel_attributes
+              .unwrap_or_default()
+              .into_iter()
+              .map(|(k, v)| (k.into(), v.into()))
+              .collect(),
+          ));
         }
 
         if is_user_worker {

crates/base/src/utils/test_utils.rs

@@ -271,6 +271,7 @@ impl TestBedBuilder {
   }
 }
 
+#[derive(Clone)]
 pub struct TestBed {
   pool_termination_token: TerminationToken,
   main_termination_token: TerminationToken,

crates/base/src/worker/supervisor/strategy_per_worker.rs

@@ -188,7 +188,7 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) {
     worker_timeout_ms
   };
 
-  let wall_clock_duration = Duration::from_millis(worker_timeout_ms);
+  let wall_clock_duration = Duration::from_millis(wall_clock_limit_ms);
 
   // Split wall clock duration into 2 intervals.
   // At the first interval, we will send a msg to retire the worker.

crates/base/src/worker/utils.rs

@@ -1,5 +1,8 @@
+use std::collections::HashMap;
+
 use anyhow::anyhow;
 use anyhow::bail;
+use deno_core::serde_json::Value;
 use ext_event_worker::events::EventMetadata;
 use ext_event_worker::events::WorkerEventWithMetadata;
 use ext_event_worker::events::WorkerEvents;
@@ -15,18 +18,39 @@ use tokio::sync::oneshot;
 use tokio_util::sync::CancellationToken;
 
 pub fn get_event_metadata(conf: &WorkerRuntimeOpts) -> EventMetadata {
+  let mut otel_attributes = HashMap::new();
   let mut event_metadata = EventMetadata {
     service_path: None,
     execution_id: None,
+    otel_attributes: None,
   };
+
+  otel_attributes.insert(
+    "edge_runtime.worker.kind".to_string(),
+    conf.to_worker_kind().to_string(),
+  );
+
   if conf.is_user_worker() {
     let conf = conf.as_user_worker().unwrap();
-    event_metadata = EventMetadata {
-      service_path: conf.service_path.clone(),
-      execution_id: conf.key,
-    };
+    let context = conf.context.clone().unwrap_or_default();
+
+    event_metadata.service_path = conf.service_path.clone();
+    event_metadata.execution_id = conf.key;
+
+    if let Some(Value::Object(attributes)) = context.get("otel") {
+      for (k, v) in attributes {
+        otel_attributes.insert(
+          k.to_string(),
+          match v {
+            Value::String(str) => str.to_string(),
+            others => others.to_string(),
+          },
+        );
+      }
+    }
   }
 
+  event_metadata.otel_attributes = Some(otel_attributes);
   event_metadata
 }