feat: add otel support (#554)

* chore: update dependencies

* chore: update `Cargo.lock`

* stamp: vendors

* chore: update `types/global.d.ts`

* chore: update `.dockerignore`

* feat: enable otel capability

* stamp: polishing runtime cleanup routine

* chore: update `examples/main`

* stamp: align with supabase otel requirement

* chore: update `Cargo.lock`

* Update cli/src/flags.rs

Co-authored-by: Ziinc <[email protected]>

* Update cli/src/flags.rs

Co-authored-by: Ziinc <[email protected]>

---------

Co-authored-by: Ziinc <[email protected]>

Author: nyannyacha

.dockerignore

@@ -2,8 +2,8 @@ target/
 k6/
 examples/
 assets/
-.vscodes/
+.vscode/
 .github/
 
 scripts/*
-!scripts/install_onnx.sh
+!scripts/install_onnx.sh

Cargo.lock

@@ -43,7 +43,6 @@ deno_package_json = { version = "0.2.1", default-features = false }
 deno_path_util = "=0.2.1"
 deno_permissions = "0.42.0"
 deno_semver = "=0.6.1"
-deno_telemetry = "0.4.0"
 
 # upstream exts
 deno_broadcast_channel = "0.176.0"
@@ -56,6 +55,7 @@ deno_fs = { version = "0.92.0", features = ["sync_fs"] }
 deno_http = "0.180.0"
 deno_io = "0.92.0"
 deno_net = "0.174.0"
+deno_telemetry = "0.17.0"
 deno_tls = "0.169.0"
 deno_url = "0.182.0"
 deno_web = "0.213.0"
@@ -108,6 +108,7 @@ anyhow = "1.0.57"
 async-trait = "0.1.73"
 base32 = "=0.5.1"
 base64 = "0.21.7"
+bencher = "0.1"
 brotli = "6.0.0"
 bytes = "1.4.0"
 cache_control = "=0.2.0"
@@ -117,29 +118,35 @@ cbc = { version = "=0.1.2", features = ["alloc"] }
 chrono = { version = "0.4", default-features = false, features = ["std", "serde"] }
 dashmap = "5.5.3"
 data-encoding = "2.3.3"
+data-url = "=0.3.0"
 dotenvy = "0.15.7"
 ecb = "=0.1.2"
 elliptic-curve = { version = "0.13.4", features = ["alloc", "arithmetic", "ecdh", "std", "pem", "jwk"] }
+fast-socks5 = "0.9.6"
 faster-hex = "0.9"
 fastwebsockets = { version = "0.8", features = ["upgrade", "unstable-split"] }
 flate2 = { version = "=1.0.30", default-features = false }
 fs3 = "0.5.0"
 futures = "0.3.21"
 glob = "0.3.1"
 h2 = "0.4.4"
+hickory-resolver = { version = "0.25.0-alpha.4", features = ["tokio-runtime", "serde"] }
 http = "1.0"
 http-body = "1.0"
 http-body-util = "0.1.2"
 http_v02 = { package = "http", version = "0.2.9" }
 httparse = "1.8.0"
 hyper = { version = "1.4.1", features = ["full"] }
+hyper-rustls = { version = "0.27.2", default-features = false, features = ["http1", "http2", "tls12", "ring"] }
 hyper-util = { version = "=0.1.7", features = ["tokio", "server", "server-auto"] }
 hyper_v014 = { package = "hyper", version = "0.14.26", features = ["runtime", "http1"] }
 indexmap = { version = "2", features = ["serde"] }
+ipnet = "2.3"
 lazy-regex = "3"
 libc = "0.2.126"
 libz-sys = { version = "1.1.20", default-features = false }
 log = "0.4.20"
+memmem = "0.1.1"
 monch = "=0.5.0"
 notify = "=6.1.1"
 num-bigint = { version = "0.4", features = ["rand"] }
@@ -149,6 +156,7 @@ p256 = { version = "0.13.2", features = ["ecdh", "jwk"] }
 p384 = { version = "0.13.0", features = ["ecdh", "jwk"] }
 parking_lot = "0.12.0"
 percent-encoding = "2.3.0"
+phf = { version = "0.11", features = ["macros"] }
 pin-project = "1.0.11" # don't pin because they yank crates from cargo
 rand = "=0.8.5"
 regex = "^1.7.0"
@@ -158,26 +166,39 @@ ring = "^0.17.14"
 rustls = { version = "0.23.11", default-features = false, features = ["logging", "std", "tls12", "ring"] }
 rustls-pemfile = "2"
 rustls-tokio-stream = "=0.3.0"
+rustls-webpki = "0.102"
 scopeguard = "1.2.0"
 sec1 = "0.7"
 serde = { version = "1.0.149", features = ["derive"] }
 serde_json = "1.0.85"
 sha1 = { version = "0.10.6", features = ["oid"] }
 sha2 = { version = "0.10.8", features = ["oid"] }
 signature = "2.1"
+smallvec = "1.8"
 spki = "0.7.2"
 tar = "=0.4.40"
 tempfile = "3.4.0"
 thiserror = "2.0.3"
 tokio = { version = "1.36.0", features = ["full"] }
+tokio-rustls = { version = "0.26.0", default-features = false, features = ["ring", "tls12"] }
+tokio-socks = "0.5.1"
 tokio-util = "0.7.4"
+tower = { version = "0.4.13", default-features = false, features = ["util"] }
+tower-http = { version = "0.6.1", features = ["decompression-br", "decompression-gzip"] }
+tower-service = "0.3.2"
 twox-hash = "=1.6.3"
 url = { version = "2.5", features = ["serde", "expose_internals"] }
 uuid = { version = "1.3.0", features = ["v4"] }
 webpki-root-certs = "0.26.5"
 webpki-roots = "0.26"
 yoke = { version = "0.7.4", features = ["derive"] }
 
+opentelemetry = "0.27.0"
+opentelemetry-http = "0.27.0"
+opentelemetry-otlp = { version = "0.27.0", features = ["logs", "http-proto", "http-json"] }
+opentelemetry-semantic-conventions = { version = "0.27.0", features = ["semconv_experimental"] }
+opentelemetry_sdk = "0.27.0"
+
 # upstream resolvers
 deno_npm_cache = "0.2.0"
 deno_resolver = "0.14.0"
@@ -203,7 +224,11 @@ windows-sys = { version = "0.59.0", features = ["Win32_Foundation", "Win32_Media
 deno_core = { git = "https://github.com/supabase/deno_core", branch = "324-supabase" }
 eszip = { git = "https://github.com/supabase/eszip", branch = "fix-pub-vis-0-80-1" }
 v8 = { git = "https://github.com/supabase/rusty_v8", tag = "v130.0.7" }
+
 deno_unsync = { path = "./vendor/deno_unsync" }
+deno_fetch = { path = "./vendor/deno_fetch" }
+deno_telemetry = { path = "./vendor/deno_telemetry" }
+deno_http = { path = "./vendor/deno_http" }
 
 [profile.dind]
 inherits = "dev"

Cargo.toml

@@ -11,6 +11,7 @@ use clap::ArgAction;
 use clap::ArgGroup;
 use clap::Command;
 use clap::ValueEnum;
+use deno::deno_telemetry;
 use deno_facade::Checksum;
 
 #[derive(ValueEnum, Default, Clone, Copy)]
@@ -28,6 +29,30 @@ impl From<EszipV2ChecksumKind> for Option<Checksum> {
   }
 }
 
+#[derive(ValueEnum, Clone, Copy)]
+pub(super) enum OtelKind {
+  Main,
+  Event,
+}
+
+#[derive(ValueEnum, Default, Clone, Copy)]
+pub(super) enum OtelConsoleConfig {
+  #[default]
+  Ignore,
+  Capture,
+  Replace,
+}
+
+impl From<OtelConsoleConfig> for deno_telemetry::OtelConsoleConfig {
+  fn from(value: OtelConsoleConfig) -> Self {
+    match value {
+      OtelConsoleConfig::Ignore => Self::Ignore,
+      OtelConsoleConfig::Capture => Self::Capture,
+      OtelConsoleConfig::Replace => Self::Replace,
+    }
+  }
+}
+
 pub(super) fn get_cli() -> Command {
   Command::new(env!("CARGO_BIN_NAME"))
     .about(env!("CARGO_PKG_DESCRIPTION"))
@@ -278,6 +303,21 @@ fn get_start_command() -> Command {
         .value_parser(value_parser!(u8).range(..=99))
         .default_value("90"),
     )
+    .arg(
+      arg!(--"enable-otel")
+        .help("Enable OpenTelemetry in the main and event workers")
+        .value_delimiter(',')
+        .value_parser(value_parser!(OtelKind))
+        .num_args(0..=1)
+        .default_missing_value("main,event")
+        .action(ArgAction::Append),
+    )
+    .arg(
+      arg!(--"otel-console" <MODE>)
+        // .env("OTEL_DENO_CONSOLE")
+        .help("Configure console auto instrumentation for OpenTelemetry Logs")
+        .value_parser(value_parser!(OtelConsoleConfig)),
+    )
 }
 
 fn get_bundle_command() -> Command {

cli/src/flags.rs

@@ -10,6 +10,7 @@ use std::sync::Arc;
 use anyhow::bail;
 use anyhow::Context;
 use anyhow::Error;
+use base::server;
 use base::server::Builder;
 use base::server::ServerFlags;
 use base::server::Tls;
@@ -19,6 +20,8 @@ use base::worker::pool::WorkerPoolPolicy;
 use base::CacheSetting;
 use base::InspectorOption;
 use base::WorkerKind;
+use deno::deno_telemetry;
+use deno::deno_telemetry::OtelConfig;
 use deno::ConfigMode;
 use deno::DenoOptionsBuilder;
 use deno_facade::extract_from_file;
@@ -28,6 +31,8 @@ use deno_facade::Metadata;
 use env::resolve_deno_runtime_env;
 use flags::get_cli;
 use flags::EszipV2ChecksumKind;
+use flags::OtelConsoleConfig;
+use flags::OtelKind;
 use log::warn;
 
 mod env;
@@ -79,6 +84,11 @@ fn main() -> Result<ExitCode, anyhow::Error> {
     #[allow(clippy::arc_with_non_send_sync)]
     let exit_code = match matches.subcommand() {
       Some(("start", sub_matches)) => {
+        deno_telemetry::init(
+          deno::versions::otel_runtime_config(),
+          OtelConfig::default(),
+        )?;
+
         let ip = sub_matches.get_one::<String>("ip").cloned().unwrap();
         let ip = IpAddr::from_str(&ip)
           .context("failed to parse the IP address to bind the server")?;
@@ -121,6 +131,17 @@ fn main() -> Result<ExitCode, anyhow::Error> {
           .cloned()
           .unwrap();
 
+        let enable_otel = sub_matches
+          .get_many::<OtelKind>("enable-otel")
+          .unwrap_or_default()
+          .cloned()
+          .collect::<Vec<_>>();
+
+        let otel_console = sub_matches
+          .get_one::<OtelConsoleConfig>("otel-console")
+          .cloned()
+          .map(Into::into);
+
         let event_service_manager_path =
           sub_matches.get_one::<String>("event-worker").cloned();
         let maybe_main_entrypoint =
@@ -203,6 +224,21 @@ fn main() -> Result<ExitCode, anyhow::Error> {
           .unwrap();
 
         let flags = ServerFlags {
+          otel: if !enable_otel.is_empty() {
+            if enable_otel.len() > 1 {
+              Some(server::OtelKind::Both)
+            } else {
+              match enable_otel.first() {
+                Some(OtelKind::Main) => Some(server::OtelKind::Main),
+                Some(OtelKind::Event) => Some(server::OtelKind::Event),
+                None => None,
+              }
+            }
+          } else {
+            None
+          },
+          otel_console,
+
           no_module_cache,
           allow_main_inspector,
           tcp_nodelay,