From 8cb9c31669908f49552d9400d5ba1a8778347409 Mon Sep 17 00:00:00 2001 From: MasterPtato Date: Thu, 4 Sep 2025 11:54:38 -0700 Subject: [PATCH] feat: add telemetry via sentry --- Cargo.lock | 316 ++++++++++++++++++ Cargo.toml | 19 +- packages/common/api-builder/Cargo.toml | 1 + packages/common/api-builder/src/middleware.rs | 44 ++- packages/common/config/src/config/mod.rs | 6 + .../common/config/src/config/telemetry.rs | 14 + packages/common/gasoline/core/Cargo.toml | 5 +- .../common/gasoline/core/src/ctx/listen.rs | 1 + .../common/gasoline/core/src/ctx/workflow.rs | 9 + packages/common/gasoline/core/src/worker.rs | 12 + packages/common/telemetry/Cargo.toml | 13 + packages/common/telemetry/README.md | 12 + packages/common/telemetry/src/lib.rs | 39 +++ packages/infra/engine/Cargo.toml | 5 +- packages/infra/engine/src/main.rs | 14 +- 15 files changed, 482 insertions(+), 28 deletions(-) create mode 100644 packages/common/config/src/config/telemetry.rs create mode 100644 packages/common/telemetry/Cargo.toml create mode 100644 packages/common/telemetry/README.md create mode 100644 packages/common/telemetry/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 36194f04ef..4831af4a5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,6 +133,9 @@ name = "anyhow" version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +dependencies = [ + "backtrace", +] [[package]] name = "approx" @@ -1109,6 +1112,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "serde", + "uuid", +] + [[package]] name = "der" version = "0.7.10" @@ -1470,6 +1483,18 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "flate2" version = "1.1.2" @@ -1650,6 +1675,7 @@ dependencies = [ "rivet-runtime", "rivet-test-deps", "rivet-util", + "sentry", "serde", "serde_json", "statrs", @@ -1721,9 +1747,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -1891,6 +1919,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "hostname" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + [[package]] name = "http" version = "0.2.12" @@ -2024,6 +2063,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.2", "tower-service", + "webpki-roots 1.0.2", ] [[package]] @@ -2515,6 +2555,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lz4" version = "1.28.1" @@ -3076,6 +3122,18 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "os_info" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0e1ac5fde8d43c34139135df8ea9ee9465394b2d8d20f032d38998f64afffc3" +dependencies = [ + "log", + "plist", + "serde", + "windows-sys 0.52.0", +] + [[package]] name = "overload" version = "0.1.1" @@ -3398,6 +3456,19 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plist" +version = "1.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3af6b589e163c5a788fab00ce0c0366f6efbb9959c2f9874b224936af7fce7e1" +dependencies = [ + "base64 0.22.1", + "indexmap 2.10.0", + "quick-xml", + "serde", + "time", +] + [[package]] name = "portable-atomic" version = "1.11.1" @@ -3580,6 +3651,70 @@ dependencies = [ "winapi", ] +[[package]] +name = "quick-xml" +version = "0.38.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" +dependencies = [ + "memchr", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.29", + "socket2 0.6.0", + "thiserror 2.0.12", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.3", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls 0.23.29", + "rustls-pki-types", + "slab", + "thiserror 2.0.12", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.0", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.40" @@ -3812,6 +3947,8 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls 0.23.29", "rustls-pki-types", "serde", "serde_json", @@ -3819,6 +3956,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-rustls 0.26.2", "tower 0.5.2", "tower-http", "tower-service", @@ -3826,6 +3964,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots 1.0.2", ] [[package]] @@ -3868,6 +4007,7 @@ dependencies = [ "rivet-error", "rivet-metrics", "rivet-pools", + "sentry", "serde", "serde_json", "thiserror 1.0.69", @@ -4115,6 +4255,7 @@ dependencies = [ "rivet-runner-protocol", "rivet-runtime", "rivet-service-manager", + "rivet-telemetry", "rivet-term", "rivet-test-deps", "rivet-util", @@ -4403,6 +4544,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "rivet-telemetry" +version = "0.0.1" +dependencies = [ + "anyhow", + "rivet-config", + "sentry", + "serde_json", + "uuid", +] + [[package]] name = "rivet-term" version = "0.1.0" @@ -4683,6 +4835,7 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ + "web-time", "zeroize", ] @@ -4882,6 +5035,127 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +[[package]] +name = "sentry" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "255914a8e53822abd946e2ce8baa41d4cded6b8e938913b7f7b9da5b7ab44335" +dependencies = [ + "httpdate", + "reqwest", + "rustls 0.23.29", + "sentry-anyhow", + "sentry-backtrace", + "sentry-contexts", + "sentry-core", + "sentry-debug-images", + "sentry-panic", + "sentry-tracing", + "tokio", + "ureq", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sentry-anyhow" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ddfaa9f4d64827e05f394c72d5628d99bf27f8979ef37b89baf057fb4b8a908" +dependencies = [ + "anyhow", + "sentry-backtrace", + "sentry-core", +] + +[[package]] +name = "sentry-backtrace" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00293cd332a859961f24fd69258f7e92af736feaeb91020cff84dac4188a4302" +dependencies = [ + "backtrace", + "once_cell", + "regex", + "sentry-core", +] + +[[package]] +name = "sentry-contexts" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "961990f9caa76476c481de130ada05614cd7f5aa70fb57c2142f0e09ad3fb2aa" +dependencies = [ + "hostname", + "libc", + "os_info", + "rustc_version", + "sentry-core", + "uname", +] + +[[package]] +name = "sentry-core" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a6409d845707d82415c800290a5d63be5e3df3c2e417b0997c60531dfbd35ef" +dependencies = [ + "once_cell", + "rand 0.8.5", + "sentry-types", + "serde", + "serde_json", +] + +[[package]] +name = "sentry-debug-images" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71ab5df4f3b64760508edfe0ba4290feab5acbbda7566a79d72673065888e5cc" +dependencies = [ + "findshlibs", + "once_cell", + "sentry-core", +] + +[[package]] +name = "sentry-panic" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "609b1a12340495ce17baeec9e08ff8ed423c337c1a84dffae36a178c783623f3" +dependencies = [ + "sentry-backtrace", + "sentry-core", +] + +[[package]] +name = "sentry-tracing" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f4e86402d5c50239dc7d8fd3f6d5e048221d5fcb4e026d8d50ab57fe4644cb" +dependencies = [ + "sentry-backtrace", + "sentry-core", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "sentry-types" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3f117b8755dbede8260952de2aeb029e20f432e72634e8969af34324591631" +dependencies = [ + "debugid", + "hex", + "rand 0.8.5", + "serde", + "serde_json", + "thiserror 1.0.69", + "time", + "url", + "uuid", +] + [[package]] name = "serde" version = "1.0.219" @@ -5961,6 +6235,15 @@ dependencies = [ "universaldb", ] +[[package]] +name = "uname" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72f89f0ca32e4db1c04e2a72f5345d59796d4866a1ee0609084569f73683dc8" +dependencies = [ + "libc", +] + [[package]] name = "unicase" version = "2.8.1" @@ -6070,6 +6353,21 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "log", + "once_cell", + "rustls 0.23.29", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "url" version = "2.5.4" @@ -6328,6 +6626,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.2", +] + +[[package]] +name = "webpki-roots" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "whoami" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index 487c15a683..7c621d6431 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] resolver = "2" -members = ["packages/common/api-builder","packages/common/api-client","packages/common/api-types","packages/common/api-util","packages/common/cache/build","packages/common/cache/result","packages/common/clickhouse-inserter","packages/common/clickhouse-user-query","packages/common/config","packages/common/env","packages/common/error/core","packages/common/error/macros","packages/common/gasoline/core","packages/common/gasoline/macros","packages/common/logs","packages/common/metrics","packages/common/pools","packages/common/runtime","packages/common/service-manager","packages/common/test-deps","packages/common/test-deps-docker","packages/common/types","packages/common/udb-util","packages/common/universaldb","packages/common/universalpubsub","packages/common/util/core","packages/common/util/id","packages/common/versioned-data-util","packages/core/actor-kv","packages/core/api-peer","packages/core/api-public","packages/core/bootstrap","packages/core/dump-openapi","packages/core/guard/core","packages/core/guard/server","packages/core/pegboard-gateway","packages/core/pegboard-runner-ws","packages/core/pegboard-tunnel","packages/core/workflow-worker","packages/infra/engine","packages/services/epoxy","packages/services/namespace","packages/services/pegboard","sdks/rust/api-full","sdks/rust/api-runtime","sdks/rust/bare_gen","sdks/rust/epoxy-protocol","sdks/rust/key-data","sdks/rust/runner-protocol","sdks/rust/tunnel-protocol"] +members = ["packages/common/api-builder","packages/common/api-client","packages/common/api-types","packages/common/api-util","packages/common/cache/build","packages/common/cache/result","packages/common/clickhouse-inserter","packages/common/clickhouse-user-query","packages/common/config","packages/common/env","packages/common/error/core","packages/common/error/macros","packages/common/gasoline/core","packages/common/gasoline/macros","packages/common/logs","packages/common/metrics","packages/common/pools","packages/common/runtime","packages/common/service-manager","packages/common/telemetry","packages/common/test-deps","packages/common/test-deps-docker","packages/common/types","packages/common/udb-util","packages/common/universaldb","packages/common/universalpubsub","packages/common/util/core","packages/common/util/id","packages/common/versioned-data-util","packages/core/actor-kv","packages/core/api-peer","packages/core/api-public","packages/core/bootstrap","packages/core/dump-openapi","packages/core/guard/core","packages/core/guard/server","packages/core/pegboard-gateway","packages/core/pegboard-runner-ws","packages/core/pegboard-tunnel","packages/core/workflow-worker","packages/infra/engine","packages/services/epoxy","packages/services/namespace","packages/services/pegboard","sdks/rust/api-full","sdks/rust/api-runtime","sdks/rust/bare_gen","sdks/rust/epoxy-protocol","sdks/rust/key-data","sdks/rust/runner-protocol","sdks/rust/tunnel-protocol"] [workspace.package] version = "0.0.1" @@ -58,6 +58,7 @@ regex = "1.4" rstest = "0.26.1" rustls-pemfile = "2.2.0" rustyline = "15.0.0" +sentry = { version = "0.37.0", default-features = false, features = ["anyhow", "backtrace", "contexts", "debug-images", "panic", "reqwest", "rustls"] } serde_bare = "0.5.0" serde_yaml = "0.9.34" sha2 = "0.10" @@ -82,14 +83,7 @@ vergen = "9.0.4" [workspace.dependencies.windows] version = "0.58" -features = [ - "Win32", - "Win32_Storage", - "Win32_Storage_FileSystem", - "Win32_System", - "Win32_System_Console", - "Win32_Security", -] +features = ["Win32","Win32_Storage","Win32_Storage_FileSystem","Win32_System","Win32_System_Console","Win32_Security"] [workspace.dependencies.pest] version = "2.7" @@ -105,12 +99,12 @@ features = ["full"] [workspace.dependencies.rustls] version = "0.23.25" default-features = false -features = ["ring", "std", "logging"] +features = ["ring","std","logging"] [workspace.dependencies.tokio-rustls] version = "0.26.2" default-features = false -features = ["ring", "logging"] +features = ["ring","logging"] [workspace.dependencies.utoipa] version = "5.4.0" @@ -303,6 +297,9 @@ path = "packages/common/runtime" [workspace.dependencies.rivet-service-manager] path = "packages/common/service-manager" +[workspace.dependencies.rivet-telemetry] +path = "packages/common/telemetry" + [workspace.dependencies.rivet-test-deps] path = "packages/common/test-deps" diff --git a/packages/common/api-builder/Cargo.toml b/packages/common/api-builder/Cargo.toml index 3701180bd8..f9ce906458 100644 --- a/packages/common/api-builder/Cargo.toml +++ b/packages/common/api-builder/Cargo.toml @@ -18,6 +18,7 @@ rivet-config.workspace = true rivet-error.workspace = true rivet-metrics.workspace = true rivet-pools.workspace = true +sentry.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true thiserror.workspace = true diff --git a/packages/common/api-builder/src/middleware.rs b/packages/common/api-builder/src/middleware.rs index 459193dcce..1ad0362dcb 100644 --- a/packages/common/api-builder/src/middleware.rs +++ b/packages/common/api-builder/src/middleware.rs @@ -128,26 +128,48 @@ pub async fn http_logging_middleware( // Log based on status if status.is_server_error() { + let group = error.as_ref().map_or("-", |x| &x.group); + let code = error.as_ref().map_or("-", |x| &x.code); + let meta = error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null); + let internal = error.as_ref().and_then(|x| x.internal.as_ref()).map_or("-", |x| x.as_ref()); + tracing::error!( - status = ?status_code, - group = %error.as_ref().map_or("-", |x| &x.group), - code = ?error.as_ref().map_or("-", |x| &x.code), - meta = %error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null), - internal = %error.as_ref().and_then(|x| x.internal.as_ref()).map_or("-", |x| x.as_ref()), + status=?status_code, + %group, + %code, + %meta, + %internal, "http server error" ); + + sentry::with_scope( + |scope| { + scope.set_tag("status", status_code); + scope.set_tag("group", group); + scope.set_tag("code", code); + scope.set_tag("meta", meta); + scope.set_tag("internal", internal); + }, + || { + sentry::capture_message(&format!("{group}.{code}"), sentry::Level::Error); + }, + ); } else if status.is_client_error() { + let group = error.as_ref().map_or("-", |x| &x.group); + let code = error.as_ref().map_or("-", |x| &x.code); + let meta = error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null); + tracing::info!( - status = ?status_code, - group = %error.as_ref().map_or("-", |x| &x.group), - code = %error.as_ref().map_or("-", |x| &x.code), - meta = %error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null), + status=?status_code, + %group, + %code, + %meta, "http client error" ); } else if status.is_redirection() { - tracing::debug!(status = ?status_code, "http redirection"); + tracing::debug!(status=?status_code, "http redirection"); } else if status.is_informational() { - tracing::debug!(status = ?status_code, "http informational"); + tracing::debug!(status=?status_code, "http informational"); } let duration = start.elapsed().as_secs_f64(); diff --git a/packages/common/config/src/config/mod.rs b/packages/common/config/src/config/mod.rs index b1bab1f223..a04dcecb97 100644 --- a/packages/common/config/src/config/mod.rs +++ b/packages/common/config/src/config/mod.rs @@ -14,6 +14,7 @@ pub mod pegboard; pub mod pegboard_gateway; pub mod pegboard_tunnel; pub mod pubsub; +pub mod telemetry; pub mod topology; pub mod vector; @@ -28,6 +29,7 @@ pub use pegboard::*; pub use pegboard_gateway::*; pub use pegboard_tunnel::*; pub use pubsub::PubSub; +pub use telemetry::*; pub use topology::*; pub use vector::*; @@ -100,6 +102,9 @@ pub struct Root { #[serde(default)] pub vector_http: Option, + + #[serde(default)] + pub telemetry: Telemetry, } impl Default for Root { @@ -118,6 +123,7 @@ impl Default for Root { cache: None, clickhouse: None, vector_http: None, + telemetry: Default::default(), } } } diff --git a/packages/common/config/src/config/telemetry.rs b/packages/common/config/src/config/telemetry.rs new file mode 100644 index 0000000000..9641232ee8 --- /dev/null +++ b/packages/common/config/src/config/telemetry.rs @@ -0,0 +1,14 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct Telemetry { + pub enabled: bool, +} + +impl Default for Telemetry { + fn default() -> Self { + // NOTE: Telemetry is opt-out + Telemetry { enabled: true } + } +} diff --git a/packages/common/gasoline/core/Cargo.toml b/packages/common/gasoline/core/Cargo.toml index f91a058232..83223ecd37 100644 --- a/packages/common/gasoline/core/Cargo.toml +++ b/packages/common/gasoline/core/Cargo.toml @@ -12,8 +12,6 @@ async-trait.workspace = true gasoline-macros.workspace = true cjson.workspace = true dirs.workspace = true -udb-util.workspace = true -universaldb.workspace = true futures-util.workspace = true hex.workspace = true include_dir.workspace = true @@ -32,6 +30,7 @@ rivet-pools.workspace = true rivet-runtime.workspace = true rivet-test-deps.workspace = true rivet-util.workspace = true +sentry.workspace = true serde_json.workspace = true serde.workspace = true strum.workspace = true @@ -40,6 +39,8 @@ tokio-util.workspace = true tokio.workspace = true tracing-logfmt.workspace = true tracing-opentelemetry.workspace = true +udb-util.workspace = true +universaldb.workspace = true tracing-subscriber = { workspace = true, features = ["env-filter"] } tracing.workspace = true universalpubsub.workspace = true diff --git a/packages/common/gasoline/core/src/ctx/listen.rs b/packages/common/gasoline/core/src/ctx/listen.rs index bbc4c71dc2..c72b7092bb 100644 --- a/packages/common/gasoline/core/src/ctx/listen.rs +++ b/packages/common/gasoline/core/src/ctx/listen.rs @@ -91,6 +91,7 @@ impl<'a> ListenCtx<'a> { KeyValue::new("signal_name", signal.signal_name.clone()), ], ); + if recv_lag > 3.0 { // We print an error here so the trace of this workflow does not get dropped tracing::error!( diff --git a/packages/common/gasoline/core/src/ctx/workflow.rs b/packages/common/gasoline/core/src/ctx/workflow.rs index 44c4c04066..a8432f406a 100644 --- a/packages/common/gasoline/core/src/ctx/workflow.rs +++ b/packages/common/gasoline/core/src/ctx/workflow.rs @@ -226,6 +226,15 @@ impl WorkflowCtx { KeyValue::new("error_code", err.to_string()), ], ); + + sentry::with_scope( + |scope| { + scope.set_tag("error", err.to_string()); + }, + || { + sentry::capture_message("workflow error", sentry::Level::Error); + }, + ); } let err_str = err.to_string(); diff --git a/packages/common/gasoline/core/src/worker.rs b/packages/common/gasoline/core/src/worker.rs index 813926747f..b940811a4f 100644 --- a/packages/common/gasoline/core/src/worker.rs +++ b/packages/common/gasoline/core/src/worker.rs @@ -258,6 +258,18 @@ impl Worker { async move { if let Err(err) = ctx.run(current_span_ctx).await { tracing::error!(?err, "unhandled workflow error"); + + sentry::with_scope( + |scope| { + scope.set_tag("error", err.to_string()); + }, + || { + sentry::capture_message( + "unhandled workflow error", + sentry::Level::Error, + ); + }, + ); } }, ); diff --git a/packages/common/telemetry/Cargo.toml b/packages/common/telemetry/Cargo.toml new file mode 100644 index 0000000000..90b21d2ce9 --- /dev/null +++ b/packages/common/telemetry/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "rivet-telemetry" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +rivet-config.workspace = true +sentry.workspace = true +serde_json.workspace = true +uuid.workspace = true diff --git a/packages/common/telemetry/README.md b/packages/common/telemetry/README.md new file mode 100644 index 0000000000..206456f466 --- /dev/null +++ b/packages/common/telemetry/README.md @@ -0,0 +1,12 @@ +# Telemetry + +Telemetry from rivet-engine is opt-out. Update your config to include: + +```json +{ + // ... + "telemetry": { + "enabled": false + } +} +``` \ No newline at end of file diff --git a/packages/common/telemetry/src/lib.rs b/packages/common/telemetry/src/lib.rs new file mode 100644 index 0000000000..d00e5577c9 --- /dev/null +++ b/packages/common/telemetry/src/lib.rs @@ -0,0 +1,39 @@ +use uuid::Uuid; + +const SENTRY_URL: &str = "https://7602663e43cb9dee8c42d1e5e70293f8@o4504307129188352.ingest.us.sentry.io/4509962797252608"; + +// We use synchronous main for Sentry. Read more: https://docs.sentry.io/platforms/rust/#async-main-function +pub fn init(config: &rivet_config::Config) -> Option { + if !config.telemetry.enabled { + return None; + } + + let guard = sentry::init(( + SENTRY_URL, + sentry::ClientOptions { + release: sentry::release_name!(), + ..Default::default() + }, + )); + + sentry::configure_scope(|scope| { + if let Ok(db) = serde_json::to_string(config.database()) { + scope.set_tag("database", db); + } + if let Ok(ps) = serde_json::to_string(config.pubsub()) { + scope.set_tag("pubsub", ps); + } + if let Ok(cache) = serde_json::to_string(config.cache()) { + scope.set_tag("cache", cache); + } + if let Ok(topo) = serde_json::to_string(config.topology()) { + scope.set_tag("topology", topo); + } + }); + + Some(guard) +} + +pub fn capture_error(err: &anyhow::Error) -> Uuid { + sentry::integrations::anyhow::capture_anyhow(err) +} diff --git a/packages/infra/engine/Cargo.toml b/packages/infra/engine/Cargo.toml index f105ada61b..2556c213f3 100644 --- a/packages/infra/engine/Cargo.toml +++ b/packages/infra/engine/Cargo.toml @@ -23,19 +23,20 @@ include_dir.workspace = true lz4_flex.workspace = true pegboard-runner-ws.workspace = true reqwest.workspace = true -rivet-api-public.workspace = true rivet-api-peer.workspace = true +rivet-api-public.workspace = true rivet-bootstrap.workspace = true rivet-cache.workspace = true rivet-config.workspace = true +rivet-guard.workspace = true rivet-logs.workspace = true rivet-pools.workspace = true rivet-runtime.workspace = true rivet-service-manager.workspace = true +rivet-telemetry.workspace = true rivet-term.workspace = true rivet-util.workspace = true rivet-workflow-worker.workspace = true -rivet-guard.workspace = true rustyline.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/packages/infra/engine/src/main.rs b/packages/infra/engine/src/main.rs index daacf12167..f1129d2f13 100644 --- a/packages/infra/engine/src/main.rs +++ b/packages/infra/engine/src/main.rs @@ -27,9 +27,19 @@ async fn main_inner() -> Result<()> { let config = rivet_config::Config::load(&cli.config).await?; tracing::info!(config = ?*config, "loaded config"); + // Initialize telemetry (does nothing if telemetry is disabled) + let _guard = rivet_telemetry::init(&config); + // Build run config - let run_config = Arc::new(run_config::config(config.clone())?); + let run_config = Arc::new(run_config::config(config.clone()).inspect_err(|err| { + rivet_telemetry::capture_error(err); + })?); // Execute command - cli.command.execute(config, run_config).await + cli.command + .execute(config, run_config) + .await + .inspect_err(|err| { + rivet_telemetry::capture_error(err); + }) }