Skip to content

Commit 8a60a1f

Browse files
MasterPtatoMasterPtato
committed
feat: add telemetry via sentry (#2865)
1 parent fe6d7a9 commit 8a60a1f

File tree

15 files changed

+482
-28
lines changed

15 files changed

+482
-28
lines changed

‎Cargo.lock‎

Lines changed: 316 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11

22
[workspace]
33
resolver = "2"
4-
members = ["packages/common/api-builder","packages/common/api-client","packages/common/api-types","packages/common/api-util","packages/common/cache/build","packages/common/cache/result","packages/common/clickhouse-inserter","packages/common/clickhouse-user-query","packages/common/config","packages/common/env","packages/common/error/core","packages/common/error/macros","packages/common/gasoline/core","packages/common/gasoline/macros","packages/common/logs","packages/common/metrics","packages/common/pools","packages/common/runtime","packages/common/service-manager","packages/common/test-deps","packages/common/test-deps-docker","packages/common/types","packages/common/udb-util","packages/common/universaldb","packages/common/universalpubsub","packages/common/util/core","packages/common/util/id","packages/common/versioned-data-util","packages/core/actor-kv","packages/core/api-peer","packages/core/api-public","packages/core/bootstrap","packages/core/dump-openapi","packages/core/guard/core","packages/core/guard/server","packages/core/pegboard-gateway","packages/core/pegboard-runner-ws","packages/core/pegboard-tunnel","packages/core/workflow-worker","packages/infra/engine","packages/services/epoxy","packages/services/namespace","packages/services/pegboard","sdks/rust/api-full","sdks/rust/api-runtime","sdks/rust/bare_gen","sdks/rust/epoxy-protocol","sdks/rust/key-data","sdks/rust/runner-protocol","sdks/rust/tunnel-protocol"]
4+
members = ["packages/common/api-builder","packages/common/api-client","packages/common/api-types","packages/common/api-util","packages/common/cache/build","packages/common/cache/result","packages/common/clickhouse-inserter","packages/common/clickhouse-user-query","packages/common/config","packages/common/env","packages/common/error/core","packages/common/error/macros","packages/common/gasoline/core","packages/common/gasoline/macros","packages/common/logs","packages/common/metrics","packages/common/pools","packages/common/runtime","packages/common/service-manager","packages/common/telemetry","packages/common/test-deps","packages/common/test-deps-docker","packages/common/types","packages/common/udb-util","packages/common/universaldb","packages/common/universalpubsub","packages/common/util/core","packages/common/util/id","packages/common/versioned-data-util","packages/core/actor-kv","packages/core/api-peer","packages/core/api-public","packages/core/bootstrap","packages/core/dump-openapi","packages/core/guard/core","packages/core/guard/server","packages/core/pegboard-gateway","packages/core/pegboard-runner-ws","packages/core/pegboard-tunnel","packages/core/workflow-worker","packages/infra/engine","packages/services/epoxy","packages/services/namespace","packages/services/pegboard","sdks/rust/api-full","sdks/rust/api-runtime","sdks/rust/bare_gen","sdks/rust/epoxy-protocol","sdks/rust/key-data","sdks/rust/runner-protocol","sdks/rust/tunnel-protocol"]
55

66
[workspace.package]
77
version = "0.0.1"
@@ -58,6 +58,7 @@ regex = "1.4"
5858
rstest = "0.26.1"
5959
rustls-pemfile = "2.2.0"
6060
rustyline = "15.0.0"
61+
sentry = { version = "0.37.0", default-features = false, features = ["anyhow", "backtrace", "contexts", "debug-images", "panic", "reqwest", "rustls"] }
6162
serde_bare = "0.5.0"
6263
serde_yaml = "0.9.34"
6364
sha2 = "0.10"
@@ -82,14 +83,7 @@ vergen = "9.0.4"
8283

8384
[workspace.dependencies.windows]
8485
version = "0.58"
85-
features = [
86-
"Win32",
87-
"Win32_Storage",
88-
"Win32_Storage_FileSystem",
89-
"Win32_System",
90-
"Win32_System_Console",
91-
"Win32_Security",
92-
]
86+
features = ["Win32","Win32_Storage","Win32_Storage_FileSystem","Win32_System","Win32_System_Console","Win32_Security"]
9387

9488
[workspace.dependencies.pest]
9589
version = "2.7"
@@ -105,12 +99,12 @@ features = ["full"]
10599
[workspace.dependencies.rustls]
106100
version = "0.23.25"
107101
default-features = false
108-
features = ["ring", "std", "logging"]
102+
features = ["ring","std","logging"]
109103

110104
[workspace.dependencies.tokio-rustls]
111105
version = "0.26.2"
112106
default-features = false
113-
features = ["ring", "logging"]
107+
features = ["ring","logging"]
114108

115109
[workspace.dependencies.utoipa]
116110
version = "5.4.0"
@@ -303,6 +297,9 @@ path = "packages/common/runtime"
303297
[workspace.dependencies.rivet-service-manager]
304298
path = "packages/common/service-manager"
305299

300+
[workspace.dependencies.rivet-telemetry]
301+
path = "packages/common/telemetry"
302+
306303
[workspace.dependencies.rivet-test-deps]
307304
path = "packages/common/test-deps"
308305

‎packages/common/api-builder/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ rivet-config.workspace = true
1818
rivet-error.workspace = true
1919
rivet-metrics.workspace = true
2020
rivet-pools.workspace = true
21+
sentry.workspace = true
2122
serde = { workspace = true, features = ["derive"] }
2223
serde_json.workspace = true
2324
thiserror.workspace = true

‎packages/common/api-builder/src/middleware.rs‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -128,26 +128,48 @@ pub async fn http_logging_middleware(
128128

129129
// Log based on status
130130
if status.is_server_error() {
131+
let group = error.as_ref().map_or("-", |x| &x.group);
132+
let code = error.as_ref().map_or("-", |x| &x.code);
133+
let meta = error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null);
134+
let internal = error.as_ref().and_then(|x| x.internal.as_ref()).map_or("-", |x| x.as_ref());
135+
131136
tracing::error!(
132-
status = ?status_code,
133-
group = %error.as_ref().map_or("-", |x| &x.group),
134-
code = ?error.as_ref().map_or("-", |x| &x.code),
135-
meta = %error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null),
136-
internal = %error.as_ref().and_then(|x| x.internal.as_ref()).map_or("-", |x| x.as_ref()),
137+
status=?status_code,
138+
%group,
139+
%code,
140+
%meta,
141+
%internal,
137142
"http server error"
138143
);
144+
145+
sentry::with_scope(
146+
|scope| {
147+
scope.set_tag("status", status_code);
148+
scope.set_tag("group", group);
149+
scope.set_tag("code", code);
150+
scope.set_tag("meta", meta);
151+
scope.set_tag("internal", internal);
152+
},
153+
|| {
154+
sentry::capture_message(&format!("{group}.{code}"), sentry::Level::Error);
155+
},
156+
);
139157
} else if status.is_client_error() {
158+
let group = error.as_ref().map_or("-", |x| &x.group);
159+
let code = error.as_ref().map_or("-", |x| &x.code);
160+
let meta = error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null);
161+
140162
tracing::info!(
141-
status = ?status_code,
142-
group = %error.as_ref().map_or("-", |x| &x.group),
143-
code = %error.as_ref().map_or("-", |x| &x.code),
144-
meta = %error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null),
163+
status=?status_code,
164+
%group,
165+
%code,
166+
%meta,
145167
"http client error"
146168
);
147169
} else if status.is_redirection() {
148-
tracing::debug!(status = ?status_code, "http redirection");
170+
tracing::debug!(status=?status_code, "http redirection");
149171
} else if status.is_informational() {
150-
tracing::debug!(status = ?status_code, "http informational");
172+
tracing::debug!(status=?status_code, "http informational");
151173
}
152174

153175
let duration = start.elapsed().as_secs_f64();

‎packages/common/config/src/config/mod.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ pub mod pegboard;
1414
pub mod pegboard_gateway;
1515
pub mod pegboard_tunnel;
1616
pub mod pubsub;
17+
pub mod telemetry;
1718
pub mod topology;
1819
pub mod vector;
1920

@@ -28,6 +29,7 @@ pub use pegboard::*;
2829
pub use pegboard_gateway::*;
2930
pub use pegboard_tunnel::*;
3031
pub use pubsub::PubSub;
32+
pub use telemetry::*;
3133
pub use topology::*;
3234
pub use vector::*;
3335

@@ -100,6 +102,9 @@ pub struct Root {
100102

101103
#[serde(default)]
102104
pub vector_http: Option<VectorHttp>,
105+
106+
#[serde(default)]
107+
pub telemetry: Telemetry,
103108
}
104109

105110
impl Default for Root {
@@ -118,6 +123,7 @@ impl Default for Root {
118123
cache: None,
119124
clickhouse: None,
120125
vector_http: None,
126+
telemetry: Default::default(),
121127
}
122128
}
123129
}

‎packages/common/config/src/config/telemetry.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
use schemars::JsonSchema;
2+
use serde::{Deserialize, Serialize};
3+
4+
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
5+
pub struct Telemetry {
6+
pub enabled: bool,
7+
}
8+
9+
impl Default for Telemetry {
10+
fn default() -> Self {
11+
// NOTE: Telemetry is opt-out
12+
Telemetry { enabled: true }
13+
}
14+
}

‎packages/common/gasoline/core/Cargo.toml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@ async-trait.workspace = true
1212
gasoline-macros.workspace = true
1313
cjson.workspace = true
1414
dirs.workspace = true
15-
udb-util.workspace = true
16-
universaldb.workspace = true
1715
futures-util.workspace = true
1816
hex.workspace = true
1917
include_dir.workspace = true
@@ -32,6 +30,7 @@ rivet-pools.workspace = true
3230
rivet-runtime.workspace = true
3331
rivet-test-deps.workspace = true
3432
rivet-util.workspace = true
33+
sentry.workspace = true
3534
serde_json.workspace = true
3635
serde.workspace = true
3736
strum.workspace = true
@@ -40,6 +39,8 @@ tokio-util.workspace = true
4039
tokio.workspace = true
4140
tracing-logfmt.workspace = true
4241
tracing-opentelemetry.workspace = true
42+
udb-util.workspace = true
43+
universaldb.workspace = true
4344
tracing-subscriber = { workspace = true, features = ["env-filter"] }
4445
tracing.workspace = true
4546
universalpubsub.workspace = true

‎packages/common/gasoline/core/src/ctx/listen.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ impl<'a> ListenCtx<'a> {
9191
KeyValue::new("signal_name", signal.signal_name.clone()),
9292
],
9393
);
94+
9495
if recv_lag > 3.0 {
9596
// We print an error here so the trace of this workflow does not get dropped
9697
tracing::error!(

‎packages/common/gasoline/core/src/ctx/workflow.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,15 @@ impl WorkflowCtx {
226226
KeyValue::new("error_code", err.to_string()),
227227
],
228228
);
229+
230+
sentry::with_scope(
231+
|scope| {
232+
scope.set_tag("error", err.to_string());
233+
},
234+
|| {
235+
sentry::capture_message("workflow error", sentry::Level::Error);
236+
},
237+
);
229238
}
230239

231240
let err_str = err.to_string();

‎packages/common/gasoline/core/src/worker.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,18 @@ impl Worker {
258258
async move {
259259
if let Err(err) = ctx.run(current_span_ctx).await {
260260
tracing::error!(?err, "unhandled workflow error");
261+
262+
sentry::with_scope(
263+
|scope| {
264+
scope.set_tag("error", err.to_string());
265+
},
266+
|| {
267+
sentry::capture_message(
268+
"unhandled workflow error",
269+
sentry::Level::Error,
270+
);
271+
},
272+
);
261273
}
262274
},
263275
);

0 commit comments

Comments
 (0)