Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
316 changes: 316 additions & 0 deletions Cargo.lock

Large diffs are not rendered by default.

19 changes: 8 additions & 11 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@

[workspace]
resolver = "2"
members = ["packages/common/api-builder","packages/common/api-client","packages/common/api-types","packages/common/api-util","packages/common/cache/build","packages/common/cache/result","packages/common/clickhouse-inserter","packages/common/clickhouse-user-query","packages/common/config","packages/common/env","packages/common/error/core","packages/common/error/macros","packages/common/gasoline/core","packages/common/gasoline/macros","packages/common/logs","packages/common/metrics","packages/common/pools","packages/common/runtime","packages/common/service-manager","packages/common/test-deps","packages/common/test-deps-docker","packages/common/types","packages/common/udb-util","packages/common/universaldb","packages/common/universalpubsub","packages/common/util/core","packages/common/util/id","packages/common/versioned-data-util","packages/core/actor-kv","packages/core/api-peer","packages/core/api-public","packages/core/bootstrap","packages/core/dump-openapi","packages/core/guard/core","packages/core/guard/server","packages/core/pegboard-gateway","packages/core/pegboard-runner-ws","packages/core/pegboard-tunnel","packages/core/workflow-worker","packages/infra/engine","packages/services/epoxy","packages/services/namespace","packages/services/pegboard","sdks/rust/api-full","sdks/rust/api-runtime","sdks/rust/bare_gen","sdks/rust/epoxy-protocol","sdks/rust/key-data","sdks/rust/runner-protocol","sdks/rust/tunnel-protocol"]
members = ["packages/common/api-builder","packages/common/api-client","packages/common/api-types","packages/common/api-util","packages/common/cache/build","packages/common/cache/result","packages/common/clickhouse-inserter","packages/common/clickhouse-user-query","packages/common/config","packages/common/env","packages/common/error/core","packages/common/error/macros","packages/common/gasoline/core","packages/common/gasoline/macros","packages/common/logs","packages/common/metrics","packages/common/pools","packages/common/runtime","packages/common/service-manager","packages/common/telemetry","packages/common/test-deps","packages/common/test-deps-docker","packages/common/types","packages/common/udb-util","packages/common/universaldb","packages/common/universalpubsub","packages/common/util/core","packages/common/util/id","packages/common/versioned-data-util","packages/core/actor-kv","packages/core/api-peer","packages/core/api-public","packages/core/bootstrap","packages/core/dump-openapi","packages/core/guard/core","packages/core/guard/server","packages/core/pegboard-gateway","packages/core/pegboard-runner-ws","packages/core/pegboard-tunnel","packages/core/workflow-worker","packages/infra/engine","packages/services/epoxy","packages/services/namespace","packages/services/pegboard","sdks/rust/api-full","sdks/rust/api-runtime","sdks/rust/bare_gen","sdks/rust/epoxy-protocol","sdks/rust/key-data","sdks/rust/runner-protocol","sdks/rust/tunnel-protocol"]

[workspace.package]
version = "0.0.1"
Expand Down Expand Up @@ -58,6 +58,7 @@ regex = "1.4"
rstest = "0.26.1"
rustls-pemfile = "2.2.0"
rustyline = "15.0.0"
sentry = { version = "0.37.0", default-features = false, features = ["anyhow", "backtrace", "contexts", "debug-images", "panic", "reqwest", "rustls"] }
serde_bare = "0.5.0"
serde_yaml = "0.9.34"
sha2 = "0.10"
Expand All @@ -82,14 +83,7 @@ vergen = "9.0.4"

[workspace.dependencies.windows]
version = "0.58"
features = [
"Win32",
"Win32_Storage",
"Win32_Storage_FileSystem",
"Win32_System",
"Win32_System_Console",
"Win32_Security",
]
features = ["Win32","Win32_Storage","Win32_Storage_FileSystem","Win32_System","Win32_System_Console","Win32_Security"]

[workspace.dependencies.pest]
version = "2.7"
Expand All @@ -105,12 +99,12 @@ features = ["full"]
[workspace.dependencies.rustls]
version = "0.23.25"
default-features = false
features = ["ring", "std", "logging"]
features = ["ring","std","logging"]

[workspace.dependencies.tokio-rustls]
version = "0.26.2"
default-features = false
features = ["ring", "logging"]
features = ["ring","logging"]

[workspace.dependencies.utoipa]
version = "5.4.0"
Expand Down Expand Up @@ -303,6 +297,9 @@ path = "packages/common/runtime"
[workspace.dependencies.rivet-service-manager]
path = "packages/common/service-manager"

[workspace.dependencies.rivet-telemetry]
path = "packages/common/telemetry"

[workspace.dependencies.rivet-test-deps]
path = "packages/common/test-deps"

Expand Down
1 change: 1 addition & 0 deletions packages/common/api-builder/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ rivet-config.workspace = true
rivet-error.workspace = true
rivet-metrics.workspace = true
rivet-pools.workspace = true
sentry.workspace = true
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
thiserror.workspace = true
Expand Down
44 changes: 33 additions & 11 deletions packages/common/api-builder/src/middleware.rs
Original file line number Diff line number Diff line change
Expand Up @@ -128,26 +128,48 @@ pub async fn http_logging_middleware(

// Log based on status
if status.is_server_error() {
let group = error.as_ref().map_or("-", |x| &x.group);
let code = error.as_ref().map_or("-", |x| &x.code);
let meta = error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null);
let internal = error.as_ref().and_then(|x| x.internal.as_ref()).map_or("-", |x| x.as_ref());

tracing::error!(
status = ?status_code,
group = %error.as_ref().map_or("-", |x| &x.group),
code = ?error.as_ref().map_or("-", |x| &x.code),
meta = %error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null),
internal = %error.as_ref().and_then(|x| x.internal.as_ref()).map_or("-", |x| x.as_ref()),
status=?status_code,
%group,
%code,
%meta,
%internal,
"http server error"
);

sentry::with_scope(
|scope| {
scope.set_tag("status", status_code);
scope.set_tag("group", group);
scope.set_tag("code", code);
scope.set_tag("meta", meta);
scope.set_tag("internal", internal);
Comment on lines +147 to +151
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There appears to be a type inconsistency in how the status code is handled. On line 131, status_code is assigned to variable status, but on line 148, status_code is used directly when setting the Sentry tag.

For proper Sentry tag serialization, consider using the string representation of the status code:

scope.set_tag("status", status.to_string());

This ensures the status code is properly serialized as a string value when sent to Sentry, rather than potentially passing the StatusCode enum directly.

Suggested change
scope.set_tag("status", status_code);
scope.set_tag("group", group);
scope.set_tag("code", code);
scope.set_tag("meta", meta);
scope.set_tag("internal", internal);
scope.set_tag("status", status.to_string());
scope.set_tag("group", group);
scope.set_tag("code", code);
scope.set_tag("meta", meta);
scope.set_tag("internal", internal);

Spotted by Diamond

Fix in Graphite


Is this helpful? React 👍 or 👎 to let us know.

},
|| {
sentry::capture_message(&format!("{group}.{code}"), sentry::Level::Error);
},
);
} else if status.is_client_error() {
let group = error.as_ref().map_or("-", |x| &x.group);
let code = error.as_ref().map_or("-", |x| &x.code);
let meta = error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null);

tracing::info!(
status = ?status_code,
group = %error.as_ref().map_or("-", |x| &x.group),
code = %error.as_ref().map_or("-", |x| &x.code),
meta = %error.as_ref().and_then(|x| x.metadata.as_ref()).unwrap_or(&serde_json::Value::Null),
status=?status_code,
%group,
%code,
%meta,
"http client error"
);
} else if status.is_redirection() {
tracing::debug!(status = ?status_code, "http redirection");
tracing::debug!(status=?status_code, "http redirection");
} else if status.is_informational() {
tracing::debug!(status = ?status_code, "http informational");
tracing::debug!(status=?status_code, "http informational");
}

let duration = start.elapsed().as_secs_f64();
Expand Down
6 changes: 6 additions & 0 deletions packages/common/config/src/config/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ pub mod pegboard;
pub mod pegboard_gateway;
pub mod pegboard_tunnel;
pub mod pubsub;
pub mod telemetry;
pub mod topology;
pub mod vector;

Expand All @@ -28,6 +29,7 @@ pub use pegboard::*;
pub use pegboard_gateway::*;
pub use pegboard_tunnel::*;
pub use pubsub::PubSub;
pub use telemetry::*;
pub use topology::*;
pub use vector::*;

Expand Down Expand Up @@ -100,6 +102,9 @@ pub struct Root {

#[serde(default)]
pub vector_http: Option<VectorHttp>,

#[serde(default)]
pub telemetry: Telemetry,
}

impl Default for Root {
Expand All @@ -118,6 +123,7 @@ impl Default for Root {
cache: None,
clickhouse: None,
vector_http: None,
telemetry: Default::default(),
}
}
}
Expand Down
14 changes: 14 additions & 0 deletions packages/common/config/src/config/telemetry.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};

#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct Telemetry {
pub enabled: bool,
}

impl Default for Telemetry {
fn default() -> Self {
// NOTE: Telemetry is opt-out
Telemetry { enabled: true }
}
Comment on lines +10 to +13
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Privacy and Security Consideration: The current implementation makes telemetry opt-out rather than opt-in (enabled: true by default). This approach may lead to unintended data collection from users who haven't explicitly consented to telemetry. Consider changing to an opt-in model where enabled defaults to false, requiring users to make a conscious choice to enable data collection. This would better align with privacy best practices and reduce the risk of inadvertent data sharing.

Suggested change
fn default() -> Self {
// NOTE: Telemetry is opt-out
Telemetry { enabled: true }
}
fn default() -> Self {
// NOTE: Telemetry is opt-in
Telemetry { enabled: false }
}

Spotted by Diamond

Fix in Graphite


Is this helpful? React 👍 or 👎 to let us know.

}
5 changes: 3 additions & 2 deletions packages/common/gasoline/core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@ async-trait.workspace = true
gasoline-macros.workspace = true
cjson.workspace = true
dirs.workspace = true
udb-util.workspace = true
universaldb.workspace = true
futures-util.workspace = true
hex.workspace = true
include_dir.workspace = true
Expand All @@ -32,6 +30,7 @@ rivet-pools.workspace = true
rivet-runtime.workspace = true
rivet-test-deps.workspace = true
rivet-util.workspace = true
sentry.workspace = true
serde_json.workspace = true
serde.workspace = true
strum.workspace = true
Expand All @@ -40,6 +39,8 @@ tokio-util.workspace = true
tokio.workspace = true
tracing-logfmt.workspace = true
tracing-opentelemetry.workspace = true
udb-util.workspace = true
universaldb.workspace = true
tracing-subscriber = { workspace = true, features = ["env-filter"] }
tracing.workspace = true
universalpubsub.workspace = true
Expand Down
1 change: 1 addition & 0 deletions packages/common/gasoline/core/src/ctx/listen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ impl<'a> ListenCtx<'a> {
KeyValue::new("signal_name", signal.signal_name.clone()),
],
);

if recv_lag > 3.0 {
// We print an error here so the trace of this workflow does not get dropped
tracing::error!(
Expand Down
9 changes: 9 additions & 0 deletions packages/common/gasoline/core/src/ctx/workflow.rs
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,15 @@ impl WorkflowCtx {
KeyValue::new("error_code", err.to_string()),
],
);

sentry::with_scope(
|scope| {
scope.set_tag("error", err.to_string());
},
|| {
sentry::capture_message("workflow error", sentry::Level::Error);
},
);
}

let err_str = err.to_string();
Expand Down
12 changes: 12 additions & 0 deletions packages/common/gasoline/core/src/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,18 @@ impl Worker {
async move {
if let Err(err) = ctx.run(current_span_ctx).await {
tracing::error!(?err, "unhandled workflow error");

sentry::with_scope(
|scope| {
scope.set_tag("error", err.to_string());
},
|| {
sentry::capture_message(
"unhandled workflow error",
sentry::Level::Error,
);
},
);
}
},
);
Expand Down
13 changes: 13 additions & 0 deletions packages/common/telemetry/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[package]
name = "rivet-telemetry"
version.workspace = true
authors.workspace = true
license.workspace = true
edition.workspace = true

[dependencies]
anyhow.workspace = true
rivet-config.workspace = true
sentry.workspace = true
serde_json.workspace = true
uuid.workspace = true
12 changes: 12 additions & 0 deletions packages/common/telemetry/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Telemetry

Telemetry from rivet-engine is opt-out. Update your config to include:

```json
{
// ...
"telemetry": {
"enabled": false
}
}
```
39 changes: 39 additions & 0 deletions packages/common/telemetry/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
use uuid::Uuid;

const SENTRY_URL: &str = "https://7602663e43cb9dee8c42d1e5e70293f8@o4504307129188352.ingest.us.sentry.io/4509962797252608";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Security Concern: The Sentry DSN URL contains an authentication token (7602663e43cb9dee8c42d1e5e70293f8) that's hardcoded in the source code. This creates a security risk as anyone with repository access can view this token.

Recommendation: Move this sensitive credential to a secure configuration mechanism:

  • Store in environment variables
  • Use a secrets management system
  • Add to a configuration file that's excluded from version control

This follows security best practices for credential management and prevents potential misuse of the Sentry instance.

Suggested change
const SENTRY_URL: &str = "https://[email protected].sentry.io/4509962797252608";
const SENTRY_URL: &str = option_env!("SENTRY_DSN").unwrap_or("https://public@sentry.example.com/1");

Spotted by Diamond

Fix in Graphite


Is this helpful? React 👍 or 👎 to let us know.


// We use synchronous main for Sentry. Read more: https://docs.sentry.io/platforms/rust/#async-main-function
pub fn init(config: &rivet_config::Config) -> Option<sentry::ClientInitGuard> {
if !config.telemetry.enabled {
return None;
}

let guard = sentry::init((
SENTRY_URL,
sentry::ClientOptions {
release: sentry::release_name!(),
..Default::default()
},
));

sentry::configure_scope(|scope| {
if let Ok(db) = serde_json::to_string(config.database()) {
scope.set_tag("database", db);
}
if let Ok(ps) = serde_json::to_string(config.pubsub()) {
scope.set_tag("pubsub", ps);
}
if let Ok(cache) = serde_json::to_string(config.cache()) {
scope.set_tag("cache", cache);
}
if let Ok(topo) = serde_json::to_string(config.topology()) {
scope.set_tag("topology", topo);
}
});

Some(guard)
}

pub fn capture_error(err: &anyhow::Error) -> Uuid {
sentry::integrations::anyhow::capture_anyhow(err)
}
5 changes: 3 additions & 2 deletions packages/infra/engine/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,20 @@ include_dir.workspace = true
lz4_flex.workspace = true
pegboard-runner-ws.workspace = true
reqwest.workspace = true
rivet-api-public.workspace = true
rivet-api-peer.workspace = true
rivet-api-public.workspace = true
rivet-bootstrap.workspace = true
rivet-cache.workspace = true
rivet-config.workspace = true
rivet-guard.workspace = true
rivet-logs.workspace = true
rivet-pools.workspace = true
rivet-runtime.workspace = true
rivet-service-manager.workspace = true
rivet-telemetry.workspace = true
rivet-term.workspace = true
rivet-util.workspace = true
rivet-workflow-worker.workspace = true
rivet-guard.workspace = true
rustyline.workspace = true
serde.workspace = true
serde_json.workspace = true
Expand Down
14 changes: 12 additions & 2 deletions packages/infra/engine/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,19 @@ async fn main_inner() -> Result<()> {
let config = rivet_config::Config::load(&cli.config).await?;
tracing::info!(config = ?*config, "loaded config");

// Initialize telemetry (does nothing if telemetry is disabled)
let _guard = rivet_telemetry::init(&config);

// Build run config
let run_config = Arc::new(run_config::config(config.clone())?);
let run_config = Arc::new(run_config::config(config.clone()).inspect_err(|err| {
rivet_telemetry::capture_error(err);
})?);

// Execute command
cli.command.execute(config, run_config).await
cli.command
.execute(config, run_config)
.await
.inspect_err(|err| {
rivet_telemetry::capture_error(err);
})
}
Loading