Skip to content
/ loom Public

Commit 24e100d

Browse files
ghuntleyclaude
andcommitted
Add self-monitoring infrastructure for Loom to monitor itself
This enables automatic crash/error monitoring for loom-server, loom-web, and loom-cli without requiring user configuration. Server changes: - Add self_monitoring.rs module that creates internal org/projects/API keys - Add routes for /api/self-monitoring/{web,cli}-config endpoints - Install panic hook for loom-server crash capture - Initialize on startup after database migrations Web changes: - Add SelfMonitoringProvider.svelte that wraps app layout - Add self-monitoring.ts that fetches config and initializes CrashClient - Install global error handlers for automatic crash capture CLI changes: - Add self_monitoring.rs module - Fetch CLI crash config from server on startup - Install panic hook for loom-cli crash capture Well-known UUIDs: - Loom Internal org: 00000000-0000-0000-0000-000000000001 - loom-server project: 00000000-0000-0000-0000-000000000002 - loom-web project: 00000000-0000-0000-0000-000000000003 - loom-cli project: 00000000-0000-0000-0000-000000000004 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 012a48d commit 24e100d

File tree

19 files changed

+914
-12
lines changed

19 files changed

+914
-12
lines changed

Cargo.lock

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.nix

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ args@{
3232
"loom-wgtunnel-engine/default"
3333
"loom-wgtunnel-conn/default"
3434
"loom-wgtunnel-derp/default"
35+
"loom-crash/default"
36+
"loom-crash-core/default"
3537
"loom-server-logs/default"
3638
"loom-redact/default"
3739
"loom-server-github-app/default"
@@ -45,7 +47,6 @@ args@{
4547
"loom-server-llm-zai/default"
4648
"loom-server/default"
4749
"loom-analytics-core/default"
48-
"loom-crash-core/default"
4950
"loom-crons-core/default"
5051
"loom-flags-core/default"
5152
"loom-server-analytics/default"
@@ -82,7 +83,6 @@ args@{
8283
"loom-flags/default"
8384
"loom-analytics/default"
8485
"loom-crons/default"
85-
"loom-crash/default"
8686
],
8787
rustPackages,
8888
buildRustPackages,
@@ -103,7 +103,7 @@ args@{
103103
cargoConfig ? {},
104104
}:
105105
let
106-
nixifiedLockHash = "cc38f25d98e3e006684d5f64a49990ced836f87891a2fb03587987859cb474e0";
106+
nixifiedLockHash = "02081f70168851874845f2e5981fcf79c2fb5278dee986f64888f08174e4f1e6";
107107
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
108108
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
109109
lockHashIgnored = if ignoreLockHash
@@ -162,6 +162,8 @@ in
162162
loom-wgtunnel-engine = rustPackages.unknown.loom-wgtunnel-engine."0.1.0";
163163
loom-wgtunnel-conn = rustPackages.unknown.loom-wgtunnel-conn."0.1.0";
164164
loom-wgtunnel-derp = rustPackages.unknown.loom-wgtunnel-derp."0.1.0";
165+
loom-crash = rustPackages.unknown.loom-crash."0.1.0";
166+
loom-crash-core = rustPackages.unknown.loom-crash-core."0.1.0";
165167
loom-server-logs = rustPackages.unknown.loom-server-logs."0.1.0";
166168
loom-redact = rustPackages.unknown.loom-redact."0.1.0";
167169
loom-server-github-app = rustPackages.unknown.loom-server-github-app."0.1.0";
@@ -175,7 +177,6 @@ in
175177
loom-server-llm-zai = rustPackages.unknown.loom-server-llm-zai."0.1.0";
176178
loom-server = rustPackages.unknown.loom-server."0.1.0";
177179
loom-analytics-core = rustPackages.unknown.loom-analytics-core."0.1.0";
178-
loom-crash-core = rustPackages.unknown.loom-crash-core."0.1.0";
179180
loom-crons-core = rustPackages.unknown.loom-crons-core."0.1.0";
180181
loom-flags-core = rustPackages.unknown.loom-flags-core."0.1.0";
181182
loom-server-analytics = rustPackages.unknown.loom-server-analytics."0.1.0";
@@ -212,7 +213,6 @@ in
212213
loom-flags = rustPackages.unknown.loom-flags."0.1.0";
213214
loom-analytics = rustPackages.unknown.loom-analytics."0.1.0";
214215
loom-crons = rustPackages.unknown.loom-crons."0.1.0";
215-
loom-crash = rustPackages.unknown.loom-crash."0.1.0";
216216
};
217217
"registry+https://github.com/rust-lang/crates.io-index".addr2line."0.25.1" = overridableMkRustCrate (profileName: rec {
218218
name = "addr2line";
@@ -6455,6 +6455,7 @@ in
64556455
loom_common_secret = (rustPackages."unknown".loom-common-secret."0.1.0" { inherit profileName; }).out;
64566456
loom_common_thread = (rustPackages."unknown".loom-common-thread."0.1.0" { inherit profileName; }).out;
64576457
loom_common_version = (rustPackages."unknown".loom-common-version."0.1.0" { inherit profileName; }).out;
6458+
loom_crash = (rustPackages."unknown".loom-crash."0.1.0" { inherit profileName; }).out;
64586459
loom_server_llm_proxy = (rustPackages."unknown".loom-server-llm-proxy."0.1.0" { inherit profileName; }).out;
64596460
loom_server_logs = (rustPackages."unknown".loom-server-logs."0.1.0" { inherit profileName; }).out;
64606461
reqwest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".reqwest."0.12.28" { inherit profileName; }).out;
@@ -7103,6 +7104,7 @@ in
71037104
loom_common_secret = (rustPackages."unknown".loom-common-secret."0.1.0" { inherit profileName; }).out;
71047105
loom_common_thread = (rustPackages."unknown".loom-common-thread."0.1.0" { inherit profileName; }).out;
71057106
loom_common_version = (rustPackages."unknown".loom-common-version."0.1.0" { inherit profileName; }).out;
7107+
loom_crash = (rustPackages."unknown".loom-crash."0.1.0" { inherit profileName; }).out;
71067108
loom_crash_core = (rustPackages."unknown".loom-crash-core."0.1.0" { inherit profileName; }).out;
71077109
loom_crons_core = (rustPackages."unknown".loom-crons-core."0.1.0" { inherit profileName; }).out;
71087110
loom_flags_core = (rustPackages."unknown".loom-flags-core."0.1.0" { inherit profileName; }).out;

crates/loom-cli/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ sha2 = { workspace = true }
2626
sys-locale = "0.3"
2727
humantime = "2"
2828
loom-cli-acp = { path = "../loom-cli-acp" }
29+
loom-crash = { path = "../loom-crash" }
2930
loom-cli-auto-commit = { path = "../loom-cli-auto-commit" }
3031
loom-cli-config = { path = "../loom-cli-config" }
3132
loom-common-core = { path = "../loom-common-core" }

crates/loom-cli/src/main.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ mod crash_client;
6767
mod credential_helper;
6868
mod crons_client;
6969
mod locale;
70+
mod self_monitoring;
7071
mod sessions_client;
7172
mod update;
7273
mod version;
@@ -1335,6 +1336,15 @@ async fn main() -> Result<()> {
13351336
"starting loom"
13361337
);
13371338

1339+
// Initialize self-monitoring for crash reporting
1340+
// This is non-blocking and failures are logged but don't prevent CLI from running
1341+
tokio::spawn({
1342+
let server_url = args.server_url.clone();
1343+
async move {
1344+
self_monitoring::initialize_self_monitoring(&server_url).await;
1345+
}
1346+
});
1347+
13381348
// Get auth token for thread sync
13391349
let auth_token = auth::load_token(&args.server_url).await;
13401350

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// Copyright (c) 2025 Geoffrey Huntley <ghuntley@ghuntley.com>. All rights reserved.
2+
// SPDX-License-Identifier: Proprietary
3+
4+
//! Self-monitoring for loom-cli.
5+
//!
6+
//! Automatically initializes crash monitoring for the CLI by fetching
7+
//! configuration from the server. This enables Loom to monitor itself
8+
//! without any user configuration.
9+
10+
use std::sync::Arc;
11+
12+
use anyhow::Result;
13+
use loom_crash::{CrashClient, CrashClientBuilder};
14+
use serde::Deserialize;
15+
use tokio::sync::OnceCell;
16+
use tracing::{debug, warn};
17+
18+
/// Well-known project ID for loom-cli (from loom-server self_monitoring.rs).
19+
const LOOM_CLI_PROJECT_ID: &str = "00000000-0000-0000-0000-000000000004";
20+
21+
/// Global crash client for loom-cli.
22+
static CLI_CRASH_CLIENT: OnceCell<Arc<CrashClient>> = OnceCell::const_new();
23+
24+
/// Configuration returned by the self-monitoring API.
25+
#[derive(Debug, Deserialize)]
26+
struct CliCrashConfig {
27+
project_id: String,
28+
api_key: String,
29+
release: String,
30+
environment: String,
31+
}
32+
33+
/// Initialize self-monitoring for loom-cli.
34+
///
35+
/// This fetches the CLI API key from the server's self-monitoring endpoint
36+
/// and initializes the crash client with a panic hook.
37+
///
38+
/// Call this once during CLI startup. Failures are logged but do not
39+
/// prevent the CLI from running.
40+
pub async fn initialize_self_monitoring(server_url: &str) -> Option<Arc<CrashClient>> {
41+
// Don't initialize twice
42+
if let Some(client) = CLI_CRASH_CLIENT.get() {
43+
return Some(client.clone());
44+
}
45+
46+
match fetch_and_initialize(server_url).await {
47+
Ok(client) => {
48+
let client = Arc::new(client);
49+
let _ = CLI_CRASH_CLIENT.set(client.clone());
50+
debug!("Self-monitoring initialized for loom-cli");
51+
Some(client)
52+
}
53+
Err(e) => {
54+
warn!("Failed to initialize self-monitoring: {}", e);
55+
None
56+
}
57+
}
58+
}
59+
60+
async fn fetch_and_initialize(server_url: &str) -> Result<CrashClient> {
61+
// Fetch CLI config from server
62+
let http = loom_common_http::new_client();
63+
let url = format!(
64+
"{}/api/self-monitoring/cli-config",
65+
server_url.trim_end_matches('/')
66+
);
67+
68+
let response = http.get(&url).send().await?;
69+
70+
if !response.status().is_success() {
71+
anyhow::bail!(
72+
"Self-monitoring not available: {} {}",
73+
response.status(),
74+
response.status().canonical_reason().unwrap_or("")
75+
);
76+
}
77+
78+
let config: CliCrashConfig = response.json().await?;
79+
80+
let client = CrashClientBuilder::new()
81+
.base_url(server_url)
82+
.project_id(&config.project_id)
83+
.auth_token(&config.api_key)
84+
.release(&config.release)
85+
.environment(&config.environment)
86+
.build()?;
87+
88+
// Install panic hook for automatic crash reporting
89+
client.install_panic_hook();
90+
91+
Ok(client)
92+
}
93+
94+
/// Get the CLI crash client for manual error capture.
95+
pub fn get_crash_client() -> Option<Arc<CrashClient>> {
96+
CLI_CRASH_CLIENT.get().cloned()
97+
}
98+
99+
/// Capture an exception using the self-monitoring crash client.
100+
///
101+
/// This is a convenience function that handles the case where
102+
/// self-monitoring is not initialized.
103+
#[allow(dead_code)]
104+
pub fn capture_exception(error: &dyn std::error::Error) -> Option<String> {
105+
if let Some(client) = get_crash_client() {
106+
// Use blocking capture in sync context
107+
// For now, just log since async capture would require runtime
108+
warn!("CLI crash captured (async send not implemented): {}", error);
109+
return None;
110+
}
111+
None
112+
}
113+
114+
/// Shutdown the CLI crash client.
115+
///
116+
/// Call this during CLI shutdown to flush pending events.
117+
pub async fn shutdown_self_monitoring() {
118+
if let Some(client) = CLI_CRASH_CLIENT.get() {
119+
if let Err(e) = client.shutdown().await {
120+
warn!("Failed to shutdown crash client: {}", e);
121+
}
122+
}
123+
}

crates/loom-server/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ loom-analytics-core = { path = "../loom-analytics-core" }
4747
loom-server-crons = { path = "../loom-server-crons" }
4848
loom-crons-core = { path = "../loom-crons-core", features = ["openapi"] }
4949
loom-server-crash = { path = "../loom-server-crash" }
50+
loom-crash = { path = "../loom-crash" }
5051
loom-crash-core = { path = "../loom-crash-core", features = ["openapi"] }
5152
loom-server-sessions = { path = "../loom-server-sessions" }
5253
loom-sessions-core = { path = "../loom-sessions-core", features = ["openapi"] }

crates/loom-server/src/api.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -904,6 +904,19 @@ pub fn create_router(state: AppState) -> Router {
904904
)
905905
// Documentation search
906906
.route("/docs/search", get(routes::docs::search_handler))
907+
// Self-monitoring configuration (public for SDK initialization)
908+
.route(
909+
"/api/self-monitoring/web-config",
910+
get(routes::self_monitoring::get_web_config),
911+
)
912+
.route(
913+
"/api/self-monitoring/cli-config",
914+
get(routes::self_monitoring::get_cli_config),
915+
)
916+
.route(
917+
"/api/self-monitoring/projects",
918+
get(routes::self_monitoring::get_internal_projects),
919+
)
907920
// Feature flags SSE streaming (SDK key auth handled in handler)
908921
.route("/api/flags/stream", get(routes::flags::stream_flags))
909922
// Analytics SDK routes (API key auth handled in handler)

crates/loom-server/src/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ pub mod query_metrics;
2626
pub mod query_security;
2727
pub mod query_tracing;
2828
pub mod routes;
29+
pub mod self_monitoring;
2930
pub mod server_query;
3031
pub mod typed_router;
3132
pub mod validation;

crates/loom-server/src/main.rs

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,41 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
8282
// Run database migrations
8383
loom_server::db::run_migrations(&pool).await?;
8484

85+
// Initialize self-monitoring for Loom itself (internal crash reporting)
86+
let base_url = format!(
87+
"{}://{}:{}",
88+
if config.http.host == "0.0.0.0" || config.http.host == "127.0.0.1" {
89+
"http"
90+
} else {
91+
"https"
92+
},
93+
&config.http.host,
94+
config.http.port
95+
);
96+
let release = env!("CARGO_PKG_VERSION");
97+
let environment = std::env::var("LOOM_ENVIRONMENT").unwrap_or_else(|_| "production".to_string());
98+
99+
match loom_server::self_monitoring::initialize_self_monitoring(
100+
&pool,
101+
&base_url,
102+
release,
103+
&environment,
104+
)
105+
.await
106+
{
107+
Ok(self_mon_config) => {
108+
tracing::info!(
109+
server_api_key = ?self_mon_config.server_api_key.as_ref().map(|k| &k[..10]),
110+
web_api_key = ?self_mon_config.web_api_key.as_ref().map(|k| &k[..10]),
111+
cli_api_key = ?self_mon_config.cli_api_key.as_ref().map(|k| &k[..10]),
112+
"Self-monitoring initialized"
113+
);
114+
}
115+
Err(e) => {
116+
tracing::warn!(error = %e, "Failed to initialize self-monitoring, continuing without it");
117+
}
118+
}
119+
85120
let repo = Arc::new(ThreadRepository::new(pool.clone()));
86121
let mut state = create_app_state(pool.clone(), repo, &config, Some(log_buffer)).await;
87122

crates/loom-server/src/routes/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ pub mod orgs;
3030
pub mod protection;
3131
pub mod repos;
3232
pub mod secrets;
33+
pub mod self_monitoring;
3334
pub mod serper;
3435
pub mod sessions;
3536
pub mod share;

0 commit comments

Comments
 (0)