feat: Add Grafana Explore links, OTLP flushing, and structured logging

- Print direct trace and metrics Grafana Explore URLs on pipeline completion.
- Implement `flush_global_otel` to ensure OTLP traces and metrics are exported before process exit.
- Save structured logs locally (`/tmp/void-box-playground-last.log` by default).
- Update relevant documentation and scripts to reflect new observability features.

Author: dpsoft

README.md

@@ -87,7 +87,8 @@ The script starts Docker Compose services, runs:
 cargo run --example playground_pipeline --features opentelemetry
 ```
 
-Then prints Grafana URL and service filter hints.
+Then prints direct Grafana Explore links for traces and metrics.
+Playground logs are also written locally to `/tmp/void-box-playground-last.log` by default.
 It also asks for provider mode (`Anthropic`, `Ollama`, `Mock`) and prepares initramfs automatically:
 - `Mock` -> `scripts/build_test_image.sh` (`/tmp/void-box-test-rootfs.cpio.gz`, claudio mock)
 - `Anthropic` / `Ollama` -> `scripts/build_guest_image.sh` (`/tmp/void-box-rootfs.cpio.gz`)

docs/GETTING_STARTED.md

@@ -85,6 +85,10 @@ This uses the OTLP-enabled example:
 cargo run --example playground_pipeline --features opentelemetry
 ```
 
+At the end of the run, the script prints:
+- direct Trace and Metrics Explore URLs
+- local log path (`/tmp/void-box-playground-last.log` by default)
+
 ## Core Test Commands
 
 ```bash

docs/observability.md

@@ -12,6 +12,7 @@
 ## OTLP Export
 
 When configured, traces and metrics are exported via OTLP.
+Structured logs remain local in the current playground flow.
 
 Required:
 
@@ -41,7 +42,8 @@ This will:
 
 1. Start Grafana LGTM via Docker Compose
 2. Run `playground_pipeline` with OTLP enabled
-3. Print Grafana URL and service filter hints
+3. Print direct Grafana Explore links for traces and metrics
+4. Write full run logs to `/tmp/void-box-playground-last.log` (default)
 
 Stop stack:
 

playground/README.md

@@ -19,3 +19,9 @@ Services:
 - Grafana: `http://localhost:3000`
 - OTLP gRPC ingest: `localhost:4317`
 - OTLP HTTP ingest: `localhost:4318`
+
+After a run, `playground/up.sh` prints direct Grafana links:
+- Traces (Tempo Explore)
+- Metrics (Prometheus Explore)
+
+Logs are stored locally at `/tmp/void-box-playground-last.log` by default.

playground/playground_pipeline.rs

@@ -11,12 +11,13 @@
 use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
 
-use void_box::observe::ObserveConfig;
+use void_box::observe::{flush_global_otel, ObserveConfig};
 use void_box::sandbox::Sandbox;
 use void_box::workflow::{Workflow, WorkflowExt};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let started_at_ms = now_ms();
     let run_id = SystemTime::now()
         .duration_since(UNIX_EPOCH)
         .unwrap_or_default()
@@ -60,6 +61,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     observe.logs.in_memory = true;
 
     let observed = workflow.observe(observe).run_in(sandbox).await?;
+    let ended_at_ms = now_ms();
+
+    if let Err(e) = flush_global_otel() {
+        eprintln!("[playground] WARN: failed to flush OTLP exporters: {e}");
+    }
+
+    let grafana_base = std::env::var("PLAYGROUND_GRAFANA_URL")
+        .unwrap_or_else(|_| "http://localhost:3000".to_string());
+    let service_name =
+        std::env::var("VOIDBOX_SERVICE_NAME").unwrap_or_else(|_| "void-box-playground".into());
+    let workflow_span = format!("workflow:{workflow_name}");
 
     println!("=== Playground Pipeline ===");
     println!("workflow: {}", workflow_name);
@@ -73,16 +85,104 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     println!();
     println!("=== Explore in Grafana ===");
-    println!("Grafana: http://localhost:3000");
+    println!("Grafana: {}", grafana_base);
+    println!("Service: {}", service_name);
+    println!("Workflow span: {}", workflow_span);
+    println!(
+        "Traces URL: {}",
+        grafana_trace_url(
+            &grafana_base,
+            &service_name,
+            &workflow_span,
+            started_at_ms,
+            ended_at_ms
+        )
+    );
     println!(
-        "Service: {}",
-        std::env::var("VOIDBOX_SERVICE_NAME").unwrap_or_else(|_| "void-box-playground".into())
+        "Metrics URL: {}",
+        grafana_metrics_url(&grafana_base, &service_name, started_at_ms, ended_at_ms)
     );
-    println!("Workflow span: workflow:{}", workflow_name);
+    if let Ok(log_path) = std::env::var("PLAYGROUND_LOG_PATH") {
+        if !log_path.is_empty() {
+            println!("Logs (local): {}", log_path);
+        }
+    }
 
     Ok(())
 }
 
+fn now_ms() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis() as u64
+}
+
+fn grafana_trace_url(
+    grafana_base: &str,
+    service_name: &str,
+    workflow_span: &str,
+    from_ms: u64,
+    to_ms: u64,
+) -> String {
+    let query = format!(
+        "{{ resource.service.name = \"{}\" && name = \"{}\" }}",
+        service_name, workflow_span
+    );
+    let left = format!(
+        "[{}, {}, \"tempo\", {{\"queryType\":\"traceql\",\"query\":\"{}\",\"refId\":\"A\"}}]",
+        from_ms,
+        to_ms.saturating_add(1000),
+        escape_json_string(&query),
+    );
+
+    format!(
+        "{}/explore?orgId=1&left={}",
+        grafana_base.trim_end_matches('/'),
+        percent_encode(&left)
+    )
+}
+
+fn grafana_metrics_url(grafana_base: &str, service_name: &str, from_ms: u64, to_ms: u64) -> String {
+    let _ = service_name;
+    let expr = String::from(
+        "sum by (__name__) ({__name__=~\"(ingest|normalize|score)_duration_ms(_bucket|_sum|_count)?\"})",
+    );
+    let left = format!(
+        "[{}, {}, \"prometheus\", {{\"refId\":\"A\",\"expr\":\"{}\"}}]",
+        from_ms,
+        to_ms.saturating_add(1000),
+        escape_json_string(&expr)
+    );
+
+    format!(
+        "{}/explore?orgId=1&left={}",
+        grafana_base.trim_end_matches('/'),
+        percent_encode(&left)
+    )
+}
+
+fn escape_json_string(input: &str) -> String {
+    input.replace('\\', "\\\\").replace('"', "\\\"")
+}
+
+fn percent_encode(input: &str) -> String {
+    let mut encoded = String::with_capacity(input.len() * 3 / 2);
+    for b in input.bytes() {
+        match b {
+            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
+                encoded.push(char::from(b));
+            }
+            _ => {
+                encoded.push('%');
+                encoded.push(char::from(b"0123456789ABCDEF"[(b >> 4) as usize]));
+                encoded.push(char::from(b"0123456789ABCDEF"[(b & 0x0F) as usize]));
+            }
+        }
+    }
+    encoded
+}
+
 fn build_sandbox() -> Result<Arc<Sandbox>, Box<dyn std::error::Error>> {
     let has_kvm = std::path::Path::new("/dev/kvm").exists();
     let has_kernel = std::env::var("VOID_BOX_KERNEL")

playground/up.sh

@@ -158,19 +158,30 @@ configure_kvm_artifacts
 
 export VOIDBOX_OTLP_ENDPOINT="${VOIDBOX_OTLP_ENDPOINT:-http://localhost:4317}"
 export VOIDBOX_SERVICE_NAME="${VOIDBOX_SERVICE_NAME:-void-box-playground}"
+export PLAYGROUND_GRAFANA_URL="${PLAYGROUND_GRAFANA_URL:-http://localhost:3000}"
+export PLAYGROUND_LOG_PATH="${PLAYGROUND_LOG_PATH:-/tmp/void-box-playground-last.log}"
 print_run_summary
 
 echo "[playground] running pipeline example..."
 (
   cd "$ROOT_DIR"
-  cargo run --example playground_pipeline --features opentelemetry
+  cargo run --example playground_pipeline --features opentelemetry 2>&1 | tee "$PLAYGROUND_LOG_PATH"
 )
 
+TRACES_URL="$(grep -E '^Traces URL:' "$PLAYGROUND_LOG_PATH" | tail -n1 | sed -E 's/^Traces URL:[[:space:]]*//')"
+METRICS_URL="$(grep -E '^Metrics URL:' "$PLAYGROUND_LOG_PATH" | tail -n1 | sed -E 's/^Metrics URL:[[:space:]]*//')"
+
 echo
 echo "[playground] wow, it's live"
-echo "  Grafana: http://localhost:3000"
+echo "  Grafana: $PLAYGROUND_GRAFANA_URL"
 echo "  Login: admin/admin"
 echo "  Service filter: service.name=$VOIDBOX_SERVICE_NAME"
 echo "  Provider: $PLAYGROUND_PROVIDER"
+echo "  Logs (local): $PLAYGROUND_LOG_PATH"
+echo
+echo "[playground] direct links"
+echo "  Traces: ${TRACES_URL:-<not emitted>}"
+echo "  Metrics: ${METRICS_URL:-<not emitted>}"
+echo "  Logs: $PLAYGROUND_LOG_PATH"
 echo
 echo "Tip: run '$0 --down' to stop the stack."

src/observe/mod.rs

@@ -30,6 +30,8 @@ pub mod tracer;
 
 use std::sync::Arc;
 #[cfg(feature = "opentelemetry")]
+use std::sync::Mutex;
+#[cfg(feature = "opentelemetry")]
 use std::sync::OnceLock;
 use std::time::Instant;
 
@@ -137,9 +139,68 @@ pub struct Observer {
     logger: Arc<StructuredLogger>,
 }
 
+#[cfg(feature = "opentelemetry")]
+#[derive(Default)]
+struct OtlpProviderState {
+    tracer: Option<opentelemetry_sdk::trace::SdkTracerProvider>,
+    meter: Option<opentelemetry_sdk::metrics::SdkMeterProvider>,
+}
+
+/// Force-flush globally configured OTLP providers.
+///
+/// This is primarily useful in short-lived binaries (examples/CLI tools) so
+/// traces and metrics are exported before process exit.
+#[cfg(feature = "opentelemetry")]
+pub fn flush_global_otel() -> crate::Result<()> {
+    let Some(state) = otlp_provider_state().get() else {
+        return Ok(());
+    };
+
+    let mut state = state
+        .lock()
+        .map_err(|e| crate::Error::Guest(format!("OTLP provider mutex poisoned: {e}")))?;
+    let tracer = state.tracer.take();
+    let meter = state.meter.take();
+    drop(state);
+
+    let mut errors = Vec::new();
+
+    if let Some(tracer) = tracer {
+        if let Err(e) = tracer.force_flush() {
+            errors.push(format!("Failed to flush OTLP tracer: {e}"));
+        }
+        if let Err(e) = tracer.shutdown() {
+            errors.push(format!("Failed to shutdown OTLP tracer: {e}"));
+        }
+    }
+
+    if let Some(meter) = meter {
+        if let Err(e) = meter.force_flush() {
+            errors.push(format!("Failed to flush OTLP metrics: {e}"));
+        }
+        if let Err(e) = meter.shutdown() {
+            errors.push(format!("Failed to shutdown OTLP metrics: {e}"));
+        }
+    }
+
+    if !errors.is_empty() {
+        return Err(crate::Error::Guest(errors.join("; ")));
+    }
+
+    Ok(())
+}
+
+/// No-op when OpenTelemetry feature is disabled.
+#[cfg(not(feature = "opentelemetry"))]
+pub fn flush_global_otel() -> crate::Result<()> {
+    Ok(())
+}
+
 impl Observer {
     /// Create a new observer with the given configuration
     pub fn new(config: ObserveConfig) -> Self {
+        #[cfg(feature = "opentelemetry")]
+        maybe_init_global_otel(&config);
         let tracer = Arc::new(Tracer::new(config.tracer.clone()));
         let metrics = Arc::new(build_metrics_collector(&config));
         let logger = Arc::new(StructuredLogger::new(config.logs.clone()));
@@ -255,15 +316,33 @@ fn maybe_init_global_otel(config: &ObserveConfig) {
             debug: false,
         };
 
-        if let Err(e) = crate::observe::otlp::init_otlp_tracer(&otlp_config) {
-            eprintln!("[observe] WARN: failed to initialize OTLP tracer export: {e}");
+        let mut state = OtlpProviderState::default();
+
+        match crate::observe::otlp::init_otlp_tracer(&otlp_config) {
+            Ok(provider) => state.tracer = Some(provider),
+            Err(e) => eprintln!("[observe] WARN: failed to initialize OTLP tracer export: {e}"),
         }
-        if let Err(e) = crate::observe::otlp::init_otlp_metrics(&otlp_config) {
-            eprintln!("[observe] WARN: failed to initialize OTLP metrics export: {e}");
+        match crate::observe::otlp::init_otlp_metrics(&otlp_config) {
+            Ok(provider) => state.meter = Some(provider),
+            Err(e) => eprintln!("[observe] WARN: failed to initialize OTLP metrics export: {e}"),
+        }
+
+        if state.tracer.is_some() || state.meter.is_some() {
+            let lock =
+                otlp_provider_state().get_or_init(|| Mutex::new(OtlpProviderState::default()));
+            if let Ok(mut slot) = lock.lock() {
+                *slot = state;
+            }
         }
     });
 }
 
+#[cfg(feature = "opentelemetry")]
+fn otlp_provider_state() -> &'static OnceLock<Mutex<OtlpProviderState>> {
+    static OTLP_PROVIDER_STATE: OnceLock<Mutex<OtlpProviderState>> = OnceLock::new();
+    &OTLP_PROVIDER_STATE
+}
+
 /// RAII guard for spans that records metrics on drop
 pub struct SpanGuard {
     span: Span,