New Example: Monitoring Data Pipelines (#230)

Author: AdepojuJeremy

axum/polars-otel-shuttle/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "polars-otel-shuttle"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+bench-cli = []          # local CLI binary
+otel-otlp = ["opentelemetry-otlp"]  # enable real OTLP exporter when you want it
+
+[dependencies]
+anyhow = "1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
+# Web (for Shuttle build)
+axum = { version = "0.8", features = ["json"] }
+tower = "0.5"
+tower-http = { version = "0.6", features = ["cors", "trace"] }
+
+# Tracing
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "json", "time"] }
+time = { version = "0.3", features = ["formatting", "macros"] }
+tracing-opentelemetry = "0.31"
+
+# OpenTelemetry (core + SDK; OTLP exporter is optional via feature)
+opentelemetry = { version = "0.30", features = ["trace"] }
+opentelemetry_sdk = { version = "0.30", features = ["trace", "rt-tokio"] }
+opentelemetry-otlp = { version = "0.30", features = ["http-proto", "tls"], optional = true }
+
+# Runtime (new enough to satisfy shuttle-runtime)
+tokio = { version = "1.47.0", features = ["rt-multi-thread", "macros"] }
+
+# ETL
+polars = { version = "0.49", features = [
+  "lazy", "csv", "parquet", "fmt", "strings", "dtype-date", "dtype-datetime"
+] }
+chrono = { version = "0.4", features = ["clock"] }
+comfy-table = "7"
+
+# Shuttle (only used on deploy)
+shuttle-runtime = { version = "0.56", features = ["setup-otel-exporter"] }
+shuttle-axum = "0.56"

axum/polars-otel-shuttle/README.md

@@ -0,0 +1,55 @@
+# Polars ETL with OpenTelemetry
+
+An ETL pipeline using Polars for data processing and OpenTelemetry for observability, deployable on Shuttle.
+
+## What it does
+
+- Processes CSV data with Polars (load → clean → aggregate → filter/sort → save)
+- Exposes HTTP endpoints via Axum
+- Emits metrics and traces via OpenTelemetry
+- Integrates with Better Stack for monitoring (when deployed on Shuttle)
+
+## Running locally
+
+**ETL benchmark:**
+```bash
+RUST_LOG=info cargo run --release --features bench-cli
+```
+
+**Web service:**
+```bash
+cargo run --release
+```
+
+Place your CSV file at `./data/yellow_tripdata_2015-01.csv` or set `DATA_PATH=/path/to/file.csv`.
+
+## Deploy to Shuttle
+
+```bash
+shuttle deploy
+```
+
+## Endpoints
+
+- `GET /` - Health check
+- `GET /health` - Health check with metrics
+- `GET /benchmark` - Returns ETL metrics and emits telemetry events
+
+## Observability
+
+The service emits OpenTelemetry metrics:
+- `monotonic_counter.http_requests_total` - Request counts
+- `histogram.request_duration_ms` - Request latency
+- `histogram.*_time_ms` - ETL stage timings
+- `monotonic_counter.rows_processed` - Data throughput
+
+When deployed on Shuttle with Better Stack integration enabled, these automatically appear in your telemetry dashboard.
+
+## Configuration
+
+**Environment variables:**
+- `DATA_PATH` - Path to CSV file (default: `data/yellow_tripdata_2015-01.csv`)
+
+**Features:**
+- `bench-cli` - Enables local ETL benchmark
+- `otel-otlp` - Enables OTLP export (for non-Shuttle deployments)

axum/polars-otel-shuttle/Shuttle.toml

@@ -0,0 +1,2 @@
+name = "polars-otel-shuttle"
+# Shuttle uses your #[shuttle_runtime::main] entry point in src/main.rs