feat(kv-router): add Prometheus metrics to standalone indexer (#7339)

Signed-off-by: PeaBrane <yanrpei@gmail.com>

Author: PeaBrane

Cargo.lock

@@ -99,6 +99,30 @@ dynamo-kv-indexer --port 8090 [--threads 4] [--block-size 16 --model-name my-mod
 
 ## HTTP API
 
+### `GET /health` — Liveness check
+
+Returns `200 OK` unconditionally.
+
+```bash
+curl http://localhost:8090/health
+```
+
+### `GET /metrics` — Prometheus metrics
+
+Returns metrics in Prometheus text exposition format. Available when the binary is built with the `metrics` feature (enabled by default via `standalone-indexer`).
+
+```bash
+curl http://localhost:8090/metrics
+```
+
+| Metric | Type | Labels | Description |
+|--------|------|--------|-------------|
+| `dynamo_kvindexer_request_duration_seconds` | Histogram | `endpoint` | HTTP request latency |
+| `dynamo_kvindexer_requests_total` | Counter | `endpoint`, `method` | Total HTTP requests |
+| `dynamo_kvindexer_errors_total` | Counter | `endpoint`, `status_class` | HTTP error responses (4xx/5xx) |
+| `dynamo_kvindexer_models` | Gauge | — | Number of active model+tenant indexers |
+| `dynamo_kvindexer_workers` | Gauge | — | Number of registered worker instances |
+
 ### `POST /register` — Register an endpoint
 
 Register a ZMQ endpoint for an instance. Each call creates or reuses the indexer for the given `(model_name, tenant_id)` pair.
@@ -307,7 +331,7 @@ graph TD
         REG[Worker Registry]
         ZMQ[ZMQ SUB Listeners]
         IDX["Indexer Map<br/>(model, tenant) → Radix Tree"]
-        HTTP[HTTP API<br/>/query /dump /register]
+        HTTP[HTTP API<br/>/query /dump /register /metrics /health]
     end
 
     CLIENT[External Client]

docs/components/router/standalone-indexer.md

@@ -132,6 +132,13 @@ class frontend_service:
     OPERATION_LABEL = "operation"
 
 
+class kv_publisher:
+    """KV Publisher metrics"""
+
+    # Total number of raw events dropped by engines before reaching publisher (detected via event_id gaps)
+    ENGINES_DROPPED_EVENTS_TOTAL = "kv_publisher_engines_dropped_events_total"
+
+
 class kvbm:
     """KVBM"""
 
@@ -167,6 +174,21 @@ class kvbm:
     OBJECT_WRITE_FAILURES = "object_write_failures"
 
 
+class kvindexer:
+    """Standalone KV indexer HTTP service metrics"""
+
+    # HTTP request latency
+    REQUEST_DURATION_SECONDS = "request_duration_seconds"
+    # Total HTTP requests
+    REQUESTS_TOTAL = "requests_total"
+    # HTTP error responses (4xx/5xx)
+    ERRORS_TOTAL = "errors_total"
+    # Number of active model+tenant indexers
+    MODELS = "models"
+    # Number of registered worker instances
+    WORKERS = "workers"
+
+
 class kvrouter:
     # Number of KV cache events applied to the index (including status)
     KV_CACHE_EVENTS_APPLIED = "kv_cache_events_applied"
@@ -225,6 +247,8 @@ class name_prefix:
     ROUTER = "dynamo_router"
     # Prefix for tokio runtime metrics
     TOKIO = "dynamo_tokio"
+    # Prefix for standalone KV indexer metrics
+    KVINDEXER = "dynamo_kvindexer"
 
 
 class router:
@@ -265,25 +289,6 @@ class routing_overhead:
     TOTAL_MS = "overhead_total_ms"
 
 
-class trtllm_additional:
-    """Additional TRT-LLM worker metrics beyond what the engine natively provides."""
-
-    # Total number of aborted/cancelled requests
-    NUM_ABORTED_REQUESTS_TOTAL = "trtllm_num_aborted_requests_total"
-    # Total number of requests containing image content
-    REQUEST_TYPE_IMAGE_TOTAL = "trtllm_request_type_image_total"
-    # Total number of requests using guided/structured decoding
-    REQUEST_TYPE_STRUCTURED_OUTPUT_TOTAL = "trtllm_request_type_structured_output_total"
-    # Total number of successful KV cache transfers
-    KV_TRANSFER_SUCCESS_TOTAL = "trtllm_kv_transfer_success_total"
-    # KV cache transfer latency per request in seconds
-    KV_TRANSFER_LATENCY_SECONDS = "trtllm_kv_transfer_latency_seconds"
-    # KV cache transfer size per request in bytes
-    KV_TRANSFER_BYTES = "trtllm_kv_transfer_bytes"
-    # KV cache transfer speed per request in GB/s
-    KV_TRANSFER_SPEED_GB_S = "trtllm_kv_transfer_speed_gb_s"
-
-
 class task_tracker:
     """Task tracker Prometheus metric name suffixes"""
 
@@ -318,6 +323,25 @@ class tokio_perf:
     ALIVE_TASKS = "alive_tasks"
 
 
+class trtllm_additional:
+    """Additional TRT-LLM worker metrics beyond what the engine natively provides."""
+
+    # Total number of aborted/cancelled requests
+    NUM_ABORTED_REQUESTS_TOTAL = "trtllm_num_aborted_requests_total"
+    # Total number of requests containing image content
+    REQUEST_TYPE_IMAGE_TOTAL = "trtllm_request_type_image_total"
+    # Total number of requests using guided/structured decoding
+    REQUEST_TYPE_STRUCTURED_OUTPUT_TOTAL = "trtllm_request_type_structured_output_total"
+    # Total number of successful KV cache transfers
+    KV_TRANSFER_SUCCESS_TOTAL = "trtllm_kv_transfer_success_total"
+    # KV cache transfer latency per request in seconds
+    KV_TRANSFER_LATENCY_SECONDS = "trtllm_kv_transfer_latency_seconds"
+    # KV cache transfer size per request in bytes
+    KV_TRANSFER_BYTES = "trtllm_kv_transfer_bytes"
+    # KV cache transfer speed per request in GB/s
+    KV_TRANSFER_SPEED_GB_S = "trtllm_kv_transfer_speed_gb_s"
+
+
 class work_handler:
     """Work handler Prometheus metric names"""
 

lib/bindings/python/src/dynamo/prometheus_names.py

@@ -12,15 +12,15 @@ repository.workspace = true
 
 [features]
 default = []
-metrics = []
+metrics = ["dep:dynamo-runtime"]
 bench = ["dep:clap", "dep:indicatif", "dep:serde_json", "dep:plotters"]
 standalone-indexer = ["metrics", "dep:axum", "dep:bytes", "dep:zeromq", "dep:serde_json", "dep:reqwest"]
 indexer-bin = ["standalone-indexer", "dep:clap", "dep:tracing-subscriber"]
 test-endpoints = ["indexer-bin"]
 
 [dependencies]
 # repo
-dynamo-runtime = { workspace = true }
+dynamo-runtime = { workspace = true, optional = true }
 dynamo-tokens = { workspace = true }
 
 # workspace