Adds Prometheus to items-service

README.md

@@ -26,13 +26,23 @@ task local:start
   🔧 Items Service:         http://localhost:8081
      - Health:              http://localhost:8081/v1/health
      - API Docs:            http://localhost:8081/docs
+     - Metrics:             http://localhost:8081/metrics
   🌐 Website App:           http://localhost:8082
      - Health:              http://localhost:8082/health
   📊 Jaeger UI:             http://localhost:16686
   📦 PostgreSQL:            localhost:5432
      - Connection:          psql postgresql://{.env.POSTGRES_USER}:{.env.POSTGRES_PASSWORD}@localhost:5432/
 ```
 
+## Production URLs
+
+- **Website**: https://roussev.com
+- **Items API**: https://app.roussev.com/items
+  - Health: https://app.roussev.com/items/v1/health
+  - Docs: https://app.roussev.com/items/docs
+  - Metrics: https://app.roussev.com/items/metrics
+- **Jaeger Tracing**: https://app.roussev.com/jaeger
+
 ## Next Steps
 
 - Set up monitoring with Prometheus/Grafana

Tiltfile

@@ -91,6 +91,7 @@ k8s_resource(
     links=[
         link('http://localhost:8081/v1/health', 'Health Check'),
         link('http://localhost:8081/docs', 'API Docs'),
+        link('http://localhost:8081/metrics', 'Prometheus Metrics'),
     ]
 )
 

apps/items-service/README.md

@@ -9,6 +9,7 @@ A simple REST API service for managing items, built with Bun and PostgreSQL.
 - **Health check** endpoint with database connectivity status
 - **OpenAPI/Swagger** documentation
 - **OpenTelemetry** distributed tracing support
+- **Prometheus** metrics collection and export
 - **Structured Logging** with Pino for high-performance logging
 - **Hot reload** in development mode
 - **Docker** support with development and production configurations
@@ -21,6 +22,7 @@ A simple REST API service for managing items, built with Bun and PostgreSQL.
 - `POST /v1/items` - Create a new item
 - `GET /docs` - Swagger UI for interactive API documentation
 - `GET /v1/openapi.json` - OpenAPI specification
+- `GET /metrics` - Prometheus metrics endpoint
 
 ## Local Development
 
@@ -34,6 +36,7 @@ Access the service at:
 - Service: http://localhost:8081
 - Health Check: http://localhost:8081/v1/health
 - API Docs: http://localhost:8081/docs
+- Metrics: http://localhost:8081/metrics
 
 
 #### PostgreSQL Database (Port 5432)
@@ -61,6 +64,9 @@ curl -X POST http://localhost:8081/v1/items \
   -H "Content-Type: application/json" \
   -d '{"name":"test-item"}'
 
+# View Prometheus metrics
+curl http://localhost:8081/metrics
+
 # Connect to PostgreSQL
 psql postgresql://{.env.POSTGRES_USER}:{.env.POSTGRES_PASSWORD}@localhost:5432/{.env.POSTGRES_DB}
 
@@ -73,7 +79,9 @@ open http://localhost:16686
 - **Runtime**: [Bun](https://bun.sh/)
 - **Database**: PostgreSQL
 - **Logging**: [Pino](https://getpino.io/) - structured logging
-- **Observability**: OpenTelemetry - distributed tracing
+- **Observability**:
+  - OpenTelemetry - distributed tracing
+  - Prometheus - metrics collection and export
 - **Container**: Docker with Alpine Linux
 - **Orchestration**: Kubernetes (k3s)
 - **Local Development**: Task and Tilt with k3d

apps/items-service/bun.lock

@@ -6,6 +6,7 @@
       "dependencies": {
         "@opentelemetry/api": "^1.9.0",
         "@opentelemetry/auto-instrumentations-node": "^0.66.0",
+        "@opentelemetry/exporter-prometheus": "^0.207.0",
         "@opentelemetry/exporter-trace-otlp-http": "^0.207.0",
         "@opentelemetry/resources": "^2.2.0",
         "@opentelemetry/sdk-node": "^0.207.0",

apps/items-service/package.json

@@ -10,6 +10,7 @@
   "dependencies": {
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/auto-instrumentations-node": "^0.66.0",
+    "@opentelemetry/exporter-prometheus": "^0.207.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.207.0",
     "@opentelemetry/resources": "^2.2.0",
     "@opentelemetry/sdk-node": "^0.207.0",

apps/items-service/src/metrics.ts

@@ -0,0 +1,131 @@
+/**
+ * Prometheus Metrics Module
+ * Provides Prometheus metrics collection using OpenTelemetry
+ * 
+ * Features:
+ * - Prometheus metrics exporter
+ * - HTTP metrics endpoint at /metrics
+ * - Integration with OpenTelemetry SDK
+ * - Custom metrics support
+ */
+
+import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
+import { MeterProvider } from "@opentelemetry/sdk-metrics";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
+import type { Histogram, Counter, Meter } from "@opentelemetry/api";
+
+const PROMETHEUS_PORT = parseInt(process.env.PROMETHEUS_PORT || "9464", 10);
+const OTEL_SERVICE_NAME = process.env.OTEL_SERVICE_NAME || "items-service";
+const OTEL_SERVICE_VERSION = process.env.OTEL_SERVICE_VERSION || "1.0.0";
+
+let prometheusExporter: PrometheusExporter | null = null;
+let meterProvider: MeterProvider | null = null;
+let meter: Meter | null = null;
+
+// HTTP metrics
+let httpRequestDuration: Histogram | null = null;
+let httpRequestCount: Counter | null = null;
+
+/**
+ * Initialize Prometheus metrics exporter
+ * This sets up the Prometheus exporter and meter provider
+ */
+export function initPrometheusMetrics() {
+  try {
+    // Configure resource with service information
+    const resource = resourceFromAttributes({
+      [ATTR_SERVICE_NAME]: OTEL_SERVICE_NAME,
+      [ATTR_SERVICE_VERSION]: OTEL_SERVICE_VERSION,
+    });
+
+    // Create Prometheus exporter
+    // Note: The exporter will NOT start its own HTTP server
+    // We'll expose metrics through our existing HTTP server
+    prometheusExporter = new PrometheusExporter({
+      // Don't start a separate HTTP server
+      preventServerStart: true,
+    });
+
+    // Create meter provider with Prometheus exporter
+    meterProvider = new MeterProvider({
+      resource,
+      readers: [prometheusExporter],
+    });
+
+    // Create meter for recording metrics
+    meter = meterProvider.getMeter(OTEL_SERVICE_NAME, OTEL_SERVICE_VERSION);
+
+    // Create HTTP metrics instruments
+    httpRequestDuration = meter.createHistogram("http_server_duration", {
+      description: "HTTP request duration in milliseconds",
+      unit: "ms",
+    });
+
+    httpRequestCount = meter.createCounter("http_server_requests_total", {
+      description: "Total number of HTTP requests",
+      unit: "1",
+    });
+
+    console.log("Prometheus metrics initialized");
+    console.log(`Service: ${OTEL_SERVICE_NAME} v${OTEL_SERVICE_VERSION}`);
+    console.log("Metrics will be available at /metrics endpoint");
+
+    return prometheusExporter;
+  } catch (error) {
+    console.error("Failed to initialize Prometheus metrics:", error);
+    return null;
+  }
+}
+
+/**
+ * Get the Prometheus exporter instance
+ */
+export function getPrometheusExporter(): PrometheusExporter | null {
+  return prometheusExporter;
+}
+
+/**
+ * Get the meter provider instance
+ */
+export function getMeterProvider(): MeterProvider | null {
+  return meterProvider;
+}
+
+/**
+ * Record HTTP request metrics
+ */
+export function recordHttpRequest(
+  method: string,
+  path: string,
+  statusCode: number,
+  durationMs: number
+) {
+  if (!httpRequestDuration || !httpRequestCount) {
+    return;
+  }
+
+  const attributes = {
+    method,
+    route: path,
+    status_code: statusCode.toString(),
+  };
+
+  httpRequestDuration.record(durationMs, attributes);
+  httpRequestCount.add(1, attributes);
+}
+
+/**
+ * Shutdown Prometheus metrics
+ */
+export async function shutdownPrometheusMetrics() {
+  if (meterProvider) {
+    try {
+      await meterProvider.shutdown();
+      console.log("Prometheus metrics shut down successfully");
+    } catch (error) {
+      console.error("Error shutting down Prometheus metrics:", error);
+    }
+  }
+}
+

apps/items-service/src/router.ts

@@ -10,6 +10,7 @@ import { openapi } from "./openapi.js";
 import { json, notFound, html } from "./http-utils.js";
 import type { ServerConfig } from "./config.js";
 import { logRequest, logResponse, logError } from "./logger.js";
+import { getPrometheusExporter, recordHttpRequest } from "./metrics.js";
 
 type RouteHandler = (req: Request) => Promise<Response> | Response;
 
@@ -59,6 +60,13 @@ export class Router {
       handler: () => this.healthController.check(),
     });
 
+    // Prometheus metrics endpoint
+    this.routes.push({
+      method: "GET",
+      path: "/metrics",
+      handler: () => this.handleMetrics(),
+    });
+
     // List items
     this.routes.push({
       method: "GET",
@@ -78,6 +86,47 @@ export class Router {
     return this.routes.find((route) => route.method === method && route.path === path);
   }
 
+  /**
+   * Handle Prometheus metrics endpoint
+   * Returns metrics in Prometheus text format
+   */
+  private async handleMetrics(): Promise<Response> {
+    const exporter = getPrometheusExporter();
+
+    if (!exporter) {
+      return new Response("Prometheus metrics not initialized", {
+        status: 503,
+        headers: { "Content-Type": "text/plain" }
+      });
+    }
+
+    try {
+      // The PrometheusExporter has a getMetricsRequestHandler method
+      // that returns a promise with the metrics in Prometheus text format
+      return new Promise((resolve) => {
+        const mockReq = {} as any;
+        const mockRes = {
+          statusCode: 200,
+          setHeader: () => {},
+          end: (data: string) => {
+            resolve(new Response(data, {
+              status: 200,
+              headers: { "Content-Type": "text/plain; version=0.0.4; charset=utf-8" }
+            }));
+          }
+        } as any;
+
+        exporter.getMetricsRequestHandler(mockReq, mockRes);
+      });
+    } catch (error) {
+      console.error("Error generating metrics:", error);
+      return new Response("Error generating metrics", {
+        status: 500,
+        headers: { "Content-Type": "text/plain" }
+      });
+    }
+  }
+
   async handle(req: Request): Promise<Response> {
     const tracer = trace.getTracer("items-service");
     const url = new URL(req.url);
@@ -113,6 +162,11 @@ export class Router {
 
         logResponse(req, response, duration);
 
+        // Record metrics
+        if (path !== "/metrics") {
+          recordHttpRequest(req.method, path, response.status, duration);
+        }
+
         return response;
       } catch (error) {
         logError(error, "Request handler error");

apps/items-service/src/telemetry.ts

@@ -5,6 +5,7 @@ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 import { resourceFromAttributes } from "@opentelemetry/resources";
 import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
 import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api";
+import { initPrometheusMetrics, shutdownPrometheusMetrics } from "./metrics.js";
 
 // Configuration from environment variables
 const OTEL_ENABLED = process.env.OTEL_ENABLED !== "false"; // Default to true
@@ -67,6 +68,9 @@ export function initTelemetry() {
     console.log(`Service: ${OTEL_SERVICE_NAME} v${OTEL_SERVICE_VERSION}`);
     console.log(`Endpoint: ${OTEL_EXPORTER_OTLP_ENDPOINT}`);
 
+    // Initialize Prometheus metrics
+    initPrometheusMetrics();
+
     // Handle graceful shutdown
     process.on("SIGTERM", async () => {
       await shutdownTelemetry();
@@ -83,6 +87,10 @@ export function initTelemetry() {
 }
 
 export async function shutdownTelemetry() {
+  // Shutdown Prometheus metrics
+  await shutdownPrometheusMetrics();
+
+  // Shutdown OpenTelemetry SDK
   if (sdk) {
     try {
       await sdk.shutdown();