diff --git a/README.md b/README.md index bcc46a0..bb45af3 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ task local:start 🔧 Items Service: http://localhost:8081 - Health: http://localhost:8081/v1/health - API Docs: http://localhost:8081/docs + - Metrics: http://localhost:8081/metrics 🌐 Website App: http://localhost:8082 - Health: http://localhost:8082/health 📊 Jaeger UI: http://localhost:16686 @@ -33,6 +34,15 @@ task local:start - Connection: psql postgresql://{.env.POSTGRES_USER}:{.env.POSTGRES_PASSWORD}@localhost:5432/ ``` +## Production URLs + +- **Website**: https://roussev.com +- **Items API**: https://app.roussev.com/items + - Health: https://app.roussev.com/items/v1/health + - Docs: https://app.roussev.com/items/docs + - Metrics: https://app.roussev.com/items/metrics +- **Jaeger Tracing**: https://app.roussev.com/jaeger + ## Next Steps - Set up monitoring with Prometheus/Grafana diff --git a/Tiltfile b/Tiltfile index a688c60..534254e 100644 --- a/Tiltfile +++ b/Tiltfile @@ -91,6 +91,7 @@ k8s_resource( links=[ link('http://localhost:8081/v1/health', 'Health Check'), link('http://localhost:8081/docs', 'API Docs'), + link('http://localhost:8081/metrics', 'Prometheus Metrics'), ] ) diff --git a/apps/items-service/README.md b/apps/items-service/README.md index d2e2a81..c60fb99 100644 --- a/apps/items-service/README.md +++ b/apps/items-service/README.md @@ -9,6 +9,7 @@ A simple REST API service for managing items, built with Bun and PostgreSQL. - **Health check** endpoint with database connectivity status - **OpenAPI/Swagger** documentation - **OpenTelemetry** distributed tracing support +- **Prometheus** metrics collection and export - **Structured Logging** with Pino for high-performance logging - **Hot reload** in development mode - **Docker** support with development and production configurations @@ -21,6 +22,7 @@ A simple REST API service for managing items, built with Bun and PostgreSQL. - `POST /v1/items` - Create a new item - `GET /docs` - Swagger UI for interactive API documentation - `GET /v1/openapi.json` - OpenAPI specification +- `GET /metrics` - Prometheus metrics endpoint ## Local Development @@ -34,6 +36,7 @@ Access the service at: - Service: http://localhost:8081 - Health Check: http://localhost:8081/v1/health - API Docs: http://localhost:8081/docs +- Metrics: http://localhost:8081/metrics #### PostgreSQL Database (Port 5432) @@ -61,6 +64,9 @@ curl -X POST http://localhost:8081/v1/items \ -H "Content-Type: application/json" \ -d '{"name":"test-item"}' +# View Prometheus metrics +curl http://localhost:8081/metrics + # Connect to PostgreSQL psql postgresql://{.env.POSTGRES_USER}:{.env.POSTGRES_PASSWORD}@localhost:5432/{.env.POSTGRES_DB} @@ -73,7 +79,9 @@ open http://localhost:16686 - **Runtime**: [Bun](https://bun.sh/) - **Database**: PostgreSQL - **Logging**: [Pino](https://getpino.io/) - structured logging -- **Observability**: OpenTelemetry - distributed tracing +- **Observability**: + - OpenTelemetry - distributed tracing + - Prometheus - metrics collection and export - **Container**: Docker with Alpine Linux - **Orchestration**: Kubernetes (k3s) - **Local Development**: Task and Tilt with k3d diff --git a/apps/items-service/bun.lock b/apps/items-service/bun.lock index cd8b5a0..c063d78 100644 --- a/apps/items-service/bun.lock +++ b/apps/items-service/bun.lock @@ -6,6 +6,7 @@ "dependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/auto-instrumentations-node": "^0.66.0", + "@opentelemetry/exporter-prometheus": "^0.207.0", "@opentelemetry/exporter-trace-otlp-http": "^0.207.0", "@opentelemetry/resources": "^2.2.0", "@opentelemetry/sdk-node": "^0.207.0", diff --git a/apps/items-service/package.json b/apps/items-service/package.json index e5f7ef2..269cdf2 100644 --- a/apps/items-service/package.json +++ b/apps/items-service/package.json @@ -10,6 +10,7 @@ "dependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/auto-instrumentations-node": "^0.66.0", + "@opentelemetry/exporter-prometheus": "^0.207.0", "@opentelemetry/exporter-trace-otlp-http": "^0.207.0", "@opentelemetry/resources": "^2.2.0", "@opentelemetry/sdk-node": "^0.207.0", diff --git a/apps/items-service/src/metrics.ts b/apps/items-service/src/metrics.ts new file mode 100644 index 0000000..e9b38c1 --- /dev/null +++ b/apps/items-service/src/metrics.ts @@ -0,0 +1,131 @@ +/** + * Prometheus Metrics Module + * Provides Prometheus metrics collection using OpenTelemetry + * + * Features: + * - Prometheus metrics exporter + * - HTTP metrics endpoint at /metrics + * - Integration with OpenTelemetry SDK + * - Custom metrics support + */ + +import { PrometheusExporter } from "@opentelemetry/exporter-prometheus"; +import { MeterProvider } from "@opentelemetry/sdk-metrics"; +import { resourceFromAttributes } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions"; +import type { Histogram, Counter, Meter } from "@opentelemetry/api"; + +const PROMETHEUS_PORT = parseInt(process.env.PROMETHEUS_PORT || "9464", 10); +const OTEL_SERVICE_NAME = process.env.OTEL_SERVICE_NAME || "items-service"; +const OTEL_SERVICE_VERSION = process.env.OTEL_SERVICE_VERSION || "1.0.0"; + +let prometheusExporter: PrometheusExporter | null = null; +let meterProvider: MeterProvider | null = null; +let meter: Meter | null = null; + +// HTTP metrics +let httpRequestDuration: Histogram | null = null; +let httpRequestCount: Counter | null = null; + +/** + * Initialize Prometheus metrics exporter + * This sets up the Prometheus exporter and meter provider + */ +export function initPrometheusMetrics() { + try { + // Configure resource with service information + const resource = resourceFromAttributes({ + [ATTR_SERVICE_NAME]: OTEL_SERVICE_NAME, + [ATTR_SERVICE_VERSION]: OTEL_SERVICE_VERSION, + }); + + // Create Prometheus exporter + // Note: The exporter will NOT start its own HTTP server + // We'll expose metrics through our existing HTTP server + prometheusExporter = new PrometheusExporter({ + // Don't start a separate HTTP server + preventServerStart: true, + }); + + // Create meter provider with Prometheus exporter + meterProvider = new MeterProvider({ + resource, + readers: [prometheusExporter], + }); + + // Create meter for recording metrics + meter = meterProvider.getMeter(OTEL_SERVICE_NAME, OTEL_SERVICE_VERSION); + + // Create HTTP metrics instruments + httpRequestDuration = meter.createHistogram("http_server_duration", { + description: "HTTP request duration in milliseconds", + unit: "ms", + }); + + httpRequestCount = meter.createCounter("http_server_requests_total", { + description: "Total number of HTTP requests", + unit: "1", + }); + + console.log("Prometheus metrics initialized"); + console.log(`Service: ${OTEL_SERVICE_NAME} v${OTEL_SERVICE_VERSION}`); + console.log("Metrics will be available at /metrics endpoint"); + + return prometheusExporter; + } catch (error) { + console.error("Failed to initialize Prometheus metrics:", error); + return null; + } +} + +/** + * Get the Prometheus exporter instance + */ +export function getPrometheusExporter(): PrometheusExporter | null { + return prometheusExporter; +} + +/** + * Get the meter provider instance + */ +export function getMeterProvider(): MeterProvider | null { + return meterProvider; +} + +/** + * Record HTTP request metrics + */ +export function recordHttpRequest( + method: string, + path: string, + statusCode: number, + durationMs: number +) { + if (!httpRequestDuration || !httpRequestCount) { + return; + } + + const attributes = { + method, + route: path, + status_code: statusCode.toString(), + }; + + httpRequestDuration.record(durationMs, attributes); + httpRequestCount.add(1, attributes); +} + +/** + * Shutdown Prometheus metrics + */ +export async function shutdownPrometheusMetrics() { + if (meterProvider) { + try { + await meterProvider.shutdown(); + console.log("Prometheus metrics shut down successfully"); + } catch (error) { + console.error("Error shutting down Prometheus metrics:", error); + } + } +} + diff --git a/apps/items-service/src/router.ts b/apps/items-service/src/router.ts index d8b20f8..65f660a 100644 --- a/apps/items-service/src/router.ts +++ b/apps/items-service/src/router.ts @@ -10,6 +10,7 @@ import { openapi } from "./openapi.js"; import { json, notFound, html } from "./http-utils.js"; import type { ServerConfig } from "./config.js"; import { logRequest, logResponse, logError } from "./logger.js"; +import { getPrometheusExporter, recordHttpRequest } from "./metrics.js"; type RouteHandler = (req: Request) => Promise | Response; @@ -59,6 +60,13 @@ export class Router { handler: () => this.healthController.check(), }); + // Prometheus metrics endpoint + this.routes.push({ + method: "GET", + path: "/metrics", + handler: () => this.handleMetrics(), + }); + // List items this.routes.push({ method: "GET", @@ -78,6 +86,47 @@ export class Router { return this.routes.find((route) => route.method === method && route.path === path); } + /** + * Handle Prometheus metrics endpoint + * Returns metrics in Prometheus text format + */ + private async handleMetrics(): Promise { + const exporter = getPrometheusExporter(); + + if (!exporter) { + return new Response("Prometheus metrics not initialized", { + status: 503, + headers: { "Content-Type": "text/plain" } + }); + } + + try { + // The PrometheusExporter has a getMetricsRequestHandler method + // that returns a promise with the metrics in Prometheus text format + return new Promise((resolve) => { + const mockReq = {} as any; + const mockRes = { + statusCode: 200, + setHeader: () => {}, + end: (data: string) => { + resolve(new Response(data, { + status: 200, + headers: { "Content-Type": "text/plain; version=0.0.4; charset=utf-8" } + })); + } + } as any; + + exporter.getMetricsRequestHandler(mockReq, mockRes); + }); + } catch (error) { + console.error("Error generating metrics:", error); + return new Response("Error generating metrics", { + status: 500, + headers: { "Content-Type": "text/plain" } + }); + } + } + async handle(req: Request): Promise { const tracer = trace.getTracer("items-service"); const url = new URL(req.url); @@ -113,6 +162,11 @@ export class Router { logResponse(req, response, duration); + // Record metrics + if (path !== "/metrics") { + recordHttpRequest(req.method, path, response.status, duration); + } + return response; } catch (error) { logError(error, "Request handler error"); diff --git a/apps/items-service/src/telemetry.ts b/apps/items-service/src/telemetry.ts index 2b96660..501e0f8 100644 --- a/apps/items-service/src/telemetry.ts +++ b/apps/items-service/src/telemetry.ts @@ -5,6 +5,7 @@ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; import { resourceFromAttributes } from "@opentelemetry/resources"; import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions"; import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api"; +import { initPrometheusMetrics, shutdownPrometheusMetrics } from "./metrics.js"; // Configuration from environment variables const OTEL_ENABLED = process.env.OTEL_ENABLED !== "false"; // Default to true @@ -67,6 +68,9 @@ export function initTelemetry() { console.log(`Service: ${OTEL_SERVICE_NAME} v${OTEL_SERVICE_VERSION}`); console.log(`Endpoint: ${OTEL_EXPORTER_OTLP_ENDPOINT}`); + // Initialize Prometheus metrics + initPrometheusMetrics(); + // Handle graceful shutdown process.on("SIGTERM", async () => { await shutdownTelemetry(); @@ -83,6 +87,10 @@ export function initTelemetry() { } export async function shutdownTelemetry() { + // Shutdown Prometheus metrics + await shutdownPrometheusMetrics(); + + // Shutdown OpenTelemetry SDK if (sdk) { try { await sdk.shutdown();