Adds Open Telemtry instrumentation and wires it to Jaeger

.env.example

@@ -6,3 +6,6 @@ POSTGRES_USER=your_postgres_user
 POSTGRES_PASSWORD=your_secure_password_here
 POSTGRES_DB=your_database_name
 
+# Jaeger Basic Auth Credentials
+# Generate with: htpasswd -nb admin yourpassword
+JAEGER_AUTH_HTPASSWD='admin:$apr1$xyz123$abc456def789...'

Taskfile.yml

@@ -226,6 +226,31 @@ tasks:
     cmds:
       - kubectl --kubeconfig={{.KUBECONFIG}} apply -f {{.K8S_DIR}}/cert-manager/cluster-issuer.yaml
 
+  deploy:jaeger:
+    desc: Deploy Jaeger for distributed tracing
+    dotenv: ['.env']
+    cmds:
+      - |
+        echo "Creating Jaeger basic auth secret..."
+        if [ -z "$JAEGER_AUTH_HTPASSWD" ]; then
+          echo "Error: JAEGER_AUTH_HTPASSWD not set in .env file"
+          echo ""
+          echo "Generate with: htpasswd -nb admin yourpassword"
+          echo "Then add to .env: JAEGER_AUTH_HTPASSWD='admin:\$apr1\$...'"
+          exit 1
+        fi
+        # Delete existing secret if it exists
+        kubectl --kubeconfig={{.KUBECONFIG}} delete secret jaeger-basic-auth -n default --ignore-not-found=true
+        # Create new secret
+        printf '%s' "$JAEGER_AUTH_HTPASSWD" | kubectl --kubeconfig={{.KUBECONFIG}} create secret generic jaeger-basic-auth --from-file=auth=/dev/stdin -n default
+        echo "Secret created"
+      - kubectl --kubeconfig={{.KUBECONFIG}} apply -f {{.K8S_DIR}}/observability/jaeger-deployment.yaml
+      - echo "Jaeger deployed"
+      - echo "Waiting for rollout to complete..."
+      - kubectl --kubeconfig={{.KUBECONFIG}} rollout status deployment/jaeger --timeout=120s
+      - echo ""
+      - echo "Jaeger UI available at https://app.roussev.com/jaeger"
+
   # Rollout tasks
   rollout:restart:items-service:
     desc: Restart items-service deployment

Tiltfile

@@ -45,6 +45,24 @@ k8s_resource(
     resource_deps=[]
 )
 
+# ============================================================================
+# Jaeger (OpenTelemetry Collector)
+# ============================================================================
+
+# Deploy Jaeger for distributed tracing
+k8s_yaml('infra/k8s/local/jaeger-local.yaml')
+
+# Configure Jaeger resource
+k8s_resource(
+    'jaeger',
+    port_forwards='16686:16686',
+    labels=['observability'],
+    resource_deps=[],
+    links=[
+        link('http://localhost:16686', 'Jaeger UI'),
+    ]
+)
+
 # ============================================================================
 # Items Service
 # ============================================================================
@@ -69,7 +87,7 @@ k8s_resource(
     'items-service',
     port_forwards='8081:8080',
     labels=['apps'],
-    resource_deps=['postgres'],
+    resource_deps=['postgres', 'jaeger'],
     links=[
         link('http://localhost:8081/v1/health', 'Health Check'),
         link('http://localhost:8081/docs', 'API Docs'),
@@ -125,6 +143,7 @@ Services will be available at:
      - API Docs:      http://localhost:8081/docs
   🌐 Website App:     http://localhost:8082
      - Health:        http://localhost:8082/health
+  📊 Jaeger UI:       http://localhost:16686
 
 Press 'space' to open Tilt UI in your browser
 Press 'q' to quit

apps/items-service/package.json

@@ -8,6 +8,12 @@
     "start": "bun run src/index.ts"
   },
   "dependencies": {
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.66.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.207.0",
+    "@opentelemetry/resources": "^2.2.0",
+    "@opentelemetry/sdk-node": "^0.207.0",
+    "@opentelemetry/semantic-conventions": "^1.37.0",
     "postgres": "^3.4.4"
   }
 }

apps/items-service/src/index.ts

@@ -1,7 +1,12 @@
 // src/index.ts
+// Initialize OpenTelemetry first, before any other imports
+import { initTelemetry } from "./telemetry";
+initTelemetry();
+
 import postgres from "postgres";
 import { getSwaggerHtml, getRootPageHtml } from "./html";
 import { openapi } from "./openapi";
+import { trace, context, SpanStatusCode } from "@opentelemetry/api";
 
 type Item = { id: number; name: string };
 
@@ -31,19 +36,27 @@ const sql = postgres({
 
 // Initialize database schema
 async function initDatabase() {
-  try {
-    await sql`
-      CREATE TABLE IF NOT EXISTS items (
-        id SERIAL PRIMARY KEY,
-        name VARCHAR(255) NOT NULL,
-        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-      )
-    `;
-    console.log("✅ Database initialized successfully");
-  } catch (error) {
-    console.error("❌ Failed to initialize database:", error);
-    throw error;
-  }
+  const tracer = trace.getTracer("items-service");
+  return await tracer.startActiveSpan("initDatabase", async (span) => {
+    try {
+      await sql`
+        CREATE TABLE IF NOT EXISTS items (
+          id SERIAL PRIMARY KEY,
+          name VARCHAR(255) NOT NULL,
+          created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+      `;
+      console.log("✅ Database initialized successfully");
+      span.setStatus({ code: SpanStatusCode.OK });
+    } catch (error) {
+      console.error("❌ Failed to initialize database:", error);
+      span.recordException(error as Error);
+      span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+      throw error;
+    } finally {
+      span.end();
+    }
+  });
 }
 
 function json(data: unknown, init: ResponseInit = {}) {
@@ -58,63 +71,109 @@ function notFound() {
 }
 
 async function handle(req: Request): Promise<Response> {
+  const tracer = trace.getTracer("items-service");
   const url = new URL(req.url);
   let path = url.pathname;
 
-  // Root page
-  if (path === "/" && req.method === "GET") return new Response(getRootPageHtml(API_PREFIX), { headers: { "content-type": "text/html; charset=utf-8" } });
-
-  // Swagger UI & OpenAPI JSON
-  if (path === "/docs" && req.method === "GET") return new Response(getSwaggerHtml(APP_PREFIX), { headers: { "content-type": "text/html; charset=utf-8" } });
-  if (path === `${API_PREFIX}/openapi.json` && req.method === "GET") return json(openapi);
-
-  // if (!path.startsWith(API_PREFIX)) return notFound();
-  path = path.slice(API_PREFIX.length) || "/";
-
-  if (path === "/health" && req.method === "GET") {
-    try {
-      // Check database connection
-      await sql`SELECT 1`;
-      return json({ status: "ok", commit: COMMIT_SHA, database: "connected" });
-    } catch (error) {
-      return json({ status: "degraded", commit: COMMIT_SHA, database: "disconnected", error: String(error) }, { status: 503 });
-    }
-  }
-
-  if (path === "/items" && req.method === "GET") {
+  // Create a span for the HTTP request
+  return await tracer.startActiveSpan(`${req.method} ${path}`, async (span) => {
     try {
-      const items = await sql<Item[]>`SELECT id, name FROM items ORDER BY id`;
-      return json({ items });
-    } catch (error) {
-      console.error("Error fetching items:", error);
-      return json({ error: "Failed to fetch items" }, { status: 500 });
-    }
-  }
+      span.setAttributes({
+        "http.method": req.method,
+        "http.url": url.toString(),
+        "http.target": path,
+        "http.scheme": url.protocol.replace(":", ""),
+        "http.host": url.host,
+      });
+
+      let response: Response;
+
+      // Root page
+      if (path === "/" && req.method === "GET") {
         response = new Response(getRootPageHtml(), { headers: { "content-type": "text/html; charset=utf-8" } });
+      }
+      // Swagger UI & OpenAPI JSON
+      else if (path === "/docs" && req.method === "GET") {
+        response = new Response(getSwaggerHtml(APP_PREFIX), { headers: { "content-type": "text/html; charset=utf-8" } });
+      }
+      else if (path === `${API_PREFIX}/openapi.json` && req.method === "GET") {
+        response = json(openapi);
+      }
+      else {
+        // if (!path.startsWith(API_PREFIX)) return notFound();
+        path = path.slice(API_PREFIX.length) || "/";
+
+        if (path === "/health" && req.method === "GET") {
+          try {
+            // Check database connection
+            await sql`SELECT 1`;
+            response = json({ status: "ok", commit: COMMIT_SHA, database: "connected" });
+          } catch (error) {
+            span.recordException(error as Error);
+            span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+            response = json({ status: "degraded", commit: COMMIT_SHA, database: "disconnected", error: String(error) }, { status: 503 });
+          }
+        }
+        else if (path === "/items" && req.method === "GET") {
+          try {
+            const items = await sql<Item[]>`SELECT id, name FROM items ORDER BY id`;
+            span.setAttribute("items.count", items.length);
+            response = json({ items });
+          } catch (error) {
+            console.error("Error fetching items:", error);
+            span.recordException(error as Error);
+            span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+            response = json({ error: "Failed to fetch items" }, { status: 500 });
+          }
+        }
+        else if (path === "/items" && req.method === "POST") {
+          try {
+            const body = (await req.json()) as Partial<Item>;
+            if (!body?.name || typeof body.name !== "string") {
+              response = json({ error: "name required" }, { status: 400 });
+            } else {
+              const [item] = await sql<Item[]>`
+                INSERT INTO items (name)
+                VALUES (${body.name})
+                RETURNING id, name
+              `;
+              span.setAttribute("item.id", item.id);
+              span.setAttribute("item.name", item.name);
+              response = json(item, { status: 201 });
+            }
+          } catch (error) {
+            console.error("Error creating item:", error);
+            span.recordException(error as Error);
+            span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+            if (error instanceof SyntaxError) {
+              response = json({ error: "invalid json" }, { status: 400 });
+            } else {
+              response = json({ error: "Failed to create item" }, { status: 500 });
+            }
+          }
+        }
+        else {
+          response = notFound();
+        }
+      }
 
-  if (path === "/items" && req.method === "POST") {
-    try {
-      const body = (await req.json()) as Partial<Item>;
-      if (!body?.name || typeof body.name !== "string") return json({ error: "name required" }, { status: 400 });
-        response = new Response(getRootPageHtml(APP_PREFIX, API_PREFIX), { headers: { "content-type": "text/html; charset=utf-8" } });
-
-      const [item] = await sql<Item[]>`
-        INSERT INTO items (name)
-        VALUES (${body.name})
-        RETURNING id, name
-      `;
+      // Set response status on span
+      span.setAttribute("http.status_code", response.status);
+      if (response.status >= 400) {
+        span.setStatus({ code: SpanStatusCode.ERROR, message: `HTTP ${response.status}` });
+      } else {
+        span.setStatus({ code: SpanStatusCode.OK });
+      }
 
-      return json(item, { status: 201 });
+      return response;
     } catch (error) {
-      console.error("Error creating item:", error);
-      if (error instanceof SyntaxError) {
-        return json({ error: "invalid json" }, { status: 400 });
-      }
-      return json({ error: "Failed to create item" }, { status: 500 });
+      span.recordException(error as Error);
+      span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) });
+      throw error;
+    } finally {
+      span.end();
     }
-  }
-
-  return notFound();
+  });
 }
 
 // Initialize database and start server

apps/items-service/src/openapi.ts

@@ -38,17 +38,6 @@ export const openapi = {
         },
       },
     },
-    "/test1": {
-      get: {
-        summary: "Test endpoint",
-        responses: {
-          "200": {
-            description: "Test response",
-            content: { "application/json": { schema: { type: "object", properties: { status: { type: "string" } } } } },
-          },
-        },
-      },
-    },
   },
   components: {
     schemas: { Item: { type: "object", properties: { id: { type: "integer" }, name: { type: "string" } }, required: ["id", "name"] } },