Add OTEL-compatible span ingestion endpoint (#2423)

## Summary

Adds a `POST /v1/traces` endpoint to the ingestion server that accepts OTLP/JSON and OTLP/Protobuf payloads, authenticates via `Authorization: Bearer <api-key>`, resolves the target project from an `X-Latitude-Project` header, and inserts the resulting spans into ClickHouse.

The endpoint extracts GenAI semantic convention attributes (`gen_ai.system`, `gen_ai.request.model`, token usage, etc.) into dedicated span fields during ingestion. Traces are derived automatically via the existing ClickHouse materialized view — no separate trace write path is needed.

API key validation reuses the same SHA-256 hash + Redis cache + Postgres lookup pattern used by the public API, with timing-safe enforcement to prevent enumeration attacks.

Author: csansoon

.husky/pre-commit

@@ -2,4 +2,5 @@
 set -e
 
 pnpm format
+pnpm check:fix
 pnpm knip

apps/api/package.json

@@ -21,7 +21,6 @@
     "@hono/node-server": "catalog:",
     "@hono/swagger-ui": "^0.5.0",
     "@hono/zod-openapi": "^1.0.0",
-    "@platform/auth-better": "workspace:*",
     "@platform/cache-redis": "workspace:*",
     "@platform/db-clickhouse": "workspace:*",
     "@platform/db-postgres": "workspace:*",

apps/api/src/clients.ts

@@ -1,17 +1,15 @@
 import type { ClickHouseClient } from "@clickhouse/client"
-import { createBetterAuth } from "@platform/auth-better"
 import { createRedisClient, createRedisConnection } from "@platform/cache-redis"
 import type { RedisClient } from "@platform/cache-redis"
 import { createClickhouseClient } from "@platform/db-clickhouse"
 import { type PostgresClient, createPostgresClient } from "@platform/db-postgres"
-import { parseEnv, parseEnvOptional } from "@platform/env"
+import { parseEnv } from "@platform/env"
 import { Effect } from "effect"
 
 let postgresClientInstance: PostgresClient | undefined
 let adminPostgresClientInstance: PostgresClient | undefined
 let clickhouseInstance: ClickHouseClient | undefined
 let redisInstance: RedisClient | undefined
-let betterAuthInstance: ReturnType<typeof createBetterAuth> | undefined
 
 export const getPostgresClient = (): PostgresClient => {
   if (!postgresClientInstance) {
@@ -54,35 +52,3 @@ export const getRedisClient = (): RedisClient => {
   }
   return redisInstance
 }
-
-/**
- * Get or create the Better Auth instance.
- *
- * This is a singleton to ensure the same auth instance is used
- * across routes and middleware.
- */
-export const getBetterAuth = () => {
-  if (!betterAuthInstance) {
-    const { db } = getPostgresClient()
-    const baseUrl = Effect.runSync(parseEnv("LAT_BETTER_AUTH_URL", "string"))
-    const betterAuthSecret = Effect.runSync(parseEnv("LAT_BETTER_AUTH_SECRET", "string"))
-    const webUrl = Effect.runSync(parseEnv("LAT_WEB_URL", "string", "http://localhost:3000"))
-
-    // Parse trusted origins from comma-separated env var or fallback to webUrl
-    const trustedOriginsEnv = Effect.runSync(parseEnvOptional("LAT_TRUSTED_ORIGINS", "string"))
-    const trustedOrigins = trustedOriginsEnv
-      ? trustedOriginsEnv
-          .split(",")
-          .map((o) => o.trim())
-          .filter(Boolean)
-      : [webUrl]
-
-    betterAuthInstance = createBetterAuth({
-      db,
-      secret: betterAuthSecret,
-      baseUrl,
-      trustedOrigins,
-    })
-  }
-  return betterAuthInstance
-}

apps/api/src/middleware/auth.ts

@@ -1,13 +1,9 @@
 import { OrganizationId, UnauthorizedError, UserId } from "@domain/shared"
-import {
-  type PostgresDb,
-  createApiKeyPostgresRepository,
-  createMembershipPostgresRepository,
-} from "@platform/db-postgres"
+import { type PostgresDb, createApiKeyPostgresRepository } from "@platform/db-postgres"
 import { hashToken } from "@repo/utils"
 import { Effect, Option } from "effect"
 import type { Context, MiddlewareHandler, Next } from "hono"
-import { getAdminPostgresClient, getBetterAuth } from "../clients.ts"
+import { getAdminPostgresClient } from "../clients.ts"
 import type { AuthContext } from "../types.ts"
 import { createTouchBuffer } from "./touch-buffer.ts"
 
@@ -160,27 +156,6 @@ const enforceMinimumTime = (startTime: number, minMs: number): Effect.Effect<voi
   return Effect.void
 }
 
-/**
- * Validate that a user is a member of the specified organization.
- * Returns true if membership is valid, false otherwise.
- */
-const validateOrganizationMembership = (
-  c: Context,
-  userId: string,
-  organizationId: string,
-): Effect.Effect<boolean, never> => {
-  const membershipRepository = createMembershipPostgresRepository(c.get("db"))
-  return membershipRepository.isMember(OrganizationId(organizationId), userId).pipe(Effect.orDie)
-}
-
-/**
- * Extract API key token from request headers.
- */
-const extractApiKeyToken = (c: Context): string | undefined => {
-  const apiKeyHeader = c.req.header("X-API-Key")
-  return apiKeyHeader || undefined
-}
-
 const extractBearerToken = (c: Context): string | undefined => {
   const authHeader = c.req.header("Authorization")
   if (!authHeader?.startsWith("Bearer ")) {
@@ -216,77 +191,31 @@ const authenticateWithApiKey = (
 }
 
 /**
- * Authenticate via JWT bearer token.
- */
-const authenticateWithJwt = (c: Context, token: string): Effect.Effect<AuthContext | null, never> => {
-  return Effect.gen(function* () {
-    const auth = getBetterAuth()
-    const headers = new Headers(c.req.raw.headers)
-    headers.set("authorization", `Bearer ${token}`)
-
-    const session = yield* Effect.tryPromise({
-      try: () => auth.api.getSession({ headers }),
-      catch: () => null,
-    }).pipe(Effect.orDie)
-
-    if (!session?.user) {
-      return null
-    }
-
-    const orgId = c.req.param("organizationId")
-    if (!orgId) {
-      return null
-    }
-
-    const isMember = yield* validateOrganizationMembership(c, session.user.id, orgId)
-    if (!isMember) {
-      return null
-    }
-
-    const authContext: AuthContext = {
-      userId: UserId(session.user.id),
-      organizationId: OrganizationId(orgId),
-      method: "jwt",
-    }
-
-    return authContext
-  }).pipe(Effect.orDie)
-}
-
-/**
- * Main authentication effect that tries all authentication methods.
+ * Authenticate via API key from the Authorization: Bearer header.
  */
 const authenticate = (c: Context, options?: AuthMiddlewareOptions): Effect.Effect<AuthContext, UnauthorizedError> => {
   return Effect.gen(function* () {
-    const apiKeyToken = extractApiKeyToken(c)
     const bearerToken = extractBearerToken(c)
 
-    let authContext: AuthContext | null = null
-
-    if (apiKeyToken) {
-      authContext = yield* authenticateWithApiKey(c, apiKeyToken, options)
-    } else if (bearerToken) {
-      authContext = yield* authenticateWithJwt(c, bearerToken)
-    }
-
-    if (!authContext) {
+    if (!bearerToken) {
       return yield* new UnauthorizedError({
         message: "Authentication required",
       })
     }
 
-    return authContext
+    const authContext = yield* authenticateWithApiKey(c, bearerToken, options)
+    if (authContext) return authContext
+
+    return yield* new UnauthorizedError({
+      message: "Invalid API key",
+    })
   })
 }
 
 /**
  * Create authentication middleware.
  *
- * This middleware validates requests using one of two methods:
- * 1. JWT Bearer token (Better Auth)
- * 2. API Key
- *
- * The middleware sets auth context on the Hono context for downstream handlers.
+ * Validates API keys sent via the Authorization: Bearer header.
  * Public routes should be excluded from this middleware.
  */
 interface AuthMiddlewareOptions {

apps/api/src/middleware/cors.ts

@@ -42,7 +42,7 @@ export const registerCorsMiddleware = (app: Hono, options: RegisterCorsMiddlewar
         return null
       },
       allowMethods: ["GET", "POST", "PUT", "DELETE"],
-      allowHeaders: ["Content-Type", "Authorization", "X-API-Key"],
+      allowHeaders: ["Content-Type", "Authorization"],
       credentials: true,
       maxAge: 86400,
       exposeHeaders: ["X-RateLimit-Limit", "X-RateLimit-Remaining"],

apps/api/src/server.ts

@@ -42,9 +42,8 @@ registerRoutes({
 
 // Register security scheme via the OpenAPI registry
 app.openAPIRegistry.registerComponent("securitySchemes", "ApiKeyAuth", {
-  type: "apiKey",
-  in: "header",
-  name: "X-API-Key",
+  type: "http",
+  scheme: "bearer",
   description: "Organization-scoped API key",
 })
 
@@ -54,7 +53,7 @@ app.doc("/openapi.json", {
   info: {
     title: "Latitude API",
     version: "1.0.0",
-    description: "The Latitude public API. Authenticate using an API key via the `X-API-Key` header.",
+    description: "The Latitude public API. Authenticate using an API key via the `Authorization: Bearer` header.",
   },
   servers: [{ url: `http://localhost:${port}`, description: "Local development" }],
 })

apps/api/src/types.ts

@@ -23,7 +23,7 @@ export interface AuthContext {
   /** The organization ID for this request (from URL param or API key) */
   readonly organizationId: OrganizationId
   /** The authentication method that was used */
-  readonly method: "jwt" | "api-key"
+  readonly method: "api-key"
 }
 
 /**

apps/ingest/package.json

@@ -14,14 +14,19 @@
   },
   "dependencies": {
     "@clickhouse/client": "catalog:",
+    "@domain/shared": "workspace:*",
+    "@domain/spans": "workspace:*",
     "@hono/node-server": "catalog:",
+    "@platform/cache-redis": "workspace:*",
     "@platform/db-clickhouse": "workspace:*",
     "@platform/db-postgres": "workspace:*",
     "@platform/env": "workspace:*",
     "@repo/observability": "workspace:*",
+    "@repo/utils": "workspace:*",
     "dotenv": "catalog:",
     "effect": "catalog:",
     "hono": "catalog:",
+    "protobufjs": "catalog:",
     "tsx": "catalog:"
   }
 }

apps/ingest/src/clients.ts

@@ -1,15 +1,33 @@
 import type { ClickHouseClient } from "@clickhouse/client"
+import { createRedisClient, createRedisConnection } from "@platform/cache-redis"
+import type { RedisClient } from "@platform/cache-redis"
 import { createClickhouseClient } from "@platform/db-clickhouse"
-import { createPostgresPool } from "@platform/db-postgres"
+import { type PostgresClient, createPostgresClient } from "@platform/db-postgres"
+import { parseEnv } from "@platform/env"
+import { Effect } from "effect"
 
-let postgresPoolInstance: ReturnType<typeof createPostgresPool> | undefined
+let postgresClientInstance: PostgresClient | undefined
+let adminPostgresClientInstance: PostgresClient | undefined
 let clickhouseInstance: ClickHouseClient | undefined
+let redisInstance: RedisClient | undefined
 
-export const getPostgresPool = (): ReturnType<typeof createPostgresPool> => {
-  if (!postgresPoolInstance) {
-    postgresPoolInstance = createPostgresPool()
+export const getPostgresClient = (): PostgresClient => {
+  if (!postgresClientInstance) {
+    postgresClientInstance = createPostgresClient()
   }
-  return postgresPoolInstance
+  return postgresClientInstance
+}
+
+/**
+ * Admin Postgres connection that bypasses RLS.
+ * Used for cross-org lookups: API key auth and project resolution.
+ */
+export const getAdminPostgresClient = (): PostgresClient => {
+  if (!adminPostgresClientInstance) {
+    const adminUrl = Effect.runSync(parseEnv("LAT_ADMIN_DATABASE_URL", "string"))
+    adminPostgresClientInstance = createPostgresClient({ databaseUrl: adminUrl })
+  }
+  return adminPostgresClientInstance
 }
 
 export const getClickhouseClient = (): ClickHouseClient => {
@@ -18,3 +36,11 @@ export const getClickhouseClient = (): ClickHouseClient => {
   }
   return clickhouseInstance
 }
+
+export const getRedisClient = (): RedisClient => {
+  if (!redisInstance) {
+    const redisConn = createRedisConnection()
+    redisInstance = createRedisClient(redisConn)
+  }
+  return redisInstance
+}