fix: improve metrics performance (#890)

Author: fenos

package-lock.json

@@ -41,19 +41,19 @@
     "@fastify/swagger-ui": "^4.1.0",
     "@isaacs/ttlcache": "^1.4.1",
     "@kubernetes/client-node": "^1.3.0",
-    "@opentelemetry/api": "^1.8.0",
-    "@opentelemetry/auto-instrumentations-node": "^0.67.3",
-    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.208.0",
-    "@opentelemetry/exporter-prometheus": "^0.208.0",
-    "@opentelemetry/host-metrics": "^0.38.2",
-    "@opentelemetry/instrumentation-aws-sdk": "^0.56.0",
-    "@opentelemetry/instrumentation-fastify": "^0.50.0",
-    "@opentelemetry/instrumentation-http": "^0.208.0",
-    "@opentelemetry/instrumentation-knex": "^0.53.0",
-    "@opentelemetry/instrumentation-pg": "^0.55.0",
-    "@opentelemetry/instrumentation-runtime-node": "^0.22.0",
-    "@opentelemetry/sdk-metrics": "^2.2.0",
-    "@opentelemetry/sdk-node": "^0.208.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.70.1",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.213.0",
+    "@opentelemetry/exporter-prometheus": "^0.213.0",
+    "@opentelemetry/host-metrics": "^0.38.3",
+    "@opentelemetry/instrumentation-aws-sdk": "^0.59.0",
+    "@opentelemetry/instrumentation-fastify": "^0.50.3",
+    "@opentelemetry/instrumentation-http": "^0.213.0",
+    "@opentelemetry/instrumentation-knex": "^0.53.1",
+    "@opentelemetry/instrumentation-pg": "^0.64.0",
+    "@opentelemetry/instrumentation-runtime-node": "^0.25.0",
+    "@opentelemetry/sdk-metrics": "^2.6.0",
+    "@opentelemetry/sdk-node": "^0.213.0",
     "@shopify/semaphore": "^3.0.2",
     "@smithy/node-http-handler": "^2.3.1",
     "@tus/file-store": "2.0.0",

package.json

@@ -16,6 +16,7 @@ const build = (opts: FastifyServerOptions = {}): FastifyInstance => {
   app.register(routes.migrations, { prefix: 'migrations' })
   app.register(routes.s3Credentials, { prefix: 's3' })
   app.register(routes.queue, { prefix: 'queue' })
+  app.register(routes.metricsConfig, { prefix: 'metrics' })
 
   // Register /metrics endpoint - uses OTel Prometheus exporter
   if (prometheusMetricsEnabled) {

src/admin-app.ts

@@ -88,19 +88,17 @@ export const httpMetrics = (options: HttpMetricsOptions = {}) =>
         const statusCode = groupStatusCodes
           ? `${Math.floor(reply.statusCode / 100)}xx`
           : String(reply.statusCode)
-        const tenantId = request.tenantId || ''
 
         const attributes = {
           method,
           route,
           operation: request.operation?.type || 'unknown',
           status_code: statusCode,
-          tenantId,
+          tenantId: request.tenantId || '',
         }
 
-        // Record metrics
+        // Record duration (histogram count replaces httpRequestsTotal)
         httpRequestDuration.record(durationSeconds, attributes)
-        httpRequestsTotal.add(1, attributes)
 
         // Record request size from content-length header
         const requestContentLength = request.headers['content-length']

src/http/plugins/metrics.ts

@@ -1,4 +1,5 @@
 export { default as jwks } from './jwks'
+export { default as metricsConfig } from './metrics'
 export { default as migrations } from './migrations'
 export { default as objects } from './objects'
 export { default as queue } from './queue'

src/http/routes/admin/index.ts

@@ -0,0 +1,49 @@
+import { getMetricsConfig, setMetricsEnabled } from '@internal/monitoring/metrics'
+import { FastifyInstance, RequestGenericInterface } from 'fastify'
+import { FromSchema } from 'json-schema-to-ts'
+import apiKey from '../../plugins/apikey'
+
+const updateMetricsConfigSchema = {
+  body: {
+    type: 'object',
+    properties: {
+      metrics: {
+        type: 'array',
+        items: {
+          type: 'object',
+          properties: {
+            name: { type: 'string' },
+            enabled: { type: 'boolean' },
+          },
+          required: ['name', 'enabled'],
+        },
+      },
+    },
+    required: ['metrics'],
+  },
+} as const
+
+interface UpdateMetricsConfigRequest extends RequestGenericInterface {
+  Body: FromSchema<typeof updateMetricsConfigSchema.body>
+}
+
+export default async function routes(fastify: FastifyInstance) {
+  fastify.register(apiKey)
+
+  fastify.get('/config', async (_request, reply) => {
+    return reply.send({
+      metrics: getMetricsConfig(),
+    })
+  })
+
+  fastify.put<UpdateMetricsConfigRequest>(
+    '/config',
+    { schema: updateMetricsConfigSchema },
+    async (request, reply) => {
+      setMetricsEnabled(request.body.metrics)
+      return reply.code(200).send({
+        metrics: getMetricsConfig(),
+      })
+    }
+  )
+}

src/http/routes/admin/metrics.ts

@@ -157,6 +157,7 @@ export class PoolManager {
  */
 class TenantPool implements PoolStrategy {
   protected pool?: Knex
+  protected monitorHandle?: ReturnType<typeof setInterval>
 
   constructor(protected readonly options: TenantConnectionOptions) {}
 
@@ -166,10 +167,12 @@ class TenantPool implements PoolStrategy {
     }
 
     this.pool = this.createKnexPool()
+    this.startMonitor()
     return this.pool
   }
 
   destroy(): Promise<void> {
+    this.stopMonitor()
     const originalPool = this.pool
 
     if (!originalPool) {
@@ -180,6 +183,27 @@ class TenantPool implements PoolStrategy {
     return this.drainPool(originalPool)
   }
 
+  protected startMonitor() {
+    this.monitorHandle = setInterval(() => {
+      const tarnPool = this.pool?.client?.pool
+      if (!tarnPool) return
+
+      dbInUseConnection.record(tarnPool.numUsed(), { tenant_id: this.options.tenantId })
+      dbActiveConnection.record(tarnPool.numUsed() + tarnPool.numFree(), {
+        tenant_id: this.options.tenantId,
+      })
+    }, 2000)
+
+    this.monitorHandle.unref()
+  }
+
+  protected stopMonitor() {
+    if (this.monitorHandle) {
+      clearInterval(this.monitorHandle)
+      this.monitorHandle = undefined
+    }
+  }
+
   getSettings() {
     const isSingleUseExternalPool = this.options.isSingleUse && this.options.isExternalPool
 
@@ -208,6 +232,7 @@ class TenantPool implements PoolStrategy {
       return
     }
 
+    this.stopMonitor()
     const originalPool = this.pool
 
     this.options.clusterSize = options.clusterSize
@@ -255,9 +280,8 @@ class TenantPool implements PoolStrategy {
     })
 
     const maxConnections = settings.maxConnections
-    const tenantId = this.options.tenantId
 
-    const pool = knex({
+    return knex({
       client: 'pg',
       version: dbPostgresVersion,
       searchPath: settings.searchPath,
@@ -276,50 +300,5 @@ class TenantPool implements PoolStrategy {
       },
       acquireConnectionTimeout: databaseConnectionTimeout,
     })
-
-    // Track total connections in pool per tenant
-    pool.client.pool.on('createSuccess', () => {
-      dbActiveConnection.add(1, { tenant_id: tenantId })
-    })
-
-    pool.client.pool.on('destroySuccess', () => {
-      dbActiveConnection.add(-1, { tenant_id: tenantId })
-    })
-
-    // Track in-use connections per tenant
-    pool.client.pool.on('acquireSuccess', () => {
-      dbInUseConnection.add(1, { tenant_id: tenantId })
-    })
-
-    pool.client.pool.on('release', () => {
-      dbInUseConnection.add(-1, { tenant_id: tenantId })
-    })
-
-    // Track connection acquisition time using eventId to correlate requests with completions
-    const pendingAcquires = new Map<number, number>()
-
-    pool.client.pool.on('acquireRequest', (eventId: number) => {
-      pendingAcquires.set(eventId, performance.now())
-    })
-
-    pool.client.pool.on('acquireSuccess', (eventId: number) => {
-      const startTime = pendingAcquires.get(eventId)
-      if (startTime !== undefined) {
-        pendingAcquires.delete(eventId)
-        const durationSeconds = (performance.now() - startTime) / 1000
-        dbConnectionAcquireTime.record(durationSeconds, { tenant_id: tenantId })
-      }
-    })
-
-    pool.client.pool.on('acquireFail', (eventId: number) => {
-      const startTime = pendingAcquires.get(eventId)
-      if (startTime !== undefined) {
-        pendingAcquires.delete(eventId)
-        const durationSeconds = (performance.now() - startTime) / 1000
-        dbConnectionAcquireTime.record(durationSeconds, { tenant_id: tenantId, failed: 'true' })
-      }
-    })
-
-    return pool
   }
 }