모니터링

Author: U-S-jun

backend/src/battles/adapters/in/battles.gateway.ts

@@ -81,8 +81,10 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
       client.data.userId = userId
       this.userIdToSocketMap.set(userId, client)
       this.metricsService.setActiveSocketConnections(this.userIdToSocketMap.size)
+      this.metricsService.recordSocketConnectionEvent('connect')
       this.logger.log(`[소켓 연결] userId: ${userId}`)
     } catch {
+      this.metricsService.recordSocketConnectionEvent('error')
       this.logger.error(`[소켓 연결 실패] - ${client.id}`)
       client.disconnect()
     }
@@ -107,6 +109,7 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
     if (userId) {
       this.userIdToSocketMap.delete(userId)
       this.metricsService.setActiveSocketConnections(this.userIdToSocketMap.size)
+      this.metricsService.recordSocketConnectionEvent('disconnect')
       this.logger.log(`[소켓 연결 해제] userId: ${userId}`)
     } else {
       this.logger.log(`[소켓 연결 해제] - ${client.id}`)
@@ -118,6 +121,7 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
   @SubscribeMessage('battle:join')
   async joinBattle(@MessageBody() battleJoinRequestDto: BattleJoinRequestDto, @ConnectedSocket() client: SocketWithUserId) {
     const stopTimer = this.metricsService.startSocketTimer('battle:join')
+    const stopActionTimer = this.metricsService.startServiceActionTimer('battle:join')
     try {
       const userId = this.getUserIdFromSocket(client)
 
@@ -145,8 +149,10 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
 
       client.emit('battle:joined', { ...res })
       stopTimer('success')
+      stopActionTimer('success')
     } catch (error) {
       stopTimer('error')
+      stopActionTimer('error')
       if (error instanceof Error) {
         client.emit('battle:join:error', {
           message: error.message,
@@ -157,19 +163,27 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
 
   @SubscribeMessage('battle:leave')
   async handleLeave(@MessageBody() dto: { battleId: string }, @ConnectedSocket() client: SocketWithUserId) {
+    const stopActionTimer = this.metricsService.startServiceActionTimer('battle:leave')
     const { battleId } = dto
     const userId = this.getUserIdFromSocket(client)
     const battleRoomId = getBattleRoomId(battleId)
 
-    const result = await this.participationUseCase.leave(userId, battleId)
-    client.data.battleId = undefined
+    try {
+      const result = await this.participationUseCase.leave(userId, battleId)
+      client.data.battleId = undefined
 
-    this.server.to(battleRoomId).emit('battle:leaved', result)
+      this.server.to(battleRoomId).emit('battle:leaved', result)
+      stopActionTimer('success')
+    } catch (error) {
+      stopActionTimer('error')
+      throw error
+    }
   }
 
   @SubscribeMessage('battle:start')
   async handleStart(@MessageBody() dto: BattleStartDto) {
     const stopTimer = this.metricsService.startSocketTimer('battle:start')
+    const stopActionTimer = this.metricsService.startServiceActionTimer('battle:start')
     try {
       const { battleId } = dto
       await this.creationUseCase.start(battleId)
@@ -178,15 +192,18 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
 
       this.server.to(battleRoomId).emit('battle:started')
       stopTimer('success')
+      stopActionTimer('success')
     } catch (error) {
       stopTimer('error')
+      stopActionTimer('error')
       throw error
     }
   }
 
   @SubscribeMessage('battle:attack')
   async handleAttack(@MessageBody() dto: AttackRequestDto, @ConnectedSocket() client: SocketWithUserId) {
     const stopTimer = this.metricsService.startSocketTimer('battle:attack')
+    const stopActionTimer = this.metricsService.startServiceActionTimer('battle:attack')
     try {
       const userId = this.getUserIdFromSocket(client)
       const battleId: string = dto.battleId
@@ -197,8 +214,10 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
 
       this.server.to(teamRoom).emit('battle:attack:created', attack)
       stopTimer('success')
+      stopActionTimer('success')
     } catch (error) {
       stopTimer('error')
+      stopActionTimer('error')
       if (error instanceof Error) {
         client.emit('battle:attack:error', {
           message: error.message,
@@ -210,6 +229,7 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
   @SubscribeMessage('battle:defense')
   async handleDefense(@MessageBody() dto: DefenseRequestDto, @ConnectedSocket() client: SocketWithUserId) {
     const stopTimer = this.metricsService.startSocketTimer('battle:defense')
+    const stopActionTimer = this.metricsService.startServiceActionTimer('battle:defense')
     try {
       const userId = this.getUserIdFromSocket(client)
       const battleId: string = dto.battleId
@@ -220,8 +240,10 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
 
       this.server.to(teamRoom).emit('battle:defense:created', defense)
       stopTimer('success')
+      stopActionTimer('success')
     } catch (error) {
       stopTimer('error')
+      stopActionTimer('error')
       if (error instanceof Error) {
         client.emit('battle:defense:error', {
           message: error.message,
@@ -233,6 +255,7 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
   @SubscribeMessage('battle:attack:vote')
   async handleAttackVote(@MessageBody() dto: AttackVoteRequestDto, @ConnectedSocket() client: SocketWithUserId) {
     const stopTimer = this.metricsService.startSocketTimer('battle:attack:vote')
+    const stopActionTimer = this.metricsService.startServiceActionTimer('battle:attack:vote')
     try {
       const userId = this.getUserIdFromSocket(client)
       const battleId: string = dto.battleId
@@ -246,8 +269,10 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
         this.server.to(teamRoom).emit('battle:attack:voted', update)
       })
       stopTimer('success')
+      stopActionTimer('success')
     } catch (error) {
       stopTimer('error')
+      stopActionTimer('error')
       if (error instanceof Error) {
         client.emit('battle:attack:vote:error', {
           message: error.message,
@@ -259,6 +284,7 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
   @SubscribeMessage('battle:defense:vote')
   async handleDefenseVote(@MessageBody() dto: DefenseVoteRequestDto, @ConnectedSocket() client: SocketWithUserId) {
     const stopTimer = this.metricsService.startSocketTimer('battle:defense:vote')
+    const stopActionTimer = this.metricsService.startServiceActionTimer('battle:defense:vote')
     try {
       const userId = this.getUserIdFromSocket(client)
       const battleId: string = dto.battleId
@@ -272,8 +298,10 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
         this.server.to(teamRoom).emit('battle:defense:voted', update)
       })
       stopTimer('success')
+      stopActionTimer('success')
     } catch (error) {
       stopTimer('error')
+      stopActionTimer('error')
       if (error instanceof Error) {
         client.emit('battle:defense:vote:error', {
           message: error.message,
@@ -285,6 +313,7 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
   @SubscribeMessage('battle:user:skip')
   async handlePhaseSkip(@MessageBody() dto: { skip: boolean; battleId: string }, @ConnectedSocket() client: SocketWithUserId) {
     const stopTimer = this.metricsService.startSocketTimer('battle:user:skip')
+    const stopActionTimer = this.metricsService.startServiceActionTimer('battle:user:skip')
     const userId = this.getUserIdFromSocket(client)
     try {
       const { skip, battleId } = dto
@@ -294,8 +323,10 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
       this.server.to(battleRoomId).emit('battle:user:skipped', { totalSkips })
 
       stopTimer('success')
+      stopActionTimer('success')
     } catch (error) {
       stopTimer('error')
+      stopActionTimer('error')
       if (error instanceof Error) {
         client.emit('battle:user:skip:error', {
           message: error.message,
@@ -362,6 +393,7 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
   @SubscribeMessage('battle:chat')
   async handleChat(@MessageBody() battleChatDto: BattleChatDto, @ConnectedSocket() client: SocketWithUserId) {
     const stopTimer = this.metricsService.startSocketTimer('battle:chat')
+    const stopActionTimer = this.metricsService.startServiceActionTimer('battle:chat')
     try {
       const userId = this.getUserIdFromSocket(client)
 
@@ -374,8 +406,10 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
       // this.server.to(roomId).emit('battle:chatted', saved)
       this.server.to(roomId).except(client.id).emit('battle:chatted', saved)
       stopTimer('success')
+      stopActionTimer('success')
     } catch (error) {
       stopTimer('error')
+      stopActionTimer('error')
       if (error instanceof Error) {
         client.emit('battle:chat:error', { message: error.message })
       }
@@ -385,15 +419,18 @@ export class BattlesGateway implements OnGatewayConnection, OnGatewayDisconnect
   @SubscribeMessage('battle:team:vote')
   async handleTeamVote(@MessageBody() dto: BattleTeamVoteDto, @ConnectedSocket() client: SocketWithUserId) {
     const stopTimer = this.metricsService.startSocketTimer('battle:team:vote')
+    const stopActionTimer = this.metricsService.startServiceActionTimer('battle:team:vote')
     try {
       const userId = this.getUserIdFromSocket(client)
       const battleId: string = dto.battleId
       const team: BattleTeam = dto.team
 
       await this.interactionUseCase.switchTeam(battleId, userId, team)
       stopTimer('success')
+      stopActionTimer('success')
     } catch (error) {
       stopTimer('error')
+      stopActionTimer('error')
       if (error instanceof Error) {
         client.emit('battle:team:vote:error', { message: error.message })
       }

backend/src/metrics/metrics.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common'
 import { Registry, collectDefaultMetrics, Counter, Histogram, Gauge } from 'prom-client'
 
 type SocketStatus = 'success' | 'error'
+type ServiceActionStatus = 'success' | 'error'
 
 @Injectable()
 export class MetricsService {
@@ -12,6 +13,8 @@ export class MetricsService {
   private readonly socketEventCount: Counter<'event' | 'status'>
   private readonly socketEventErrors: Counter<'event'>
   private readonly socketActiveConnections: Gauge<'scope'>
+  private readonly socketConnectionEvents: Counter<'type'>
+  private readonly serviceActionDuration: Histogram<'action' | 'status'>
 
   constructor() {
     collectDefaultMetrics({ register: this.registry })
@@ -59,6 +62,21 @@ export class MetricsService {
       labelNames: ['scope'],
       registers: [this.registry],
     })
+
+    this.socketConnectionEvents = new Counter({
+      name: 'socket_connection_events_total',
+      help: 'Socket connection events',
+      labelNames: ['type'],
+      registers: [this.registry],
+    })
+
+    this.serviceActionDuration = new Histogram({
+      name: 'service_action_duration_seconds',
+      help: 'Service action handling duration in seconds',
+      labelNames: ['action', 'status'],
+      buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5],
+      registers: [this.registry],
+    })
   }
 
   get contentType() {
@@ -95,6 +113,18 @@ export class MetricsService {
     this.socketActiveConnections.set({ scope: 'user' }, count)
   }
 
+  recordSocketConnectionEvent(type: 'connect' | 'disconnect' | 'error') {
+    this.socketConnectionEvents.inc({ type })
+  }
+
+  startServiceActionTimer(action: string) {
+    const start = process.hrtime()
+    return (status: ServiceActionStatus) => {
+      const durationSeconds = this.getDurationSeconds(start)
+      this.serviceActionDuration.observe({ action, status }, durationSeconds)
+    }
+  }
+
   private getDurationSeconds(start: [number, number]) {
     const diff = process.hrtime(start)
     return diff[0] + diff[1] / 1e9

monitoring/grafana/provisioning/dashboards/socket-events.json

@@ -38,14 +38,22 @@
     {
       "id": 2,
       "type": "timeseries",
-      "title": "p95 Handling Time by event",
+      "title": "Handling Time p50/p95/p99 by event",
       "datasource": "Prometheus",
       "gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 },
       "fieldConfig": { "defaults": { "unit": "s" } },
       "targets": [
+        {
+          "expr": "histogram_quantile(0.5, sum(rate(socket_event_duration_seconds_bucket{event=~\"$event\"}[5m])) by (le, event))",
+          "legendFormat": "p50 {{event}}"
+        },
         {
           "expr": "histogram_quantile(0.95, sum(rate(socket_event_duration_seconds_bucket{event=~\"$event\"}[5m])) by (le, event))",
-          "legendFormat": "{{event}}"
+          "legendFormat": "p95 {{event}}"
+        },
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(socket_event_duration_seconds_bucket{event=~\"$event\"}[5m])) by (le, event))",
+          "legendFormat": "p99 {{event}}"
         }
       ]
     },
@@ -77,19 +85,89 @@
         }
       ]
     },
+    {
+      "id": 8,
+      "type": "timeseries",
+      "title": "Service Action p95 by action",
+      "datasource": "Prometheus",
+      "gridPos": { "x": 0, "y": 16, "w": 12, "h": 8 },
+      "fieldConfig": { "defaults": { "unit": "s" } },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(service_action_duration_seconds_bucket[5m])) by (le, action))",
+          "legendFormat": "{{action}}"
+        }
+      ]
+    },
+    {
+      "id": 10,
+      "type": "timeseries",
+      "title": "Service Action Average by action",
+      "datasource": "Prometheus",
+      "gridPos": { "x": 12, "y": 16, "w": 12, "h": 8 },
+      "fieldConfig": { "defaults": { "unit": "s" } },
+      "targets": [
+        {
+          "expr": "sum(rate(service_action_duration_seconds_sum[5m])) by (action) / sum(rate(service_action_duration_seconds_count[5m])) by (action)",
+          "legendFormat": "{{action}}"
+        }
+      ]
+    },
     {
       "id": 5,
       "type": "stat",
       "title": "Active Socket Connections",
       "datasource": "Prometheus",
-      "gridPos": { "x": 0, "y": 16, "w": 24, "h": 4 },
+      "gridPos": { "x": 0, "y": 24, "w": 24, "h": 4 },
       "fieldConfig": { "defaults": { "unit": "none", "decimals": 0 } },
       "targets": [
         {
           "expr": "socket_active_connections",
           "legendFormat": "connections"
         }
       ]
+    },
+    {
+      "id": 6,
+      "type": "timeseries",
+      "title": "Connection Events (rate)",
+      "datasource": "Prometheus",
+      "gridPos": { "x": 0, "y": 28, "w": 24, "h": 6 },
+      "fieldConfig": { "defaults": { "unit": "ops" } },
+      "targets": [
+        {
+          "expr": "sum(rate(socket_connection_events_total[1m])) by (type)",
+          "legendFormat": "{{type}}"
+        }
+      ]
+    },
+    {
+      "id": 9,
+      "type": "timeseries",
+      "title": "Service Action p99 by action",
+      "datasource": "Prometheus",
+      "gridPos": { "x": 0, "y": 34, "w": 12, "h": 6 },
+      "fieldConfig": { "defaults": { "unit": "s" } },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.99, sum(rate(service_action_duration_seconds_bucket[5m])) by (le, action))",
+          "legendFormat": "{{action}}"
+        }
+      ]
+    },
+    {
+      "id": 11,
+      "type": "timeseries",
+      "title": "Service Action Error Rate by action",
+      "datasource": "Prometheus",
+      "gridPos": { "x": 12, "y": 34, "w": 12, "h": 6 },
+      "fieldConfig": { "defaults": { "unit": "percent" } },
+      "targets": [
+        {
+          "expr": "100 * sum(rate(service_action_duration_seconds_count{status=\"error\"}[5m])) by (action) / sum(rate(service_action_duration_seconds_count[5m])) by (action)",
+          "legendFormat": "{{action}}"
+        }
+      ]
     }
   ]
 }