feat: report metrics about rate-limiter (#1875)

* feat: report metrics about rate-limiter

* refactor: apply PR feedback

Author: aleortega

content/src/components.ts

@@ -136,7 +136,7 @@ export async function initComponentsWithEnv(env: Environment): Promise<AppCompon
   const failedDeployments = await createFailedDeployments({ metrics, database })
 
   const deployRateLimiter = createDeployRateLimiter(
-    { logs },
+    { logs, metrics },
     {
       defaultTtl: env.getConfig(EnvironmentConfig.DEPLOYMENTS_DEFAULT_RATE_LIMIT_TTL) ?? ms('1m'),
       defaultMax: env.getConfig(EnvironmentConfig.DEPLOYMENTS_DEFAULT_RATE_LIMIT_MAX) ?? 300,
@@ -184,7 +184,7 @@ export async function initComponentsWithEnv(env: Environment): Promise<AppCompon
 
   const validator = { validate }
 
-  const serverValidator = createServerValidator({ failedDeployments, metrics, clock })
+  const serverValidator = createServerValidator({ failedDeployments, clock })
 
   const deployedEntitiesBloomFilter = createDeployedEntitiesBloomFilter({ database, logs, clock })
   const deployments = createDeploymentsComponent({ database, logs })

content/src/metrics.ts

@@ -72,6 +72,18 @@ export const metricsDeclaration = validateMetricsDeclaration({
   dcl_content_rate_limited_deployments_total: {
     help: 'Total failed deployments due rate limit',
     type: 'counter',
+    labelNames: ['entity_type', 'reason']
+  },
+
+  dcl_content_rate_limiter_cache_keys: {
+    help: 'Current number of keys in rate limiter caches',
+    type: 'gauge',
+    labelNames: ['entity_type', 'cache_type']
+  },
+
+  dcl_content_rate_limiter_cache_max_size: {
+    help: 'Configured max size for deployment rate limiter cache',
+    type: 'gauge',
     labelNames: ['entity_type']
   },
 

content/src/ports/deployRateLimiterComponent.ts

@@ -20,7 +20,7 @@ export type DeploymentRateLimitConfig = {
 }
 
 export function createDeployRateLimiter(
-  components: Pick<AppComponents, 'logs'>,
+  components: Pick<AppComponents, 'logs' | 'metrics'>,
   rateLimitConfig: DeploymentRateLimitConfig
 ): IDeployRateLimiterComponent {
   const logs: ILoggerComponent.ILogger = components.logs.getLogger('DeployRateLimiterComponent')
@@ -35,6 +35,11 @@ export function createDeployRateLimiter(
     rateLimitConfig
   )
 
+  // Set static max size gauge for each entity type
+  for (const [entityType, { maxSize }] of deploymentCacheMap) {
+    components.metrics.observe('dcl_content_rate_limiter_cache_max_size', { entity_type: entityType }, maxSize)
+  }
+
   function getCacheFromEntityType(entityType: EntityType): { cache: NodeCache; maxSize: number } {
     const cache = deploymentCacheMap.get(entityType)
     if (!cache) {
@@ -57,28 +62,52 @@ export function createDeployRateLimiter(
       for (const pointer of pointers) {
         cacheByEntityType.cache.set(pointer, localTimestamp)
       }
+      components.metrics.observe(
+        'dcl_content_rate_limiter_cache_keys',
+        { entity_type: entityType, cache_type: 'deployment' },
+        cacheByEntityType.cache.stats.keys
+      )
     },
 
     /** Check if the entity should be rate limit: no deployment has been made for the same pointer in the last ttl
      * and no more than max size of deployments were made either   */
     isRateLimited(entityType: EntityType, pointers: string[]): boolean {
       const cacheByEntityType = getCacheFromEntityType(entityType)
-      return (
-        pointers.some((p) => !!cacheByEntityType.cache.get(p)) ||
-        cacheByEntityType.cache.stats.keys > cacheByEntityType.maxSize
-      )
+      const ttlHit = pointers.some((p) => !!cacheByEntityType.cache.get(p))
+      const maxSizeHit = cacheByEntityType.cache.stats.keys > cacheByEntityType.maxSize
+
+      if (ttlHit || maxSizeHit) {
+        components.metrics.increment('dcl_content_rate_limited_deployments_total', {
+          entity_type: entityType,
+          reason: ttlHit ? 'ttl' : 'max_size'
+        })
+      }
+
+      return ttlHit || maxSizeHit
     },
 
     newUnchangedDeployment(entityType: EntityType, pointers: string[], localTimestamp: number): void {
       const cache = getUnchangedCacheFromEntityType(entityType)
       for (const pointer of pointers) {
         cache.set(pointer, localTimestamp)
       }
+      components.metrics.observe(
+        'dcl_content_rate_limiter_cache_keys',
+        { entity_type: entityType, cache_type: 'unchanged' },
+        cache.stats.keys
+      )
     },
 
     isUnchangedDeploymentRateLimited(entityType: EntityType, pointers: string[]): boolean {
       const cache = getUnchangedCacheFromEntityType(entityType)
-      return pointers.some((p) => !!cache.get(p))
+      const limited = pointers.some((p) => !!cache.get(p))
+      if (limited) {
+        components.metrics.increment('dcl_content_rate_limited_deployments_total', {
+          entity_type: entityType,
+          reason: 'unchanged_ttl'
+        })
+      }
+      return limited
     }
   }
 }

content/src/service/validations/server.ts

@@ -32,7 +32,7 @@ function isRequestTtlForwards(components: Pick<AppComponents, 'clock'>, entity:
 const localChecks = async (
   entity: Entity,
   serviceCalls: ServiceCalls,
-  components: Pick<AppComponents, 'metrics' | 'clock'>
+  components: Pick<AppComponents, 'clock'>
 ): Promise<string | undefined> => {
   /** Validate that there are no newer deployments on the entity's pointers */
   if (await serviceCalls.areThereNewerEntities(entity))
@@ -46,7 +46,6 @@ const localChecks = async (
 
   /** Validate the deployment is not rate limited */
   if (await serviceCalls.isEntityRateLimited(entity)) {
-    components.metrics.increment('dcl_content_rate_limited_deployments_total', { entity_type: entity.type })
     return `Entity rate limited (entityId=${entity.id} pointers=${entity.pointers.join(',')}).`
   }
 
@@ -78,7 +77,7 @@ export const IGNORING_FIX_ERROR = 'Ignoring fix for failed deployment since ther
  * Server side validations for current deploying entity for LOCAL and FIX_ATTEMPT contexts
  */
 export const createServerValidator = (
-  components: Pick<AppComponents, 'failedDeployments' | 'metrics' | 'clock'>
+  components: Pick<AppComponents, 'failedDeployments' | 'clock'>
 ): ServerValidator => ({
   validate: async (entity, context, serviceCalls) => {
     // these contexts doesn't validate anything in this side

content/test/integration/ports/deployer/rate-limiting.spec.ts

@@ -32,7 +32,7 @@ describe('Rate limiting E2E', () => {
    */
   function applyRealRateLimiter(): void {
     const realRateLimiter = createDeployRateLimiter(
-      { logs: server.components.logs },
+      { logs: server.components.logs, metrics: server.components.metrics },
       {
         defaultTtl: NORMAL_TTL_MS,
         defaultMax: 10000,