Merge pull request #562 from codex-team/feat/rate-limits-settings

Feat/rate limits settings

Author: Dobrunia

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hawk.api",
-  "version": "1.2.4",
+  "version": "1.2.7",
   "main": "index.ts",
   "license": "BUSL-1.1",
   "scripts": {

src/index.ts

@@ -26,8 +26,8 @@ import PlansFactory from './models/plansFactory';
 import BusinessOperationsFactory from './models/businessOperationsFactory';
 import schema from './schema';
 import { graphqlUploadExpress } from 'graphql-upload';
-import morgan from 'morgan';
 import { metricsMiddleware, createMetricsServer, graphqlMetricsPlugin } from './metrics';
+import { requestLogger } from './utils/logger';
 
 /**
  * Option to enable playground
@@ -85,19 +85,17 @@ class HawkAPI {
       next();
     });
 
-    /**
-     * Setup request logger.
-     * Uses 'combined' format in production for Apache-style logging,
-     * and 'dev' format in development for colored, concise output.
-     */
-    this.app.use(morgan(process.env.NODE_ENV === 'production' ? 'combined' : 'dev'));
-
     /**
      * Add metrics middleware to track HTTP requests
      */
     this.app.use(metricsMiddleware);
 
     this.app.use(express.json());
+
+    /**
+     * Setup request logger with custom formatters (GraphQL operation name support)
+     */
+    this.app.use(requestLogger);
     this.app.use(bodyParser.urlencoded({ extended: false }));
     this.app.use('/static', express.static(`./static`));
 

src/metrics/mongodb.ts

@@ -1,5 +1,6 @@
 import promClient from 'prom-client';
 import { MongoClient, MongoClientOptions } from 'mongodb';
+import { Effect, sgr } from '../utils/ansi';
 
 /**
  * MongoDB command duration histogram
@@ -113,12 +114,155 @@ export function withMongoMetrics(options: MongoClientOptions = {}): MongoClientO
   };
 }
 
+/**
+ * Format filter/update parameters for logging
+ * @param params - Parameters to format
+ * @returns Formatted string
+ */
+function formatParams(params: any): string {
+  if (!params || Object.keys(params).length === 0) {
+    return '';
+  }
+
+  try {
+    return JSON.stringify(params);
+  } catch (e) {
+    return String(params);
+  }
+}
+
+/**
+ * Colorize duration based on performance thresholds
+ * @param duration - Duration in milliseconds
+ * @returns Colorized duration string
+ */
+function colorizeDuration(duration: number): string {
+  let color: Effect;
+
+  if (duration < 50) {
+    color = Effect.ForegroundGreen;
+  } else if (duration < 100) {
+    color = Effect.ForegroundYellow;
+  } else {
+    color = Effect.ForegroundRed;
+  }
+
+  return sgr(`${duration}ms`, color);
+}
+
+/**
+ * Interface for storing command information with timestamp
+ */
+interface StoredCommandInfo {
+  formattedCommand: string;
+  timestamp: number;
+}
+
+/**
+ * Map to store formatted command information by requestId
+ */
+const commandInfoMap = new Map<number, StoredCommandInfo>();
+
+/**
+ * Timeout for cleaning up stale command info (30 seconds)
+ */
+const COMMAND_INFO_TIMEOUT_MS = 30000;
+
+/**
+ * Cleanup stale command info to prevent memory leaks
+ * Removes entries older than COMMAND_INFO_TIMEOUT_MS
+ */
+function cleanupStaleCommandInfo(): void {
+  const now = Date.now();
+  const keysToDelete: number[] = [];
+
+  for (const [requestId, info] of commandInfoMap.entries()) {
+    if (now - info.timestamp > COMMAND_INFO_TIMEOUT_MS) {
+      keysToDelete.push(requestId);
+    }
+  }
+
+  if (keysToDelete.length > 0) {
+    console.warn(`Cleaning up ${keysToDelete.length} stale MongoDB command info entries (possible memory leak)`);
+    for (const key of keysToDelete) {
+      commandInfoMap.delete(key);
+    }
+  }
+}
+
+/**
+ * Periodic cleanup interval
+ */
+setInterval(cleanupStaleCommandInfo, COMMAND_INFO_TIMEOUT_MS);
+
+/**
+ * Store MongoDB command details for later logging
+ * @param event - MongoDB command event
+ */
+function storeCommandInfo(event: any): void {
+  const collectionRaw = extractCollectionFromCommand(event.command, event.commandName);
+  const collection = sgr(normalizeCollectionName(collectionRaw), Effect.ForegroundGreen);
+  const db = event.databaseName || 'unknown db';
+  const commandName = sgr(event.commandName, Effect.ForegroundRed);
+  const filter = event.command.filter;
+  const update = event.command.update;
+  const pipeline = event.command.pipeline;
+  const projection = event.command.projection;
+  const params = filter || update || pipeline;
+  const paramsStr = formatParams(params);
+  const projectionStr = projection ? ` projection: ${formatParams(projection)}` : '';
+
+  const formattedCommand = `[${event.requestId}] ${db}.${collection}.${commandName}(${paramsStr})${projectionStr}`;
+
+  commandInfoMap.set(event.requestId, {
+    formattedCommand,
+    timestamp: Date.now(),
+  });
+}
+
+/**
+ * Log MongoDB command success to console
+ * Format: [requestId] db.collection.command(params) ✓ duration
+ * @param event - MongoDB command event
+ */
+function logCommandSucceeded(event: any): void {
+  const info = commandInfoMap.get(event.requestId);
+  const durationStr = colorizeDuration(event.duration);
+
+  if (info) {
+    console.log(`${info.formattedCommand} ✓ ${durationStr}`);
+    commandInfoMap.delete(event.requestId);
+  } else {
+    console.log(`[${event.requestId}] ${event.commandName} ✓ ${durationStr}`);
+  }
+}
+
+/**
+ * Log MongoDB command failure to console
+ * Format: [requestId] db.collection.command(params) ✗ error duration
+ * @param event - MongoDB command event
+ */
+function logCommandFailed(event: any): void {
+  const errorMsg = event.failure?.message || event.failure?.errmsg || 'Unknown error';
+  const info = commandInfoMap.get(event.requestId);
+  const durationStr = colorizeDuration(event.duration);
+
+  if (info) {
+    console.error(`${info.formattedCommand} ✗ ${errorMsg} ${durationStr}`);
+    commandInfoMap.delete(event.requestId);
+  } else {
+    console.error(`[${event.requestId}] ${event.commandName} ✗ ${errorMsg} ${durationStr}`);
+  }
+}
+
 /**
  * Setup MongoDB metrics monitoring on a MongoClient
  * @param client - MongoDB client to monitor
  */
 export function setupMongoMetrics(client: MongoClient): void {
   client.on('commandStarted', (event) => {
+    storeCommandInfo(event);
+
     // Store start time and metadata for this command
     const metadataKey = `${event.requestId}`;
 
@@ -139,6 +283,8 @@ export function setupMongoMetrics(client: MongoClient): void {
   });
 
   client.on('commandSucceeded', (event) => {
+    logCommandSucceeded(event);
+
     const metadataKey = `${event.requestId}`;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const metadata = (client as any)[metadataKey];
@@ -157,6 +303,8 @@ export function setupMongoMetrics(client: MongoClient): void {
   });
 
   client.on('commandFailed', (event) => {
+    logCommandFailed(event);
+
     const metadataKey = `${event.requestId}`;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const metadata = (client as any)[metadataKey];

src/models/eventsFactory.js

@@ -665,7 +665,7 @@ class EventsFactory extends Factory {
   async getEventRelease(eventId) {
     const eventOriginal = await this.findById(eventId);
 
-    if (!eventOriginal) {
+    if (!eventOriginal || !eventOriginal.payload.release) {
       return null;
     }
 

src/mongo.ts

@@ -75,7 +75,9 @@ export async function setupConnections(): Promise<void> {
     databases.hawk = hawkMongoClient.db();
     databases.events = eventsMongoClient.db();
 
-    // Setup metrics monitoring for both clients
+    /**
+     * Log and and measure MongoDB metrics
+     */
     setupMongoMetrics(hawkMongoClient);
     setupMongoMetrics(eventsMongoClient);
   } catch (e) {

src/resolvers/billingNew.ts

@@ -111,7 +111,6 @@ export default {
 
       let isCardLinkOperation = false;
 
-
       /**
        * We need to only link card and not pay for the whole plan in case
        * 1. We are paying for the same plan and

src/resolvers/project.js

@@ -176,6 +176,37 @@ module.exports = {
       }
     },
 
+    /**
+     * Update project rate limits settings
+     *
+     * @param {ResolverObj} _obj
+     * @param {string} id - project id
+     * @param {Object | null} rateLimitSettings - rate limit settings (null to remove)
+     * @param {UserInContext} user - current authorized user {@see ../index.js}
+     * @param {ContextFactories} factories - factories for working with models
+     *
+     * @returns {Project}
+     */
+    async updateProjectRateLimits(_obj, { id, rateLimitSettings }, { user, factories }) {
+      const project = await factories.projectsFactory.findById(id);
+
+      if (!project) {
+        throw new ApolloError('There is no project with that id');
+      }
+
+      if (project.workspaceId.toString() === '6213b6a01e6281087467cc7a') {
+        throw new ApolloError('Unable to update demo project');
+      }
+
+      try {
+        return project.updateProject({
+          rateLimitSettings: rateLimitSettings || null,
+        });
+      } catch (err) {
+        throw new ApolloError('Something went wrong');
+      }
+    },
+
     /**
      * Generates new project integration token by id
      *

src/resolvers/user.ts

@@ -65,7 +65,9 @@ export default {
           priority: TaskPriorities.IMPORTANT,
         });
 
-        telegram.sendMessage(`🚶 User "${email}" signed up`);
+        const source = user.utm?.source;
+
+        telegram.sendMessage(`🚶 User "${email}" signed up` + (source ? `, from ${source}` : ''));
 
         return isE2E ? password : true;
       } catch (e) {

src/typeDefs/project.ts

@@ -1,6 +1,35 @@
 import { gql } from 'apollo-server-express';
 
 export default gql`
+  """
+  Rate limits configuration input
+  """
+  input RateLimitSettingsInput {
+    """
+    Rate limit threshold (N events)
+    """
+    N: Int!
+
+    """
+    Rate limit period in seconds (T seconds)
+    """
+    T: Int!
+  }
+
+  """
+  Rate limits configuration
+  """
+  type RateLimitSettings {
+    """
+    Rate limit threshold (N events)
+    """
+    N: Int!
+
+    """
+    Rate limit period in seconds (T seconds)
+    """
+    T: Int!
+  }
 
 """
 Possible events order
@@ -253,6 +282,11 @@ type Project {
   Event grouping patterns
   """
   eventGroupingPatterns: [ProjectEventGroupingPattern]
+
+  """
+  Rate limits configuration
+  """
+  rateLimitSettings: RateLimitSettings
 }
 
 extend type Query {
@@ -305,6 +339,26 @@ extend type Mutation {
     Project image
     """
     image: Upload @uploadImage
+
+    """
+    Rate limits configuration
+    """
+    rateLimitSettings: RateLimitSettingsInput
+  ): Project! @requireAdmin
+
+  """
+  Update project rate limits settings
+  """
+  updateProjectRateLimits(
+    """
+    What project to update
+    """
+    id: ID!
+
+    """
+    Rate limits configuration. Pass null to remove rate limits.
+    """
+    rateLimitSettings: RateLimitSettingsInput
   ): Project! @requireAdmin
 
   """