add reacher throttle requests (#2014)

Author: malek10xdev

.env.master.dev

@@ -72,7 +72,7 @@ LOAD_BALANCE_VERIFIERS = false # true to load balance between multiple email ver
 
 # Reacher: https://help.reacher.email/
 REACHER_HOST = https://stoplight.io/mocks/reacher/backend/68673 # ( REQUIRED )
-# REACHER_API_KEY =
+REACHER_API_KEY = 'test-key'
 # REACHER_HEADER_SECRET =
 # REACHER_SMTP_FROM =
 # REACHER_SMTP_HELLO =
@@ -87,7 +87,8 @@ REACHER_HOTMAIL_USE_HEADLESS =true
 # REACHER_MICROSOFT365_USE_API =
 REACHER_GMAIL_USE_API =true
 # REACHER_YAHOO_USE_API =
-
+REACHER_RATE_LIMITER_REQUESTS = 60
+REACHER_RATE_LIMITER_INTERVAL = 60 * 1000
 # Mailercheck: https://developers.mailercheck.com/
 # MAILERCHECK_API_KEY =
 # Zerobounce:  https://www.zerobounce.net/docs/email-validation-api-quickstart/

.env.master.prod

@@ -87,6 +87,8 @@ REACHER_HOTMAIL_USE_HEADLESS = true
 # REACHER_MICROSOFT365_USE_API =
 REACHER_GMAIL_USE_API = true
 # REACHER_YAHOO_USE_API =
+REACHER_RATE_LIMITER_REQUESTS = 60
+REACHER_RATE_LIMITER_INTERVAL = 60 * 1000
 
 # Mailercheck:  https://developers.mailercheck.com/
 MAILERCHECK_API_KEY =

backend/src/config/schema.ts

@@ -81,6 +81,8 @@ const schema = z.object({
   REACHER_MICROSOFT365_USE_API: boolean().optional().default('true'),
   REACHER_GMAIL_USE_API: boolean().optional().default('false'),
   REACHER_YAHOO_USE_API: boolean().optional().default('false'),
+  REACHER_RATE_LIMITER_REQUESTS: number(),
+  REACHER_RATE_LIMITER_INTERVAL: number(),
 
   /* MAILERCHECK */
   MAILERCHECK_API_KEY: z.string().min(1).optional(),

backend/src/services/email-status/EmailStatusVerifierFactory.ts

@@ -28,6 +28,8 @@ interface ReacherConfig {
   REACHER_MICROSOFT365_USE_API: boolean;
   REACHER_GMAIL_USE_API: boolean;
   REACHER_YAHOO_USE_API: boolean;
+  REACHER_RATE_LIMITER_REQUESTS: number;
+  REACHER_RATE_LIMITER_INTERVAL: number;
 }
 
 interface MailerCheckConfig {
@@ -210,6 +212,11 @@ export default class EmailStatusVerifierFactory {
                 password: config.REACHER_PROXY_PASSWORD
               }
             : undefined
+      },
+      rateLimiter: {
+        requests: config.REACHER_RATE_LIMITER_REQUESTS,
+        interval: config.REACHER_RATE_LIMITER_INTERVAL,
+        spaced: false
       }
     });
 

backend/src/services/email-status/reacher/client.ts

@@ -1,5 +1,6 @@
 import axios, { AxiosInstance } from 'axios';
 import { Logger } from 'winston';
+import throttledQueue from 'throttled-queue';
 import { logError } from '../../../utils/axios';
 
 interface BulkSubmitResponse {
@@ -67,6 +68,12 @@ interface BulkVerificationResultsResponse {
   results: EmailCheckOutput[];
 }
 
+interface RateLimiterOptions {
+  requests: number;
+  interval: number;
+  spaced: boolean;
+}
+
 interface ReacherConfig {
   host?: string;
   timeoutMs?: number;
@@ -79,6 +86,7 @@ interface ReacherConfig {
   gmailUseApi?: boolean;
   yahooUseApi?: boolean;
   hotmailUseHeadless?: string;
+  rateLimiter: RateLimiterOptions;
 }
 
 interface SMTPConfig {
@@ -97,12 +105,14 @@ interface ValidationOptions {
 }
 
 export default class ReacherClient {
-  private static readonly SINGLE_VERIFICATION_PATH = '/v1/check_email';
+  static readonly SINGLE_VERIFICATION_PATH = '/v1/check_email';
 
-  private static readonly BULK_VERIFICATION_PATH = '/v0/bulk';
+  static readonly BULK_VERIFICATION_PATH = '/v0/bulk';
 
   private readonly api: AxiosInstance;
 
+  private readonly rate_limit_handler;
+
   private readonly smtpConfig: {
     from_email?: string;
     hello_name?: string;
@@ -133,6 +143,12 @@ export default class ReacherClient {
     this.api = axios.create({
       baseURL: config.host
     });
+    this.rate_limit_handler = throttledQueue(
+      config.rateLimiter.requests,
+      config.rateLimiter.interval,
+      config.rateLimiter.spaced
+    );
+
     if (config.timeoutMs) {
       this.api.defaults.timeout = config.timeoutMs;
     }
@@ -190,17 +206,19 @@ export default class ReacherClient {
     validationOptions?: ValidationOptions
   ): Promise<EmailCheckOutput> {
     try {
-      const { data } = await this.api.post<EmailCheckOutput>(
-        ReacherClient.SINGLE_VERIFICATION_PATH,
-        {
-          to_email: email,
-          ...this.additionalSettings,
-          ...this.smtpConfig,
-          from_email: validationOptions
-            ? validationOptions.fromEmail
-            : this.smtpConfig.from_email
-        },
-        { signal: abortSignal }
+      const { data } = await this.rate_limit_handler(() =>
+        this.api.post<EmailCheckOutput>(
+          ReacherClient.SINGLE_VERIFICATION_PATH,
+          {
+            to_email: email,
+            ...this.additionalSettings,
+            ...this.smtpConfig,
+            from_email: validationOptions
+              ? validationOptions.fromEmail
+              : this.smtpConfig.from_email
+          },
+          { signal: abortSignal }
+        )
       );
       return { ...data, input: email };
     } catch (error) {

backend/test/unit/email-status/EmailStatusVerifierFactory.test.ts

@@ -16,7 +16,9 @@ const reacherDefaultConfig = {
   REACHER_MICROSOFT365_USE_API: true,
   REACHER_GMAIL_USE_API: true,
   REACHER_YAHOO_USE_API: true,
-  REACHER_REQUEST_TIMEOUT_MS: 5
+  REACHER_REQUEST_TIMEOUT_MS: 5,
+  REACHER_RATE_LIMITER_REQUESTS: 5,
+  REACHER_RATE_LIMITER_INTERVAL: 1000
 };
 
 const outlookEmails = [