fix: add a more aggressive rate limit for spammy users

.env.example

@@ -11,6 +11,7 @@ PINEAPPLE_URL=https://.../upload # optional
 SCORE_API_URL=https://score.snapshot.org
 # If you need unlimted access to score-api, use `https://score.snapshot.org?apiKey=...`
 RATE_LIMIT_DATABASE_URL= # optional
+RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer: #optional
 BROVIDER_URL=https://rpc.snapshot.org # optional
 # Secret for laser
 AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional

.github/workflows/test.yml

@@ -9,3 +9,4 @@ jobs:
     with:
       mysql_database_name: snapshot_sequencer_test
       mysql_schema_path: 'test/schema.sql'
+      redis: true

src/helpers/rateLimit.ts

@@ -1,30 +1,11 @@
 import rateLimit from 'express-rate-limit';
 import RedisStore from 'rate-limit-redis';
-import { createClient } from 'redis';
+import redisClient from './redis';
 import { getIp, sendError, sha256 } from './utils';
-import log from './log';
-
-let client;
-
-(async () => {
-  if (!process.env.RATE_LIMIT_DATABASE_URL) return;
-
-  log.info('[redis-rl] Connecting to Redis');
-  client = createClient({ url: process.env.RATE_LIMIT_DATABASE_URL });
-  client.on('connect', () => log.info('[redis-rl] Redis connect'));
-  client.on('ready', () => log.info('[redis-rl] Redis ready'));
-  client.on('reconnecting', err => log.info('[redis-rl] Redis reconnecting', err));
-  client.on('error', err => log.info('[redis-rl] Redis error', err));
-  client.on('end', err => log.info('[redis-rl] Redis end', err));
-  await client.connect();
-})();
 
 const hashedIp = (req): string => sha256(getIp(req)).slice(0, 7);
 
-export default rateLimit({
-  windowMs: 60 * 1e3,
-  max: 100,
-  keyGenerator: req => hashedIp(req),
+const rateLimitConfig = {
   standardHeaders: true,
   legacyHeaders: false,
   handler: (req, res) => {
@@ -34,10 +15,27 @@ export default rateLimit({
       429
     );
   },
-  store: client
+  store: redisClient
     ? new RedisStore({
-        sendCommand: (...args: string[]) => client.sendCommand(args),
-        prefix: 'snapshot-sequencer:'
+        sendCommand: (...args: string[]) => redisClient.sendCommand(args),
+        prefix: process.env.RATE_LIMIT_KEYS_PREFIX || 'snapshot-sequencer:'
       })
     : undefined
+};
+
+const regularRateLimit = rateLimit({
+  keyGenerator: req => `rl:${hashedIp(req)}`,
+  windowMs: 60 * 1e3,
+  max: 100,
+  ...rateLimitConfig
 });
+
+const highErroredRateLimit = rateLimit({
+  keyGenerator: req => `rl-s:${hashedIp(req)}`,
+  windowMs: 15 * 1e3,
+  max: 15,
+  skipSuccessfulRequests: true,
+  ...rateLimitConfig
+});
+
+export { regularRateLimit, highErroredRateLimit };

src/helpers/redis.ts

@@ -0,0 +1,19 @@
+import { createClient } from 'redis';
+import log from './log';
+
+let client;
+
+(async () => {
+  if (!process.env.RATE_LIMIT_DATABASE_URL) return;
+
+  log.info('[redis] Connecting to Redis');
+  client = createClient({ url: process.env.RATE_LIMIT_DATABASE_URL });
+  client.on('connect', () => log.info('[redis] Redis connect'));
+  client.on('ready', () => log.info('[redis] Redis ready'));
+  client.on('reconnecting', err => log.info('[redis] Redis reconnecting', err));
+  client.on('error', err => log.info('[redis] Redis error', err));
+  client.on('end', err => log.info('[redis] Redis end', err));
+  await client.connect();
+})();
+
+export default client;

src/index.ts

@@ -3,7 +3,7 @@ import cors from 'cors';
 import { initLogger, fallbackLogger } from '@snapshot-labs/snapshot-sentry';
 import express from 'express';
 import api from './api';
-import rateLimit from './helpers/rateLimit';
+import { regularRateLimit, highErroredRateLimit } from './helpers/rateLimit';
 import shutter from './helpers/shutter';
 import log from './helpers/log';
 import refreshModeration from './helpers/moderation';
@@ -19,7 +19,7 @@ app.disable('x-powered-by');
 app.use(express.json({ limit: '20mb' }));
 app.use(express.urlencoded({ limit: '20mb', extended: false }));
 app.use(cors({ maxAge: 86400 }));
-app.use(rateLimit);
+app.use(regularRateLimit, highErroredRateLimit);
 app.set('trust proxy', 1);
 app.use('/', api);
 app.use('/shutter', shutter);

test/.env.test

@@ -3,3 +3,5 @@ SEQ_DATABASE_URL=mysql://root:[email protected]:3306/snapshot_sequencer_test
 NETWORK=main
 RELAYER_PK=01686849e86499c1860ea0afc97f29c11018cbac049abf843df875c60054076e
 NODE_ENV=test
+RATE_LIMIT_DATABASE_URL=redis://localhost:6379
+RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer-test:

test/e2e/api.test.ts

@@ -1,21 +1,71 @@
 import fetch from 'node-fetch';
 import proposalInput from '../fixtures/ingestor-payload/proposal.json';
+import redis from '../../src/helpers/redis';
 
 const HOST = `http://localhost:${process.env.PORT || 3003}`;
 
+function successRequest() {
+  return fetch(`${HOST}`);
+}
+
+function failRequest() {
+  return fetch(HOST, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(proposalInput)
+  });
+}
+
 describe('POST /', () => {
+  beforeEach(async () => {
+    const keyPrefix = process.env.RATE_LIMIT_KEYS_PREFIX || 'snapshot-sequencer:';
+    await redis.del(`${keyPrefix}rl:3e48ef9`);
+    await redis.del(`${keyPrefix}rl-s:3e48ef9`);
+  });
+
+  afterAll(async () => {
+    await redis.quit();
+  });
+
   describe('on invalid client input', () => {
     it('returns a 400 error', async () => {
-      const response = await fetch(HOST, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify(proposalInput)
-      });
+      const response = await failRequest();
       const body = await response.json();
 
       expect(response.status).toBe(400);
       expect(body.error).toBe('client_error');
       expect(body.error_description).toBe('wrong timestamp');
     });
   });
+
+  describe('rate limit', () => {
+    describe('on a mix of success and failed requests', () => {
+      it('should return a 429 errors only after 100 requests / min', async () => {
+        for (let i = 1; i <= 100; i++) {
+          // 2% of failing requests
+          const response = await (Math.random() < 0.02 ? failRequest() : successRequest());
+          expect(response.status).not.toEqual(429);
+        }
+
+        const response = await fetch(HOST, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify(proposalInput)
+        });
+        expect(response.status).toBe(429);
+      });
+    });
+
+    describe('on multiple failed requests', () => {
+      it('should return a 429 errors after 15 requests / 15s', async () => {
+        for (let i = 1; i <= 15; i++) {
+          const response = await failRequest();
+          expect(response.status).toBe(400);
+        }
+
+        const response = await fetch(`${HOST}/scores/proposal-id`);
+        expect(response.status).toBe(429);
+      });
+    });
+  });
 });

yarn.lock

@@ -2760,9 +2760,9 @@ express-prom-bundle@^6.6.0:
     url-value-parser "^2.0.0"
 
 express-rate-limit@^6.9.0:
-  version "6.9.0"
-  resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-6.9.0.tgz#afecb23936d9cd1d133a3c20056708b9955cad0f"
-  integrity sha512-AnISR3V8qy4gpKM62/TzYdoFO9NV84fBx0POXzTryHU/qGUJBWuVGd+JhbvtVmKBv37t8/afmqdnv16xWoQxag==
+  version "6.11.2"
+  resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-6.11.2.tgz#6c42035603d3b52e4e2fb59f6ebaa89e628ef980"
+  integrity sha512-a7uwwfNTh1U60ssiIkuLFWHt4hAC5yxlLGU2VP0X4YNlyEDZAqF4tK3GD3NSitVBrCQmQ0++0uOyFOgC2y4DDw==
 
 express@^4.18.1:
   version "4.18.1"