DX-2218: Refactor rate limit test cases and update request timing in TestHarness (#142)

* Refactor rate limit test cases and update request timing in TestHarness

* fix: use rate in cachedFixedWindow correctly

* chore: update node version to 20 in workflow configurations

* fix: change wrangler publish to wrangler deploy in workflow

* fix: update test cases to use a single limit value of 16

* fix: update package.json dependencies to fix nextjs deployed errors

* fix: add delays before and after resetting tokens in tests to fix flaky tests

Author: CahidArda

.github/workflows/tests.yaml

@@ -89,7 +89,7 @@ jobs:
       - name: Setup nodejs
         uses: actions/setup-node@v3
         with:
-          node-version: 18
+          node-version: 20
 
       - name: Setup Bun
         uses: oven-sh/setup-bun@v1
@@ -114,7 +114,7 @@ jobs:
         working-directory: examples/cloudflare-workers
 
       - name: Deploy
-        run: wrangler publish
+        run: wrangler deploy
         working-directory: examples/cloudflare-workers
         env:
           CLOUDFLARE_API_TOKEN: ${{secrets.CLOUDFLARE_API_TOKEN}}
@@ -238,7 +238,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v2
         with:
-          node-version: 18
+          node-version: 20
 
       - name: Set package version
         run: echo $(jq --arg v "${{ steps.version.outputs.version }}" '(.version) = $v' package.json) > package.json

examples/nextjs/bun.lockb

@@ -1,5 +1,6 @@
 /// <reference types="next" />
 /// <reference types="next/image-types/global" />
+/// <reference types="next/navigation-types/compat/navigation" />
 
 // NOTE: This file should not be edited
-// see https://nextjs.org/docs/basic-features/typescript for more information.
+// see https://nextjs.org/docs/app/building-your-application/configuring/typescript for more information.

examples/nextjs/next-env.d.ts

@@ -9,7 +9,7 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@upstash/ratelimit": "latest",
+    "@upstash/ratelimit": "^2.0.6",
     "@upstash/redis": "latest",
     "@vercel/functions": "^1.0.2",
     "next": "14.2.23",
@@ -19,8 +19,8 @@
   "devDependencies": {
     "@types/bun": "^1.1.10",
     "@types/node": "^20",
-    "@types/react": "^18",
-    "@types/react-dom": "^18",
+    "@types/react": "^19.2.2",
+    "@types/react-dom": "^19",
     "bun-types": "latest",
     "eslint": "^8",
     "eslint-config-next": "14.2.3",

examples/nextjs/package.json

@@ -34,9 +34,9 @@ export class Cache implements EphemeralCache {
     return this.cache.get(key) || null;
   }
 
-  public incr(key: string): number {
+  public incr(key: string, incrementAmount: number = 1): number {
     let value = this.cache.get(key) ?? 0;
-    value += 1;
+    value += incrementAmount;
     this.cache.set(key, value);
     return value;
   }

src/cache.ts

@@ -11,75 +11,101 @@ import { TestHarness } from "./test_utils";
 import type { Context, MultiRegionContext, RegionContext } from "./types";
 
 type TestCase = {
-  // requests per second
-  rps: number;
   /**
-   * Multiplier for rate
-   *
-   * rate = 10, load = 0.5 -> attack rate will be 5
+   * Limit allowed during window
+   */
+  limit: number;
+  /**
+   * Request load
+   * 
+   * E.g., 0.5 means 50% of the limit in each window will be consumed,
+   * so all requests will succeed (assuming rate=1)
+   * 
+   * E.g., 2 means 200% of the limit in each window will be consumed,
+   * so half of the requests will be rejected (assuming rate=1)
    */
   load: number;
   /**
-   * rate at which the tokens will be added or consumed, default should be 1
-   * @default 1
+   * rate at which the tokens will be added or consumed
    */
-  rate?: number;
+  rate: number;
 };
-const attackDuration = 10;
-const window = 5;
+const attackDuration = 8;
+const window = 4;
 const windowString: Duration = `${window} s`;
 
 const testcases: TestCase[] = [];
 
-for (const rps of [10, 100]) {
-  for (const load of [0.5, 0.7]) {
-    for (const rate of [undefined, 10]) {
-      testcases.push({ load, rps, rate });
+for (const limit of [16]) {
+  for (const load of [0.8, 1.6]) {
+    for (const rate of [1, 3]) {
+      testcases.push({ load, limit, rate });
     }
   }
 }
 
-function run<TContext extends Context>(builder: (tc: TestCase) => Ratelimit<TContext>) {
+function run<TContext extends Context>(
+  builder: (tc: TestCase) => Ratelimit<TContext>
+) {
   for (const tc of testcases) {
-    const name = `${tc.rps.toString().padStart(4, " ")}/s - Load: ${(tc.load * 100)
-      .toString()
-      .padStart(3, " ")}% -> Sending ${(tc.rps * tc.load)
-        .toString()
-        .padStart(4, " ")}req/s at the rate of ${tc.rate ?? 1}`;
-    const ratelimit = builder(tc);
+
+    const windowCount = attackDuration / window;
+    /**
+     * Total number of requests sent during the attack
+     */
+    const attackRequestCount = windowCount * tc.limit * tc.load;
+    /**
+     * Number of requests the simulated attacker shall attempt
+     */
+    const attackRequestPerSecond = attackRequestCount / attackDuration;
+    /**
+     * Maximum number of requests that can be allowed per second
+     */
+    const maxSuccessRequestCount = windowCount * tc.limit / tc.rate;
+    /**
+     * Number of successful requests expected during the attack
+     */
+    const expectedSuccessRequestCount = Number.parseFloat(Math.min(maxSuccessRequestCount, attackRequestCount).toFixed(2));
 
     const limits = {
-      lte: ((attackDuration * tc.rps * (tc.rate ?? 1)) / window) * 1.5,
-      gte: ((attackDuration * tc.rps) / window) * 0.5,
+      lte: Number.parseFloat((expectedSuccessRequestCount * 1.5).toFixed(2)),
+      gte: Number.parseFloat((expectedSuccessRequestCount * 0.5).toFixed(2)),
     };
+
+    const name = `${tc.limit} Limit, ${tc.load * 100}% Load, ${attackRequestPerSecond} req/s (with rate=${tc.rate})`;
+    const range = `Range:  ${limits.gte} - ${limits.lte} Success`
+    
+    const ratelimit = builder(tc);
+
     describe(name, () => {
       test(
-        `should be within ${limits.gte} - ${limits.lte}`,
+        range,
         async () => {
-          log(name);
+          log();
+          log(`  Config: ${name}`);
+          log(`  ${range} (Expected: ${expectedSuccessRequestCount})`);
           const harness = new TestHarness(ratelimit);
-          await harness.attack(tc.rps * tc.load, attackDuration, tc.rate).catch((error) => {
-            console.error(error);
-          });
+          await harness
+            .attack(attackRequestPerSecond, attackDuration, tc.rate)
+            .catch((error) => {
+              console.error(error);
+            });
           log(
-            "success:",
-            harness.metrics.success,
-            ", blocked:",
-            harness.metrics.rejected,
-            "out of:",
-            harness.metrics.requests,
+            `  Result: success: ${harness.metrics.success}, blocked: ${harness.metrics.rejected} (out of: ${harness.metrics.requests})`
           );
 
           expect(harness.metrics.success).toBeLessThanOrEqual(limits.lte);
           expect(harness.metrics.success).toBeGreaterThanOrEqual(limits.gte);
         },
-        attackDuration * 1000 * 4,
+        attackDuration * 1000 * 4
       );
     });
   }
 }
 
-function newMultiRegion(limiter: Algorithm<MultiRegionContext>): Ratelimit<MultiRegionContext> {
+function newMultiRegion(
+  limiter: Algorithm<MultiRegionContext>
+): Ratelimit<MultiRegionContext> {
   // eslint-disable-next-line unicorn/consistent-function-scoping
   function ensureEnv(key: string): string {
     const value = process.env[key];
@@ -109,7 +135,9 @@ function newMultiRegion(limiter: Algorithm<MultiRegionContext>): Ratelimit<Multi
   });
 }
 
-function newRegion(limiter: Algorithm<RegionContext>): Ratelimit<RegionContext> {
+function newRegion(
+  limiter: Algorithm<RegionContext>
+): Ratelimit<RegionContext> {
   return new RegionRatelimit({
     prefix: crypto.randomUUID(),
     redis: Redis.fromEnv(),
@@ -146,32 +174,44 @@ describe("timeout", () => {
 
 describe("fixedWindow", () => {
   describe("region", () =>
-    run((tc) => newRegion(RegionRatelimit.fixedWindow(tc.rps * (tc.rate ?? 1), windowString))));
+    run((tc) =>
+      newRegion(RegionRatelimit.fixedWindow(tc.limit, windowString))
+    ));
 
   describe("multiRegion", () =>
     run((tc) =>
-      newMultiRegion(MultiRegionRatelimit.fixedWindow(tc.rps * (tc.rate ?? 1), windowString)),
+      newMultiRegion(
+        MultiRegionRatelimit.fixedWindow(tc.limit, windowString)
+      )
     ));
 });
 describe("slidingWindow", () => {
   describe("region", () =>
-    run((tc) => newRegion(RegionRatelimit.slidingWindow(tc.rps * (tc.rate ?? 1), windowString))));
+    run((tc) =>
+      newRegion(RegionRatelimit.slidingWindow(tc.limit, windowString))
+    ));
   describe("multiRegion", () =>
     run((tc) =>
-      newMultiRegion(MultiRegionRatelimit.slidingWindow(tc.rps * (tc.rate ?? 1), windowString)),
+      newMultiRegion(
+        MultiRegionRatelimit.slidingWindow(tc.limit, windowString)
+      )
     ));
 });
 
 describe("tokenBucket", () => {
   describe("region", () =>
     run((tc) =>
-      newRegion(RegionRatelimit.tokenBucket(tc.rps, windowString, tc.rps * (tc.rate ?? 1))),
+      newRegion(
+        RegionRatelimit.tokenBucket(tc.limit, windowString, tc.limit)
+      )
     ));
 });
 
 describe("cachedFixedWindow", () => {
   describe("region", () =>
     run((tc) =>
-      newRegion(RegionRatelimit.cachedFixedWindow(tc.rps * (tc.rate ?? 1), windowString)),
+      newRegion(
+        RegionRatelimit.cachedFixedWindow(tc.limit, windowString)
+      )
     ));
 });

src/ratelimit.test.ts

@@ -23,8 +23,11 @@ function run<TContext extends Context>(builder: Ratelimit<TContext>) {
       }
       await Promise.all(pendings)
 
+      await new Promise(r => setTimeout(r, 300));
+
       // reset tokens
       await builder.resetUsedTokens(id);
+      await new Promise(r => setTimeout(r, 300));
       const { remaining } = await builder.getRemaining(id);
       expect(remaining).toBe(limit);
     }, 10_000);

src/resetUsedTokens.test.ts

@@ -9,7 +9,7 @@ import type { Algorithm, RegionContext } from "./types";
 import type { Redis as RedisCore } from "./types";
 
 // Fix for https://github.com/upstash/ratelimit-js/issues/125
-type Redis = Pick<RedisCore, "get" | "set">
+type Redis = Pick<RedisCore, "evalsha" | "get" | "set">
 
 export type RegionRatelimitConfig = {
   /**
@@ -490,7 +490,7 @@ export class RegionRatelimit extends Ratelimit<RegionContext> {
 
         const hit = typeof ctx.cache.get(key) === "number";
         if (hit) {
-          const cachedTokensAfterUpdate = ctx.cache.incr(key);
+          const cachedTokensAfterUpdate = ctx.cache.incr(key, incrementBy);
           const success = cachedTokensAfterUpdate < tokens;
 
           const pending = success

src/single.ts

@@ -46,7 +46,7 @@ export class TestHarness<TContext extends Context> {
             return res;
           }),
         );
-        await new Promise((r) => setTimeout(r, 500 / rps));
+        await new Promise((r) => setTimeout(r, 1000 / rps));
       }
     }
 

src/test_utils.ts

@@ -11,7 +11,7 @@ export type EphemeralCache = {
   set: (key: string, value: number) => void;
   get: (key: string) => number | null;
 
-  incr: (key: string) => number;
+  incr: (key: string, incrementAmount?: number) => number;
 
   pop: (key: string) => void;
   empty: () => void;