Rework client router filter handling (#49741)

This simplifies the implementation for the client router filter and
makes the error rate configuring more accurate. Additional tests haven't
been added as this is still a matter of probability and existing tests
cover the client router behavior.

Example scenario of filter size/false positives with an error rate of
`0.01%`:

```sh
entries: 128 filter size: 4986 bytes, gzip size: 627 bytes
false positives with 100000 samples: 26 0.026%
```

x-ref: [slack
thread](https://vercel.slack.com/archives/C04K237UHCP/p1683129488543039)

Author: ijjk

packages/next/src/lib/create-client-router-filter.ts

@@ -5,12 +5,10 @@ import { removeTrailingSlash } from '../shared/lib/router/utils/remove-trailing-
 import { Redirect } from './load-custom-routes'
 import { tryToParsePath } from './try-to-parse-path'
 
-const POTENTIAL_ERROR_RATE = 0.01
-
 export function createClientRouterFilter(
   paths: string[],
   redirects: Redirect[],
-  allowedErrorRate: number = POTENTIAL_ERROR_RATE
+  allowedErrorRate?: number
 ): {
   staticFilter: ReturnType<BloomFilter['export']>
   dynamicFilter: ReturnType<BloomFilter['export']>

packages/next/src/server/lib/incremental-cache/index.ts

@@ -8,7 +8,7 @@ import {
   IncrementalCacheValue,
   IncrementalCacheEntry,
 } from '../../response-cache'
-import { encode } from '../../../shared/lib/bloom-filter/base64-arraybuffer'
+import { encode } from '../../../shared/lib/base64-arraybuffer'
 import { encodeText } from '../../stream-utils/encode-decode'
 import {
   CACHE_ONE_YEAR,

packages/next/src/server/lib/patch-fetch.ts

@@ -446,7 +446,7 @@ export function patchFetch({
 
               if (process.env.NEXT_RUNTIME === 'edge') {
                 const { decode } =
-                  require('../../shared/lib/bloom-filter/base64-arraybuffer') as typeof import('../../shared/lib/bloom-filter/base64-arraybuffer')
+                  require('../../shared/lib/base64-arraybuffer') as typeof import('../../shared/lib/base64-arraybuffer')
                 decodedBody = decode(resData.body)
               } else {
                 decodedBody = Buffer.from(resData.body, 'base64').subarray()

…d/lib/bloom-filter/base64-arraybuffer.ts …ext/src/shared/lib/base64-arraybuffer.tspackages/next/src/shared/lib/bloom-filter/base64-arraybuffer.ts renamed to packages/next/src/shared/lib/base64-arraybuffer.ts packages/next/src/shared/lib/bloom-filter/base64-arraybuffer.ts renamed to packages/next/src/shared/lib/base64-arraybuffer.ts

@@ -0,0 +1,94 @@
+// minimal implementation MurmurHash2 hash function
+function murmurhash2(str: string) {
+  let h = 0
+  for (let i = 0; i < str.length; i++) {
+    const c = str.charCodeAt(i)
+    h = Math.imul(h ^ c, 0x5bd1e995)
+    h ^= h >>> 13
+    h = Math.imul(h, 0x5bd1e995)
+  }
+  return h >>> 0
+}
+
+export class BloomFilter {
+  numItems: number
+  errorRate: number
+  numBits: number
+  numHashes: number
+  bitArray: number[]
+
+  constructor(numItems: number, errorRate: number) {
+    this.numItems = numItems
+    this.errorRate = errorRate
+    this.numBits = Math.ceil(
+      -(numItems * Math.log(errorRate)) / (Math.log(2) * Math.log(2))
+    )
+    this.numHashes = Math.ceil((this.numBits / numItems) * Math.log(2))
+    this.bitArray = new Array(this.numBits).fill(0)
+  }
+
+  static from(items: string[], errorRate = 0.01) {
+    const filter = new BloomFilter(items.length, errorRate)
+
+    for (const item of items) {
+      filter.add(item)
+    }
+    return filter
+  }
+
+  export() {
+    const data = {
+      numItems: this.numItems,
+      errorRate: this.errorRate,
+      numBits: this.numBits,
+      numHashes: this.numHashes,
+      bitArray: this.bitArray,
+    }
+
+    if (typeof window === 'undefined' && process.env.NEXT_RUNTIME !== 'edge') {
+      if (this.errorRate < 0.01) {
+        const filterData = JSON.stringify(data)
+        const gzipSize = require('next/dist/compiled/gzip-size').sync(
+          filterData
+        )
+
+        if (gzipSize > 1024) {
+          console.warn(
+            `Creating filter with error rate less than 1% (0.01) can increase the size dramatically proceed with caution. Received error rate ${this.errorRate} resulted in size ${filterData.length} bytes, ${gzipSize} bytes (gzip)`
+          )
+        }
+      }
+    }
+
+    return data
+  }
+
+  import(data: ReturnType<typeof this['export']>) {
+    this.numItems = data.numItems
+    this.errorRate = data.errorRate
+    this.numBits = data.numBits
+    this.numHashes = data.numHashes
+    this.bitArray = data.bitArray
+  }
+
+  add(item: string) {
+    const hashValues = this.getHashValues(item)
+    hashValues.forEach((hash) => {
+      this.bitArray[hash] = 1
+    })
+  }
+
+  contains(item: string) {
+    const hashValues = this.getHashValues(item)
+    return hashValues.every((hash) => this.bitArray[hash])
+  }
+
+  getHashValues(item: string) {
+    const hashValues = []
+    for (let i = 1; i <= this.numHashes; i++) {
+      const hash = murmurhash2(`${item}${i}`) % this.numBits
+      hashValues.push(hash)
+    }
+    return hashValues
+  }
+}