Make want batches smaller (#758)

* add simple test

* wip

* ranges work

* fix bugs

* fix tests

* deopt instead of failing wants in bad cases

* and we are done

* format

* Add unit tests for `LocalWants` & `RemoteWants` (#762)

Includes the following:

- `LocalWants`'s `.add()`
- `LocalWants`'s `.remove()`
- `LocalWants`'s `.destroy()`
- `RemoteWants`'s `.add()`
- `RemoteWants`'s `.remove()`
- `RemoteWants`'s `.hasRange()`

* Update lib/replicator.js

Co-authored-by: Sean Zellmer <[email protected]>

* Update lib/replicator.js

Co-authored-by: Sean Zellmer <[email protected]>

* Update lib/replicator.js

Co-authored-by: Sean Zellmer <[email protected]>

* Update lib/wants.js

Co-authored-by: Sean Zellmer <[email protected]>

* Update lib/wants.js

Co-authored-by: Sean Zellmer <[email protected]>

* lint

* Update lib/replicator.js

Co-authored-by: Sean Zellmer <[email protected]>

* fix some issues

* Update lib/replicator.js

Co-authored-by: Sean Zellmer <[email protected]>

---------

Co-authored-by: Sean Zellmer <[email protected]>

Author: mafintosh

lib/bitfield.js

@@ -420,6 +420,26 @@ module.exports = class Bitfield {
     return this.findLast(false, position)
   }
 
+  hasUnset(start, length) {
+    const end = start + length
+
+    let j = start & (BITS_PER_SEGMENT - 1)
+    let i = (start - j) / BITS_PER_SEGMENT
+
+    while (i < this._segments.maxLength) {
+      const s = this._segments.get(i)
+      const index = s ? s.findFirst(false, j) : j
+      if (index !== -1) return i * BITS_PER_SEGMENT + index < end
+
+      j = 0
+      i++
+
+      if (i * BITS_PER_SEGMENT >= end) return false
+    }
+
+    return true
+  }
+
   hasSet(start, length) {
     const end = start + length
 
@@ -475,23 +495,23 @@ module.exports = class Bitfield {
   }
 
   *want(start, length) {
-    const j = start & (BITS_PER_SEGMENT - 1)
+    let j = start & (BITS_PER_SEGMENT - 1)
     let i = (start - j) / BITS_PER_SEGMENT
 
     while (length > 0) {
       const s = this._segments.get(i)
 
       if (s) {
-        // We always send at least 4 KiB worth of bitfield in a want, rounding
-        // to the nearest 4 KiB.
-        const end = ceilTo(clamp(length / 8, 4096, BYTES_PER_SEGMENT), 4096)
+        let end = Math.min(j + length, BITS_PER_SEGMENT)
+        if (end & 31) end += 32 - (end & 31)
 
         yield {
-          start: i * BITS_PER_SEGMENT,
-          bitfield: s.bitfield.subarray(0, end / 4)
+          start: i * BITS_PER_SEGMENT + j,
+          bitfield: s.bitfield.subarray(j / 32, end / 32)
         }
       }
 
+      j = 0
       i++
       length -= BITS_PER_SEGMENT
     }
@@ -521,13 +541,3 @@ module.exports = class Bitfield {
     return new Bitfield(buffer)
   }
 }
-
-function clamp(n, min, max) {
-  return Math.min(Math.max(n, min), max)
-}
-
-function ceilTo(n, multiple = 1) {
-  const remainder = n % multiple
-  if (remainder === 0) return n
-  return n + multiple - remainder
-}

lib/messages.js

@@ -543,19 +543,32 @@ wire.noData = {
   }
 }
 
+// TODO: we should move to having these have a handle like in data, easier since they
+// allocate state in practice
 wire.want = {
   preencode(state, m) {
     c.uint.preencode(state, m.start)
     c.uint.preencode(state, m.length)
+
+    const flags = m.any ? 1 : 0
+    if (flags !== 0) c.uint.preencode(state, flags)
   },
   encode(state, m) {
     c.uint.encode(state, m.start)
     c.uint.encode(state, m.length)
+
+    const flags = m.any ? 1 : 0
+    if (flags !== 0) c.uint.encode(state, flags)
   },
   decode(state) {
+    const start = c.uint.decode(state)
+    const length = c.uint.decode(state)
+    const flags = state.start < state.end ? c.uint.decode(state) : 0
+
     return {
-      start: c.uint.decode(state),
-      length: c.uint.decode(state)
+      start,
+      length,
+      any: (flags & 1) !== 0
     }
   }
 }
@@ -564,15 +577,26 @@ wire.unwant = {
   preencode(state, m) {
     c.uint.preencode(state, m.start)
     c.uint.preencode(state, m.length)
+
+    const flags = m.any ? 1 : 0
+    if (flags !== 0) c.uint.preencode(state, flags)
   },
   encode(state, m) {
     c.uint.encode(state, m.start)
     c.uint.encode(state, m.length)
+
+    const flags = m.any ? 1 : 0
+    if (flags !== 0) c.uint.encode(state, flags)
   },
   decode(state, m) {
+    const start = c.uint.decode(state)
+    const length = c.uint.decode(state)
+    const flags = state.start < state.end ? c.uint.decode(state) : 0
+
     return {
-      start: c.uint.decode(state),
-      length: c.uint.decode(state)
+      start,
+      length,
+      any: (flags & 1) !== 0
     }
   }
 }

lib/remote-bitfield.js

@@ -147,6 +147,7 @@ class RemoteBitfieldSegment {
 
 module.exports = class RemoteBitfield {
   static BITS_PER_PAGE = BITS_PER_PAGE
+  static BITS_PER_SEGMENT = BITS_PER_SEGMENT
 
   constructor() {
     this._pages = new BigSparseArray()