Add sparse option, sans core.contiguousByteLength (#100)

* Add `sparse` option, sans `core.contiguousByteLength`

* Don't inherit `sparse`

* Start continuous replication in non-sparse mode

* Assert tree length

* Adjust description

* Fix typo

Author: kasperisager

README.md

@@ -62,6 +62,7 @@ Note that `tree`, `data`, and `bitfield` are normally heavily sparse files.
 {
   createIfMissing: true, // create a new Hypercore key pair if none was present in storage
   overwrite: false, // overwrite any old Hypercore that might already exist
+  sparse: true, // enable sparse mode, counting unavailable blocks towards core.length and core.byteLength
   valueEncoding: 'json' | 'utf-8' | 'binary', // defaults to binary
   encodeBatch: batch => { ... }, // optionally apply an encoding to complete batches
   keyPair: kp, // optionally pass the public key and secret key as a key pair
@@ -235,13 +236,13 @@ Buffer containing the optional block encryption key of this core. Will be `null`
 
 #### `core.length`
 
-How many blocks of data are available on this core?
+How many blocks of data are available on this core? If `sparse: false`, this will equal `core.contiguousLength`.
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
 #### `core.byteLength`
 
-How much data is available on this core in bytes?
+How much data is available on this core in bytes? If `sparse: false`, this will equal `core.contiguousByteLength`.
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
@@ -251,6 +252,12 @@ How many blocks are contiguously available starting from the first block of this
 
 Populated after `ready` has been emitted. Will be `0` before the event.
 
+#### `core.contiguousByteLength`
+
+How much data is contiguously available starting from the first block of this core?
+
+Populated after `ready` has been emitted. Will be `0` before the event.
+
 #### `core.fork`
 
 What is the current fork id of this core?

index.js

@@ -66,6 +66,7 @@ module.exports = class Hypercore extends EventEmitter {
     this.opened = false
     this.closed = false
     this.snapshotted = !!opts.snapshot
+    this.sparse = opts.sparse !== false
     this.sessions = opts._sessions || [this]
     this.auth = opts.auth || null
     this.autoClose = !!opts.autoClose
@@ -271,6 +272,9 @@ module.exports = class Hypercore extends EventEmitter {
       this.encodeBatch = opts.encodeBatch
     }
 
+    // Start continous replication if not in sparse mode.
+    if (!this.sparse) this.download({ start: 0, end: -1 })
+
     // This is a hidden option that's only used by Corestore.
     // It's required so that corestore can load a name from userData before 'ready' is emitted.
     if (opts._preready) await opts._preready(this)
@@ -411,21 +415,27 @@ module.exports = class Hypercore extends EventEmitter {
   }
 
   get length () {
-    return this._snapshot
-      ? this._snapshot.length
-      : (this.core === null ? 0 : this.core.tree.length)
+    if (this._snapshot) return this._snapshot.length
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousLength
+    return this.core.tree.length
   }
 
   get byteLength () {
-    return this._snapshot
-      ? this._snapshot.byteLength
-      : (this.core === null ? 0 : this.core.tree.byteLength - (this.core.tree.length * this.padding))
+    if (this._snapshot) return this._snapshot.byteLength
+    if (this.core === null) return 0
+    if (!this.sparse) return this.contiguousByteLength
+    return this.core.tree.byteLength - (this.core.tree.length * this.padding)
   }
 
   get contiguousLength () {
     return this.core === null ? 0 : this.core.header.contiguousLength
   }
 
+  get contiguousByteLength () {
+    return 0
+  }
+
   get fork () {
     return this.core === null ? 0 : this.core.tree.fork
   }

test/replicate.js

@@ -598,3 +598,20 @@ test('one inflight request to a peer per block', async function (t) {
 
   t.is(uploads, 1)
 })
+
+test('non-sparse replication', async function (t) {
+  t.plan(6)
+
+  const a = await create()
+  const b = await create(a.key, { sparse: false })
+
+  await a.append(['a', 'b', 'c', 'd', 'e'])
+
+  replicate(a, b, t)
+
+  b
+    .once('download', () => t.is(b.core.tree.length, 5))
+    .on('download', (i) => {
+      t.is(b.length, b.contiguousLength, `block ${i}`)
+    })
+})