Major update with Hyperdrive API.

Author: andrewosh

index.js

@@ -85,7 +85,7 @@ class HyperdriveDaemon extends EventEmitter {
 }
 
 module.exports = async function start (opts = {}) {
-  const metadata = await new Promise((resolve, reject) => {
+  const metadata = opts.metadata || await new Promise((resolve, reject) => {
     loadMetadata((err, metadata) => {
       if (err) return reject(err)
       return resolve(metadata)
@@ -96,6 +96,7 @@ module.exports = async function start (opts = {}) {
 
   const daemonOpts = {}
   const bootstrapOpts = opts.bootstrap || argv.bootstrap
+  console.log('BOOTSTRAP OPTS:', opts.bootstrap)
   if (bootstrapOpts.length) {
     if (bootstrapOpts === false && bootstrapOpts[0] === 'false') {
       daemonOpts.network = { bootstrap: false }

lib/drives/index.js

@@ -11,7 +11,9 @@ const {
   fromHyperdriveOptions,
   toHyperdriveOptions,
   fromStat,
-  toStat
+  toStat,
+  toMount,
+  fromMount
 } = require('hyperdrive-daemon-client/lib/common')
 const { rpc } = require('hyperdrive-daemon-client')
 
@@ -151,6 +153,10 @@ class DriveManager extends EventEmitter {
 
     if (drive.writable) {
       await this._configureDrive(drive, opts && opts.configure)
+    } else {
+      // All read-only drives are currently published by default.
+      console.log('PUBLISHING READ-ONLY DRIVE')
+      await this.publish(drive)
     }
     if (this.opts.stats) {
       this._collectStats(drive)
@@ -228,6 +234,32 @@ function createDriveHandlers (driveManager) {
       return rsp
     },
 
+    publish: async (call) => {
+      console.error('IN SERVER PUBLISH')
+      const id = call.request.getId()
+
+      if (!id) throw new Error('A publish request must specify a session ID.')
+      const drive = driveManager.driveForSession(id)
+
+      console.log('PUBLISHING DRIVE WITH KEY:', drive.key)
+      await driveManager.publish(drive)
+
+      const rsp = new rpc.drive.messages.PublishDriveResponse()
+      return rsp
+    },
+
+    unpublish: async (call) => {
+      const id = call.request.getId()
+
+      if (!id) throw new Error('An unpublish request must specify a session ID.')
+      const drive = driveManager.driveForSession(id)
+
+      await driveManager.unpublish(drive)
+
+      const rsp = new rpc.drive.messages.UnpublishDriveResponse()
+      return rsp
+    },
+
     readFile: async (call) => {
       const id = call.request.getId()
       const path = call.request.getPath()
@@ -294,22 +326,41 @@ function createDriveHandlers (driveManager) {
       const recursive = call.request.getRecursive()
 
       if (!id) throw new Error('A readdir request must specify a session ID.')
-      if (!path) throw new Error('A readdir request must specify a path. ')
+      if (!path) throw new Error('A readdir request must specify a path.')
       const drive = driveManager.driveForSession(id)
 
       return new Promise((resolve, reject) => {
         drive.readdir(path, { recursive }, (err, files) => {
           if (err) return reject(err)
 
           const rsp = new rpc.drive.messages.ReadDirectoryResponse()
-          console.log('FILES:', files)
           rsp.setFilesList(files)
 
           return resolve(rsp)
         })
       })
     },
 
+    mount: async (call) => {
+      const id = call.request.getId()
+      const path = call.request.getPath()
+      const opts = fromMount(call.request.getOpts())
+
+      if (!id) throw new Error('A mount request must specify a session ID.')
+      if (!path) throw new Error('A mount request must specify a path.')
+      if (!opts) throw new Error('A mount request must specify mount options.')
+      const drive = driveManager.driveForSession(id)
+
+      return new Promise((resolve, reject) => {
+        drive.mount(path, opts.key, opts, err => {
+          console.error('after server mount, err:', err)
+          if (err) return reject(err)
+          const rsp = new rpc.drive.messages.MountDriveResponse()
+          return resolve(rsp)
+        })
+      })
+    },
+
     watch: async (call) => {
 
     },
@@ -322,7 +373,7 @@ function createDriveHandlers (driveManager) {
 
     },
 
-    closeSession: async (call) => {
+    close: async (call) => {
       const id = call.request.getId()
 
       const drive = driveManager.driveForSession(id)

lib/fuse/index.js

@@ -291,7 +291,6 @@ class FuseManager extends EventEmitter {
       await this.unmount()
       return mountRoot(drive)
     }
-    console.error('MOUNTING SUBDRIVE HERE')
     return mountSubdrive(relativePath, drive)
 
     async function mountSubdrive (relativePath, drive) {

test/all.js renamed to test/hyperdrive.js

@@ -1,18 +1,9 @@
-const { spawn } = require('child_process')
-
-const pify = require('pify')
 const test = require('tape')
-const tmp = require('tmp-promise')
-
-const PORT = 3101
-process.env['HYPERDRIVE_TOKEN'] = 'test-token'
-process.env['HYPERDRIVE_ENDPOINT'] = `localhost:${PORT}`
 
-const loadClient = require('hyperdrive-daemon-client/lib/loader')
-const start = require('..')
+const { createOne } = require('./util/create')
 
 test('can write/read a file from a remote hyperdrive', async t => {
-  const { client, cleanup } = await create()
+  const { client, cleanup } = await createOne()
 
   try {
     const { opts, id } = await client.drive.get()
@@ -24,7 +15,7 @@ test('can write/read a file from a remote hyperdrive', async t => {
     const contents = await client.drive.readFile(id, 'hello')
     t.same(contents, Buffer.from('world'))
 
-    await client.drive.closeSession(id)
+    await client.drive.close(id)
   } catch (err) {
     t.fail(err)
   }
@@ -34,7 +25,7 @@ test('can write/read a file from a remote hyperdrive', async t => {
 })
 
 test('can stat a file from a remote hyperdrive', async t => {
-  const { client, cleanup } = await create()
+  const { client, cleanup } = await createOne()
 
   try {
     const { opts, id } = await client.drive.get()
@@ -48,7 +39,7 @@ test('can stat a file from a remote hyperdrive', async t => {
     t.same(stat.uid, 0)
     t.same(stat.gid, 0)
 
-    await client.drive.closeSession(id)
+    await client.drive.close(id)
   } catch (err) {
     t.fail(err)
   }
@@ -58,7 +49,7 @@ test('can stat a file from a remote hyperdrive', async t => {
 })
 
 test('can list a directory from a remote hyperdrive', async t => {
-  const { client, cleanup } = await create()
+  const { client, cleanup } = await createOne()
 
   try {
     const { opts, id } = await client.drive.get()
@@ -67,7 +58,7 @@ test('can list a directory from a remote hyperdrive', async t => {
 
     await client.drive.writeFile(id, 'hello', 'world')
     await client.drive.writeFile(id, 'goodbye', 'dog')
-    await client.drive.writeFile(id, 'adios', 'friend')
+    await client.drive.writeFile(id, 'adios', 'amigo')
 
     const files = await client.drive.readdir(id, '')
     t.same(files.length, 4)
@@ -76,7 +67,7 @@ test('can list a directory from a remote hyperdrive', async t => {
     t.notEqual(files.indexOf('adios'), -1)
     t.notEqual(files.indexOf('.key'), -1)
 
-    await client.drive.closeSession(id)
+    await client.drive.close(id)
   } catch (err) {
     t.fail(err)
   }
@@ -86,7 +77,7 @@ test('can list a directory from a remote hyperdrive', async t => {
 })
 
 test('can read/write multiple remote hyperdrives on one server', async t => {
-  const { client, cleanup } = await create()
+  const { client, cleanup } = await createOne()
   var startingId = 1
 
   const files = [
@@ -116,37 +107,39 @@ test('can read/write multiple remote hyperdrives on one server', async t => {
   t.end()
 })
 
-async function create () {
-  const { path, cleanup: dirCleanup } = await tmp.dir({ unsafeCleanup: true })
-  const stop = await start({
-    storage: path,
-    bootstrap: false,
-    port: 3101
-  })
-
-  return new Promise((resolve, reject) => {
-    return loadClient((err, client) => {
-      if (err) return reject(err)
-      return resolve({
-        client: {
-          drive: promisifyClass(client.drive),
-          fuse: promisifyClass(client.fuse)
-        },
-        cleanup
-      })
-    })
-  })
-
-  async function cleanup () {
-    await stop()
-    await dirCleanup()
+test('can mount a drive within a remote hyperdrive', async t => {
+  const { client, cleanup } = await createOne()
+
+  try {
+    const { opts: opts1, id: id1 } = await client.drive.get()
+    t.true(opts1.key)
+    t.same(id1, 1)
+
+    const { opts: opts2, id: id2 } = await client.drive.get()
+    t.true(opts2.key)
+    t.same(id2, 2)
+    t.notEqual(opts1.key, opts2.key)
+
+    const noVersion = { ...opts2, version: null }
+
+    await client.drive.mount(id1, 'a', noVersion)
+
+    await client.drive.writeFile(id1, 'a/hello', 'world')
+    await client.drive.writeFile(id1, 'a/goodbye', 'dog')
+    await client.drive.writeFile(id1, 'adios', 'amigo')
+    await client.drive.writeFile(id2, 'hamster', 'wheel')
+
+    t.same(await client.drive.readFile(id1, 'adios'), Buffer.from('amigo'))
+    t.same(await client.drive.readFile(id1, 'a/hello'), Buffer.from('world'))
+    t.same(await client.drive.readFile(id2, 'hello'), Buffer.from('world'))
+    t.same(await client.drive.readFile(id2, 'hamster'), Buffer.from('wheel'))
+
+    await client.drive.close(id1)
+    await client.drive.close(id2)
+  } catch (err) {
+    t.fail(err)
   }
-}
-
-function promisifyClass (clazz) {
-  const methods = Object.getOwnPropertyNames(Object.getPrototypeOf(clazz)).filter(name => name !== 'constructor')
-  methods.forEach(name => {
-    clazz[name] = pify(clazz[name])
-  })
-  return clazz
-}
+
+  await cleanup()
+  t.end()
+})

test/replication.js

@@ -0,0 +1,91 @@
+const test = require('tape')
+
+const { create } = require('./util/create')
+
+test('can replicate a single drive between daemons', async t => {
+  const { clients, cleanup } = await create(2)
+  const firstClient = clients[0]
+  const secondClient = clients[1]
+
+  try {
+    const { opts, id: id1 } = await firstClient.drive.get()
+    await firstClient.drive.publish(id1)
+
+    const { id: id2 } = await secondClient.drive.get({ key: opts.key })
+
+    await firstClient.drive.writeFile(id1, 'hello', 'world')
+
+    // 100 ms delay for replication.
+    await delay(100)
+
+    const replicatedContent = await secondClient.drive.readFile(id2, 'hello')
+    t.same(replicatedContent, Buffer.from('world'))
+  } catch (err) {
+    t.fail(err)
+  }
+
+  await cleanup()
+  t.end()
+})
+
+test('can replicate many mounted drives between daemons', async t => {
+  const { clients, cleanup } = await create(2)
+  const firstClient = clients[0]
+  const secondClient = clients[1]
+
+  const NUM_MOUNTS = 100
+
+  try {
+    const mounts = await createFirst()
+    const second = await createSecond(mounts)
+
+    // 100 ms delay for replication.
+    console.log('VALIDATING IN 10s')
+    await delay(10000)
+
+    await validate(mounts, second)
+  } catch (err) {
+    t.fail(err)
+  }
+
+  await cleanup()
+  t.end()
+
+  async function createFirst () {
+    const { opts: rootOpts, id: rootId } = await firstClient.drive.get()
+    const mounts = []
+    for (let i = 0; i < NUM_MOUNTS; i++) {
+      const key = '' + i
+      const { opts: mountOpts, id: mountId } = await firstClient.drive.get()
+      await firstClient.drive.mount(rootId, key, { ...mountOpts, version: null })
+      await firstClient.drive.writeFile(mountId, key, key)
+      await firstClient.drive.publish(mountId)
+      mounts.push({ key: mountOpts.key, path: key + '/' + key, content: key })
+    }
+    return mounts
+  }
+
+  async function createSecond (mounts) {
+    const { opts: rootOpts, id: rootId } = await secondClient.drive.get()
+    for (const { key, path, content } of mounts) {
+      const { id } = await secondClient.drive.get({ key })
+      await secondClient.drive.mount(rootId, content, { key })
+    }
+    return rootId
+  }
+
+  async function validate (mounts, id) {
+    for (const { key, path, content } of mounts) {
+      const readContent = await secondClient.drive.readFile(id, path)
+      t.same(readContent, Buffer.from(content))
+    }
+  }
+})
+
+test('can replicate recursive mounts between daemons', async t => {
+  t.end()
+})
+
+function delay (ms) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}