Move to corestore-only + add bootstrap opts

Author: andrewosh

bin/start.js

@@ -22,6 +22,11 @@ exports.builder = {
     description: 'The log level',
     type: 'string',
     default: 'info'
+  },
+  'bootstrap': {
+    description: 'Comma-separated bootstrap servers to use.',
+    type: 'array',
+    default: []
   }
 }
 exports.handler = async function (argv) {
@@ -51,7 +56,7 @@ async function start (argv) {
     logFile: './hyperdrive.log',
     outFile: './hyperdrive.log',
     errFile: './hyperdrive.log',
-    args: ['--port', argv.port, '--storage', argv.storage, '--log-level', argv['log-level']]
+    args: ['--port', argv.port, '--storage', argv.storage, '--log-level', argv['log-level'], '--bootstrap', argv.bootstrap.join(',')]
   })
   console.log(chalk.green(`Daemon started at ${endpoint}`))
 }

index.js

@@ -5,11 +5,10 @@ const mkdirp = require('mkdirp')
 const raf = require('random-access-file')
 const level = require('level')
 const sub = require('subleveldown')
-const argv = require('yargs').argv
 const grpc = require('grpc')
-const { rpc, loadMetadata } = require('hyperdrive-daemon-client')
 
-const Megastore = require('mini-megastore')
+const { rpc, loadMetadata } = require('hyperdrive-daemon-client')
+const corestore = require('random-access-corestore')
 const SwarmNetworker = require('megastore-swarm-networking')
 
 const { DriveManager, createDriveHandlers } = require('./lib/drives')
@@ -23,6 +22,8 @@ try {
 }
 const log = require('./lib/log').child({ component: 'server' })
 
+const argv = extractArguments()
+
 class HyperdriveDaemon extends EventEmitter {
   constructor (storage, opts = {}) {
     super()
@@ -35,14 +36,16 @@ class HyperdriveDaemon extends EventEmitter {
       drives: sub(this.db, 'drives', { valueEncoding: 'json' })
     }
 
-    const megastoreOpts = {
+    const corestoreOpts = {
       storage: path => raf(`${storage}/cores/${path}`),
       sparse: true
     }
+    this.corestore = corestore(corestoreOpts.storage, corestoreOpts)
+    // The root corestore should be bootstrapped with an empty default feed.
+    this.corestore.default()
 
-    this.megastore = new Megastore(megastoreOpts.storage, megastoreOpts)
-    this.networking = new SwarmNetworker(this.megastore, opts.network)
-    this.drives = new DriveManager(this.megastore, this.networking, dbs.drives, this.opts)
+    this.networking = new SwarmNetworker(this.corestore, opts.network)
+    this.drives = new DriveManager(this.corestore, this.networking, dbs.drives, this.opts)
     this.fuse = hyperfuse ? new FuseManager(this.megastore, this.drives, dbs.fuse, this.opts) : null
 
     this.drives.on('error', err => this.emit('error', err))
@@ -97,7 +100,13 @@ async function start () {
   const storageRoot = argv.storage
   await ensureStorage()
 
-  const daemon = new HyperdriveDaemon(storageRoot)
+  const daemonOpts = {}
+  if (argv.bootstrap.length) {
+    daemonOpts.network = {
+      bootstrap: argv.bootstrap
+    }
+  }
+  const daemon = new HyperdriveDaemon(storageRoot, daemonOpts)
   await daemon.ready()
 
   const server = new grpc.Server();
@@ -137,6 +146,23 @@ async function start () {
   }
 }
 
+function extractArguments () {
+  return require('yargs')
+    .options({
+      bootstrap: {
+        array: true,
+        default: []
+      },
+      storage: {
+        string: true
+      },
+      port: {
+        number: true
+      }
+    })
+    .argv
+}
+
 function wrap (metadata, methods) {
   const promisified = promisify(methods)
   let authenticated = authenticate(metadata, methods)

lib/drives/index.js

@@ -4,23 +4,23 @@ const { EventEmitter } = require('events')
 const hyperdrive = require('hyperdrive')
 const sub = require('subleveldown')
 const collectStream = require('stream-collector')
+const bjson = require('buffer-json-encoding')
 const datEncoding = require('dat-encoding')
 
 const { fromHyperdriveOptions, toHyperdriveOptions } = require('hyperdrive-daemon-client/lib/common')
 const log = require('../log').child({ component: 'drive-manager' })
 
 class DriveManager extends EventEmitter {
-  constructor (megastore, networking, db, opts) {
+  constructor (corestore, networking, db, opts) {
     super()
 
-    this.megastore = megastore
+    this.corestore = corestore
     this.networking = networking
     this.db = db
     this.opts = opts || {}
 
-    this._driveIndex = sub(this.db, 'drives', { valueEncoding: 'json' })
+    this._driveIndex = sub(this.db, 'drives', { valueEncoding: bjson })
     this._seedIndex = sub(this.db, 'seeding', { valueEncoding: 'utf8' })
-    this._nameIndex = sub(this.db, 'names', { valueEncoding: 'utf8' })
 
     if (this.opts.stats) {
       this._statsIndex = sub(this.db, 'stats', { valueEncoding: 'json' })
@@ -29,6 +29,8 @@ class DriveManager extends EventEmitter {
 
     // TODO: Replace with an LRU cache.
     this._drives = new Map()
+    this._sessions = new Map()
+
     this._ready = new Promise(async resolve => {
       await this._reseed()
       return resolve()
@@ -56,14 +58,13 @@ class DriveManager extends EventEmitter {
         drive.writeFile('.key', drive.key.toString('hex'), err => {
           if (err) return reject(err)
           if (opts && opts.rootDrive) {
-            drive.mkdir('/home', { uid: process.getuid(), gid: process.getgid() }, err => {
+            return drive.mkdir('/home', { uid: process.getuid(), gid: process.getgid() }, err => {
               if (err) return reject(err)
               return resolve(err)
             })
           } else {
             return resolve()
           }
-
         })
       }
     })
@@ -89,12 +90,6 @@ class DriveManager extends EventEmitter {
     }, this.opts.statsInterval || 2000)
   }
 
-  _generateName (key) {
-    if (key) return key.toString('hex')
-    // TODO: check collisions
-    return crypto.randomBytes(64).toString('hex')
-  }
-
   _generateKeyString (key, opts) {
     var keyString = (key instanceof Buffer) ? key.toString('hex') : key
     if (opts && opts.version) keyString = keyString + '+' + opts.version
@@ -104,53 +99,40 @@ class DriveManager extends EventEmitter {
 
   async get (key, opts) {
     key = (key instanceof Buffer) ? datEncoding.decode(key) : key
-    const keyString = this._generateKeyString(key, opts)
-    var newDrive = false
+    var keyString = this._generateKeyString(key, opts)
 
     if (key) {
       // TODO: cache checkouts
       const existing = this._drives.get(keyString)
       if (existing) return existing
-      try {
-        var name = await this._nameIndex.get(key)
-      } catch (err) {
-        if (!err.notFound) throw err
-        // If a name was not found for this key, then the drive is not writable and must be synced.
-        name = this._generateName(key)
-        newDrive = true
-      }
-    } else {
-      name = this._generateName()
-      newDrive = true
     }
 
-    console.error('GETTING CORESTORE WITH OPTS:', { ...this.opts, ...opts })
-    const corestore = this.megastore.get(name, { ...this.opts, ...opts })
     const driveOpts = {
       ...opts,
       sparse: opts.sparse !== false,
       sparseMetadata: opts.sparseMetadata !== false
     }
-    const drive = hyperdrive(corestore, key, driveOpts)
-
+    const drive = hyperdrive(this.corestore, key, driveOpts)
     await new Promise((resolve, reject) => {
       drive.ready(err => {
         if (err) return reject(err)
         return resolve()
       })
     })
+
     key = datEncoding.encode(drive.key)
+    keyString = this._generateKeystring(key, opts)
+
 
     if (drive.writable) {
       await this._configureDrive(drive, opts && opts.configure)
     }
-    if (newDrive && this.opts.stats) {
+    if (this.opts.stats) {
       this._collectStats(drive)
     }
 
     // TODO: This should all be in one batch.
     await Promise.all([
-      this._nameIndex.put(key, name),
       this._driveIndex.put(key, driveOpts)
     ])
     this._drives.set(key, drive)

lib/fuse/index.js

@@ -249,6 +249,7 @@ class FuseManager extends EventEmitter {
   async _driveForPath (path, opts = {}) {
     const self = this
     console.error('GETTING DRIVE FOR PATH:', path)
+    if (!this._rootDrive && path !== '/hyperdrive') throw new Error('You can only mount the root hyperdrive at /hyperdrive')
 
     if (!this._rootDrive) {
       console.log('NO ROOT DRIVE')