Various fixes to make the hyperdrive-daemon work

Author: pfrazee

dat/daemon.js

@@ -60,17 +60,28 @@ exports.setup = async function () {
   // fetch daemon metadata from disk
   var metadata
   try {
-    metadata = await loadMetadata()
+    metadata = await new Promise((resolve, reject) => {
+      loadMetadata((err, metadata) => {
+        if (err) reject(err)
+        else resolve(metadata)
+      })
+    })
   } catch (e) {
     await createMetadata(`localhost:${DAEMON_PORT}`)
-    metadata = await loadMetadata()
+    metadata = await new Promise((resolve, reject) => {
+      loadMetadata((err, metadata) => {
+        if (err) reject(err)
+        else resolve(metadata)
+      })
+    })
   }
 
   // instantiate the daemon
   // TODO the daemon should be managed in an external promise
   await startDaemon({
     storage: DAEMON_STORAGE_PATH,
     port: DAEMON_PORT,
+    bootstrap: [],
     metadata
   })
 
@@ -91,11 +102,11 @@ exports.setup = async function () {
 exports.createDatArchiveSession = async function (opts) {
   const session = await client.drive.get(opts)
   const sessionId = session.id
-  const key = datEncoding.toStr(opts.key)
+  const key = datEncoding.toStr(session.opts.key)
   var datArchive = {
     key: datEncoding.toBuf(key),
     url: `dat://${key}`,
-    writable: false, // TODO
+    writable: true, // TODO
 
     session: {
       async close () {
@@ -127,8 +138,13 @@ exports.createDatArchiveSession = async function (opts) {
       })
       client.drive.stat(sessionId, ...args)
     },
-    readFile: (...args) => client.drive.readFile(sessionId, ...args),
-    writeFile: (...args) => client.drive.writeFile(sessionId, ...args),
+    // readFile: (...args) => client.drive.readFile(sessionId, ...args), TODO opts not accepted by daemon yet
+    readFile: (path, opts, cb) => {
+      client.drive.readFile(sessionId, path, cb ? cb : opts)
+    },
+    // writeFile: (...args) => client.drive.writeFile(sessionId, ...args), TODO encoding/opts not accepted by daemon yet
+    writeFile: (path, content, encoding, cb) => client.drive.writeFile(sessionId, path, content, cb),
+
     readdir: (...args) => client.drive.readdir(sessionId, ...args),
     // ready: makeArchiveProxyCbFn(key, version, 'ready'),
     // download: makeArchiveProxyCbFn(key, version, 'download'),

dat/library.js

@@ -112,14 +112,15 @@ exports.setup = async function setup ({rpcAPI, disallowedSavePaths}) {
   // daemonEvents.on('folder-sync-error', evt => archivesEvents.emit('folder-sync-error', evt))
 
   // configure the bandwidth throttle
-  settingsDb.getAll().then(({dat_bandwidth_limit_up, dat_bandwidth_limit_down}) => {
-    daemon.setBandwidthThrottle({
-      up: dat_bandwidth_limit_up,
-      down: dat_bandwidth_limit_down
-    })
-  })
-  settingsDb.on('set:dat_bandwidth_limit_up', up => daemon.setBandwidthThrottle({up}))
-  settingsDb.on('set:dat_bandwidth_limit_down', down => daemon.setBandwidthThrottle({down}))
+  // TODO
+  // settingsDb.getAll().then(({dat_bandwidth_limit_up, dat_bandwidth_limit_down}) => {
+  //   daemon.setBandwidthThrottle({
+  //     up: dat_bandwidth_limit_up,
+  //     down: dat_bandwidth_limit_down
+  //   })
+  // })
+  // settingsDb.on('set:dat_bandwidth_limit_up', up => daemon.setBandwidthThrottle({up}))
+  // settingsDb.on('set:dat_bandwidth_limit_down', down => daemon.setBandwidthThrottle({down}))
 
   // start the GC manager
   datGC.setup()
@@ -178,9 +179,6 @@ const pullLatestArchiveMeta = exports.pullLatestArchiveMeta = async function pul
   try {
     var key = archive.key.toString('hex')
 
-    // ready() just in case (we need .blocks)
-    await pify(archive.ready.bind(archive))()
-
     // read the archive meta and size on disk
     var [manifest, oldMeta, size] = await Promise.all([
       archive.pda.readManifest().catch(_ => {}),
@@ -299,7 +297,6 @@ exports.forkArchive = async function forkArchive (srcArchiveUrl, manifest = {},
 
 const loadArchive = exports.loadArchive = async function loadArchive (key, userSettings = null) {
   // validate key
-  var secretKey
   if (key) {
     if (!Buffer.isBuffer(key)) {
       // existing dat
@@ -309,35 +306,32 @@ const loadArchive = exports.loadArchive = async function loadArchive (key, userS
       }
       key = datEncoding.toBuf(key)
     }
-  } else {
-    // new dat, generate keys
-    var kp = signatures.keyPair()
-    key = kp.publicKey
-    secretKey = kp.secretKey
   }
 
   // fallback to the promise, if possible
-  var keyStr = datEncoding.toStr(key)
-  if (keyStr in archiveLoadPromises) {
+  var keyStr = key ? datEncoding.toStr(key) : null
+  if (keyStr && keyStr in archiveLoadPromises) {
     return archiveLoadPromises[keyStr]
   }
 
   // run and cache the promise
-  var p = loadArchiveInner(key, secretKey, userSettings)
-  archiveLoadPromises[keyStr] = p
+  var p = loadArchiveInner(key, userSettings)
+  if (key) archiveLoadPromises[keyStr] = p
   p.catch(err => {
     console.error('Failed to load archive', keyStr, err.toString())
   })
 
   // when done, clear the promise
-  const clear = () => delete archiveLoadPromises[keyStr]
-  p.then(clear, clear)
+  if (key) {
+    const clear = () => delete archiveLoadPromises[keyStr]
+    p.then(clear, clear)
+  }
 
   return p
 }
 
 // main logic, separated out so we can capture the promise
-async function loadArchiveInner (key, secretKey, userSettings = null) {
+async function loadArchiveInner (key, userSettings = null) {
   // load the user settings as needed
   if (!userSettings) {
     try {
@@ -351,8 +345,9 @@ async function loadArchiveInner (key, secretKey, userSettings = null) {
   }
 
   // ensure the folders exist
-  var metaPath = archivesDb.getArchiveMetaPath(key)
-  mkdirp.sync(metaPath)
+  // TODO needed?
+  // var metaPath = archivesDb.getArchiveMetaPath(key)
+  // mkdirp.sync(metaPath)
 
   // create the archive session with the daemon
   var archive = await daemon.createDatArchiveSession({key})
@@ -361,22 +356,22 @@ async function loadArchiveInner (key, secretKey, userSettings = null) {
   archive.session.publish()
 
   // update db
-  archivesDb.touch(key).catch(err => console.error('Failed to update lastAccessTime for archive', key, err))
+  archivesDb.touch(archive.key).catch(err => console.error('Failed to update lastAccessTime for archive', archive.key, err))
   await pullLatestArchiveMeta(archive)
   datAssets.update(archive)
-
   // configure subsystems
   folderSync.reconfigureArchive(archive, userSettings)
 
   // wire up events
   archive.pullLatestArchiveMeta = _debounce(opts => pullLatestArchiveMeta(archive, opts), 1e3)
-  archive.fileActStream = archive.pda.watch()
-  archive.fileActStream.on('data', ([event, {path}]) => {
-    if (event === 'changed') {
-      archive.pullLatestArchiveMeta({updateMTime: true})
-      datAssets.update(archive, [path])
-    }
-  })
+  // TODO
+  // archive.fileActStream = archive.pda.watch()
+  // archive.fileActStream.on('data', ([event, {path}]) => {
+  //   if (event === 'changed') {
+  //     archive.pullLatestArchiveMeta({updateMTime: true})
+  //     datAssets.update(archive, [path])
+  //   }
+  // })
   // TODO
   // archive.fileActStream = pda.watch(archive)
   // archive.fileActStream.on('data', ([event, {path}]) => {

dat/protocol.js

@@ -344,6 +344,16 @@ exports.electronHandler = async function (request, respond) {
     })
   }
 
+  // TODO
+  // replace this with createReadStream when that method is available
+  var content = await checkoutFS.pda.readFile(entry.path)
+  Object.assign(headers, {
+    'Content-Type': mime.identify(entry.path)
+  })
+  respond({statusCode, headers, data: intoStream(content)})
+  cleanup()
+  return
+
   // fetch the entry and stream the response
   fileReadStream = checkoutFS.createReadStream(entry.path, range)
   var dataStream = fileReadStream

package.json

@@ -49,7 +49,7 @@
     "hypercore-protocol": "^6.11.0",
     "hyperdrive": "^9.14.5",
     "hyperdrive-daemon": "^0.9.13",
-    "hyperdrive-daemon-client": "^0.9.9",
+    "hyperdrive-daemon-client": "^0.9.10",
     "hyperdrive-network-speed": "^2.1.0",
     "icojs": "^0.12.3",
     "identify-filetype": "^1.0.0",

users/index.js

@@ -416,6 +416,7 @@ async function validateUserUrl (url) {
  * @returns {void}
  */
 function startWatch (user) {
+  return // TODO
   /* dont await */crawler.watchSite(user.archive)
   watchThumb(user)
   watchAndSyncBookmarks(user)