feat: ipns over dht (#1725)

Author: vasco-santos

package.json

@@ -127,10 +127,10 @@
     "joi": "^13.4.0",
     "joi-browser": "^13.4.0",
     "joi-multiaddr": "^3.0.0",
-    "libp2p": "~0.24.0",
+    "libp2p": "~0.24.1",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-crypto": "~0.14.1",
-    "libp2p-kad-dht": "~0.11.1",
+    "libp2p-kad-dht": "~0.12.1",
     "libp2p-keychain": "~0.3.3",
     "libp2p-mdns": "~0.12.0",
     "libp2p-mplex": "~0.8.4",

src/core/components/init.js

@@ -117,7 +117,7 @@ module.exports = function init (self) {
       (_, cb) => {
         const offlineDatastore = new OfflineDatastore(self._repo)
 
-        self._ipns = new IPNS(offlineDatastore, self._repo, self._peerInfo, self._keychain, self._options)
+        self._ipns = new IPNS(offlineDatastore, self._repo.datastore, self._peerInfo, self._keychain, self._options)
         cb(null, true)
       },
       // add empty unixfs dir object (go-ipfs assumes this exists)

src/core/components/libp2p.js

@@ -3,6 +3,7 @@
 const promisify = require('promisify-es6')
 const get = require('lodash/get')
 const defaultsDeep = require('@nodeutils/defaults-deep')
+const ipnsUtils = require('../ipns/routing/utils')
 
 module.exports = function libp2p (self) {
   return {
@@ -16,6 +17,7 @@ module.exports = function libp2p (self) {
 
         const defaultBundle = (opts) => {
           const libp2pDefaults = {
+            datastore: opts.datastore,
             peerInfo: opts.peerInfo,
             peerBook: opts.peerBook,
             config: {
@@ -43,6 +45,14 @@ module.exports = function libp2p (self) {
                     get(opts.config, 'relay.hop.active', false))
                 }
               },
+              dht: {
+                validators: {
+                  ipns: ipnsUtils.validator
+                },
+                selectors: {
+                  ipns: ipnsUtils.selector
+                }
+              },
               EXPERIMENTAL: {
                 dht: get(opts.options, 'EXPERIMENTAL.dht', false),
                 pubsub: get(opts.options, 'EXPERIMENTAL.pubsub', false)
@@ -72,6 +82,7 @@ module.exports = function libp2p (self) {
         self._libp2pNode = libp2pBundle({
           options: self._options,
           config: config,
+          datastore: self._repo.datastore,
           peerInfo: self._peerInfo,
           peerBook: self._peerInfoBook
         })

src/core/components/start.js

@@ -54,14 +54,18 @@ module.exports = (self) => {
           ipnsStores.push(pubsubDs)
         }
 
-        // NOTE: IPNS routing is being replaced by the local repo datastore while the IPNS over DHT is not ready
-        // When DHT is added, if local option enabled, should receive offlineDatastore as well
-        const offlineDatastore = new OfflineDatastore(self._repo)
-        ipnsStores.push(offlineDatastore)
+        // DHT should be added as routing if we are not running with local flag
+        // TODO: Need to change this logic once DHT is enabled by default, for now fallback to Offline datastore
+        if (get(self._options, 'EXPERIMENTAL.dht', false) && !self._options.local) {
+          ipnsStores.push(self._libp2pNode.dht)
+        } else {
+          const offlineDatastore = new OfflineDatastore(self._repo)
+          ipnsStores.push(offlineDatastore)
+        }
 
         // Create ipns routing with a set of datastores
         const routing = new TieredDatastore(ipnsStores)
-        self._ipns = new IPNS(routing, self._repo, self._peerInfo, self._keychain, self._options)
+        self._ipns = new IPNS(routing, self._repo.datastore, self._peerInfo, self._keychain, self._options)
 
         self._bitswap = new Bitswap(
           self._libp2pNode,

src/core/ipns/index.js

@@ -17,9 +17,9 @@ const path = require('./path')
 const defaultRecordTtl = 60 * 1000
 
 class IPNS {
-  constructor (routing, repo, peerInfo, keychain, options) {
-    this.publisher = new IpnsPublisher(routing, repo)
-    this.republisher = new IpnsRepublisher(this.publisher, repo, peerInfo, keychain, options)
+  constructor (routing, datastore, peerInfo, keychain, options) {
+    this.publisher = new IpnsPublisher(routing, datastore)
+    this.republisher = new IpnsRepublisher(this.publisher, datastore, peerInfo, keychain, options)
     this.resolver = new IpnsResolver(routing)
     this.cache = new Receptacle({ max: 1000 }) // Create an LRU cache with max 1000 items
     this.routing = routing

src/core/ipns/publisher.js

@@ -15,9 +15,9 @@ const defaultRecordTtl = 60 * 60 * 1000
 
 // IpnsPublisher is capable of publishing and resolving names to the IPFS routing system.
 class IpnsPublisher {
-  constructor (routing, repo) {
+  constructor (routing, datastore) {
     this._routing = routing
-    this._repo = repo
+    this._datastore = datastore
   }
 
   // publish record with a eol
@@ -56,7 +56,6 @@ class IpnsPublisher {
       log.error(errMsg)
       return callback(errcode(new Error(errMsg), 'ERR_INVALID_PEER_ID'))
     }
-
     const publicKey = peerId._pubKey
 
     ipns.embedPublicKey(publicKey, record, (err, embedPublicKeyRecord) => {
@@ -74,9 +73,10 @@ class IpnsPublisher {
 
       series([
         (cb) => this._publishEntry(keys.routingKey, embedPublicKeyRecord || record, peerId, cb),
-        // Publish the public key if a public key cannot be extracted from the ID
-        // We will be able to deprecate this part in the future, since the public keys will be only in the peerId
-        (cb) => embedPublicKeyRecord ? this._publishPublicKey(keys.routingPubKey, publicKey, peerId, cb) : cb()
+        // Publish the public key to support old go-ipfs nodes that are looking for it in the routing
+        // We will be able to deprecate this part in the future, since the public keys will be only
+        // in IPNS record and the peerId.
+        (cb) => this._publishPublicKey(keys.routingPubKey, publicKey, peerId, cb)
       ], (err) => {
         if (err) {
           log.error(err)
@@ -159,50 +159,57 @@ class IpnsPublisher {
     }
 
     options = options || {}
-    const checkRouting = !(options.checkRouting === false)
-
-    this._repo.datastore.get(ipns.getLocalKey(peerId.id), (err, dsVal) => {
-      let result
+    const checkRouting = options.checkRouting !== false
 
+    this._datastore.get(ipns.getLocalKey(peerId.id), (err, dsVal) => {
       if (err) {
         if (err.code !== 'ERR_NOT_FOUND') {
           const errMsg = `unexpected error getting the ipns record ${peerId.id} from datastore`
 
           log.error(errMsg)
           return callback(errcode(new Error(errMsg), 'ERR_UNEXPECTED_DATASTORE_RESPONSE'))
-        } else {
-          if (!checkRouting) {
-            return callback(null, null)
-          } else {
-            // TODO ROUTING - get from DHT
-            return callback(new Error('not implemented yet'))
-          }
         }
-      }
 
-      if (Buffer.isBuffer(dsVal)) {
-        result = dsVal
-      } else {
-        const errMsg = `found ipns record that we couldn't convert to a value`
+        if (!checkRouting) {
+          return callback((errcode(err)))
+        }
 
-        log.error(errMsg)
-        return callback(errcode(new Error(errMsg), 'ERR_INVALID_IPNS_RECORD'))
-      }
+        // Try to get from routing
+        let keys
+        try {
+          keys = ipns.getIdKeys(peerId.toBytes())
+        } catch (err) {
+          log.error(err)
+          return callback(err)
+        }
 
-      // unmarshal data
-      try {
-        result = ipns.unmarshal(dsVal)
-      } catch (err) {
-        const errMsg = `found ipns record that we couldn't convert to a value`
+        this._routing.get(keys.routingKey.toBuffer(), (err, res) => {
+          if (err) {
+            return callback(err)
+          }
 
-        log.error(errMsg)
-        return callback(null, null)
+          // unmarshal data
+          this._unmarshalData(res, callback)
+        })
+      } else {
+        // unmarshal data
+        this._unmarshalData(dsVal, callback)
       }
-
-      callback(null, result)
     })
   }
 
+  _unmarshalData (data, callback) {
+    let result
+    try {
+      result = ipns.unmarshal(data)
+    } catch (err) {
+      log.error(err)
+      return callback(errcode(err, 'ERR_INVALID_RECORD_DATA'))
+    }
+
+    callback(null, result)
+  }
+
   _updateOrCreateRecord (privKey, value, validity, peerId, callback) {
     if (!(PeerId.isPeerId(peerId))) {
       const errMsg = `peerId received is not valid`
@@ -212,12 +219,17 @@ class IpnsPublisher {
     }
 
     const getPublishedOptions = {
-      checkRouting: false // TODO ROUTING - change to true
+      checkRouting: true
     }
 
     this._getPublished(peerId, getPublishedOptions, (err, record) => {
       if (err) {
-        return callback(err)
+        if (err.code !== 'ERR_NOT_FOUND') {
+          const errMsg = `unexpected error when determining the last published IPNS record for ${peerId.id}`
+
+          log.error(errMsg)
+          return callback(errcode(new Error(errMsg), 'ERR_DETERMINING_PUBLISHED_RECORD'))
+        }
       }
 
       // Determinate the record sequence number
@@ -241,7 +253,7 @@ class IpnsPublisher {
         const data = ipns.marshal(entryData)
 
         // Store the new record
-        this._repo.datastore.put(ipns.getLocalKey(peerId.id), data, (err, res) => {
+        this._datastore.put(ipns.getLocalKey(peerId.id), data, (err, res) => {
           if (err) {
             const errMsg = `ipns record for ${value} could not be stored in the datastore`
 

src/core/ipns/republisher.js

@@ -18,9 +18,9 @@ const defaultBroadcastInterval = 4 * hour
 const defaultRecordLifetime = 24 * hour
 
 class IpnsRepublisher {
-  constructor (publisher, repo, peerInfo, keychain, options) {
+  constructor (publisher, datastore, peerInfo, keychain, options) {
     this._publisher = publisher
-    this._repo = repo
+    this._datastore = datastore
     this._peerInfo = peerInfo
     this._keychain = keychain
     this._options = options
@@ -160,7 +160,7 @@ class IpnsRepublisher {
       return callback(errcode(new Error(errMsg), 'ERR_INVALID_PEER_ID'))
     }
 
-    this._repo.datastore.get(ipns.getLocalKey(peerId.id), (err, dsVal) => {
+    this._datastore.get(ipns.getLocalKey(peerId.id), (err, dsVal) => {
       // error handling
       // no need to republish
       if (err && err.notFound) {

src/core/ipns/resolver.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const ipns = require('ipns')
+const crypto = require('libp2p-crypto')
 const PeerId = require('peer-id')
 const errcode = require('err-code')
 
@@ -96,13 +97,9 @@ class IpnsResolver {
       return callback(err)
     }
 
-    const { routingKey } = ipns.getIdKeys(peerId.toBytes())
+    const { routingKey, routingPubKey } = ipns.getIdKeys(peerId.toBytes())
 
-    // TODO DHT - get public key from routing?
-    // https://github.com/ipfs/go-ipfs/blob/master/namesys/routing.go#L70
-    // https://github.com/libp2p/go-libp2p-routing/blob/master/routing.go#L99
-
-    this._routing.get(routingKey.toBuffer(), (err, res) => {
+    this._routing.get(routingKey.toBuffer(), (err, record) => {
       if (err) {
         if (err.code !== 'ERR_NOT_FOUND') {
           const errMsg = `unexpected error getting the ipns record ${peerId.id}`
@@ -116,29 +113,66 @@ class IpnsResolver {
         return callback(errcode(new Error(errMsg), 'ERR_NO_RECORD_FOUND'))
       }
 
+      // IPNS entry
       let ipnsEntry
       try {
-        ipnsEntry = ipns.unmarshal(res)
+        ipnsEntry = ipns.unmarshal(record)
       } catch (err) {
         const errMsg = `found ipns record that we couldn't convert to a value`
 
         log.error(errMsg)
         return callback(errcode(new Error(errMsg), 'ERR_INVALID_RECORD_RECEIVED'))
       }
 
-      ipns.extractPublicKey(peerId, ipnsEntry, (err, pubKey) => {
+      // if the record has a public key validate it
+      if (ipnsEntry.pubKey) {
+        return this._validateRecord(peerId, ipnsEntry, callback)
+      }
+
+      // Otherwise, try to get the public key from routing
+      this._routing.get(routingKey.toBuffer(), (err, pubKey) => {
         if (err) {
-          return callback(err)
-        }
+          if (err.code !== 'ERR_NOT_FOUND') {
+            const errMsg = `unexpected error getting the public key for the ipns record ${peerId.id}`
 
-        // IPNS entry validation
-        ipns.validate(pubKey, ipnsEntry, (err) => {
-          if (err) {
-            return callback(err)
+            log.error(errMsg)
+            return callback(errcode(new Error(errMsg), 'ERR_UNEXPECTED_ERROR_GETTING_PUB_KEY'))
           }
+          const errMsg = `public key requested was not found for ${name} (${routingPubKey}) in the network`
+
+          log.error(errMsg)
+          return callback(errcode(new Error(errMsg), 'ERR_NO_RECORD_FOUND'))
+        }
+
+        try {
+          // Insert it into the peer id, in order to be validated by IPNS validator
+          peerId.pubKey = crypto.keys.unmarshalPublicKey(pubKey)
+        } catch (err) {
+          const errMsg = `found public key record that we couldn't convert to a value`
+
+          log.error(errMsg)
+          return callback(errcode(new Error(errMsg), 'ERR_INVALID_PUB_KEY_RECEIVED'))
+        }
+
+        this._validateRecord(peerId, ipnsEntry, callback)
+      })
+    })
+  }
+
+  // validate a resolved record
+  _validateRecord (peerId, ipnsEntry, callback) {
+    ipns.extractPublicKey(peerId, ipnsEntry, (err, pubKey) => {
+      if (err) {
+        return callback(err)
+      }
+
+      // IPNS entry validation
+      ipns.validate(pubKey, ipnsEntry, (err) => {
+        if (err) {
+          return callback(err)
+        }
 
-          callback(null, ipnsEntry.value.toString())
-        })
+        callback(null, ipnsEntry.value.toString())
       })
     })
   }

src/core/ipns/routing/utils.js

@@ -1,9 +1,16 @@
 'use strict'
 
 const multibase = require('multibase')
+const ipns = require('ipns')
 
-module.exports.encodeBase32 = (buf) => {
-  const m = multibase.encode('base32', buf).slice(1) // slice off multibase codec
+module.exports = {
+  encodeBase32: (buf) => {
+    const m = multibase.encode('base32', buf).slice(1) // slice off multibase codec
 
-  return m.toString().toUpperCase() // should be uppercase for interop with go
+    return m.toString().toUpperCase() // should be uppercase for interop with go
+  },
+  validator: {
+    func: (key, record, cb) => ipns.validator.validate(record, key, cb)
+  },
+  selector: (k, records) => ipns.validator.select(records[0], records[1])
 }