Merge branch 'v0.13.6-rc'

* v0.13.6-rc:
  Bump version to v0.13.6
  [#1800] Extract ipfs.GetCachedIPNSRecord and ipfs.DeleteCachedIPNSRecord
  [#1800] Make ipfs.getFromDatastore IpnsEntry unmarshal fail gracefully
  [#1800] Don't drop /ipns/persistentcache records from datastore cache
  [#1800] Clarify difference between IPFS prefix and OB prefix
  [#1800] Internalize key prefix handling to ipfs package for PubSub
  [#1800] Extract ipfs.Get|PutCachedPubkey from app
  [#1800] Clean all IPNS records from IPFS datastore cache
  [#1800] Remove APIRouter from IPFS routing stack

Author: placer14

api/jsonapi.go

@@ -24,12 +24,12 @@ import (
 	ipnspath "gx/ipfs/QmQAgv6Gaoe2tQpcabqwKXKChp2MZ7i3UXv9DqTTaxCaTR/go-path"
 	files "gx/ipfs/QmQmhotPUzVrMEWNK3x1R5jQ5ZHWyL7tVUrmRPjrBrvyCb/go-ipfs-files"
 	cid "gx/ipfs/QmTbxNB1NwDesLmKTscr4udL2tVP7MaxvXnD1D9yX7g3PN/go-cid"
-	datastore "gx/ipfs/QmUadX5EcvrBmxAV9sE7wUWtWSqxns5K84qKJBixmcT1w9/go-datastore"
 	ipns "gx/ipfs/QmUwMnKKjH3JwGKNVZ3TcP37W93xzqNA4ECFFiMo6sXkkc/go-ipns"
 	iface "gx/ipfs/QmXLwxifxwfc2bAwq6rdjbYqAsGzWsDE9RM5TWMGtykyj6/interface-go-ipfs-core"
 	peer "gx/ipfs/QmYVXrKrKHDC9FobgmcmshCDyWwdrfwfanNQN4oxJ9Fk3h/go-libp2p-peer"
 	routing "gx/ipfs/QmYxUdYY9S6yg5tSPVin5GFTvtfsLauVcr7reHDD3dM8xf/go-libp2p-routing"
 	ps "gx/ipfs/QmaCTz9RkrU13bm9kMB54f7atgqM4qkjDZpRwRoJiWXEqs/go-libp2p-peerstore"
+	ggproto "gx/ipfs/QmddjPSGZb3ieihSseFeCfVRpZzcqczPNsD2DvarSwnjJB/gogo-protobuf/proto"
 	mh "gx/ipfs/QmerPMzPk1mJVowm8KgmoknWa4yCYvvugMPsgWmDNUvDLW/go-multihash"
 
 	"github.com/OpenBazaar/jsonpb"
@@ -45,7 +45,6 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
 	"github.com/ipfs/go-ipfs/core/coreapi"
-	"github.com/ipfs/go-ipfs/namesys"
 	"github.com/ipfs/go-ipfs/repo/fsrepo"
 )
 
@@ -3733,6 +3732,7 @@ func (i *jsonAPIHandler) GETWalletStatus(w http.ResponseWriter, r *http.Request)
 }
 
 func (i *jsonAPIHandler) GETIPNS(w http.ResponseWriter, r *http.Request) {
+	ipfsStore := i.node.IpfsNode.Repo.Datastore()
 	_, peerID := path.Split(r.URL.Path)
 
 	pid, err := peer.IDB58Decode(peerID)
@@ -3741,7 +3741,7 @@ func (i *jsonAPIHandler) GETIPNS(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	val, err := i.node.IpfsNode.Repo.Datastore().Get(namesys.IpnsDsKey(pid))
+	peerIPNSRecord, err := ipfs.GetCachedIPNSRecord(ipfsStore, pid)
 	if err != nil { // No record in datastore
 		ErrorResponse(w, http.StatusNotFound, err.Error())
 		return
@@ -3750,7 +3750,7 @@ func (i *jsonAPIHandler) GETIPNS(w http.ResponseWriter, r *http.Request) {
 	var keyBytes []byte
 	pubkey := i.node.IpfsNode.Peerstore.PubKey(pid)
 	if pubkey == nil || !pid.MatchesPublicKey(pubkey) {
-		keyval, err := i.node.IpfsNode.Repo.Datastore().Get(datastore.NewKey(core.KeyCachePrefix + peerID))
+		keyval, err := ipfs.GetCachedPubkey(ipfsStore, peerID)
 		if err != nil {
 			ErrorResponse(w, http.StatusNotFound, err.Error())
 			return
@@ -3768,8 +3768,13 @@ func (i *jsonAPIHandler) GETIPNS(w http.ResponseWriter, r *http.Request) {
 		Pubkey string `json:"pubkey"`
 		Record string `json:"record"`
 	}
+	peerIPNSBytes, err := ggproto.Marshal(peerIPNSRecord)
+	if err != nil {
+		ErrorResponse(w, http.StatusInternalServerError, fmt.Sprintf("marshaling IPNS record: %s", err.Error()))
+		return
+	}
 
-	ret := KeyAndRecord{hex.EncodeToString(keyBytes), string(val)}
+	ret := KeyAndRecord{hex.EncodeToString(keyBytes), string(peerIPNSBytes)}
 	retBytes, err := json.MarshalIndent(ret, "", "    ")
 	if err != nil {
 		ErrorResponse(w, http.StatusInternalServerError, err.Error())
@@ -3794,9 +3799,14 @@ func (i *jsonAPIHandler) GETResolveIPNS(w http.ResponseWriter, r *http.Request)
 	var response = respType{PeerID: peerID}
 
 	if i.node.IpfsNode.Identity.Pretty() == peerID {
-		ipnsBytes, err := i.node.IpfsNode.Repo.Datastore().Get(namesys.IpnsDsKey(i.node.IpfsNode.Identity))
+		rec, err := ipfs.GetCachedIPNSRecord(i.node.IpfsNode.Repo.Datastore(), i.node.IpfsNode.Identity)
+		if err != nil {
+			ErrorResponse(w, http.StatusInternalServerError, fmt.Sprintf("retrieving self: %s", err))
+			return
+		}
+		ipnsBytes, err := proto.Marshal(rec)
 		if err != nil {
-			ErrorResponse(w, http.StatusInternalServerError, fmt.Sprintf("retrieving self from datastore: %s", err))
+			ErrorResponse(w, http.StatusInternalServerError, fmt.Sprintf("marshaling self: %s", err))
 			return
 		}
 		response.Record.Hex = hex.EncodeToString(ipnsBytes)

cmd/start.go

@@ -23,12 +23,10 @@ import (
 	dht "gx/ipfs/QmSY3nkMNLzh9GdbFKK5tT7YMfLpf52iUZ8ZRkr29MJaa5/go-libp2p-kad-dht"
 	ma "gx/ipfs/QmTZBfrPJmjWsCvHEtX5FE6KimVJhsJg5sBbqEFYf4UZtL/go-multiaddr"
 	config "gx/ipfs/QmUAuYuiafnJRZxDDX7MuruMNsicYNuyub5vUeAcupUBNs/go-ipfs-config"
-	ipnspb "gx/ipfs/QmUwMnKKjH3JwGKNVZ3TcP37W93xzqNA4ECFFiMo6sXkkc/go-ipns/pb"
 	peer "gx/ipfs/QmYVXrKrKHDC9FobgmcmshCDyWwdrfwfanNQN4oxJ9Fk3h/go-libp2p-peer"
 	oniontp "gx/ipfs/QmYv2MbwHn7qcvAPFisZ94w85crQVpwUuv8G7TuUeBnfPb/go-onion-transport"
 	ipfslogging "gx/ipfs/QmbkT7eMTyXfpeyB3ZMxxcxg7XH8t6uXp49jqzz4HB7BGF/go-log/writer"
 	manet "gx/ipfs/Qmc85NSvmSG4Frn9Vb2cBc1rMyULH6D3TNVEfCzSKoUpip/go-multiaddr-net"
-	proto "gx/ipfs/QmddjPSGZb3ieihSseFeCfVRpZzcqczPNsD2DvarSwnjJB/gogo-protobuf/proto"
 
 	"github.com/OpenBazaar/openbazaar-go/api"
 	"github.com/OpenBazaar/openbazaar-go/core"
@@ -53,7 +51,6 @@ import (
 	"github.com/ipfs/go-ipfs/commands"
 	ipfscore "github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/core/corehttp"
-	"github.com/ipfs/go-ipfs/namesys"
 	"github.com/ipfs/go-ipfs/repo/fsrepo"
 	"github.com/natefinch/lumberjack"
 	"github.com/op/go-logging"
@@ -407,34 +404,28 @@ func (x *Start) Execute(args []string) error {
 	}
 	var dhtRouting *dht.IpfsDHT
 	for _, router := range tiered.Routers {
-		if r, ok := router.(*ipfs.CachingRouter); ok {
-			r.APIRouter().Start(torDialer)
-			dhtRouting, err = r.DHT()
-			if err != nil {
-				return err
-			}
+		if r, ok := router.(*dht.IpfsDHT); ok {
+			dhtRouting = r
 		}
 	}
 	if dhtRouting == nil {
 		return errors.New("IPFS DHT routing is not configured")
 	}
 
 	// Get current directory root hash
-	ipnskey := namesys.IpnsDsKey(nd.Identity)
-	ival, hasherr := nd.Repo.Datastore().Get(ipnskey)
+	cachedIPNSRecord, hasherr := ipfs.GetCachedIPNSRecord(nd.Repo.Datastore(), nd.Identity)
 	if hasherr != nil {
-		log.Error("get ipns key:", hasherr)
-	}
-	ourIpnsRecord := new(ipnspb.IpnsEntry)
-	err = proto.Unmarshal(ival, ourIpnsRecord)
-	if err != nil {
-		log.Error("unmarshal record value", err)
-		nd.Repo.Datastore().Delete(ipnskey)
+		log.Warning(err)
+		if err := ipfs.DeleteCachedIPNSRecord(nd.Repo.Datastore(), nd.Identity); err != nil {
+			log.Errorf("deleting invalid IPNS record: %s", err.Error())
+		}
 	}
 
 	if x.ForceKeyCachePurge {
 		log.Infof("forcing key purge from namesys cache...")
-		nd.Repo.Datastore().Delete(ipnskey)
+		if err := ipfs.DeleteCachedIPNSRecord(nd.Repo.Datastore(), nd.Identity); err != nil {
+			log.Errorf("force-purging IPNS record: %s", err.Error())
+		}
 	}
 
 	// Wallet
@@ -587,6 +578,11 @@ func (x *Start) Execute(args []string) error {
 	subscriber := ipfs.NewPubsubSubscriber(context.Background(), nd.PeerHost, nd.Routing, nd.Repo.Datastore(), nd.PubSub)
 	ps := ipfs.Pubsub{Publisher: publisher, Subscriber: subscriber}
 
+	var rootHash string
+	if cachedIPNSRecord != nil {
+		rootHash = string(cachedIPNSRecord.Value)
+	}
+
 	// OpenBazaar node setup
 	core.Node = &core.OpenBazaarNode{
 		AcceptStoreRequests:           dataSharing.AcceptStoreRequests,
@@ -601,7 +597,7 @@ func (x *Start) Execute(args []string) error {
 		PushNodes:                     pushNodes,
 		RegressionTestEnable:          x.Regtest,
 		RepoPath:                      repoPath,
-		RootHash:                      string(ourIpnsRecord.Value),
+		RootHash:                      rootHash,
 		TestnetEnable:                 x.Testnet,
 		TorDialer:                     torDialer,
 		UserAgent:                     core.USERAGENT,

core/core.go

@@ -10,7 +10,6 @@ import (
 	libp2p "gx/ipfs/QmTW4SdgBWq9GjsBsHeUx8WuGxzhgzAf88UMH2w62PC8yK/go-libp2p-crypto"
 	ma "gx/ipfs/QmTZBfrPJmjWsCvHEtX5FE6KimVJhsJg5sBbqEFYf4UZtL/go-multiaddr"
 	cid "gx/ipfs/QmTbxNB1NwDesLmKTscr4udL2tVP7MaxvXnD1D9yX7g3PN/go-cid"
-	"gx/ipfs/QmUadX5EcvrBmxAV9sE7wUWtWSqxns5K84qKJBixmcT1w9/go-datastore"
 	peer "gx/ipfs/QmYVXrKrKHDC9FobgmcmshCDyWwdrfwfanNQN4oxJ9Fk3h/go-libp2p-peer"
 	routing "gx/ipfs/QmYxUdYY9S6yg5tSPVin5GFTvtfsLauVcr7reHDD3dM8xf/go-libp2p-routing"
 
@@ -35,7 +34,7 @@ import (
 
 const (
 	// VERSION - current version
-	VERSION = "0.13.5"
+	VERSION = "0.13.6"
 	// USERAGENT - user-agent header string
 	USERAGENT = "/openbazaar-go:" + VERSION + "/"
 )
@@ -332,8 +331,11 @@ func (n *OpenBazaarNode) EncryptMessage(peerID peer.ID, peerKey *libp2p.PubKey,
 	ctx, cancel := context.WithTimeout(context.Background(), n.OfflineMessageFailoverTimeout)
 	defer cancel()
 	if peerKey == nil {
-		var pubKey libp2p.PubKey
-		keyval, err := n.IpfsNode.Repo.Datastore().Get(datastore.NewKey(KeyCachePrefix + peerID.Pretty()))
+		var (
+			pubKey libp2p.PubKey
+			store  = n.IpfsNode.Repo.Datastore()
+		)
+		keyval, err := ipfs.GetCachedPubkey(store, peerID.Pretty())
 		if err != nil {
 			pubKey, err = routing.GetPublicKey(n.IpfsNode.Routing, ctx, peerID)
 			if err != nil {

core/net.go

@@ -122,7 +122,7 @@ func (n *OpenBazaarNode) SendOfflineMessage(p peer.ID, k *libp2p.PubKey, m *pb.M
 	go func() {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
-		err := n.Pubsub.Publisher.Publish(ctx, ipfs.MessageTopicPrefix+pointer.Cid.String(), ciphertext)
+		err := n.Pubsub.Publisher.Publish(ctx, pointer.Cid.String(), ciphertext)
 		if err != nil {
 			log.Error(err)
 		}

core/profile.go

@@ -24,9 +24,6 @@ import (
 	"github.com/imdario/mergo"
 )
 
-// KeyCachePrefix - cache prefix for public key
-const KeyCachePrefix = "/pubkey/"
-
 // ErrorProfileNotFound - profile not found error
 var ErrorProfileNotFound = errors.New("profile not found")
 

docs/install-linux.md

@@ -54,7 +54,7 @@ It will use git to checkout the source code into `$GOPATH/src/github.com/OpenBaz
 
 Checkout a release version:
 ```
-git checkout v0.13.5
+git checkout v0.13.6
 ```
 
 Note: `go get` leaves the repo pointing at `master` which is a branch used for active development. Running OpenBazaar from `master` is NOT recommended. Check the [release versions](https://github.com/OpenBazaar/openbazaar-go/releases) for the available versions that you use in checkout.

docs/install-osx.md

@@ -56,7 +56,7 @@ It will use git to checkout the source code into `$GOPATH/src/github.com/OpenBaz
 
 Checkout a release version:
 ```
-git checkout v0.13.5
+git checkout v0.13.6
 ```
 
 Note: `go get` leaves the repo pointing at `master` which is a branch used for active development. Running OpenBazaar from `master` is NOT recommended. Check the [release versions](https://github.com/OpenBazaar/openbazaar-go/releases) for the available versions that you use in checkout.

docs/install-pi3.md

@@ -57,7 +57,7 @@ It will use git to checkout the source code into `$GOPATH/src/github.com/OpenBaz
 
 Checkout a release version:
 ```
-git checkout v0.13.5
+git checkout v0.13.6
 ```
 
 Note: `go get` leaves the repo pointing at `master` which is a branch used for active development. Running OpenBazaar from `master` is NOT recommended. Check the [release versions](https://github.com/OpenBazaar/openbazaar-go/releases) for the available versions that you use iin checkout.

docs/install-windows.md

@@ -27,7 +27,7 @@ Create a directory to store all your Go projects (e.g. `C:\goprojects`):
 - Install `openbazaar-go`:
     + Open the command prompt and run: `go get github.com/OpenBazaar/openbazaar-go`. This will use git to checkout the source code into `%GOPATH%\src\github.com\OpenBazaar\openbazaar-go`.
 - Checkout an OpenBazaar release:
-    + Run `git checkout v0.13.5` to checkout a release version.
+    + Run `git checkout v0.13.6` to checkout a release version.
     + Note: `go get` leaves the repo pointing at `master` which is a branch used for active development. Running OpenBazaar from `master` is NOT recommended. Check the [release versions](https://github.com/OpenBazaar/openbazaar-go/releases) for the available versions that you use in checkout.
 - To compile and run `openbazaar-go`:
     + Open the command prompt and navigate the source directory: `cd %GOPATH%\src\github.com\OpenBazaar\openbazaar-go` 

ipfs/config.go

@@ -55,7 +55,7 @@ func PrepareIPFSConfig(r repo.Repo, routerAPIEndpoint string, testEnable, regtes
 }
 
 func constructRouting(ctx context.Context, host p2phost.Host, dstore ds.Batching, validator record.Validator) (routing.IpfsRouting, error) {
-	dhtRouting, err := dht.New(
+	return dht.New(
 		ctx, host,
 		dhtopts.Datastore(dstore),
 		dhtopts.Validator(validator),
@@ -64,12 +64,6 @@ func constructRouting(ctx context.Context, host p2phost.Host, dstore ds.Batching
 			IPFSProtocolDHTMainnetLegacy,
 		),
 	)
-	if err != nil {
-		return nil, err
-	}
-	apiRouter := NewAPIRouter(routerCacheURI, dhtRouting.Validator)
-	cachingRouter := NewCachingRouter(dhtRouting, &apiRouter)
-	return cachingRouter, nil
 }
 
 func constructRegtestRouting(ctx context.Context, host p2phost.Host, dstore ds.Batching, validator record.Validator) (routing.IpfsRouting, error) {
@@ -85,21 +79,13 @@ func constructRegtestRouting(ctx context.Context, host p2phost.Host, dstore ds.B
 }
 
 func constructTestnetRouting(ctx context.Context, host p2phost.Host, dstore ds.Batching, validator record.Validator) (routing.IpfsRouting, error) {
-	var (
-		dhtRouting, err = dht.New(
-			ctx, host,
-			dhtopts.Datastore(dstore),
-			dhtopts.Validator(validator),
-			dhtopts.Protocols(
-				IPFSProtocolKademliaTestnetOne,
-				IPFSProtocolAppTestnetOne,
-			),
-		)
+	return dht.New(
+		ctx, host,
+		dhtopts.Datastore(dstore),
+		dhtopts.Validator(validator),
+		dhtopts.Protocols(
+			IPFSProtocolKademliaTestnetOne,
+			IPFSProtocolAppTestnetOne,
+		),
 	)
-	if err != nil {
-		return nil, err
-	}
-	apiRouter := NewAPIRouter(routerCacheURI, dhtRouting.Validator)
-	cachingRouter := NewCachingRouter(dhtRouting, &apiRouter)
-	return cachingRouter, nil
 }