Integrate Arbo-based StateDB in vochain

Replace the IAVL-based StateDB used to store the vochain state by the new
Arbo-based StateDB.  This change doesn't introduce new features.

Due to the change of the vochain state underlying structure, even if the same
data is stored in the StateDB, the calculated hash of the state will be
completely different and incompatible with the previous one.  This means that
this change is not compatible with a chain built with the old StateDB, and
requires a chain reset to work.

Change notes:
- In statedb, remove SubTreeSingleConfig type and SubTreeSingle method
  in favour of a single config type (renamed to TreeConfig) and method.
  Introduced a new type TreeNonSingletonConfig that becomes a TreeConfig
  with the method WithKey, in order to keep different types for
  singleton trees configuration and non-singleton trees configuration.
  So now, we open a singleton tree like this:
  `mainTree.SubTree(OraclesCfg)` and a non-singleton tree like this:
  `processesTree.SubTree(CensusCfg.WithKey(pid))`.  This type unification
  allows creating a slice of trees configuration (no matter if they are
  singleton or non-singleton) to easily access nested subtrees.
- Added new methods to the TreeUpdate/TreeView in statedb to make it
  simpler to query nested trees: `DeepSubTree`, `DeepGet`, `DeepAdd` and
  `DeepSet`.
- Replace VoteID = processID + nullifier by hash(processID + nullifier).  The
  old VoteID was longer than 32 bytes, and that made it not friendly to be used
  as a key in Arbo.  By hashing we obtain a 32byte that suits an Arbo key.
- In VotesTree, replace storage format from (key:VoteID, value:VoteHash) to
  (key:VoteID, value:StateDBVote) where StateDBVote is a new protobuf model
  which contains { VoteHash, ProcessID, Nullifier }.  This is required because
  now the VoteID is a hash of processID and nullifier, and thus from the VoteID
  we can't recover the nullifier, and we need a way to obtain the nullifier of
  a stored vote.
- In ProcessesTree, replace storage format from (key: ProcessID, value:
  Process) to (key: ProcessID, value: StateDBProcess) where StateDBProcess is a
  new protobuf model which contains { Process, VotesRoot }.  This is required
  because now there is a VotesTree for each Processes stored in the StateDB.
  To query the VotesTree at diferent blocks we need to store it's root next to
  the Process.
- New hierarchy in the StateDB. In bold are trees.
    - Before:
        - 1 **AppTree**
            - 1 OracleList
            - 1 ValidatorList
            - 1 Header
        - 1 **ProcessTree**
            - N Processes indexed by ProcessID
        - 1 **VoteTree**
            - N Votes indexed by ProcessID + Nullifier
    - After:
        - 1 **MainTree**
            - 1 **OracleTree**
                - N Oracles indexed by address
            - 1 **ValidatorTree**
                - N Validators indexed by address
            - 1 Header
            - 1 **ProcessesTree**
                - N Processes indexed by ProcessID
                    - 1 **VotesTree**
                        - N Votes indexed by VoteID

Author: ed255

go.mod

@@ -48,11 +48,11 @@ require (
 	github.com/tendermint/tendermint v0.34.10
 	github.com/tendermint/tm-db v0.6.4
 	github.com/timshannon/badgerhold/v3 v3.0.0-20210415132401-e7c90fb5919f
-	github.com/vocdoni/arbo v0.0.0-20210831141531-de5914f453cf
+	github.com/vocdoni/arbo v0.0.0-20210909102959-f09b0b039255
 	github.com/vocdoni/go-external-ip v0.0.0-20210705122950-fae6195a1d44
 	github.com/vocdoni/storage-proofs-eth-go v0.1.6
 	go.uber.org/zap v1.18.1
-	go.vocdoni.io/proto v1.0.4-0.20210726091234-bceaf416353b
+	go.vocdoni.io/proto v1.0.4-0.20210910085433-e7c056b7c23a
 	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97
 	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d
 	google.golang.org/protobuf v1.27.1

go.sum

@@ -1969,6 +1969,8 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI
 github.com/vocdoni/arbo v0.0.0-20210616072504-a8c7ea980892/go.mod h1:Pikn2YIz/Rt05HWs35QHdpx7IWk4E2G0KdnCX+KRsNs=
 github.com/vocdoni/arbo v0.0.0-20210831141531-de5914f453cf h1:8hZXnUdi3v6hsarvQ6Qv7b0+j8UaqBrfLlKvn6JpNSI=
 github.com/vocdoni/arbo v0.0.0-20210831141531-de5914f453cf/go.mod h1:qJqeB/91mJfvNHaJf04iyLpCZoroqrrjOrQFlDUn0ko=
+github.com/vocdoni/arbo v0.0.0-20210909102959-f09b0b039255 h1:Gabst/0jIl3CgiywQWS5VpWsqaPuBwz0XAuMws5Pcyw=
+github.com/vocdoni/arbo v0.0.0-20210909102959-f09b0b039255/go.mod h1:qJqeB/91mJfvNHaJf04iyLpCZoroqrrjOrQFlDUn0ko=
 github.com/vocdoni/badgerhold/v3 v3.0.0-20210514115050-2d704df3456f h1:z7CK3k1yIutKycPY8s2ZbtwUBKMy3xPcLMh12QukLRY=
 github.com/vocdoni/badgerhold/v3 v3.0.0-20210514115050-2d704df3456f/go.mod h1:BoN7UMTA/JcgmNPaoUQq6RwuixhgSrk5PmCMVMKLxpc=
 github.com/vocdoni/blind-ca v0.1.4/go.mod h1:4ouWDqlvXrrNS0Csf3hKA3cuDTmKh6nP7kSXF39nT4s=
@@ -2102,8 +2104,11 @@ go.vocdoni.io/dvote v1.0.4-0.20210806163627-9494efbc5382/go.mod h1:kl66EQmAjR252
 go.vocdoni.io/proto v0.1.7/go.mod h1:cyITrt7+sHmUJH06WLu69xB7LBY9c9FakFaBOe8gs/M=
 go.vocdoni.io/proto v0.1.8/go.mod h1:cyITrt7+sHmUJH06WLu69xB7LBY9c9FakFaBOe8gs/M=
 go.vocdoni.io/proto v0.1.9-0.20210304214308-6f7363b52750/go.mod h1:cyITrt7+sHmUJH06WLu69xB7LBY9c9FakFaBOe8gs/M=
-go.vocdoni.io/proto v1.0.4-0.20210726091234-bceaf416353b h1:Xcc4GRtegTWKCwn2VKQTATrkmhy9f+Ju5teULJCRYg0=
 go.vocdoni.io/proto v1.0.4-0.20210726091234-bceaf416353b/go.mod h1:QV3gKc9Zf0xHW3o8wEaqSn8iZ94UTl8gOzekxoz3kWs=
+go.vocdoni.io/proto v1.0.4-0.20210909161946-f3158498ba88 h1:EJnuknmHjwPQL44d5TVOZJhzVA4oFzZT41etyaRervk=
+go.vocdoni.io/proto v1.0.4-0.20210909161946-f3158498ba88/go.mod h1:QV3gKc9Zf0xHW3o8wEaqSn8iZ94UTl8gOzekxoz3kWs=
+go.vocdoni.io/proto v1.0.4-0.20210910085433-e7c056b7c23a h1:7QimTKlkCY0O5KRFi0ejafFkLQze7zN0qOE/BuZosbE=
+go.vocdoni.io/proto v1.0.4-0.20210910085433-e7c056b7c23a/go.mod h1:oi/WtiBFJ6QwNDv2aUQYwOnUKzYuS/fBqXF8xDNwcGo=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU=
 go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg=

statedb/statedb.go

@@ -95,84 +95,106 @@ func subReadTx(tx db.ReadTx, path string) db.ReadTx {
 	return prefixeddb.NewPrefixedReadTx(tx, []byte(path+"/"))
 }
 
-// treeConfig is a unified configuration for a subTree.
-type treeConfig struct {
-	parentLeafKey []byte
-	prefix        string
-	*SubTreeConfig
-}
-
 // GetRootFn is a function type that takes a leaf value and returns the contained root.
 type GetRootFn func(value []byte) ([]byte, error)
 
 // SetRootFn is a function type that takes a leaf value and a root, updates the
 // leaf value with the new root and returns it.
 type SetRootFn func(value []byte, root []byte) ([]byte, error)
 
-// SubTreeConfig
-type SubTreeConfig struct {
+// TreeParams are the parameters used in the constructor for a tree configuration.
+type TreeParams struct {
+	// HashFunc is the hash function used in the merkle tree
+	HashFunc arbo.HashFunction
+	// KindID is a unique identifier that specifies what kind of tree this
+	// is.  Diferent kind of trees under the same parent tree must have
+	// different KindIDs, as this parameter is used to build a disjoint
+	// database prefix for each subTree under the same parent tree.
+	KindID string
+	// MaxLevels is the maximum number of merkle tree levels allowed.
+	MaxLevels int
+	// ParentLeafGetRoot is the function that takes a leaf value of the
+	// parent at which this tree hangs, and returns the root of this tree.
+	ParentLeafGetRoot GetRootFn
+	// ParentLeafSetRoot is the function that takes a leaf value of the
+	// parent at which this tree hangs, and updates it (returning it) with
+	// the new root of this tree.
+	ParentLeafSetRoot SetRootFn
+}
+
+// TreeNonSingletonConfig contains the configuration used for a non-singleton subTree.
+type TreeNonSingletonConfig struct {
 	hashFunc          arbo.HashFunction
 	kindID            string
 	parentLeafGetRoot GetRootFn
 	parentLeafSetRoot SetRootFn
 	maxLevels         int
 }
 
-// SubTreeSingleConfig contains the configuration used for a non-singleton subTree.
-func NewSubTreeConfig(hashFunc arbo.HashFunction, kindID string, maxLevels int,
-	parentLeafGetRoot GetRootFn, parentLeafSetRoot SetRootFn) *SubTreeConfig {
-	return &SubTreeConfig{
-		hashFunc:          hashFunc,
-		kindID:            kindID,
-		parentLeafGetRoot: parentLeafGetRoot,
-		parentLeafSetRoot: parentLeafSetRoot,
-		maxLevels:         maxLevels,
+// NewTreeNonSingletonConfig creates a new configuration for a non-singleton subTree.
+func NewTreeNonSingletonConfig(params TreeParams) *TreeNonSingletonConfig {
+	return &TreeNonSingletonConfig{
+		hashFunc:          params.HashFunc,
+		kindID:            params.KindID,
+		parentLeafGetRoot: params.ParentLeafGetRoot,
+		parentLeafSetRoot: params.ParentLeafSetRoot,
+		maxLevels:         params.MaxLevels,
 	}
 }
 
-// treeConfig returns a unified configuration type for opening a singleton
-// subTree that is identified by `id`.  `id` is the path in the parent tree to
-// the leaf that contains the subTree root.
-func (c *SubTreeConfig) treeConfig(id []byte) *treeConfig {
-	return &treeConfig{
-		parentLeafKey: id,
-		prefix:        c.kindID + string(id),
-		SubTreeConfig: c,
+// HashFunc returns the hashFunc set for this SubTreeConfig
+func (c *TreeNonSingletonConfig) HashFunc() arbo.HashFunction {
+	return c.hashFunc
+}
+
+// WithKey returns a unified subTree configuration type for opening a singleton
+// subTree that is identified by `key`.  `key` is the path in the parent tree
+// to the leaf that contains the subTree root.
+func (c *TreeNonSingletonConfig) WithKey(key []byte) TreeConfig {
+	return TreeConfig{
+		parentLeafKey:          key,
+		prefix:                 c.kindID + string(key),
+		TreeNonSingletonConfig: c,
 	}
 }
 
-// SubTreeSingleConfig contains the configuration used for a singleton subTree.
-type SubTreeSingleConfig struct {
-	*SubTreeConfig
+// TreeConfig contains the configuration used for a subTree.
+type TreeConfig struct {
+	parentLeafKey []byte
+	prefix        string
+	*TreeNonSingletonConfig
 }
 
-// NewSubTreeSingleConfig creates a new SubTreeSingleConfig.
-func NewSubTreeSingleConfig(hashFunc arbo.HashFunction, kindID string, maxLevels int,
-	parentLeafGetRoot GetRootFn, parentLeafSetRoot SetRootFn) *SubTreeSingleConfig {
-	return &SubTreeSingleConfig{
-		NewSubTreeConfig(hashFunc, kindID, maxLevels, parentLeafGetRoot, parentLeafSetRoot),
+// NewTreeSingletonConfig creates a new configuration for a singleton subTree.
+func NewTreeSingletonConfig(params TreeParams) TreeConfig {
+	return TreeConfig{
+		parentLeafKey:          []byte(params.KindID),
+		prefix:                 params.KindID,
+		TreeNonSingletonConfig: NewTreeNonSingletonConfig(params),
 	}
 }
 
 // Key returns the key used in the parent tree in which the value that contains
 // the subTree root is stored.  The key is the path of the parent leaf with the root.
-func (c *SubTreeSingleConfig) Key() []byte {
+func (c *TreeConfig) Key() []byte {
 	return []byte(c.kindID)
 }
 
-// treeConfig returns a unified configuration type for opening a subTree.
-func (c *SubTreeSingleConfig) treeConfig() *treeConfig {
-	return &treeConfig{
-		parentLeafKey: []byte(c.kindID),
-		prefix:        c.kindID,
-		SubTreeConfig: c.SubTreeConfig,
-	}
+// HashFunc returns the hashFunc set for this SubTreeSingleConfig
+func (c *TreeConfig) HashFunc() arbo.HashFunction {
+	return c.hashFunc
 }
 
 // mainTreeCfg is the subTree configuration of the mainTree.  It doesn't have a
 // kindID because it's the top level tree.  For the same reason, it doesn't
 // contain functions to work with the parent leaf: it doesn't have a parent.
-var mainTreeCfg = NewSubTreeSingleConfig(arbo.HashFunctionSha256, "", 256, nil, nil).treeConfig()
+var mainTreeCfg = NewTreeSingletonConfig(TreeParams{
+	HashFunc:          arbo.HashFunctionSha256,
+	KindID:            "",
+	MaxLevels:         256,
+	ParentLeafGetRoot: nil,
+	ParentLeafSetRoot: nil,
+})
 
 // StateDB is a database backed structure that holds a dynamic hierarchy of
 // linked merkle trees with the property that the keys and values of all merkle

statedb/statedb_test.go

@@ -101,7 +101,7 @@ func TestStateDB(t *testing.T) {
 		qt.Assert(t, rootView, qt.DeepEquals, root0)
 
 		_, err = mainTreeView.Get(keys[0])
-		qt.Assert(t, err, qt.Equals, db.ErrKeyNotFound)
+		qt.Assert(t, err, qt.Equals, arbo.ErrKeyNotFound)
 	}
 
 	// dumpPrint(sdb.db)
@@ -142,65 +142,65 @@ func TestStateDB(t *testing.T) {
 }
 
 // singleCfg is a test configuration for a singleton subTree
-var singleCfg = NewSubTreeSingleConfig(
-	arbo.HashFunctionSha256,
-	"single",
-	256,
-	func(value []byte) ([]byte, error) {
+var singleCfg = NewTreeSingletonConfig(TreeParams{
+	HashFunc:  arbo.HashFunctionSha256,
+	KindID:    "single",
+	MaxLevels: 256,
+	ParentLeafGetRoot: func(value []byte) ([]byte, error) {
 		if len(value) != 32 {
 			return nil, fmt.Errorf("len(value) = %v != 32", len(value))
 		}
 		return value, nil
 	},
-	func(value []byte, root []byte) ([]byte, error) {
+	ParentLeafSetRoot: func(value []byte, root []byte) ([]byte, error) {
 		if len(value) != 32 {
 			return nil, fmt.Errorf("len(value) = %v != 32", len(value))
 		}
 		return root, nil
 	},
-)
+})
 
 // multiACfg is a test configuration for a non-singleton subtTree whose root is
 // stored in a leaf alongside another root (multiB)
-var multiACfg = NewSubTreeConfig(
-	arbo.HashFunctionSha256,
-	"multia",
-	256,
-	func(value []byte) ([]byte, error) {
+var multiACfg = NewTreeNonSingletonConfig(TreeParams{
+	HashFunc:  arbo.HashFunctionSha256,
+	KindID:    "multia",
+	MaxLevels: 256,
+	ParentLeafGetRoot: func(value []byte) ([]byte, error) {
 		if len(value) != 64 {
 			return nil, fmt.Errorf("len(value) = %v != 64", len(value))
 		}
 		return value[:32], nil
 	},
-	func(value []byte, root []byte) ([]byte, error) {
+	ParentLeafSetRoot: func(value []byte, root []byte) ([]byte, error) {
 		if len(value) != 64 {
 			return nil, fmt.Errorf("len(value) = %v != 64", len(value))
 		}
 		copy(value[:32], root)
 		return value, nil
 	},
-)
+})
 
 // multiBCfg is a test configuration for a non-singleton subtTree whose root is
 // stored in a leaf alongside another root (multiA)
-var multiBCfg = NewSubTreeConfig(
-	arbo.HashFunctionPoseidon,
-	"multib",
-	256,
-	func(value []byte) ([]byte, error) {
+var multiBCfg = NewTreeNonSingletonConfig(TreeParams{
+	HashFunc:  arbo.HashFunctionPoseidon,
+	KindID:    "multib",
+	MaxLevels: 256,
+	ParentLeafGetRoot: func(value []byte) ([]byte, error) {
 		if len(value) != 64 {
 			return nil, fmt.Errorf("len(value) = %v != 64", len(value))
 		}
 		return value[32:], nil
 	},
-	func(value []byte, root []byte) ([]byte, error) {
+	ParentLeafSetRoot: func(value []byte, root []byte) ([]byte, error) {
 		if len(value) != 64 {
 			return nil, fmt.Errorf("len(value) = %v != 64", len(value))
 		}
 		copy(value[32:], root)
 		return value, nil
 	},
-)
+})
 
 func TestSubTree(t *testing.T) {
 	// In this test we have:
@@ -218,7 +218,7 @@ func TestSubTree(t *testing.T) {
 	// Crete the leaf in mainTree which contains the root of the single
 	qt.Assert(t, mainTree.Add(singleCfg.Key(), emptyHash), qt.IsNil)
 	// treePrint(mainTree.tree, mainTree.txTree, "main")
-	single, err := mainTree.SubTreeSingle(singleCfg)
+	single, err := mainTree.SubTree(singleCfg)
 	qt.Assert(t, err, qt.IsNil)
 	// We add two key-values to the single tree
 	qt.Assert(t, single.Add([]byte("key0"), []byte("value0")), qt.IsNil)
@@ -231,11 +231,11 @@ func TestSubTree(t *testing.T) {
 	// (multiA_id, multiB_id)
 	qt.Assert(t, mainTree.Add(id, make([]byte, 32*2)), qt.IsNil)
 	// treePrint(mainTree.tree, mainTree.txTree, "main")
-	multiA, err := mainTree.SubTree(id, multiACfg)
+	multiA, err := mainTree.SubTree(multiACfg.WithKey(id))
 	qt.Assert(t, err, qt.IsNil)
 	// We add one key-value to multiA
 	qt.Assert(t, multiA.Add([]byte("key2"), []byte("value2")), qt.IsNil)
-	multiB, err := mainTree.SubTree(id, multiBCfg)
+	multiB, err := mainTree.SubTree(multiBCfg.WithKey(id))
 	qt.Assert(t, err, qt.IsNil)
 	// We add one key-value to multiB
 	qt.Assert(t, multiB.Add([]byte("key3"), []byte("value3")), qt.IsNil)
@@ -251,7 +251,7 @@ func TestSubTree(t *testing.T) {
 		qt.Assert(t, err, qt.IsNil)
 
 		// Expect the two key-values in single
-		single, err := mainTreeView.SubTreeSingle(singleCfg)
+		single, err := mainTreeView.SubTree(singleCfg)
 		qt.Assert(t, err, qt.IsNil)
 		v0, err := single.Get([]byte("key0"))
 		qt.Assert(t, err, qt.IsNil)
@@ -261,7 +261,7 @@ func TestSubTree(t *testing.T) {
 		qt.Assert(t, v1, qt.DeepEquals, []byte("value1"))
 
 		// Expect the key-value in multiA
-		multiA, err := mainTreeView.SubTree(id, multiACfg)
+		multiA, err := mainTreeView.SubTree(multiACfg.WithKey(id))
 		qt.Assert(t, err, qt.IsNil)
 		// dumpPrint(multiA.tree.DB())
 		// treePrint(multiA.tree, nil, "multiA")
@@ -270,7 +270,7 @@ func TestSubTree(t *testing.T) {
 		qt.Assert(t, v2, qt.DeepEquals, []byte("value2"))
 
 		// Expect the key-value in multiB
-		multiB, err := mainTreeView.SubTree(id, multiBCfg)
+		multiB, err := mainTreeView.SubTree(multiBCfg.WithKey(id))
 		qt.Assert(t, err, qt.IsNil)
 		v3, err := multiB.Get([]byte("key3"))
 		qt.Assert(t, err, qt.IsNil)
@@ -288,20 +288,20 @@ func TestSubTree(t *testing.T) {
 		qt.Assert(t, err, qt.IsNil)
 
 		// Expect that mainTree's leaf for single == single.Root
-		single, err := mainTreeView.SubTreeSingle(singleCfg)
+		single, err := mainTreeView.SubTree(singleCfg)
 		qt.Assert(t, err, qt.IsNil)
 		singleRoot, err := single.Root()
 		qt.Assert(t, err, qt.IsNil)
 		qt.Assert(t, singleParentLeaf, qt.DeepEquals, singleRoot)
 
 		// Expect that mainTree's leaf for id == multiA.Root | multiB.Root
-		multiA, err := mainTreeView.SubTree(id, multiACfg)
+		multiA, err := mainTreeView.SubTree(multiACfg.WithKey(id))
 		qt.Assert(t, err, qt.IsNil)
 		multiARoot, err := multiA.Root()
 		qt.Assert(t, err, qt.IsNil)
 		qt.Assert(t, multiParentLeaf[:32], qt.DeepEquals, multiARoot)
 
-		multiB, err := mainTreeView.SubTree(id, multiBCfg)
+		multiB, err := mainTreeView.SubTree(multiBCfg.WithKey(id))
 		qt.Assert(t, err, qt.IsNil)
 		multiBRoot, err := multiB.Root()
 		qt.Assert(t, err, qt.IsNil)
@@ -327,7 +327,7 @@ func TestNoState(t *testing.T) {
 	mainTreeRoot, err := mainTree.Root()
 	qt.Assert(t, err, qt.IsNil)
 
-	single, err := mainTree.SubTreeSingle(singleCfg)
+	single, err := mainTree.SubTree(singleCfg)
 	qt.Assert(t, err, qt.IsNil)
 	singleRoot, err := single.Root()
 	qt.Assert(t, err, qt.IsNil)
@@ -358,7 +358,7 @@ func TestNoState(t *testing.T) {
 		qt.Assert(t, v0, qt.DeepEquals, []byte("value0"))
 
 		// single root hasn't changed
-		single, err := mainTreeView.SubTreeSingle(singleCfg)
+		single, err := mainTreeView.SubTree(singleCfg)
 		qt.Assert(t, err, qt.IsNil)
 		singleRoot1, err := single.Root()
 		qt.Assert(t, err, qt.IsNil)