GetProofItems now get the proof values from the tree (#361)

A remnant of the time that the pre-state values were taken from
the witness, the data to be passed to the proof serialization code
was taken from it. But since the witness does not currently hold
the initial values, and since the data is taken from the tree, this
commit changes the interface for simplification.

This is enshrining a slower way to do things, but given the complexity
of the previous version, it's wiser to keep things simple at first,
and worry about optimization later.

Author: gballet

proof_ipa.go

@@ -79,7 +79,7 @@ func GetCommitmentsForMultiproof(root VerkleNode, keys [][]byte) (*ProofElements
 	return root.GetProofItems(keylist(keys))
 }
 
-func MakeVerkleMultiProof(root VerkleNode, keys [][]byte, keyvals map[string][]byte) (*Proof, []*Point, []byte, []*Fr, error) {
+func MakeVerkleMultiProof(root VerkleNode, keys [][]byte) (*Proof, []*Point, []byte, []*Fr, error) {
 	// go-ipa won't accept no key as an input, catch this corner case
 	// and return an empty result.
 	if len(keys) == 0 {
@@ -94,13 +94,18 @@ func MakeVerkleMultiProof(root VerkleNode, keys [][]byte, keyvals map[string][]b
 		return nil, nil, nil, nil, err
 	}
 
-	var vals [][]byte
-	for _, k := range keys {
-		// TODO at the moment, do not include the post-data
-		// val, _ := root.Get(k, nil)
-		// vals = append(vals, val)
-		vals = append(vals, keyvals[string(k)])
-	}
+	// NOTE this is leftover code from the time the proof was
+	// made against the POST state. Since proofs are expected
+	// to prove PRE and POST state in the future, I'm leaving
+	// this for reference - eventhough it's unlikely that the
+	// final version will look like this, but you never know.
+	// var vals [][]byte
+	// for _, k := range keys {
+	// 	// TODO at the moment, do not include the post-data
+	// 	// val, _ := root.Get(k, nil)
+	// 	// vals = append(vals, val)
+	// 	vals = append(vals, keyvals[string(k)])
+	// }
 
 	cfg := GetConfig()
 	mpArg := ipa.CreateMultiProof(tr, cfg.conf, pe.Cis, pe.Fis, pe.Zis)
@@ -127,7 +132,7 @@ func MakeVerkleMultiProof(root VerkleNode, keys [][]byte, keyvals map[string][]b
 		ExtStatus:  es,
 		PoaStems:   poas,
 		Keys:       keys,
-		Values:     vals,
+		Values:     pe.Vals,
 	}
 	return proof, pe.Cis, pe.Zis, pe.Yis, nil
 }
@@ -353,7 +358,7 @@ func TreeFromProof(proof *Proof, rootC *Point) (VerkleNode, error) {
 				continue
 			}
 
-			if bytes.Equal(k, info[string(p)].stem) {
+			if bytes.Equal(k[:31], info[string(p)].stem) {
 				values[k[31]] = proof.Values[i]
 			}
 		}

proof_test.go

@@ -41,7 +41,7 @@ func TestProofVerifyTwoLeaves(t *testing.T) {
 	root.Insert(ffx32KeyTest, zeroKeyTest, nil)
 	root.Commit()
 
-	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{ffx32KeyTest}, map[string][]byte{string(ffx32KeyTest): zeroKeyTest})
+	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{ffx32KeyTest})
 
 	cfg := GetConfig()
 	if !VerifyVerkleProof(proof, cis, zis, yis, cfg) {
@@ -61,7 +61,7 @@ func TestProofVerifyMultipleLeaves(t *testing.T) {
 		root.Insert(key, fourtyKeyTest, nil)
 	}
 
-	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{keys[0]}, map[string][]byte{string(keys[0]): fourtyKeyTest})
+	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{keys[0]})
 
 	cfg := GetConfig()
 	if !VerifyVerkleProof(proof, cis, zis, yis, cfg) {
@@ -73,17 +73,15 @@ func TestMultiProofVerifyMultipleLeaves(t *testing.T) {
 	const leafCount = 1000
 
 	keys := make([][]byte, leafCount)
-	kv := make(map[string][]byte)
 	root := New()
 	for i := 0; i < leafCount; i++ {
 		key := make([]byte, 32)
 		rand.Read(key)
 		keys[i] = key
 		root.Insert(key, fourtyKeyTest, nil)
-		kv[string(key)] = fourtyKeyTest
 	}
 
-	proof, _, _, _, _ := MakeVerkleMultiProof(root, keys[0:2], kv)
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, keys[0:2])
 
 	pe, _, _, err := GetCommitmentsForMultiproof(root, keys[0:2])
 	if err != nil {
@@ -100,12 +98,10 @@ func TestMultiProofVerifyMultipleLeavesWithAbsentStem(t *testing.T) {
 
 	var keys [][]byte
 	var absentstem [31]byte
-	kv := make(map[string][]byte)
 	root := New()
 	for i := 0; i < leafCount; i++ {
 		key := make([]byte, 32)
 		key[2] = byte(i)
-		kv[string(key)] = fourtyKeyTest
 		root.Insert(key, fourtyKeyTest, nil)
 		if i%2 == 0 {
 			keys = append(keys, key)
@@ -119,7 +115,7 @@ func TestMultiProofVerifyMultipleLeavesWithAbsentStem(t *testing.T) {
 	absent[3] = 1 // and the stem differs
 	keys = append(keys, absent)
 
-	proof, _, _, _, _ := MakeVerkleMultiProof(root, keys, kv)
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, keys)
 
 	pe, _, isabsent, err := GetCommitmentsForMultiproof(root, keys)
 	if err != nil {
@@ -139,17 +135,14 @@ func TestMultiProofVerifyMultipleLeavesWithAbsentStem(t *testing.T) {
 }
 
 func TestMultiProofVerifyMultipleLeavesCommitmentRedundancy(t *testing.T) {
-	kv := make(map[string][]byte)
 	keys := make([][]byte, 2)
 	root := New()
 	keys[0] = zeroKeyTest
-	kv[string(zeroKeyTest)] = fourtyKeyTest
 	root.Insert(keys[0], fourtyKeyTest, nil)
 	keys[1] = oneKeyTest
-	kv[string(oneKeyTest)] = fourtyKeyTest
 	root.Insert(keys[1], fourtyKeyTest, nil)
 
-	proof, _, _, _, _ := MakeVerkleMultiProof(root, keys, kv)
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, keys)
 
 	pe, _, _, err := GetCommitmentsForMultiproof(root, keys)
 	if err != nil {
@@ -166,7 +159,7 @@ func TestProofOfAbsenceInternalVerify(t *testing.T) {
 	root.Insert(zeroKeyTest, zeroKeyTest, nil)
 	root.Insert(oneKeyTest, zeroKeyTest, nil)
 
-	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{ffx32KeyTest}, map[string][]byte{})
+	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{ffx32KeyTest})
 
 	cfg := GetConfig()
 	if !VerifyVerkleProof(proof, cis, zis, yis, cfg) {
@@ -179,7 +172,7 @@ func TestProofOfAbsenceLeafVerify(t *testing.T) {
 	root.Insert(zeroKeyTest, zeroKeyTest, nil)
 	root.Insert(ffx32KeyTest, zeroKeyTest, nil)
 
-	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{oneKeyTest}, map[string][]byte{})
+	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{oneKeyTest})
 
 	cfg := GetConfig()
 	if !VerifyVerkleProof(proof, cis, zis, yis, cfg) {
@@ -196,7 +189,7 @@ func TestProofOfAbsenceLeafVerifyOtherSuffix(t *testing.T) {
 		return ret
 	}()
 
-	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{key}, map[string][]byte{})
+	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{key})
 
 	cfg := GetConfig()
 	if !VerifyVerkleProof(proof, cis, zis, yis, cfg) {
@@ -213,7 +206,7 @@ func TestProofOfAbsenceStemVerify(t *testing.T) {
 		return ret
 	}()
 
-	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{key}, map[string][]byte{})
+	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{key})
 
 	cfg := GetConfig()
 	if !VerifyVerkleProof(proof, cis, zis, yis, cfg) {
@@ -235,7 +228,7 @@ func BenchmarkProofCalculation(b *testing.B) {
 	b.ReportAllocs()
 
 	for i := 0; i < b.N; i++ {
-		MakeVerkleMultiProof(root, [][]byte{keys[len(keys)/2]}, map[string][]byte{})
+		MakeVerkleMultiProof(root, [][]byte{keys[len(keys)/2]})
 	}
 }
 
@@ -250,7 +243,7 @@ func BenchmarkProofVerification(b *testing.B) {
 	}
 
 	root.Commit()
-	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{keys[len(keys)/2]}, map[string][]byte{})
+	proof, cis, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{keys[len(keys)/2]})
 
 	b.ResetTimer()
 	b.ReportAllocs()
@@ -273,7 +266,7 @@ func TestProofSerializationNoAbsentStem(t *testing.T) {
 		root.Insert(key, fourtyKeyTest, nil)
 	}
 
-	proof, _, _, _, _ := MakeVerkleMultiProof(root, [][]byte{keys[0]}, map[string][]byte{})
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, [][]byte{keys[0]})
 
 	vp, statediff, err := SerializeProof(proof)
 	if err != nil {
@@ -307,7 +300,7 @@ func TestProofSerializationWithAbsentStem(t *testing.T) {
 	absentkey[2] = 2
 	absentkey[3] = 1
 
-	proof, _, _, _, _ := MakeVerkleMultiProof(root, [][]byte{absentkey[:]}, map[string][]byte{})
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, [][]byte{absentkey[:]})
 
 	vp, statediff, err := SerializeProof(proof)
 	if err != nil {
@@ -343,7 +336,7 @@ func TestProofDeserialize(t *testing.T) {
 	absentkey[2] = 2
 	absentkey[3] = 1
 
-	proof, _, _, _, _ := MakeVerkleMultiProof(root, [][]byte{absentkey[:]}, map[string][]byte{})
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, [][]byte{absentkey[:]})
 
 	vp, statediff, err := SerializeProof(proof)
 	if err != nil {
@@ -371,7 +364,7 @@ func TestProofOfAbsenceEdgeCase(t *testing.T) {
 	root.Commit()
 
 	ret, _ := hex.DecodeString("0303030303030303030303030303030303030303030303030303030303030303")
-	proof, cs, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{ret}, map[string][]byte{string(ret): nil})
+	proof, cs, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{ret})
 	cfg := GetConfig()
 	if !VerifyVerkleProof(proof, cs, zis, yis, cfg) {
 		t.Fatal("could not verify proof")
@@ -388,7 +381,7 @@ func TestProofOfAbsenceOtherMultipleLeaves(t *testing.T) {
 
 	ret1, _ := hex.DecodeString("0303030303030303030303030303030303030303030303030303030303030300")
 	ret2, _ := hex.DecodeString("0303030303030303030303030303030303030303030303030303030303030301")
-	proof, cs, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{ret1, ret2}, map[string][]byte{string(ret1): nil, string(ret2): nil})
+	proof, cs, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{ret1, ret2})
 	cfg := GetConfig()
 	if !VerifyVerkleProof(proof, cs, zis, yis, cfg) {
 		t.Fatal("could not verify proof")
@@ -407,7 +400,7 @@ func TestProofOfAbsenceNoneMultipleStems(t *testing.T) {
 
 	ret1, _ := hex.DecodeString("0303030303030303030303030303030303030303030303030303030303030300")
 	ret2, _ := hex.DecodeString("0303030303030303030303030303030303030303030303030303030303030200")
-	proof, cs, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{ret1, ret2}, map[string][]byte{string(ret1): nil, string(ret2): nil})
+	proof, cs, zis, yis, _ := MakeVerkleMultiProof(root, [][]byte{ret1, ret2})
 	cfg := GetConfig()
 	if !VerifyVerkleProof(proof, cs, zis, yis, cfg) {
 		t.Fatal("could not verify proof")
@@ -576,7 +569,7 @@ func TestStatelessDeserialize(t *testing.T) {
 		root.Insert(k, fourtyKeyTest, nil)
 	}
 
-	proof, _, _, _, _ := MakeVerkleMultiProof(root, keylist{zeroKeyTest, fourtyKeyTest}, map[string][]byte{string(zeroKeyTest): fourtyKeyTest, string(fourtyKeyTest): fourtyKeyTest})
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, keylist{zeroKeyTest, fourtyKeyTest})
 
 	serialized, statediff, err := SerializeProof(proof)
 	if err != nil {
@@ -607,13 +600,13 @@ func TestStatelessDeserialize(t *testing.T) {
 	}
 }
 
-func TestStatelessDeserializeMissginChildNode(t *testing.T) {
+func TestStatelessDeserializeMissingChildNode(t *testing.T) {
 	root := New()
 	for _, k := range [][]byte{zeroKeyTest, oneKeyTest, ffx32KeyTest} {
 		root.Insert(k, fourtyKeyTest, nil)
 	}
 
-	proof, _, _, _, _ := MakeVerkleMultiProof(root, keylist{zeroKeyTest, fourtyKeyTest}, map[string][]byte{string(zeroKeyTest): fourtyKeyTest, string(fourtyKeyTest): nil})
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, keylist{zeroKeyTest, fourtyKeyTest})
 
 	serialized, statediff, err := SerializeProof(proof)
 	if err != nil {
@@ -649,7 +642,7 @@ func TestStatelessDeserializeDepth2(t *testing.T) {
 		root.Insert(k, fourtyKeyTest, nil)
 	}
 
-	proof, _, _, _, _ := MakeVerkleMultiProof(root, keylist{zeroKeyTest, key1}, map[string][]byte{string(zeroKeyTest): fourtyKeyTest, string(key1): nil})
+	proof, _, _, _, _ := MakeVerkleMultiProof(root, keylist{zeroKeyTest, key1})
 
 	serialized, statediff, err := SerializeProof(proof)
 	if err != nil {

tree.go

@@ -99,6 +99,7 @@ type ProofElements struct {
 	Yis    []*Fr
 	Fis    [][]Fr
 	ByPath map[string]*Point // Gather commitments by path
+	Vals   [][]byte          // list of values read from the tree
 
 	// dedups flags the presence of each (Ci,zi) tuple
 	dedups map[*Point]map[byte]struct{}
@@ -146,6 +147,8 @@ func (pe *ProofElements) Merge(other *ProofElements) {
 			pe.ByPath[path] = C
 		}
 	}
+
+	pe.Vals = append(pe.Vals, other.Vals...)
 }
 
 const (
@@ -810,6 +813,7 @@ func (n *InternalNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]
 			// repeated as many times? It would be wasteful, so the
 			// decoding code has to be aware of this corner case.
 			esses = append(esses, extStatusAbsentEmpty|((n.depth+1)<<3))
+			pe.Vals = append(pe.Vals, nil)
 			continue
 		}
 
@@ -1271,6 +1275,7 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte
 			Zis:    []byte{0, 1},
 			Yis:    []*Fr{&poly[0], &poly[1]}, // Should be 0
 			Fis:    [][]Fr{poly[:], poly[:]},
+			Vals:   make([][]byte, 0, len(keys)),
 			ByPath: map[string]*Point{},
 		}
 
@@ -1321,6 +1326,7 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte
 			if len(esses) == 0 {
 				esses = append(esses, extStatusAbsentOther|(n.depth<<3))
 				poass = append(poass, n.stem)
+				pe.Vals = append(pe.Vals, nil)
 			}
 			continue
 		}
@@ -1357,6 +1363,7 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte
 			// as all the information is available before fillSuffixTreePoly
 			// has to be called, save the count.
 			esses = append(esses, extStatusAbsentEmpty|(n.depth<<3))
+			pe.Vals = append(pe.Vals, nil)
 			continue
 		}
 
@@ -1384,6 +1391,7 @@ func (n *LeafNode) GetProofItems(keys keylist) (*ProofElements, []byte, [][]byte
 		pe.Zis = append(pe.Zis, 2*suffix, 2*suffix+1)
 		pe.Yis = append(pe.Yis, &leaves[0], &leaves[1])
 		pe.Fis = append(pe.Fis, suffPoly[:], suffPoly[:])
+		pe.Vals = append(pe.Vals, n.values[key[31]])
 		if len(esses) == 0 || esses[len(esses)-1] != extStatusPresent|(n.depth<<3) {
 			esses = append(esses, extStatusPresent|(n.depth<<3))
 		}

tree_test.go

@@ -1122,7 +1122,7 @@ func TestRustBanderwagonBlock48(t *testing.T) {
 
 	r := tree.Commit()
 
-	proof, cis, zis, yis, _ := MakeVerkleMultiProof(tree, keys, initialVals)
+	proof, cis, zis, yis, _ := MakeVerkleMultiProof(tree, keys)
 	vp, statediff, err := SerializeProof(proof)
 	if err != nil {
 		t.Fatal(err)