MPT StorageDoesNotExist (#1699)

### Description

A couple of `StorageDoesNotExist` issues to be addressed:

1.  Witness generation fails for `getProof` proofs of length 0.

2. For `StorageDoesNotExist`, when the trie is empty, the key
occupies `33 (33 = 161 - 128)` bytes, as in the example below:
```
[227 161 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
```
Currently, the length for the RLP items is set `valueLen = 34`, so we
don't have space for all 33 bytes.

3. It seems that some constraints of the `StorageDoesNotExist` proof
were lost on the way - currently there are only constraints for the
`wrong` leaf, but the constraints for the case when the node in branch
is `nil` are missing.

### Type of change

- [x] Bug fix (non-breaking change which fixes an issue)

### How Has This Been Tested?

`TestStorageDoesNotExistOnlySProof` in
`mpt-witness-generator/witness/gen_witness_from_infura_blockchain_test.go`.

Author: miha-stopar

mpt-witness-generator/witness/branch.go

@@ -65,8 +65,8 @@ func prepareBranchWitness(rows [][]byte, branch []byte, branchStart int, branchR
 	}
 }
 
-func prepareBranchNode(branch1, branch2, extNode1, extNode2, extListRlpBytes []byte, extValues [][]byte, key, driftedInd,
-	branchC16, branchC1 byte, isBranchSPlaceholder, isBranchCPlaceholder, isExtension bool) Node {
+func prepareBranchNode(branch1, branch2, extNode1, extNode2, extListRlpBytes []byte, extValues [][]byte, key, driftedInd byte,
+	isBranchSPlaceholder, isBranchCPlaceholder, isExtension bool) Node {
 	extensionNode := ExtensionNode{
 		ListRlpBytes: extListRlpBytes,
 	}
@@ -197,7 +197,7 @@ func addBranchAndPlaceholder(proof1, proof2,
 	leafRow0, key, neighbourNode []byte,
 	keyIndex, extensionNodeInd int,
 	additionalBranch, isAccountProof, nonExistingAccountProof,
-	isShorterProofLastLeaf bool, branchC16, branchC1 byte, toBeHashed *[][]byte) (bool, bool, int, byte, Node) {
+	isShorterProofLastLeaf bool, toBeHashed *[][]byte) (bool, bool, int, Node) {
 	len1 := len(proof1)
 	len2 := len(proof2)
 
@@ -211,32 +211,14 @@ func addBranchAndPlaceholder(proof1, proof2,
 	}
 
 	isExtension := (len1 == len2+2) || (len2 == len1+2)
-	if !isExtension {
-		if branchC16 == 1 {
-			branchC16 = 0
-			branchC1 = 1
-		} else {
-			branchC16 = 1
-			branchC1 = 0
-		}
-	} else {
+	if isExtension {
 		var numNibbles byte
 		if len1 > len2 {
 			numNibbles, extListRlpBytes, extValues = prepareExtensions(extNibblesS, extensionNodeInd, proof1[len1-3], proof1[len1-3])
 		} else {
 			numNibbles, extListRlpBytes, extValues = prepareExtensions(extNibblesC, extensionNodeInd, proof2[len2-3], proof2[len2-3])
 		}
 		numberOfNibbles = int(numNibbles)
-
-		if numberOfNibbles%2 == 0 {
-			if branchC16 == 1 {
-				branchC16 = 0
-				branchC1 = 1
-			} else {
-				branchC16 = 1
-				branchC1 = 0
-			}
-		}
 	}
 
 	/*
@@ -290,8 +272,7 @@ func addBranchAndPlaceholder(proof1, proof2,
 		driftedInd := getDriftedPosition(leafRow0, numberOfNibbles)
 
 		node = prepareBranchNode(proof1[len1-2], proof1[len1-2], extNode, extNode, extListRlpBytes, extValues,
-			key[keyIndex+numberOfNibbles], driftedInd,
-			branchC16, branchC1, false, true, isExtension)
+			key[keyIndex+numberOfNibbles], driftedInd, false, true, isExtension)
 
 		// We now get the first nibble of the leaf that was turned into branch.
 		// This first nibble presents the position of the leaf once it moved
@@ -303,9 +284,8 @@ func addBranchAndPlaceholder(proof1, proof2,
 		driftedInd := getDriftedPosition(leafRow0, numberOfNibbles)
 
 		node = prepareBranchNode(proof2[len2-2], proof2[len2-2], extNode, extNode, extListRlpBytes, extValues,
-			key[keyIndex+numberOfNibbles], driftedInd,
-			branchC16, branchC1, true, false, isExtension)
+			key[keyIndex+numberOfNibbles], driftedInd, true, false, isExtension)
 	}
 
-	return isModifiedExtNode, isExtension, numberOfNibbles, branchC16, node
+	return isModifiedExtNode, isExtension, numberOfNibbles, node
 }

mpt-witness-generator/witness/gen_witness_from_infura_blockchain_test.go

@@ -2302,3 +2302,34 @@ func TestWrongAccount(t *testing.T) {
 
 	prepareWitness("WrongAccount", trieModifications, statedb)
 }
+
+func TestStorageDoesNotExistOnlySProof(t *testing.T) {
+	blockNum := 2000003
+	blockNumberParent := big.NewInt(int64(blockNum))
+	blockHeaderParent := oracle.PrefetchBlock(blockNumberParent, true, nil)
+	database := state.NewDatabase(blockHeaderParent)
+	statedb, _ := state.New(blockHeaderParent.Root, database, nil)
+	addr := common.HexToAddress("0xcaac46d9bd68bffb533320545a90cd92c6e98e58")
+
+	// Implicitly create account:
+	trieMod1 := TrieModification{
+		Type:    BalanceChanged,
+		Balance: big.NewInt(98),
+		Address: addr,
+	}
+
+	key1 := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
+	// leave the same
+	val1 := common.Hash{}
+
+	trieMod2 := TrieModification{
+		Type:    StorageDoesNotExist,
+		Key:     key1,
+		Value:   val1,
+		Address: addr,
+	}
+
+	trieModifications := []TrieModification{trieMod1, trieMod2}
+
+	prepareWitness("StorageDoesNotExistOnlySProof", trieModifications, statedb)
+}

mpt-witness-generator/witness/leaf.go

@@ -14,7 +14,7 @@ func prepareEmptyNonExistingStorageRow() []byte {
 	return nonExistingStorageRow
 }
 
-func prepareNonExistingStorageRow(leafC, keyNibbles []byte, noLeaf bool) ([]byte, []byte) {
+func prepareNonExistingStorageRow(leafC, keyNibbles []byte) ([]byte, []byte) {
 	// nonExistingStorageRow is used only for proof that nothing is stored at a particular storage key
 	nonExistingStorageRow := prepareEmptyNonExistingStorageRow()
 
@@ -307,7 +307,7 @@ func prepareAccountLeafNode(addr common.Address, addrh []byte, leafS, leafC, nei
 
 // prepareLeafAndPlaceholderNode prepares a leaf node and its placeholder counterpart
 // (used when one of the proofs does not have a leaf).
-func prepareLeafAndPlaceholderNode(addr common.Address, addrh []byte, proof1, proof2 [][]byte, storage_key common.Hash, key []byte, nonExistingAccountProof, isAccountProof, isSModExtension, isCModExtension bool) Node {
+func prepareLeafAndPlaceholderNode(addr common.Address, addrh []byte, proof1, proof2 [][]byte, storage_key common.Hash, key []byte, isAccountProof, isSModExtension, isCModExtension bool) Node {
 	len1 := len(proof1)
 	len2 := len(proof2)
 
@@ -407,7 +407,10 @@ func prepareAccountLeafPlaceholderNode(addr common.Address, addrh, key []byte, k
 }
 
 func prepareStorageLeafPlaceholderNode(storage_key common.Hash, key []byte, keyIndex int) Node {
-	leaf := make([]byte, valueLen)
+	// valueLen + 1 because the placeholder leaf in the empty trie occupies 35 bytes:
+	// [227 161 32 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+	// 33 (33 = 161 - 128) bytes for path, as in the example above
+	leaf := make([]byte, valueLen+1)
 	setStorageLeafKeyRLP(&leaf, key, keyIndex)
 	keyLen := getLeafKeyLen(keyIndex)
 	leaf[0] = 192 + 1 + byte(keyLen) + 1
@@ -529,8 +532,7 @@ func prepareStorageLeafNode(leafS, leafC, neighbourNode []byte, storage_key comm
 	var nonExistingStorageRow []byte
 	var wrongRlpBytes []byte
 	if nonExistingStorageProof {
-		noLeaf := false
-		wrongRlpBytes, nonExistingStorageRow = prepareNonExistingStorageRow(leafC, key, noLeaf)
+		wrongRlpBytes, nonExistingStorageRow = prepareNonExistingStorageRow(leafC, key)
 	} else {
 		nonExistingStorageRow = prepareEmptyNonExistingStorageRow()
 	}

mpt-witness-generator/witness/modified_extension_node.go

@@ -15,7 +15,7 @@ func equipLeafWithModExtensionNode(statedb *state.StateDB, leafNode Node, addr c
 	key, neighbourNode []byte,
 	keyIndex, extensionNodeInd, numberOfNibbles int,
 	additionalBranch, isAccountProof, nonExistingAccountProof,
-	isShorterProofLastLeaf bool, branchC16, branchC1 byte, toBeHashed *[][]byte) Node {
+	isShorterProofLastLeaf bool, toBeHashed *[][]byte) Node {
 	len1 := len(proof1)
 	len2 := len(proof2)
 

mpt-witness-generator/witness/prepare_witness.go

@@ -369,11 +369,18 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
 
 	var nodes []Node
 
-	branchC16 := byte(0)
-	branchC1 := byte(1)
 	for i := 0; i < upTo; i++ {
 		if !isBranch(proof1[i]) {
-			if i != len1-1 { // extension node
+			isNonExistingProof := (isAccountProof && nonExistingAccountProof) || (!isAccountProof && nonExistingStorageProof)
+			areThereNibbles := len(extNibblesS) != 0 || len(extNibblesC) != 0
+			// If i < upTo-1, it means it's not a leaf, so it's an extension node.
+			// There is no any special relation between isNonExistingProof and isExtension,
+			// except that in the non-existing proof the extension node can appear in `i == upTo-1`.
+			// For non-existing proof, the last node in the proof could be an extension node (we have
+			// nil in the underlying branch). For the non-existing proof with the wrong leaf
+			// (non-existing proofs can be with a nil leaf or with a wrong leaf),
+			// we don't need to worry because it appears in i = upTo-1).
+			if (i != upTo-1) || (areThereNibbles && isNonExistingProof) { // extension node
 				var numberOfNibbles byte
 				isExtension = true
 				numberOfNibbles, extListRlpBytes, extValues = prepareExtensions(extNibblesS, extensionNodeInd, proof1[i], proof2[i])
@@ -393,27 +400,6 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
 
 			nodes = append(nodes, node)
 		} else {
-			switchC16 := true // If not extension node, switchC16 = true.
-			if isExtension {
-				keyLen := getExtensionNodeKeyLen(proof1[i-1])
-				if keyLen == 1 {
-					switchC16 = false
-				} else {
-					if proof1[i-1][2] != 0 { // If even, switch16 = true.
-						switchC16 = false
-					}
-				}
-			}
-			if switchC16 {
-				if branchC16 == 1 {
-					branchC16 = 0
-					branchC1 = 1
-				} else {
-					branchC16 = 1
-					branchC1 = 0
-				}
-			}
-
 			var extNode1 []byte = nil
 			var extNode2 []byte = nil
 			if isExtension {
@@ -422,7 +408,7 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
 			}
 
 			bNode := prepareBranchNode(proof1[i], proof2[i], extNode1, extNode2, extListRlpBytes, extValues,
-				key[keyIndex], key[keyIndex], branchC16, branchC1, false, false, isExtension)
+				key[keyIndex], key[keyIndex], false, false, isExtension)
 			nodes = append(nodes, bNode)
 
 			keyIndex += 1
@@ -438,10 +424,10 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
 				leafRow0 = proof2[len2-1]
 			}
 
-			isModifiedExtNode, _, numberOfNibbles, branchC16, bNode := addBranchAndPlaceholder(proof1, proof2, extNibblesS, extNibblesC,
+			isModifiedExtNode, _, numberOfNibbles, bNode := addBranchAndPlaceholder(proof1, proof2, extNibblesS, extNibblesC,
 				leafRow0, key, neighbourNode,
 				keyIndex, extensionNodeInd, additionalBranch,
-				isAccountProof, nonExistingAccountProof, isShorterProofLastLeaf, branchC16, branchC1, &toBeHashed)
+				isAccountProof, nonExistingAccountProof, isShorterProofLastLeaf, &toBeHashed)
 
 			nodes = append(nodes, bNode)
 
@@ -458,7 +444,7 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
 					} else {
 						isCModExtension = true
 					}
-					leafNode = prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, nonExistingAccountProof, isAccountProof, isSModExtension, isCModExtension)
+					leafNode = prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, isAccountProof, isSModExtension, isCModExtension)
 				}
 			} else {
 				// Add storage leaf after branch placeholder
@@ -472,7 +458,7 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
 					} else {
 						isCModExtension = true
 					}
-					leafNode = prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, nonExistingAccountProof, isAccountProof, isSModExtension, isCModExtension)
+					leafNode = prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, isAccountProof, isSModExtension, isCModExtension)
 				}
 			}
 
@@ -482,14 +468,14 @@ func convertProofToWitness(statedb *state.StateDB, addr common.Address, addrh []
 			if isModifiedExtNode {
 				leafNode = equipLeafWithModExtensionNode(statedb, leafNode, addr, proof1, proof2, extNibblesS, extNibblesC, key, neighbourNode,
 					keyIndex, extensionNodeInd, numberOfNibbles, additionalBranch,
-					isAccountProof, nonExistingAccountProof, isShorterProofLastLeaf, branchC16, branchC1, &toBeHashed)
+					isAccountProof, nonExistingAccountProof, isShorterProofLastLeaf, &toBeHashed)
 			}
 			nodes = append(nodes, leafNode)
 		} else {
-			node := prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, nonExistingAccountProof, isAccountProof, false, false)
+			node := prepareLeafAndPlaceholderNode(addr, addrh, proof1, proof2, storage_key, key, isAccountProof, false, false)
 			nodes = append(nodes, node)
 		}
-	} else if isBranch(proof2[len(proof2)-1]) {
+	} else if (len1 == 0 && len2 == 0) || isBranch(proof2[len(proof2)-1]) {
 		// Account proof has drifted leaf as the last row, storage proof has non-existing-storage row
 		// as the last row.
 		// When non existing proof and only the branches are returned, we add a placeholder leaf.