From 2f8ed9835bfbad8e0cca3baf36bdcc673ec9f5c8 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 11 Sep 2019 17:09:26 +0200 Subject: [PATCH 01/18] Add turboproof generation and verification --- src/multiproof.ts | 315 +++++++++++++++++++++++++++++++++++++++++++++ test/multiproof.js | 180 ++++++++++++++++++++++++++ 2 files changed, 495 insertions(+) create mode 100644 src/multiproof.ts create mode 100644 test/multiproof.js diff --git a/src/multiproof.ts b/src/multiproof.ts new file mode 100644 index 0000000..6292899 --- /dev/null +++ b/src/multiproof.ts @@ -0,0 +1,315 @@ +import * as assert from 'assert' +import { decode, encode } from 'rlp' +import { keccak256 } from 'ethereumjs-util' +import { Trie } from './baseTrie' +import { BranchNode, ExtensionNode, LeafNode, EmbeddedNode, decodeRawNode } from './trieNode' +import { stringToNibbles, nibblesToBuffer, matchingNibbleLength } from './util/nibbles' +import { addHexPrefix } from './util/hex' +const promisify = require('util.promisify') + +export enum Opcode { + Branch = 0, + Hasher = 1, + Leaf = 2, + Extension = 3, + Add = 4 +} + +export enum NodeType { + Branch = 0, + Leaf = 1, + Extension = 2, + Hash = 3 +} + +export interface Instruction { + kind: Opcode + value: number | number[] +} + +export interface Multiproof { + hashes: Buffer[] + keyvals: Buffer[] + instructions: Instruction[] +} + +export function verifyMultiproof( + root: Buffer, + proof: Multiproof, +): boolean { + const stack: any[] = [] + + const leaves = proof.keyvals.map((l: Buffer) => decode(l)) + let leafIdx = 0 + let hashIdx = 0 + + for (const instr of proof.instructions) { + if (instr.kind === Opcode.Hasher) { + const h = proof.hashes[hashIdx++] + if (!h) { + throw new Error('Not enough hashes in multiproof') + } + stack.push([NodeType.Hash, [h, instr.value as number]]) + } else if (instr.kind === Opcode.Leaf) { + const l = leaves[leafIdx++] + if (!l) { + throw new Error('Expected leaf in multiproof') + } + // TODO: Nibble from prefix `digit` + // @ts-ignore + //stack.push([NodeType.Leaf, [l[0].slice(l[0].length - instr.value), l[1]]]) + // Disregard leaf operand + stack.push([NodeType.Leaf, [l[0], l[1]]]) + } else if (instr.kind === Opcode.Branch) { + const n = stack.pop() + if (!n) { + throw new Error('Stack underflow') + } + const children = new Array(16).fill(null) + children[instr.value as number] = n + stack.push([NodeType.Branch, children]) + } else if (instr.kind === Opcode.Extension) { + const n = stack.pop() + if (!n) { + throw new Error('Stack underflow') + } + stack.push([NodeType.Extension, [instr.value, n]]) + } else if (instr.kind === Opcode.Add) { + const n1 = stack.pop() + const n2 = stack.pop() + if (!n1 || !n2) { + throw new Error('Stack underflow') + } + assert(n2[0] === NodeType.Branch, 'expected branch node on stack') + assert(instr.value < 17) + n2[1][instr.value as number] = n1 + stack.push(n2) + } else { + throw new Error('Invalid opcode') + } + } + + const r = stack.pop() + if (!r) { + throw new Error('Expected root node on top of stack') + } + let h = hashTrie(r) + // Special case, if trie contains only one leaf + // and that leaf has length < 32 + if (h.length < 32) { + h = keccak256(encode(h)) + } + return h.equals(root) +} + +function hashTrie(node: any): Buffer { + const typ = node[0] + node = node[1] + if (typ === NodeType.Branch) { + const res = new Array(17).fill(Buffer.alloc(0)) + for (let i = 0; i < 16; i++) { + if (node[i] === null) { + continue + } + res[i] = hashTrie(node[i]) + } + const e = encode(res) + if (e.length > 32) { + return keccak256(e) + } else { + return e + } + } else if (typ === NodeType.Leaf) { + const e = encode(node) + if (e.length > 32) { + return keccak256(e) + } else { + return node + } + } else if (typ === NodeType.Hash) { + // TODO: What if it's an embedded node with length === 32? + // Maybe try decoding and if it fails assume it's a hash + if (node[0].length < 32) { + // Embedded node, decode to get correct serialization for parent node + return decode(node[0]) + } + return node[0] + } else if (typ === NodeType.Extension) { + const hashedNode = hashTrie(node[1]) + node = [nibblesToBuffer(addHexPrefix(node[0], false)), hashedNode] + const e = encode(node) + if (e.length > 32) { + return keccak256(e) + } else { + return e + } + } else { + throw new Error('Invalid node') + } +} + +export async function makeMultiproof(trie: Trie, keys: Buffer[]): Promise { + const proof: Multiproof = { + hashes: [], + keyvals: [], + instructions: [] + } + + if (keys.length === 0) { + // TODO: should add root as hash? + return proof + } + + const keysNibbles = [] + for (const k of keys) { + keysNibbles.push(stringToNibbles(k)) + } + + return _makeMultiproof(trie, trie.root, keysNibbles) +} + +async function _makeMultiproof(trie: Trie, rootHash: EmbeddedNode, keys: number[][]): Promise { + let proof: Multiproof = { + hashes: [], + keyvals: [], + instructions: [] + } + + let root + if (Buffer.isBuffer(rootHash)) { + root = await promisify(trie._lookupNode.bind(trie))(rootHash) + } else if (Array.isArray(rootHash)) { + // Embedded node + root = decodeRawNode(rootHash) + } else { + throw new Error('Unexpected root') + } + + if (root instanceof BranchNode) { + // Truncate first nibble of keys + const table = new Array(16).fill(undefined) + // Group target keys based by their first nibbles. + // Also implicitly sorts the keys. + for (const k of keys) { + const idx = k[0] + if (!table[idx]) table[idx] = [] + table[idx].push(k.slice(1)) + } + + let addBranchOp = true + for (let i = 0; i < 16; i++) { + if (table[i] === undefined) { + // Empty subtree, hash it and add a HASHER op + const child = root.getBranch(i) + if (child) { + proof.instructions.push({ kind: Opcode.Hasher, value: 0 }) + // TODO: Make sure child is a hash + // what to do if embedded? + if (Buffer.isBuffer(child)) { + proof.hashes.push(child) + } else if (Array.isArray(child)) { + proof.hashes.push(encode(child)) + } else { + throw new Error('Invalid branch child') + } + if (addBranchOp) { + proof.instructions.push({ kind: Opcode.Branch, value: i }) + addBranchOp = false + } else { + proof.instructions.push({ kind: Opcode.Add, value: i }) + } + } + } else { + const child = root.getBranch(i) as Buffer + const p = await _makeMultiproof(trie, child, table[i]) + proof.hashes.push(...p.hashes) + proof.keyvals.push(...p.keyvals) + proof.instructions.push(...p.instructions) + + if (addBranchOp) { + proof.instructions.push({ kind: Opcode.Branch, value: i }) + addBranchOp = false + } else { + proof.instructions.push({ kind: Opcode.Add, value: i }) + } + } + } + } else if (root instanceof ExtensionNode) { + const extkey = root.key + // Make sure all keys follow the extension node + // and truncate them. + for (let i = 0; i < keys.length; i++) { + const k = keys[i] + if (matchingNibbleLength(k, extkey) !== extkey.length) { + throw new Error('key doesn\'t follow extension') + } + keys[i] = k.slice(extkey.length) + } + const p = await _makeMultiproof(trie, root.value, keys) + proof.hashes.push(...p.hashes) + proof.keyvals.push(...p.keyvals) + proof.instructions.push(...p.instructions) + proof.instructions.push({ kind: Opcode.Extension, value: extkey }) + } else if (root instanceof LeafNode) { + if (keys.length !== 1) { + throw new Error('Expected 1 remaining key') + } + // TODO: Check key matches leaf's key + proof = { + hashes: [], + keyvals: [root.serialize()], + instructions: [{ kind: Opcode.Leaf, value: root.key.length }] + } + } else { + throw new Error('Unexpected node type') + } + + return proof +} + +export function decodeMultiproof(raw: Buffer): Multiproof { + const dec = decode(raw) + assert(dec.length === 3) + + return { + // @ts-ignore + hashes: dec[0], + // @ts-ignore + keyvals: dec[1], + // @ts-ignore + instructions: decodeInstructions(dec[2]) + } +} + +export function decodeInstructions(instructions: Buffer[][]) { + const res = [] + for (const op of instructions) { + switch (bufToU8(op[0])) { + case Opcode.Branch: + res.push({ kind: Opcode.Branch, value: bufToU8(op[1]) }) + break + case Opcode.Hasher: + res.push({ kind: Opcode.Hasher, value: bufToU8(op[1]) }) + break + case Opcode.Leaf: + res.push({ kind: Opcode.Leaf, value: bufToU8(op[1]) }) + break + case Opcode.Extension: + // @ts-ignore + res.push({ kind: Opcode.Extension, value: op[1].map((v) => bufToU8(v)) }) + break + case Opcode.Add: + res.push({ kind: Opcode.Add, value: bufToU8(op[1]) }) + break + } + } + return res +} + +function bufToU8(b: Buffer): number { + // RLP decoding of 0 is empty buffer + if (b.length === 0) { + return 0 + } + return b.readUInt8(0) +} diff --git a/test/multiproof.js b/test/multiproof.js new file mode 100644 index 0000000..c4d094d --- /dev/null +++ b/test/multiproof.js @@ -0,0 +1,180 @@ +const tape = require('tape') +const promisify = require('util.promisify') +const rlp = require('rlp') +const { decodeMultiproof, decodeInstructions, verifyMultiproof, makeMultiproof, Instruction, Opcode } = require('../dist/multiproof') +const { Trie } = require('../dist/baseTrie') +const { LeafNode } = require('../dist/trieNode') +const { stringToNibbles } = require('../dist/util/nibbles') + +tape('decode instructions', (t) => { + const raw = Buffer.from('d0c20201c20405c603c403030303c28006', 'hex') + const expected = [ + { kind: Opcode.Leaf, value: 1 }, + { kind: Opcode.Add, value: 5 }, + { kind: Opcode.Extension, value: [3, 3, 3, 3] }, + { kind: Opcode.Branch, value: 6 } + ] + const res = decodeInstructions(rlp.decode(raw)) + t.deepEqual(expected, res) + t.end() +}) + +tape('decode multiproof', (t) => { + const raw = Buffer.from('ebe1a00101010101010101010101010101010101010101010101010101010101010101c483c20102c3c20280', 'hex') + const expected = { + hashes: [Buffer.alloc(32, 1)], + instructions: [{ kind: Opcode.Leaf, value: 0 }], + keyvals: [Buffer.from('c20102', 'hex')] + } + const proof = decodeMultiproof(raw) + t.deepEqual(expected, proof) + + t.end() +}) + +tape('multiproof tests', (t) => { + t.skip('hash before nested nodes in branch', (st) => { + // TODO: Replace with valid multiproof + const raw = Buffer.from('f876e1a01bbb8445ba6497d9a4642a114cb06b3a61ea8e49ca3853991b4f07b7e1e04892f845b843f8419f02020202020202020202020202020202020202020202020202020202020202a00000000000000000000000000000000000000000000000000000000000000000ccc20180c28001c2021fc20402', 'hex') + const expectedRoot = Buffer.from('0d76455583723bb10c56d34cfad1fb218e692299ae2edb5dd56a950f7062a6e0', 'hex') + const expectedInstructions = [ + { kind: Opcode.Hasher, value: 0 }, + { kind: Opcode.Branch, value: 1 }, + { kind: Opcode.Leaf, value: 31 }, + { kind: Opcode.Add, value: 2 }, + ] + const proof = decodeMultiproof(raw) + st.deepEqual(proof.instructions, expectedInstructions) + st.assert(verifyMultiproof(expectedRoot, proof)) + st.end() + }) + + t.skip('two values', (st) => { + // TODO: Replace with valid multiproof + const raw = Buffer.from('f8c1e1a09afbad9ae00ded5a066bd6f0ec67a45d51f31c258066b997e9bb8336bc13eba8f88ab843f8419f01010101010101010101010101010101010101010101010101010101010101a00101010101010101010101010101010101010101010101010101010101010101b843f8419f02020202020202020202020202020202020202020202020202020202020202a00000000000000000000000000000000000000000000000000000000000000000d2c2021fc28001c2021fc20402c20180c20408', 'hex') + const expectedRoot = Buffer.from('32291409ceb27a3b68b6beff58cfc41c084c0bde9e6aca03a20ce9aa795bb248', 'hex') + const expectedInstructions = [ + { kind: Opcode.Leaf, value: 31 }, + { kind: Opcode.Branch, value: 1 }, + { kind: Opcode.Leaf, value: 31 }, + { kind: Opcode.Add, value: 2 }, + { kind: Opcode.Hasher, value: 0 }, + { kind: Opcode.Add, value: 8 } + ] + const proof = decodeMultiproof(raw) + st.deepEqual(proof.instructions, expectedInstructions) + st.assert(verifyMultiproof(expectedRoot, proof)) + st.end() + }) + + t.end() +}) + +tape('make multiproof', (t) => { + t.test('trie with one leaf', async (st) => { + const t = new Trie() + const put = promisify(t.put.bind(t)) + const lookupNode = promisify(t._lookupNode.bind(t)) + const key = Buffer.from('1'.repeat(40), 'hex') + await put(key, Buffer.from('ffff', 'hex')) + const leaf = await lookupNode(t.root) + + const proof = await makeMultiproof(t, [key]) + st.deepEqual(proof, { + hashes: [], + keyvals: [leaf.serialize()], + instructions: [{ kind: Opcode.Leaf, value: 40 }] + }) + st.assert(verifyMultiproof(t.root, proof)) + st.end() + }) + + t.test('prove one of two leaves in trie', async (st) => { + const t = new Trie() + const put = promisify(t.put.bind(t)) + const lookupNode = promisify(t._lookupNode.bind(t)) + const key1 = Buffer.from('1'.repeat(40), 'hex') + const key2 = Buffer.from('2'.repeat(40), 'hex') + await put(key1, Buffer.from('f'.repeat(64), 'hex')) + await put(key2, Buffer.from('e'.repeat(64), 'hex')) + + const proof = await makeMultiproof(t, [key1]) + st.equal(proof.hashes.length, 1) + st.equal(proof.keyvals.length, 1) + st.equal(proof.instructions.length, 4) + st.assert(verifyMultiproof(t.root, proof)) + st.end() + }) + + t.test('prove two of three leaves in trie', async (st) => { + const t = new Trie() + const put = promisify(t.put.bind(t)) + const lookupNode = promisify(t._lookupNode.bind(t)) + const key1 = Buffer.from('1'.repeat(40), 'hex') + const key2 = Buffer.from('2'.repeat(40), 'hex') + const key3 = Buffer.from('3'.repeat(40), 'hex') + await put(key1, Buffer.from('f'.repeat(64), 'hex')) + await put(key2, Buffer.from('e'.repeat(64), 'hex')) + await put(key3, Buffer.from('d'.repeat(64), 'hex')) + + const proof = await makeMultiproof(t, [key3, key1]) + st.assert(verifyMultiproof(t.root, proof)) + st.end() + }) + + t.test('prove two of three leaves (with extension) in trie', async (st) => { + const t = new Trie() + const put = promisify(t.put.bind(t)) + const lookupNode = promisify(t._lookupNode.bind(t)) + const key1 = Buffer.from('1'.repeat(40), 'hex') + const key2 = Buffer.from('2'.repeat(40), 'hex') + const key3 = Buffer.from('1'.repeat(10).concat('3'.repeat(30)), 'hex') + await put(key1, Buffer.from('f'.repeat(64), 'hex')) + await put(key2, Buffer.from('e'.repeat(64), 'hex')) + await put(key3, Buffer.from('d'.repeat(64), 'hex')) + + const proof = await makeMultiproof(t, [key3, key1]) + st.assert(verifyMultiproof(t.root, proof)) + st.end() + }) + + t.test('two embedded leaves in branch', async (st) => { + const t = new Trie() + const put = promisify(t.put.bind(t)) + const lookupNode = promisify(t._lookupNode.bind(t)) + const key1 = Buffer.from('1'.repeat(40), 'hex') + const key2 = Buffer.from('2'.repeat(40), 'hex') + await put(key1, Buffer.from('f'.repeat(4), 'hex')) + await put(key2, Buffer.from('e'.repeat(4), 'hex')) + + const proof = await makeMultiproof(t, [key1]) + st.assert(verifyMultiproof(t.root, proof)) + st.end() + }) +}) + +tape('multiproof generation/verification with official jeff tests', async (t) => { + const jsonTest = require('./fixture/trietest.json').tests.jeff + const inputs = jsonTest.in + const expect = jsonTest.root + + const trie = new Trie() + for (let input of inputs) { + for (i = 0; i < 2; i++) { + if (input[i] && input[i].slice(0, 2) === '0x') { + input[i] = Buffer.from(input[i].slice(2), 'hex') + } + } + await promisify(trie.put.bind(trie))(Buffer.from(input[0]), input[1]) + } + t.assert(trie.root.equals(Buffer.from(expect.slice(2), 'hex'))) + + const keys = [ + Buffer.from('000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1', 'hex'), + Buffer.from('000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6', 'hex'), + Buffer.from('0000000000000000000000000000000000000000000000000000000000000046', 'hex') + ] + const proof = await makeMultiproof(trie, keys) + t.assert(verifyMultiproof(trie.root, proof)) + t.end() +}) From f73657ee458330fbdc99c10325f4581c03fca804 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 11 Sep 2019 17:10:16 +0200 Subject: [PATCH 02/18] Fix linting error --- src/multiproof.ts | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index 6292899..7fa049d 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -1,7 +1,7 @@ import * as assert from 'assert' import { decode, encode } from 'rlp' import { keccak256 } from 'ethereumjs-util' -import { Trie } from './baseTrie' +import { Trie } from './baseTrie' import { BranchNode, ExtensionNode, LeafNode, EmbeddedNode, decodeRawNode } from './trieNode' import { stringToNibbles, nibblesToBuffer, matchingNibbleLength } from './util/nibbles' import { addHexPrefix } from './util/hex' @@ -12,14 +12,14 @@ export enum Opcode { Hasher = 1, Leaf = 2, Extension = 3, - Add = 4 + Add = 4, } export enum NodeType { Branch = 0, - Leaf = 1, - Extension = 2, - Hash = 3 + Leaf = 1, + Extension = 2, + Hash = 3, } export interface Instruction { @@ -33,10 +33,7 @@ export interface Multiproof { instructions: Instruction[] } -export function verifyMultiproof( - root: Buffer, - proof: Multiproof, -): boolean { +export function verifyMultiproof(root: Buffer, proof: Multiproof): boolean { const stack: any[] = [] const leaves = proof.keyvals.map((l: Buffer) => decode(l)) @@ -152,7 +149,7 @@ export async function makeMultiproof(trie: Trie, keys: Buffer[]): Promise { +async function _makeMultiproof( + trie: Trie, + rootHash: EmbeddedNode, + keys: number[][], +): Promise { let proof: Multiproof = { hashes: [], keyvals: [], - instructions: [] + instructions: [], } let root @@ -225,7 +226,7 @@ async function _makeMultiproof(trie: Trie, rootHash: EmbeddedNode, keys: number[ proof.hashes.push(...p.hashes) proof.keyvals.push(...p.keyvals) proof.instructions.push(...p.instructions) - + if (addBranchOp) { proof.instructions.push({ kind: Opcode.Branch, value: i }) addBranchOp = false @@ -241,7 +242,7 @@ async function _makeMultiproof(trie: Trie, rootHash: EmbeddedNode, keys: number[ for (let i = 0; i < keys.length; i++) { const k = keys[i] if (matchingNibbleLength(k, extkey) !== extkey.length) { - throw new Error('key doesn\'t follow extension') + throw new Error("key doesn't follow extension") } keys[i] = k.slice(extkey.length) } @@ -258,7 +259,7 @@ async function _makeMultiproof(trie: Trie, rootHash: EmbeddedNode, keys: number[ proof = { hashes: [], keyvals: [root.serialize()], - instructions: [{ kind: Opcode.Leaf, value: root.key.length }] + instructions: [{ kind: Opcode.Leaf, value: root.key.length }], } } else { throw new Error('Unexpected node type') @@ -277,7 +278,7 @@ export function decodeMultiproof(raw: Buffer): Multiproof { // @ts-ignore keyvals: dec[1], // @ts-ignore - instructions: decodeInstructions(dec[2]) + instructions: decodeInstructions(dec[2]), } } @@ -296,7 +297,7 @@ export function decodeInstructions(instructions: Buffer[][]) { break case Opcode.Extension: // @ts-ignore - res.push({ kind: Opcode.Extension, value: op[1].map((v) => bufToU8(v)) }) + res.push({ kind: Opcode.Extension, value: op[1].map(v => bufToU8(v)) }) break case Opcode.Add: res.push({ kind: Opcode.Add, value: bufToU8(op[1]) }) From cff57e683a2c280f044639b6c45eadbd18f497c5 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 11 Sep 2019 17:15:47 +0200 Subject: [PATCH 03/18] Add promisify dep, drop node v6 travis add v11,12 --- .travis.yml | 3 ++- package.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 984811d..6f6afc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,10 @@ language: node_js node_js: - - "6" - "8" - "9" - "10" + - "11" + - "12" env: - CXX=g++-4.8 services: diff --git a/package.json b/package.json index 63d7ace..c268448 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,8 @@ "level-ws": "^1.0.0", "readable-stream": "^3.0.6", "rlp": "^2.2.3", - "semaphore": ">=1.0.1" + "semaphore": ">=1.0.1", + "util.promisify": "^1.0.0" }, "devDependencies": { "@ethereumjs/config-nyc": "^1.1.1", From c0a9af159c92ca2d940a30da9ddf0ba3cafb2ba5 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 12 Sep 2019 15:52:22 +0200 Subject: [PATCH 04/18] Minor improvements --- src/multiproof.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index 7fa049d..e2e4fee 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -146,15 +146,12 @@ function hashTrie(node: any): Buffer { } export async function makeMultiproof(trie: Trie, keys: Buffer[]): Promise { - const proof: Multiproof = { - hashes: [], - keyvals: [], - instructions: [], - } - if (keys.length === 0) { - // TODO: should add root as hash? - return proof + return { + hashes: [trie.root], + keyvals: [], + instructions: [{ kind: Opcode.Hasher, value: 0 }] + } } const keysNibbles = [] @@ -222,6 +219,9 @@ async function _makeMultiproof( } } else { const child = root.getBranch(i) as Buffer + if (!child) { + throw new Error('Key not in trie') + } const p = await _makeMultiproof(trie, child, table[i]) proof.hashes.push(...p.hashes) proof.keyvals.push(...p.keyvals) @@ -242,7 +242,8 @@ async function _makeMultiproof( for (let i = 0; i < keys.length; i++) { const k = keys[i] if (matchingNibbleLength(k, extkey) !== extkey.length) { - throw new Error("key doesn't follow extension") + // TODO: Maybe allow proving non-existent keys + throw new Error('Key not in trie') } keys[i] = k.slice(extkey.length) } From cace42ab35d10836657059ece04c809ae1c309d9 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 12 Sep 2019 15:52:55 +0200 Subject: [PATCH 05/18] Fix linting error --- src/multiproof.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index e2e4fee..b2be78e 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -150,7 +150,7 @@ export async function makeMultiproof(trie: Trie, keys: Buffer[]): Promise Date: Thu, 12 Sep 2019 15:53:33 +0200 Subject: [PATCH 06/18] Fuzz turboproof against multiple official test suites --- test/multiproof.js | 104 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 85 insertions(+), 19 deletions(-) diff --git a/test/multiproof.js b/test/multiproof.js index c4d094d..1e60f84 100644 --- a/test/multiproof.js +++ b/test/multiproof.js @@ -1,8 +1,10 @@ const tape = require('tape') const promisify = require('util.promisify') const rlp = require('rlp') +const { keccak256 } = require('ethereumjs-util') const { decodeMultiproof, decodeInstructions, verifyMultiproof, makeMultiproof, Instruction, Opcode } = require('../dist/multiproof') const { Trie } = require('../dist/baseTrie') +const { SecureTrie } = require('../dist/secure') const { LeafNode } = require('../dist/trieNode') const { stringToNibbles } = require('../dist/util/nibbles') @@ -153,28 +155,92 @@ tape('make multiproof', (t) => { }) }) -tape('multiproof generation/verification with official jeff tests', async (t) => { - const jsonTest = require('./fixture/trietest.json').tests.jeff - const inputs = jsonTest.in - const expect = jsonTest.root +tape('fuzz multiproof generation/verification with official tests', async (t) => { + const trietest = require('./fixture/trietest.json').tests + const trietestSecure = require('./fixture/trietest_secureTrie.json').tests + const hexEncodedTests = require('./fixture/hex_encoded_securetrie_test.json').tests + // Inputs of hex encoded tests are objects instead of arrays + Object.keys(hexEncodedTests).map((k) => { + hexEncodedTests[k].in = Object.keys(hexEncodedTests[k].in).map((key) => [key, hexEncodedTests[k].in[key]]) + }) + const testCases = [ + { name: 'jeff', secure: false, input: trietest.jeff.in, root: trietest.jeff.root }, + { name: 'jeffSecure', secure: true, input: trietestSecure.jeff.in, root: trietestSecure.jeff.root }, + { name: 'test1', secure: true, input: hexEncodedTests.test1.in, root: hexEncodedTests.test1.root }, + { name: 'test2', secure: true, input: hexEncodedTests.test2.in, root: hexEncodedTests.test2.root }, + { name: 'test3', secure: true, input: hexEncodedTests.test3.in, root: hexEncodedTests.test3.root } + ] + for (const testCase of testCases) { + const testName = testCase.name + t.comment(testName) + const expect = Buffer.from(testCase.root.slice(2), 'hex') + // Clean inputs + let inputs = testCase.input.map((input) => { + for (i = 0; i < 2; i++) { + if (!input[i]) continue + if (input[i].slice(0, 2) === '0x') { + input[i] = Buffer.from(input[i].slice(2), 'hex') + } else { + input[i] = Buffer.from(input[i]) + } + } + return input + }) - const trie = new Trie() - for (let input of inputs) { - for (i = 0; i < 2; i++) { - if (input[i] && input[i].slice(0, 2) === '0x') { - input[i] = Buffer.from(input[i].slice(2), 'hex') + let trie + if (testCase.secure) { + trie = new SecureTrie() + } else { + trie = new Trie() + } + for (let input of inputs) { + await promisify(trie.put.bind(trie))(input[0], input[1]) + } + t.assert(trie.root.equals(expect)) + + const keyCombinations = getCombinations(inputs.map((i) => i[0])) + for (let combination of keyCombinations) { + // If using secure make sure to hash keys + if (testCase.secure) { + combination = combination.map((k) => keccak256(k)) + } + try { + const proof = await makeMultiproof(trie, combination) + t.assert(verifyMultiproof(trie.root, proof)) + } catch (e) { + if (e.message !== 'Key not in trie') { + t.fail(e) + } + t.comment('skipped combination because key is not in trie') } } - await promisify(trie.put.bind(trie))(Buffer.from(input[0]), input[1]) } - t.assert(trie.root.equals(Buffer.from(expect.slice(2), 'hex'))) - - const keys = [ - Buffer.from('000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1', 'hex'), - Buffer.from('000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6', 'hex'), - Buffer.from('0000000000000000000000000000000000000000000000000000000000000046', 'hex') - ] - const proof = await makeMultiproof(trie, keys) - t.assert(verifyMultiproof(trie.root, proof)) t.end() }) + +// Given array [a, b, c], produce combinations +// with all lengths [1, arr.length]: +// [[a], [b], [c], [a, b], [a, c], [b, c], [a, b, c]] +function getCombinations(arr) { + // Make sure there are no duplicates + for (let i = 0; i < arr.length; i++) { + for (let j = i + 1; j < arr.length; j++) { + if (arr[i].equals(arr[j])) { + arr.splice(j, 1) + } + } + } + + const res = [] + const numCombinations = Math.pow(2, arr.length) + for (let i = 0; i < numCombinations; i++) { + const tmp = [] + for (let j = 0; j < arr.length; j++) { + if ((i & Math.pow(2, j))) { + tmp.push(arr[j]) + } + } + res.push(tmp) + } + return res +} From 85a0d6e9d73c17212c1aa19bcf8f20435e43c6c0 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Fri, 13 Sep 2019 12:32:16 +0200 Subject: [PATCH 07/18] Fix mutability issue in multiproof test --- test/multiproof.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/multiproof.js b/test/multiproof.js index 1e60f84..5681357 100644 --- a/test/multiproof.js +++ b/test/multiproof.js @@ -156,9 +156,9 @@ tape('make multiproof', (t) => { }) tape('fuzz multiproof generation/verification with official tests', async (t) => { - const trietest = require('./fixture/trietest.json').tests - const trietestSecure = require('./fixture/trietest_secureTrie.json').tests - const hexEncodedTests = require('./fixture/hex_encoded_securetrie_test.json').tests + const trietest = Object.assign({}, require('./fixture/trietest.json').tests) + const trietestSecure = Object.assign({}, require('./fixture/trietest_secureTrie.json').tests) + const hexEncodedTests = Object.assign({}, require('./fixture/hex_encoded_securetrie_test.json').tests) // Inputs of hex encoded tests are objects instead of arrays Object.keys(hexEncodedTests).map((k) => { hexEncodedTests[k].in = Object.keys(hexEncodedTests[k].in).map((key) => [key, hexEncodedTests[k].in[key]]) @@ -176,15 +176,16 @@ tape('fuzz multiproof generation/verification with official tests', async (t) => const expect = Buffer.from(testCase.root.slice(2), 'hex') // Clean inputs let inputs = testCase.input.map((input) => { + const res = [null, null] for (i = 0; i < 2; i++) { if (!input[i]) continue if (input[i].slice(0, 2) === '0x') { - input[i] = Buffer.from(input[i].slice(2), 'hex') + res[i] = Buffer.from(input[i].slice(2), 'hex') } else { - input[i] = Buffer.from(input[i]) + res[i] = Buffer.from(input[i]) } } - return input + return res }) let trie From 949d9e85305b4cbe89568a0ec962182965c73529 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Fri, 13 Sep 2019 15:42:34 +0200 Subject: [PATCH 08/18] Exclude removed keys from multiproof fuzzer combinations --- src/multiproof.ts | 3 +++ test/multiproof.js | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index b2be78e..b0b5856 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -256,6 +256,9 @@ async function _makeMultiproof( if (keys.length !== 1) { throw new Error('Expected 1 remaining key') } + if (matchingNibbleLength(keys[0], root.key) !== root.key.length) { + throw new Error('Leaf key doesn\'t match target key') + } // TODO: Check key matches leaf's key proof = { hashes: [], diff --git a/test/multiproof.js b/test/multiproof.js index 5681357..90c8828 100644 --- a/test/multiproof.js +++ b/test/multiproof.js @@ -166,6 +166,7 @@ tape('fuzz multiproof generation/verification with official tests', async (t) => const testCases = [ { name: 'jeff', secure: false, input: trietest.jeff.in, root: trietest.jeff.root }, { name: 'jeffSecure', secure: true, input: trietestSecure.jeff.in, root: trietestSecure.jeff.root }, + { name: 'emptyValuesSecure', secure: true, input: trietestSecure.emptyValues.in, root: trietestSecure.emptyValues.root }, { name: 'test1', secure: true, input: hexEncodedTests.test1.in, root: hexEncodedTests.test1.root }, { name: 'test2', secure: true, input: hexEncodedTests.test2.in, root: hexEncodedTests.test2.root }, { name: 'test3', secure: true, input: hexEncodedTests.test3.in, root: hexEncodedTests.test3.root } @@ -174,6 +175,7 @@ tape('fuzz multiproof generation/verification with official tests', async (t) => const testName = testCase.name t.comment(testName) const expect = Buffer.from(testCase.root.slice(2), 'hex') + const removedKeys = {} // Clean inputs let inputs = testCase.input.map((input) => { const res = [null, null] @@ -185,6 +187,9 @@ tape('fuzz multiproof generation/verification with official tests', async (t) => res[i] = Buffer.from(input[i]) } } + if (res[1] === null) { + removedKeys[res[0].toString('hex')] = true + } return res }) @@ -199,7 +204,10 @@ tape('fuzz multiproof generation/verification with official tests', async (t) => } t.assert(trie.root.equals(expect)) - const keyCombinations = getCombinations(inputs.map((i) => i[0])) + // TODO: include keys that have been removed from trie + const keyCombinations = getCombinations( + inputs.map((i) => i[0]).filter((i) => removedKeys[i.toString('hex')] !== true) + ) for (let combination of keyCombinations) { // If using secure make sure to hash keys if (testCase.secure) { From 1d356ee8367974639aa66d0986d1d839da439090 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Fri, 13 Sep 2019 15:43:08 +0200 Subject: [PATCH 09/18] Fix linting error --- src/multiproof.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index b0b5856..80bb254 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -257,7 +257,7 @@ async function _makeMultiproof( throw new Error('Expected 1 remaining key') } if (matchingNibbleLength(keys[0], root.key) !== root.key.length) { - throw new Error('Leaf key doesn\'t match target key') + throw new Error("Leaf key doesn't match target key") } // TODO: Check key matches leaf's key proof = { From 78810c5711b2e0512079bcfc0bfdb44cca58e664 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 16 Sep 2019 11:50:02 +0200 Subject: [PATCH 10/18] Verify paths in verifyMultiproof --- src/multiproof.ts | 32 ++++++++++++++++++++++++++------ test/multiproof.js | 18 +++++++++++------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index 80bb254..2b10ac2 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -4,7 +4,7 @@ import { keccak256 } from 'ethereumjs-util' import { Trie } from './baseTrie' import { BranchNode, ExtensionNode, LeafNode, EmbeddedNode, decodeRawNode } from './trieNode' import { stringToNibbles, nibblesToBuffer, matchingNibbleLength } from './util/nibbles' -import { addHexPrefix } from './util/hex' +import { addHexPrefix, removeHexPrefix } from './util/hex' const promisify = require('util.promisify') export enum Opcode { @@ -33,12 +33,14 @@ export interface Multiproof { instructions: Instruction[] } -export function verifyMultiproof(root: Buffer, proof: Multiproof): boolean { +export function verifyMultiproof(root: Buffer, proof: Multiproof, keys: Buffer[]): boolean { const stack: any[] = [] const leaves = proof.keyvals.map((l: Buffer) => decode(l)) + assert(leaves.length === keys.length) let leafIdx = 0 let hashIdx = 0 + const paths = new Array(leaves.length).fill(undefined) for (const instr of proof.instructions) { if (instr.kind === Opcode.Hasher) { @@ -46,7 +48,7 @@ export function verifyMultiproof(root: Buffer, proof: Multiproof): boolean { if (!h) { throw new Error('Not enough hashes in multiproof') } - stack.push([NodeType.Hash, [h, instr.value as number]]) + stack.push([NodeType.Hash, [h, instr.value as number], []]) } else if (instr.kind === Opcode.Leaf) { const l = leaves[leafIdx++] if (!l) { @@ -56,7 +58,9 @@ export function verifyMultiproof(root: Buffer, proof: Multiproof): boolean { // @ts-ignore //stack.push([NodeType.Leaf, [l[0].slice(l[0].length - instr.value), l[1]]]) // Disregard leaf operand - stack.push([NodeType.Leaf, [l[0], l[1]]]) + stack.push([NodeType.Leaf, [l[0], l[1]], [leafIdx-1]]) + // @ts-ignore + paths[leafIdx-1] = removeHexPrefix(stringToNibbles(l[0])) } else if (instr.kind === Opcode.Branch) { const n = stack.pop() if (!n) { @@ -64,13 +68,19 @@ export function verifyMultiproof(root: Buffer, proof: Multiproof): boolean { } const children = new Array(16).fill(null) children[instr.value as number] = n - stack.push([NodeType.Branch, children]) + stack.push([NodeType.Branch, children, n[2].slice()]) + for (let i = 0; i < n[2].length; i++) { + paths[n[2][i]] = [instr.value as number, ...paths[n[2][i]]] + } } else if (instr.kind === Opcode.Extension) { const n = stack.pop() if (!n) { throw new Error('Stack underflow') } - stack.push([NodeType.Extension, [instr.value, n]]) + stack.push([NodeType.Extension, [instr.value, n], n[2].slice()]) + for (let i = 0; i < n[2].length; i++) { + paths[n[2][i]] = [...instr.value as number[], ...paths[n[2][i]]] + } } else if (instr.kind === Opcode.Add) { const n1 = stack.pop() const n2 = stack.pop() @@ -80,7 +90,11 @@ export function verifyMultiproof(root: Buffer, proof: Multiproof): boolean { assert(n2[0] === NodeType.Branch, 'expected branch node on stack') assert(instr.value < 17) n2[1][instr.value as number] = n1 + n2[2] = Array.from(new Set([...n1[2], ...n2[2]])) stack.push(n2) + for (let i = 0; i < n1[2].length; i++) { + paths[n1[2][i]] = [instr.value as number, ...paths[n1[2][i]]] + } } else { throw new Error('Invalid opcode') } @@ -96,6 +110,12 @@ export function verifyMultiproof(root: Buffer, proof: Multiproof): boolean { if (h.length < 32) { h = keccak256(encode(h)) } + + // Assuming sorted keys + for (let i = 0; i < paths.length; i++) { + const addr = nibblesToBuffer(paths[i]) + assert(addr.equals(keys[i])) + } return h.equals(root) } diff --git a/test/multiproof.js b/test/multiproof.js index 90c8828..12e35e7 100644 --- a/test/multiproof.js +++ b/test/multiproof.js @@ -87,7 +87,7 @@ tape('make multiproof', (t) => { keyvals: [leaf.serialize()], instructions: [{ kind: Opcode.Leaf, value: 40 }] }) - st.assert(verifyMultiproof(t.root, proof)) + st.assert(verifyMultiproof(t.root, proof, [key])) st.end() }) @@ -104,7 +104,7 @@ tape('make multiproof', (t) => { st.equal(proof.hashes.length, 1) st.equal(proof.keyvals.length, 1) st.equal(proof.instructions.length, 4) - st.assert(verifyMultiproof(t.root, proof)) + st.assert(verifyMultiproof(t.root, proof, [key1])) st.end() }) @@ -120,7 +120,7 @@ tape('make multiproof', (t) => { await put(key3, Buffer.from('d'.repeat(64), 'hex')) const proof = await makeMultiproof(t, [key3, key1]) - st.assert(verifyMultiproof(t.root, proof)) + st.assert(verifyMultiproof(t.root, proof, [key1, key3])) st.end() }) @@ -135,8 +135,10 @@ tape('make multiproof', (t) => { await put(key2, Buffer.from('e'.repeat(64), 'hex')) await put(key3, Buffer.from('d'.repeat(64), 'hex')) - const proof = await makeMultiproof(t, [key3, key1]) - st.assert(verifyMultiproof(t.root, proof)) + const keys = [key3, key1] + const proof = await makeMultiproof(t, keys) + keys.sort(Buffer.compare) + st.assert(verifyMultiproof(t.root, proof, keys)) st.end() }) @@ -150,7 +152,7 @@ tape('make multiproof', (t) => { await put(key2, Buffer.from('e'.repeat(4), 'hex')) const proof = await makeMultiproof(t, [key1]) - st.assert(verifyMultiproof(t.root, proof)) + st.assert(verifyMultiproof(t.root, proof, [key1])) st.end() }) }) @@ -215,7 +217,9 @@ tape('fuzz multiproof generation/verification with official tests', async (t) => } try { const proof = await makeMultiproof(trie, combination) - t.assert(verifyMultiproof(trie.root, proof)) + // Verification expects a sorted array of keys + combination.sort(Buffer.compare) + t.assert(verifyMultiproof(trie.root, proof, combination)) } catch (e) { if (e.message !== 'Key not in trie') { t.fail(e) From 4d4bd7fa822366834545fa49ef5d6ad39e0022ac Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 16 Sep 2019 11:50:35 +0200 Subject: [PATCH 11/18] Fix linting error --- src/multiproof.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index 2b10ac2..214ba57 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -58,9 +58,9 @@ export function verifyMultiproof(root: Buffer, proof: Multiproof, keys: Buffer[] // @ts-ignore //stack.push([NodeType.Leaf, [l[0].slice(l[0].length - instr.value), l[1]]]) // Disregard leaf operand - stack.push([NodeType.Leaf, [l[0], l[1]], [leafIdx-1]]) + stack.push([NodeType.Leaf, [l[0], l[1]], [leafIdx - 1]]) // @ts-ignore - paths[leafIdx-1] = removeHexPrefix(stringToNibbles(l[0])) + paths[leafIdx - 1] = removeHexPrefix(stringToNibbles(l[0])) } else if (instr.kind === Opcode.Branch) { const n = stack.pop() if (!n) { @@ -79,7 +79,7 @@ export function verifyMultiproof(root: Buffer, proof: Multiproof, keys: Buffer[] } stack.push([NodeType.Extension, [instr.value, n], n[2].slice()]) for (let i = 0; i < n[2].length; i++) { - paths[n[2][i]] = [...instr.value as number[], ...paths[n[2][i]]] + paths[n[2][i]] = [...(instr.value as number[]), ...paths[n[2][i]]] } } else if (instr.kind === Opcode.Add) { const n1 = stack.pop() From 62ffb8a0f9ec5ada37b6730002adcc63426842f5 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 17 Sep 2019 14:48:00 +0200 Subject: [PATCH 12/18] Add multiproof encoding --- src/multiproof.ts | 14 ++++++++++++++ test/multiproof.js | 46 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index 214ba57..a4cee24 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -306,6 +306,20 @@ export function decodeMultiproof(raw: Buffer): Multiproof { } } +export function encodeMultiproof(proof: Multiproof): Buffer { + return encode(rawMultiproof(proof)) +} + +export function rawMultiproof(proof: Multiproof): any { + return [ + proof.hashes, + proof.keyvals, + proof.instructions.map((i) => { + return [i.kind, i.value] + }) + ] +} + export function decodeInstructions(instructions: Buffer[][]) { const res = [] for (const op of instructions) { diff --git a/test/multiproof.js b/test/multiproof.js index 12e35e7..45ff828 100644 --- a/test/multiproof.js +++ b/test/multiproof.js @@ -2,7 +2,7 @@ const tape = require('tape') const promisify = require('util.promisify') const rlp = require('rlp') const { keccak256 } = require('ethereumjs-util') -const { decodeMultiproof, decodeInstructions, verifyMultiproof, makeMultiproof, Instruction, Opcode } = require('../dist/multiproof') +const { decodeMultiproof, encodeMultiproof, decodeInstructions, verifyMultiproof, makeMultiproof, Instruction, Opcode } = require('../dist/multiproof') const { Trie } = require('../dist/baseTrie') const { SecureTrie } = require('../dist/secure') const { LeafNode } = require('../dist/trieNode') @@ -21,17 +21,41 @@ tape('decode instructions', (t) => { t.end() }) -tape('decode multiproof', (t) => { - const raw = Buffer.from('ebe1a00101010101010101010101010101010101010101010101010101010101010101c483c20102c3c20280', 'hex') - const expected = { - hashes: [Buffer.alloc(32, 1)], - instructions: [{ kind: Opcode.Leaf, value: 0 }], - keyvals: [Buffer.from('c20102', 'hex')] - } - const proof = decodeMultiproof(raw) - t.deepEqual(expected, proof) +tape('decode and encode multiproof', (t) => { + t.test('decode and encode one leaf', (st) => { + const raw = Buffer.from('ebe1a00101010101010101010101010101010101010101010101010101010101010101c483c20102c3c20280', 'hex') + const expected = { + hashes: [Buffer.alloc(32, 1)], + instructions: [{ kind: Opcode.Leaf, value: 0 }], + keyvals: [Buffer.from('c20102', 'hex')] + } + const proof = decodeMultiproof(raw) + st.deepEqual(expected, proof) - t.end() + const encoded = encodeMultiproof(expected) + st.assert(raw.equals(encoded)) + + st.end() + }) + + t.test('decode and encode two out of three leaves with extension', async (st) => { + const t = new Trie() + const put = promisify(t.put.bind(t)) + const lookupNode = promisify(t._lookupNode.bind(t)) + const key1 = Buffer.from('1'.repeat(40), 'hex') + const key2 = Buffer.from('2'.repeat(40), 'hex') + const key3 = Buffer.from('1'.repeat(10).concat('3'.repeat(30)), 'hex') + await put(key1, Buffer.from('f'.repeat(64), 'hex')) + await put(key2, Buffer.from('e'.repeat(64), 'hex')) + await put(key3, Buffer.from('d'.repeat(64), 'hex')) + + const keys = [key3, key1] + const proof = await makeMultiproof(t, keys) + const encoded = encodeMultiproof(proof) + const decoded = decodeMultiproof(encoded) + st.deepEqual(proof, decoded) + st.end() + }) }) tape('multiproof tests', (t) => { From e0280249b405e0f71bea1d0d3c3dbdea18281b90 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 17 Sep 2019 14:49:27 +0200 Subject: [PATCH 13/18] Fix linting error --- src/multiproof.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index a4cee24..f520a11 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -311,13 +311,7 @@ export function encodeMultiproof(proof: Multiproof): Buffer { } export function rawMultiproof(proof: Multiproof): any { - return [ - proof.hashes, - proof.keyvals, - proof.instructions.map((i) => { - return [i.kind, i.value] - }) - ] + return [proof.hashes, proof.keyvals, proof.instructions.map(i => [i.kind, i.value])] } export function decodeInstructions(instructions: Buffer[][]) { From 63b49a31d22aea6bbc1b0f5d2b499062050d8401 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 18 Sep 2019 14:46:30 +0200 Subject: [PATCH 14/18] Fix mutability issue in addHexPrefix and removeHexPrefix --- src/util/hex.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/util/hex.ts b/src/util/hex.ts index 17aa25a..c594db0 100644 --- a/src/util/hex.ts +++ b/src/util/hex.ts @@ -5,20 +5,21 @@ * @returns {Array} - returns buffer of encoded data **/ export function addHexPrefix(key: number[], terminator: boolean): number[] { + const res = key.slice() // odd - if (key.length % 2) { - key.unshift(1) + if (res.length % 2) { + res.unshift(1) } else { // even - key.unshift(0) - key.unshift(0) + res.unshift(0) + res.unshift(0) } if (terminator) { - key[0] += 2 + res[0] += 2 } - return key + return res } /** @@ -28,13 +29,15 @@ export function addHexPrefix(key: number[], terminator: boolean): number[] { * @private */ export function removeHexPrefix(val: number[]): number[] { - if (val[0] % 2) { - val = val.slice(1) + let res = val.slice() + + if (res[0] % 2) { + res = val.slice(1) } else { - val = val.slice(2) + res = val.slice(2) } - return val + return res } /** From ba2fc2e9b6ed5246ed865cb273100738fb871876 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 24 Sep 2019 11:56:55 +0200 Subject: [PATCH 15/18] Remove hasher's operand --- src/multiproof.ts | 17 ++++++++++------- test/multiproof.js | 7 ++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index f520a11..440f0ea 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -24,7 +24,7 @@ export enum NodeType { export interface Instruction { kind: Opcode - value: number | number[] + value?: number | number[] } export interface Multiproof { @@ -48,7 +48,7 @@ export function verifyMultiproof(root: Buffer, proof: Multiproof, keys: Buffer[] if (!h) { throw new Error('Not enough hashes in multiproof') } - stack.push([NodeType.Hash, [h, instr.value as number], []]) + stack.push([NodeType.Hash, [h], []]) } else if (instr.kind === Opcode.Leaf) { const l = leaves[leafIdx++] if (!l) { @@ -88,7 +88,7 @@ export function verifyMultiproof(root: Buffer, proof: Multiproof, keys: Buffer[] throw new Error('Stack underflow') } assert(n2[0] === NodeType.Branch, 'expected branch node on stack') - assert(instr.value < 17) + assert(instr.value as number < 17) n2[1][instr.value as number] = n1 n2[2] = Array.from(new Set([...n1[2], ...n2[2]])) stack.push(n2) @@ -170,7 +170,7 @@ export async function makeMultiproof(trie: Trie, keys: Buffer[]): Promise [i.kind, i.value])] + return [proof.hashes, proof.keyvals, proof.instructions.map(i => { + if (i.value !== undefined) return [i.kind, i.value] + return [i.kind] + })] } export function decodeInstructions(instructions: Buffer[][]) { @@ -322,7 +325,7 @@ export function decodeInstructions(instructions: Buffer[][]) { res.push({ kind: Opcode.Branch, value: bufToU8(op[1]) }) break case Opcode.Hasher: - res.push({ kind: Opcode.Hasher, value: bufToU8(op[1]) }) + res.push({ kind: Opcode.Hasher }) break case Opcode.Leaf: res.push({ kind: Opcode.Leaf, value: bufToU8(op[1]) }) diff --git a/test/multiproof.js b/test/multiproof.js index 45ff828..e9536e0 100644 --- a/test/multiproof.js +++ b/test/multiproof.js @@ -2,7 +2,7 @@ const tape = require('tape') const promisify = require('util.promisify') const rlp = require('rlp') const { keccak256 } = require('ethereumjs-util') -const { decodeMultiproof, encodeMultiproof, decodeInstructions, verifyMultiproof, makeMultiproof, Instruction, Opcode } = require('../dist/multiproof') +const { decodeMultiproof, rawMultiproof, encodeMultiproof, decodeInstructions, verifyMultiproof, makeMultiproof, Instruction, Opcode } = require('../dist/multiproof') const { Trie } = require('../dist/baseTrie') const { SecureTrie } = require('../dist/secure') const { LeafNode } = require('../dist/trieNode') @@ -33,6 +33,7 @@ tape('decode and encode multiproof', (t) => { st.deepEqual(expected, proof) const encoded = encodeMultiproof(expected) + console.log(encoded.toString('hex')) st.assert(raw.equals(encoded)) st.end() @@ -64,7 +65,7 @@ tape('multiproof tests', (t) => { const raw = Buffer.from('f876e1a01bbb8445ba6497d9a4642a114cb06b3a61ea8e49ca3853991b4f07b7e1e04892f845b843f8419f02020202020202020202020202020202020202020202020202020202020202a00000000000000000000000000000000000000000000000000000000000000000ccc20180c28001c2021fc20402', 'hex') const expectedRoot = Buffer.from('0d76455583723bb10c56d34cfad1fb218e692299ae2edb5dd56a950f7062a6e0', 'hex') const expectedInstructions = [ - { kind: Opcode.Hasher, value: 0 }, + { kind: Opcode.Hasher }, { kind: Opcode.Branch, value: 1 }, { kind: Opcode.Leaf, value: 31 }, { kind: Opcode.Add, value: 2 }, @@ -84,7 +85,7 @@ tape('multiproof tests', (t) => { { kind: Opcode.Branch, value: 1 }, { kind: Opcode.Leaf, value: 31 }, { kind: Opcode.Add, value: 2 }, - { kind: Opcode.Hasher, value: 0 }, + { kind: Opcode.Hasher }, { kind: Opcode.Add, value: 8 } ] const proof = decodeMultiproof(raw) From 1f2ffea554669e236778ac554a624d07bbcd43aa Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 24 Sep 2019 12:02:34 +0200 Subject: [PATCH 16/18] Remove leaf's operand --- src/multiproof.ts | 4 ++-- test/multiproof.js | 15 +++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index 440f0ea..b9c7b04 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -283,7 +283,7 @@ async function _makeMultiproof( proof = { hashes: [], keyvals: [root.serialize()], - instructions: [{ kind: Opcode.Leaf, value: root.key.length }], + instructions: [{ kind: Opcode.Leaf }], } } else { throw new Error('Unexpected node type') @@ -328,7 +328,7 @@ export function decodeInstructions(instructions: Buffer[][]) { res.push({ kind: Opcode.Hasher }) break case Opcode.Leaf: - res.push({ kind: Opcode.Leaf, value: bufToU8(op[1]) }) + res.push({ kind: Opcode.Leaf }) break case Opcode.Extension: // @ts-ignore diff --git a/test/multiproof.js b/test/multiproof.js index e9536e0..2c0d8cf 100644 --- a/test/multiproof.js +++ b/test/multiproof.js @@ -11,7 +11,7 @@ const { stringToNibbles } = require('../dist/util/nibbles') tape('decode instructions', (t) => { const raw = Buffer.from('d0c20201c20405c603c403030303c28006', 'hex') const expected = [ - { kind: Opcode.Leaf, value: 1 }, + { kind: Opcode.Leaf }, { kind: Opcode.Add, value: 5 }, { kind: Opcode.Extension, value: [3, 3, 3, 3] }, { kind: Opcode.Branch, value: 6 } @@ -23,17 +23,16 @@ tape('decode instructions', (t) => { tape('decode and encode multiproof', (t) => { t.test('decode and encode one leaf', (st) => { - const raw = Buffer.from('ebe1a00101010101010101010101010101010101010101010101010101010101010101c483c20102c3c20280', 'hex') + const raw = Buffer.from('eae1a00101010101010101010101010101010101010101010101010101010101010101c483c20102c2c102', 'hex') const expected = { hashes: [Buffer.alloc(32, 1)], - instructions: [{ kind: Opcode.Leaf, value: 0 }], + instructions: [{ kind: Opcode.Leaf }], keyvals: [Buffer.from('c20102', 'hex')] } const proof = decodeMultiproof(raw) st.deepEqual(expected, proof) const encoded = encodeMultiproof(expected) - console.log(encoded.toString('hex')) st.assert(raw.equals(encoded)) st.end() @@ -67,7 +66,7 @@ tape('multiproof tests', (t) => { const expectedInstructions = [ { kind: Opcode.Hasher }, { kind: Opcode.Branch, value: 1 }, - { kind: Opcode.Leaf, value: 31 }, + { kind: Opcode.Leaf }, { kind: Opcode.Add, value: 2 }, ] const proof = decodeMultiproof(raw) @@ -81,9 +80,9 @@ tape('multiproof tests', (t) => { const raw = Buffer.from('f8c1e1a09afbad9ae00ded5a066bd6f0ec67a45d51f31c258066b997e9bb8336bc13eba8f88ab843f8419f01010101010101010101010101010101010101010101010101010101010101a00101010101010101010101010101010101010101010101010101010101010101b843f8419f02020202020202020202020202020202020202020202020202020202020202a00000000000000000000000000000000000000000000000000000000000000000d2c2021fc28001c2021fc20402c20180c20408', 'hex') const expectedRoot = Buffer.from('32291409ceb27a3b68b6beff58cfc41c084c0bde9e6aca03a20ce9aa795bb248', 'hex') const expectedInstructions = [ - { kind: Opcode.Leaf, value: 31 }, + { kind: Opcode.Leaf }, { kind: Opcode.Branch, value: 1 }, - { kind: Opcode.Leaf, value: 31 }, + { kind: Opcode.Leaf }, { kind: Opcode.Add, value: 2 }, { kind: Opcode.Hasher }, { kind: Opcode.Add, value: 8 } @@ -110,7 +109,7 @@ tape('make multiproof', (t) => { st.deepEqual(proof, { hashes: [], keyvals: [leaf.serialize()], - instructions: [{ kind: Opcode.Leaf, value: 40 }] + instructions: [{ kind: Opcode.Leaf }] }) st.assert(verifyMultiproof(t.root, proof, [key])) st.end() From cb63d5643f3f17ae475ad1b7afc04064490ad183 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 24 Sep 2019 12:45:40 +0200 Subject: [PATCH 17/18] Add flat encoding for instructions --- src/multiproof.ts | 64 +++++++++++++++++++++++++++++++++++++++++----- test/multiproof.js | 45 +++++++++++++++++++++++--------- 2 files changed, 90 insertions(+), 19 deletions(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index b9c7b04..7054852 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -306,15 +306,65 @@ export function decodeMultiproof(raw: Buffer): Multiproof { } } -export function encodeMultiproof(proof: Multiproof): Buffer { - return encode(rawMultiproof(proof)) +export function encodeMultiproof(proof: Multiproof, flatInstructions: boolean = false): Buffer { + return encode(rawMultiproof(proof, flatInstructions)) } -export function rawMultiproof(proof: Multiproof): any { - return [proof.hashes, proof.keyvals, proof.instructions.map(i => { - if (i.value !== undefined) return [i.kind, i.value] - return [i.kind] - })] +export function rawMultiproof(proof: Multiproof, flatInstructions: boolean = false): any { + if (flatInstructions) { + return [proof.hashes, proof.keyvals, flatEncodeInstructions(proof.instructions)] + } else { + return [proof.hashes, proof.keyvals, proof.instructions.map(i => { + if (i.value !== undefined) return [i.kind, i.value] + return [i.kind] + })] + } +} + +export function flatEncodeInstructions(instructions: Instruction[]): Buffer { + const res: number[] = [] + for (const instr of instructions) { + res.push(instr.kind) + if (instr.kind === Opcode.Branch || instr.kind === Opcode.Add) { + res.push(instr.value as number) + } else if (instr.kind === Opcode.Extension) { + const nibbles = instr.value as number[] + res.push(nibbles.length) + res.push(...nibbles) + } + } + return Buffer.from(new Uint8Array(res)) +} + +export function flatDecodeInstructions(raw: Buffer): Instruction[] { + const res = [] + let i = 0 + while (i < raw.length) { + const op = raw[i++] + switch (op) { + case Opcode.Branch: + res.push({ kind: Opcode.Branch, value: raw[i++] }) + break + case Opcode.Hasher: + res.push({ kind: Opcode.Hasher }) + break + case Opcode.Leaf: + res.push({ kind: Opcode.Leaf }) + break + case Opcode.Extension: + const length = raw.readUInt8(i++) + const nibbles = [] + for (let j = 0; j < length; j++) { + nibbles.push(raw[i++]) + } + res.push({ kind: Opcode.Extension, value: nibbles }) + break + case Opcode.Add: + res.push({ kind: Opcode.Add, value: raw[i++] }) + break + } + } + return res } export function decodeInstructions(instructions: Buffer[][]) { diff --git a/test/multiproof.js b/test/multiproof.js index 2c0d8cf..095202d 100644 --- a/test/multiproof.js +++ b/test/multiproof.js @@ -2,23 +2,44 @@ const tape = require('tape') const promisify = require('util.promisify') const rlp = require('rlp') const { keccak256 } = require('ethereumjs-util') -const { decodeMultiproof, rawMultiproof, encodeMultiproof, decodeInstructions, verifyMultiproof, makeMultiproof, Instruction, Opcode } = require('../dist/multiproof') const { Trie } = require('../dist/baseTrie') const { SecureTrie } = require('../dist/secure') const { LeafNode } = require('../dist/trieNode') const { stringToNibbles } = require('../dist/util/nibbles') +const { + decodeMultiproof, rawMultiproof, encodeMultiproof, + decodeInstructions, flatEncodeInstructions, flatDecodeInstructions, + verifyMultiproof, makeMultiproof, Instruction, Opcode +} = require('../dist/multiproof') -tape('decode instructions', (t) => { - const raw = Buffer.from('d0c20201c20405c603c403030303c28006', 'hex') - const expected = [ - { kind: Opcode.Leaf }, - { kind: Opcode.Add, value: 5 }, - { kind: Opcode.Extension, value: [3, 3, 3, 3] }, - { kind: Opcode.Branch, value: 6 } - ] - const res = decodeInstructions(rlp.decode(raw)) - t.deepEqual(expected, res) - t.end() +tape('decode and encode instructions', (t) => { + t.test('rlp encoding', (st) => { + const raw = Buffer.from('d0c20201c20405c603c403030303c28006', 'hex') + const expected = [ + { kind: Opcode.Leaf }, + { kind: Opcode.Add, value: 5 }, + { kind: Opcode.Extension, value: [3, 3, 3, 3] }, + { kind: Opcode.Branch, value: 6 } + ] + const res = decodeInstructions(rlp.decode(raw)) + st.deepEqual(expected, res) + st.end() + }) + + t.test('flat encoding', (st) => { + const raw = Buffer.from('0204050304030303030006', 'hex') + const instructions = [ + { kind: Opcode.Leaf }, + { kind: Opcode.Add, value: 5 }, + { kind: Opcode.Extension, value: [3, 3, 3, 3] }, + { kind: Opcode.Branch, value: 6 } + ] + const encoded = flatEncodeInstructions(instructions) + st.assert(raw.equals(encoded)) + const decoded = flatDecodeInstructions(raw) + st.deepEqual(instructions, decoded) + st.end() + }) }) tape('decode and encode multiproof', (t) => { From d125f368fe2ceb4221c6e9c5ff780df0833e8376 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 24 Sep 2019 12:46:09 +0200 Subject: [PATCH 18/18] Fix linting error --- src/multiproof.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/multiproof.ts b/src/multiproof.ts index 7054852..0b25fd7 100644 --- a/src/multiproof.ts +++ b/src/multiproof.ts @@ -88,7 +88,7 @@ export function verifyMultiproof(root: Buffer, proof: Multiproof, keys: Buffer[] throw new Error('Stack underflow') } assert(n2[0] === NodeType.Branch, 'expected branch node on stack') - assert(instr.value as number < 17) + assert((instr.value as number) < 17) n2[1][instr.value as number] = n1 n2[2] = Array.from(new Set([...n1[2], ...n2[2]])) stack.push(n2) @@ -314,10 +314,14 @@ export function rawMultiproof(proof: Multiproof, flatInstructions: boolean = fal if (flatInstructions) { return [proof.hashes, proof.keyvals, flatEncodeInstructions(proof.instructions)] } else { - return [proof.hashes, proof.keyvals, proof.instructions.map(i => { - if (i.value !== undefined) return [i.kind, i.value] - return [i.kind] - })] + return [ + proof.hashes, + proof.keyvals, + proof.instructions.map(i => { + if (i.value !== undefined) return [i.kind, i.value] + return [i.kind] + }), + ] } }