trie -> new CP mechanism: added DB revert operations for put, del and batch, added DB checkpointing tests

Author: holgerd77

packages/trie/src/checkpointTrie.ts

@@ -50,7 +50,7 @@ export class CheckpointTrie extends BaseTrie {
 
     await this.lock.wait()
     // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-    this.root = this.db.revert()
+    this.root = await this.db.revert()
     this.lock.signal()
   }
 

packages/trie/src/db.ts

@@ -5,7 +5,7 @@ export const ENCODING_OPTS = { keyEncoding: 'binary', valueEncoding: 'binary' }
 
 export type Checkpoint = {
   root: Buffer
-  operations: any[]
+  revertOps: BatchDBOp[]
 }
 
 export type BatchDBOp = PutBatch | DelBatch
@@ -40,30 +40,47 @@ export class DB {
     this._leveldb = leveldb || level()
   }
 
+  /**
+   * Is the DB during a checkpoint phase?
+   */
+  get isCheckpoint() {
+    return this.checkpoints.length > 0
+  }
+
   /**
    * Adds a new checkpoint to the stack
    * @param root
    */
   checkpoint(root: Buffer) {
-    this.checkpoints.push({ root, operations: [] })
+    this.checkpoints.push({ root, revertOps: [] })
   }
 
   /**
    * Commits the latest checkpoint
    */
   commit() {
-    const { root } = this.checkpoints.pop()!
+    const { root, revertOps } = this.checkpoints.pop()!
+    // On nested checkpoints put the revertOps on the parent
+    // stack in case there is a revert
+    if (this.isCheckpoint) {
+      this.checkpoints[this.checkpoints.length - 1].revertOps.concat(revertOps)
+    }
     return root
   }
 
   /**
    * Reverts the latest checkpoint
    */
-  revert() {
-    const { root } = this.checkpoints.pop()!
+  async revert() {
+    const { root, revertOps } = this.checkpoints.pop()!
+    await this.batch(revertOps.reverse())
     return root
   }
 
+  private addCPRevertOperation(op: BatchDBOp) {
+    this.checkpoints[this.checkpoints.length - 1].revertOps.push(op)
+  }
+
   /**
    * Retrieves a raw value from leveldb.
    * @param key
@@ -89,23 +106,79 @@ export class DB {
    * @param value The value to be stored
    */
   async put(key: Buffer, val: Buffer): Promise<void> {
+    const revertOps: BatchDBOp[] = []
+    // In CP mode check for an old value to be put
+    // on the revert ops stack
+    if (this.isCheckpoint) {
+      const oldValue = await this.get(key)
+      if (oldValue !== null) {
+        revertOps.push({
+          type: 'put',
+          key: key,
+          value: oldValue,
+        })
+      }
+    }
     await this._leveldb.put(key, val, ENCODING_OPTS)
+    // In CP mode add del to the revert ops stack
+    if (this.isCheckpoint) {
+      revertOps.push({
+        type: 'del',
+        key: key,
+      })
+      for (const revertOp of revertOps) {
+        this.addCPRevertOperation(revertOp)
+      }
+    }
   }
 
   /**
    * Removes a raw value in the underlying leveldb.
    * @param keys
    */
   async del(key: Buffer): Promise<void> {
+    const revertOps: BatchDBOp[] = []
+    // In CP mode check for an old value to be put
+    // on the revert ops stack
+    if (this.isCheckpoint) {
+      const oldValue = await this.get(key)
+      if (oldValue !== null) {
+        revertOps.push({
+          type: 'put',
+          key: key,
+          value: oldValue,
+        })
+      }
+    }
     await this._leveldb.del(key, ENCODING_OPTS)
+    if (this.isCheckpoint) {
+      for (const revertOp of revertOps) {
+        this.addCPRevertOperation(revertOp)
+      }
+    }
   }
 
   /**
    * Performs a batch operation on db.
    * @param opStack A stack of levelup operations
    */
   async batch(opStack: BatchDBOp[]): Promise<void> {
-    await this._leveldb.batch(opStack, ENCODING_OPTS)
+    if (this.isCheckpoint) {
+      for (const op of opStack) {
+        if (op.type === 'put') {
+          // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+          if (!op.value) {
+            continue
+          }
+          await this.put(op.key, op.value)
+          // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+        } else if (op.type === 'del') {
+          await this.del(op.key)
+        }
+      }
+    } else {
+      await this._leveldb.batch(opStack, ENCODING_OPTS)
+    }
   }
 
   /**

packages/trie/test/checkpoint.spec.ts

@@ -104,4 +104,18 @@ tape('testing checkpoints', function (tester) {
     t.equal(trie.root.toString('hex'), root.toString('hex'))
     t.end()
   })
+
+  it('revert -> put', async function (t) {
+    trie = new CheckpointTrie()
+
+    trie.checkpoint()
+    const k1 = Buffer.from('k1')
+    const v1 = Buffer.from('v1')
+    await trie.put(k1, v1)
+    t.deepEqual(await trie.get(k1), v1, 'before revert: v1 in trie')
+    await trie.revert()
+    t.deepEqual(await trie.get(k1), null, 'after revert: v1 removed')
+
+    t.end()
+  })
 })

packages/trie/test/db.spec.ts

@@ -1,29 +1,31 @@
 import tape from 'tape'
 import { DB, BatchDBOp } from '../src/db'
 
-tape('DB basic functionality', (t) => {
+tape('DB tests', (t) => {
   const db = new DB()
 
-  const k = Buffer.from('foo')
-  const v = Buffer.from('bar')
+  const k = Buffer.from('k1')
+  const v = Buffer.from('v1')
+  const k2 = Buffer.from('k2')
+  const v2 = Buffer.from('v2')
+  //const k3 = Buffer.from('k3')
+  const v3 = Buffer.from('v3')
 
-  t.test('puts and gets value', async (st) => {
+  t.test('Operations: puts and gets value', async (st) => {
     await db.put(k, v)
     const res = await db.get(k)
     st.ok(v.equals(res!))
     st.end()
   })
 
-  t.test('deletes value', async (st) => {
+  t.test('Operations: deletes value', async (st) => {
     await db.del(k)
     const res = await db.get(k)
     st.notOk(res)
     st.end()
   })
 
-  t.test('batch ops', async (st) => {
-    const k2 = Buffer.from('bar')
-    const v2 = Buffer.from('baz')
+  t.test('Operations: batch ops', async (st) => {
     const ops = [
       { type: 'put', key: k, value: v },
       { type: 'put', key: k2, value: v2 },
@@ -33,4 +35,68 @@ tape('DB basic functionality', (t) => {
     st.ok(v2.equals(res!))
     st.end()
   })
+
+  t.test('Checkpointing: revert -> put (add)', async (st) => {
+    const db = new DB()
+    db.checkpoint(Buffer.from('1', 'hex'))
+    await db.put(k, v)
+    st.deepEqual(await db.get(k), v, 'before revert: v1')
+    await db.revert()
+    st.deepEqual(await db.get(k), null, 'after revert: null')
+    st.end()
+  })
+
+  t.test('Checkpointing: revert -> put (update)', async (st) => {
+    const db = new DB()
+    await db.put(k, v)
+    st.deepEqual(await db.get(k), v, 'before CP: v1')
+    db.checkpoint(Buffer.from('1', 'hex'))
+    await db.put(k, v2)
+    await db.put(k, v3)
+    await db.revert()
+    st.deepEqual(await db.get(k), v, 'after revert: v1')
+    st.end()
+  })
+
+  t.test('Checkpointing: revert -> put (update) batched', async (st) => {
+    const db = new DB()
+    await db.put(k, v)
+    st.deepEqual(await db.get(k), v, 'before CP: v1')
+    db.checkpoint(Buffer.from('1', 'hex'))
+    const ops = [
+      { type: 'put', key: k, value: v2 },
+      { type: 'put', key: k, value: v3 },
+    ] as BatchDBOp[]
+    await db.batch(ops)
+    await db.revert()
+    st.deepEqual(await db.get(k), v, 'after revert: v1')
+    st.end()
+  })
+
+  t.test('Checkpointing: revert -> del', async (st) => {
+    const db = new DB()
+    await db.put(k, v)
+    st.deepEqual(await db.get(k), v, 'before CP: v1')
+    db.checkpoint(Buffer.from('1', 'hex'))
+    await db.del(k)
+    st.deepEqual(await db.get(k), null, 'before revert: null')
+    await db.revert()
+    st.deepEqual(await db.get(k), v, 'after revert: v1')
+    st.end()
+  })
+
+  t.test('Checkpointing: nested checkpoints -> commit -> revert', async (st) => {
+    const db = new DB()
+    await db.put(k, v)
+    st.deepEqual(await db.get(k), v, 'before CP: v1')
+    db.checkpoint(Buffer.from('1', 'hex'))
+    await db.put(k, v2)
+    db.checkpoint(Buffer.from('2', 'hex'))
+    await db.put(k, v3)
+    db.commit()
+    st.deepEqual(await db.get(k), v3, 'after commit (second CP): v3')
+    await db.revert()
+    st.deepEqual(await db.get(k), v, 'after revert (first CP): v1')
+    st.end()
+  })
 })

packages/vm/lib/state/stateManager.ts

@@ -161,7 +161,8 @@ export default class DefaultStateManager implements StateManager {
     const account = await this.getAccount(address)
     const storageTrie = this._trie.copy(false)
     storageTrie.root = account.stateRoot
-    storageTrie._checkpoints = []
+    //@ts-ignore
+    storageTrie.db.checkpoints = []
     return storageTrie
   }