Skip to content

Commit 1a26e39

Browse files
jochem-brouwerjochem-brouwer
committed
trie: partial checkpointing fix
1 parent 8e4f060 commit 1a26e39

File tree

4 files changed

+95
-98
lines changed

4 files changed

+95
-98
lines changed

‎packages/trie/src/checkpointDb.ts‎

Lines changed: 20 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,15 @@
11
import { LevelUp } from 'levelup'
2-
import { DB } from './db'
3-
const level = require('level-mem')
2+
import { DB, BatchDBOp } from './db'
43

54
export const ENCODING_OPTS = { keyEncoding: 'binary', valueEncoding: 'binary' }
65

76
export type Checkpoint = {
8-
keyValueMap: Map<Buffer, Buffer | null>,
7+
// We cannot use a Buffer => Buffer map directly. If you create two Buffers with the same internal value,
8+
// then when setting a value on the Map, it actually creates two indices.
9+
keyValueMap: Map<string, Buffer | null>
910
root: Buffer
1011
}
1112

12-
export type BatchDBOp = PutBatch | DelBatch
13-
export interface PutBatch {
14-
type: 'put'
15-
key: Buffer
16-
value: Buffer
17-
}
18-
export interface DelBatch {
19-
type: 'del'
20-
key: Buffer
21-
}
22-
2313
/**
2414
* DB is a thin wrapper around the underlying levelup db,
2515
* which validates inputs and sets encoding type.
@@ -50,7 +40,7 @@ export class CheckpointDB extends DB {
5040
* @param root
5141
*/
5242
checkpoint(root: Buffer) {
53-
this.checkpoints.push({ keyValueMap: new Map<Buffer, Buffer>(), root })
43+
this.checkpoints.push({ keyValueMap: new Map<string, Buffer>(), root })
5444
}
5545

5646
/**
@@ -61,38 +51,36 @@ export class CheckpointDB extends DB {
6151
if (!this.isCheckpoint) {
6252
// This was the final checkpoint, we should now commit and flush everything to disk
6353
const batchOp: BatchDBOp[] = []
64-
keyValueMap.forEach(function(value, key) {
54+
keyValueMap.forEach(function (value, key) {
6555
if (value === null) {
6656
batchOp.push({
6757
type: 'del',
68-
key
58+
key: Buffer.from(key),
6959
})
7060
} else {
7161
batchOp.push({
7262
type: 'put',
73-
key,
74-
value
63+
key: Buffer.from(key),
64+
value,
7565
})
7666
}
7767
})
7868
await this.batch(batchOp)
7969
} else {
8070
// dump everything into the current (higher level) cache
8171
const currentKeyValueMap = this.checkpoints[this.checkpoints.length - 1].keyValueMap
82-
keyValueMap.forEach(function(value, key){
72+
keyValueMap.forEach(function (value, key) {
8373
currentKeyValueMap.set(key, value)
8474
})
85-
8675
}
87-
8876
}
8977

9078
/**
9179
* Reverts the latest checkpoint
9280
*/
9381
async revert() {
94-
const { root } = this.checkpoints.pop()!
95-
return root
82+
const { root } = this.checkpoints.pop()!
83+
return root
9684
}
9785

9886
/**
@@ -102,14 +90,14 @@ export class CheckpointDB extends DB {
10290
*/
10391
async get(key: Buffer): Promise<Buffer | null> {
10492
// Lookup the value in our cache. We return the latest checkpointed value (which should be the value on disk)
105-
for (let index = this.checkpoints.length; index >= 0; index--) {
106-
const value = this.checkpoints[index].keyValueMap.get(key)
107-
if (value != undefined) {
93+
for (let index = this.checkpoints.length - 1; index >= 0; index--) {
94+
const value = this.checkpoints[index].keyValueMap.get(key.toString())
95+
if (value !== undefined) {
10896
return value
10997
}
11098
}
11199
// Nothing has been found in cache, look up from disk
112-
// TODO: put value in cache.
100+
// TODO: put value in cache if we are a checkpoint.
113101
return await super.get(key)
114102
}
115103

@@ -120,9 +108,9 @@ export class CheckpointDB extends DB {
120108
*/
121109
async put(key: Buffer, val: Buffer): Promise<void> {
122110
if (this.isCheckpoint) {
123-
// put value in cache
124-
this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(key, val)
125-
} else {
111+
// put value in cache
112+
this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(key.toString(), val)
113+
} else {
126114
await super.put(key, val)
127115
}
128116
}
@@ -134,7 +122,7 @@ export class CheckpointDB extends DB {
134122
async del(key: Buffer): Promise<void> {
135123
if (this.isCheckpoint) {
136124
// delete the value in the current cache
137-
this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(key, null)
125+
this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(key.toString(), null)
138126
} else {
139127
// delete the value on disk
140128
await this._leveldb.del(key, ENCODING_OPTS)

‎packages/trie/src/checkpointTrie.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { CheckpointDB } from './checkpointDb'
66
*/
77
export class CheckpointTrie extends BaseTrie {
88
db: CheckpointDB
9-
9+
1010
constructor(...args: any) {
1111
super(...args)
1212
this.db = new CheckpointDB(...args)

‎packages/trie/test/checkpointDb.spec.ts‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import tape from 'tape'
2+
import { CheckpointDB as DB } from '../src/checkpointDb'
3+
import { BatchDBOp } from '../src/db'
4+
5+
tape('DB tests', (t) => {
6+
const k = Buffer.from('k1')
7+
const v = Buffer.from('v1')
8+
const v2 = Buffer.from('v2')
9+
const v3 = Buffer.from('v3')
10+
11+
t.test('Checkpointing: revert -> put (add)', async (st) => {
12+
const db = new DB()
13+
db.checkpoint(Buffer.from('1', 'hex'))
14+
await db.put(k, v)
15+
st.deepEqual(await db.get(k), v, 'before revert: v1')
16+
await db.revert()
17+
st.deepEqual(await db.get(k), null, 'after revert: null')
18+
st.end()
19+
})
20+
21+
t.test('Checkpointing: revert -> put (update)', async (st) => {
22+
const db = new DB()
23+
await db.put(k, v)
24+
st.deepEqual(await db.get(k), v, 'before CP: v1')
25+
db.checkpoint(Buffer.from('1', 'hex'))
26+
await db.put(k, v2)
27+
await db.put(k, v3)
28+
await db.revert()
29+
st.deepEqual(await db.get(k), v, 'after revert: v1')
30+
st.end()
31+
})
32+
33+
t.test('Checkpointing: revert -> put (update) batched', async (st) => {
34+
const db = new DB()
35+
await db.put(k, v)
36+
st.deepEqual(await db.get(k), v, 'before CP: v1')
37+
db.checkpoint(Buffer.from('1', 'hex'))
38+
const ops = [
39+
{ type: 'put', key: k, value: v2 },
40+
{ type: 'put', key: k, value: v3 },
41+
] as BatchDBOp[]
42+
await db.batch(ops)
43+
await db.revert()
44+
st.deepEqual(await db.get(k), v, 'after revert: v1')
45+
st.end()
46+
})
47+
48+
t.test('Checkpointing: revert -> del', async (st) => {
49+
const db = new DB()
50+
await db.put(k, v)
51+
st.deepEqual(await db.get(k), v, 'before CP: v1')
52+
db.checkpoint(Buffer.from('1', 'hex'))
53+
await db.del(k)
54+
st.deepEqual(await db.get(k), null, 'before revert: null')
55+
await db.revert()
56+
st.deepEqual(await db.get(k), v, 'after revert: v1')
57+
st.end()
58+
})
59+
60+
t.test('Checkpointing: nested checkpoints -> commit -> revert', async (st) => {
61+
const db = new DB()
62+
await db.put(k, v)
63+
st.deepEqual(await db.get(k), v, 'before CP: v1')
64+
db.checkpoint(Buffer.from('1', 'hex'))
65+
await db.put(k, v2)
66+
db.checkpoint(Buffer.from('2', 'hex'))
67+
await db.put(k, v3)
68+
await db.commit()
69+
st.deepEqual(await db.get(k), v3, 'after commit (second CP): v3')
70+
await db.revert()
71+
st.deepEqual(await db.get(k), v, 'after revert (first CP): v1')
72+
st.end()
73+
})
74+
})

‎packages/trie/test/db.spec.ts‎

Lines changed: 0 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ tape('DB tests', (t) => {
88
const v = Buffer.from('v1')
99
const k2 = Buffer.from('k2')
1010
const v2 = Buffer.from('v2')
11-
const v3 = Buffer.from('v3')
1211

1312
t.test('Operations: puts and gets value', async (st) => {
1413
await db.put(k, v)
@@ -34,68 +33,4 @@ tape('DB tests', (t) => {
3433
st.ok(v2.equals(res!))
3534
st.end()
3635
})
37-
38-
t.test('Checkpointing: revert -> put (add)', async (st) => {
39-
const db = new DB()
40-
db.checkpoint(Buffer.from('1', 'hex'))
41-
await db.put(k, v)
42-
st.deepEqual(await db.get(k), v, 'before revert: v1')
43-
await db.revert()
44-
st.deepEqual(await db.get(k), null, 'after revert: null')
45-
st.end()
46-
})
47-
48-
t.test('Checkpointing: revert -> put (update)', async (st) => {
49-
const db = new DB()
50-
await db.put(k, v)
51-
st.deepEqual(await db.get(k), v, 'before CP: v1')
52-
db.checkpoint(Buffer.from('1', 'hex'))
53-
await db.put(k, v2)
54-
await db.put(k, v3)
55-
await db.revert()
56-
st.deepEqual(await db.get(k), v, 'after revert: v1')
57-
st.end()
58-
})
59-
60-
t.test('Checkpointing: revert -> put (update) batched', async (st) => {
61-
const db = new DB()
62-
await db.put(k, v)
63-
st.deepEqual(await db.get(k), v, 'before CP: v1')
64-
db.checkpoint(Buffer.from('1', 'hex'))
65-
const ops = [
66-
{ type: 'put', key: k, value: v2 },
67-
{ type: 'put', key: k, value: v3 },
68-
] as BatchDBOp[]
69-
await db.batch(ops)
70-
await db.revert()
71-
st.deepEqual(await db.get(k), v, 'after revert: v1')
72-
st.end()
73-
})
74-
75-
t.test('Checkpointing: revert -> del', async (st) => {
76-
const db = new DB()
77-
await db.put(k, v)
78-
st.deepEqual(await db.get(k), v, 'before CP: v1')
79-
db.checkpoint(Buffer.from('1', 'hex'))
80-
await db.del(k)
81-
st.deepEqual(await db.get(k), null, 'before revert: null')
82-
await db.revert()
83-
st.deepEqual(await db.get(k), v, 'after revert: v1')
84-
st.end()
85-
})
86-
87-
t.test('Checkpointing: nested checkpoints -> commit -> revert', async (st) => {
88-
const db = new DB()
89-
await db.put(k, v)
90-
st.deepEqual(await db.get(k), v, 'before CP: v1')
91-
db.checkpoint(Buffer.from('1', 'hex'))
92-
await db.put(k, v2)
93-
db.checkpoint(Buffer.from('2', 'hex'))
94-
await db.put(k, v3)
95-
db.commit()
96-
st.deepEqual(await db.get(k), v3, 'after commit (second CP): v3')
97-
await db.revert()
98-
st.deepEqual(await db.get(k), v, 'after revert (first CP): v1')
99-
st.end()
100-
})
10136
})

0 commit comments

Comments
 (0)