Skip to content

Commit 605d689

Browse files
cb1kenobicb1kenobi
authored
Merge pull request #293 from HarperFast/benchmark-updates
Benchmark updates
2 parents d9b5184 + 32f3d86 commit 605d689

File tree

12 files changed

+122
-15
lines changed

12 files changed

+122
-15
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,15 @@ on:
1717
- 'LICENSE'
1818
- '.gitignore'
1919
workflow_dispatch:
20+
inputs:
21+
mode:
22+
description: 'Do you want to run the full or essential benchmark suite?'
23+
default: 'full'
24+
required: false
25+
type: choice
26+
options:
27+
- 'full'
28+
- 'essential'
2029

2130
concurrency:
2231
cancel-in-progress: true
@@ -66,6 +75,8 @@ jobs:
6675
run: pnpm build
6776

6877
- name: Run benchmarks
78+
env:
79+
BENCHMARK_MODE: ${{ github.event.inputs.mode || 'essential' }}
6980
run: pnpm bench
7081

7182
- name: Upload benchmark results as artifact
@@ -136,14 +147,13 @@ jobs:
136147
comment += '_No benchmark files found_\n';
137148
} else {
138149
for (const file of results.files) {
139-
const filename = file.filepath.split('/').pop();
140-
comment += `### ${filename}\n\n`;
141-
142150
if (!file.groups || file.groups.length === 0) {
143-
comment += '_No benchmark groups found_\n\n';
144151
continue;
145152
}
146153
154+
const filename = file.filepath.split('/').pop();
155+
comment += `### ${filename}\n\n`;
156+
147157
for (const group of file.groups) {
148158
// Extract test name from fullName (remove file path prefix)
149159
const testName = group.fullName.replace(/^[^>]+>\s*/, '');

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
benchmark-results.json
2+
benchmark/data
23
build
34
/coverage
45
deps

‎benchmark/data/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
This directory contains benchmark database files for RocksDB-JS. It is important to run benchmarks on a real filesystem (not a tempfs). These files should be deleted after benchmarking.

‎benchmark/get-sync.bench.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
const SMALL_DATASET = 100;
1111

1212
describe('getSync()', () => {
13-
describe.only('random keys - small key size (100 records)', () => {
13+
describe('random keys - small key size (100 records)', () => {
1414
function setup(ctx) {
1515
ctx.data = generateRandomKeys(SMALL_DATASET);
1616
for (const key of ctx.data) {
@@ -19,6 +19,7 @@ describe('getSync()', () => {
1919
}
2020

2121
benchmark('rocksdb', {
22+
mode: 'essential',
2223
setup,
2324
bench({ data, db }) {
2425
for (const key of data) {
@@ -28,6 +29,7 @@ describe('getSync()', () => {
2829
});
2930

3031
benchmark('lmdb', {
32+
mode: 'essential',
3133
setup,
3234
bench({ data, db }) {
3335
for (const key of data) {
@@ -46,6 +48,7 @@ describe('getSync()', () => {
4648
}
4749

4850
benchmark('rocksdb', {
51+
mode: 'essential',
4952
setup,
5053
bench({ data, db }) {
5154
for (const key of data) {
@@ -55,6 +58,7 @@ describe('getSync()', () => {
5558
});
5659

5760
benchmark('lmdb', {
61+
mode: 'essential',
5862
setup,
5963
bench({ data, db }) {
6064
for (const key of data) {

‎benchmark/ranges.bench.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ function setupRangeTestData(ctx: any, datasetSize: number) {
2323
describe('getRange()', () => {
2424
describe('small range (100 records, 50 range)', () => {
2525
benchmark('rocksdb', {
26+
mode: 'essential',
2627
setup(ctx) {
2728
setupRangeTestData(ctx, SMALL_DATASET);
2829
},
@@ -32,6 +33,7 @@ describe('getRange()', () => {
3233
});
3334

3435
benchmark('lmdb', {
36+
mode: 'essential',
3537
setup(ctx) {
3638
setupRangeTestData(ctx, SMALL_DATASET);
3739
},

‎benchmark/realistic-load.bench.ts‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import {
2+
concurrent,
3+
workerDescribe as describe,
4+
workerBenchmark as benchmark,
5+
type BenchmarkContext,
6+
type LMDBDatabase
7+
} from './setup.js';
8+
import type { RocksDatabase } from '../dist/index.mjs';
9+
10+
const DELETE_RATIO = 0.2;
11+
const NUM_KEYS = 5_000;
12+
describe('Realistic write load with workers', () => {
13+
const aaaa = Buffer.alloc(1500, 'a');
14+
const ITERATIONS = 100;
15+
describe('write variable records with transaction log', () => {
16+
benchmark('rocksdb', concurrent({
17+
mode: 'essential',
18+
numWorkers: 4,
19+
concurrencyMaximum: 32,
20+
dbOptions: { disableWAL: true },
21+
setup(ctx: BenchmarkContext<RocksDatabase>) {
22+
const db = ctx.db;
23+
const log = db.useLog('0');
24+
ctx.log = log;
25+
},
26+
async bench({ db, log }) {
27+
for (let i = 0; i < ITERATIONS; i++) {
28+
await db.transaction((txn) => {
29+
const key = Math.floor(Math.random() * NUM_KEYS).toString();
30+
if (Math.random() < DELETE_RATIO) {
31+
log.addEntry(aaaa.subarray(0, 30), txn.id);
32+
db.removeSync(key, { transaction: txn });
33+
} else {
34+
const data = aaaa.subarray(0, Math.random() * 1500);
35+
log.addEntry(data, txn.id);
36+
db.putSync(key, data, { transaction: txn });
37+
}
38+
}).catch((error) => {
39+
if (error.code !== 'ERR_BUSY') {
40+
console.error('Error occurred during transaction:', error);
41+
}
42+
})
43+
};
44+
},
45+
}));
46+
47+
benchmark('lmdb', concurrent({
48+
mode: 'essential',
49+
numWorkers: 4,
50+
concurrencyMaximum: 32,
51+
async setup(ctx: BenchmarkContext<LMDBDatabase>) {
52+
let start = Date.now();
53+
ctx.index = start;
54+
ctx.lastTime = Date.now();
55+
},
56+
async bench(ctx: BenchmarkContext<LMDBDatabase>) {
57+
const { db } = ctx;
58+
for (let i = 0; i < ITERATIONS; i++) {
59+
let auditTime = ctx.lastTime = Math.max(ctx.lastTime + 0.001, Date.now());
60+
const key = Math.floor(Math.random() * NUM_KEYS).toString();
61+
if (Math.random() < DELETE_RATIO) {
62+
db.put('audit' + auditTime, aaaa.subarray(0, 30));
63+
await db.remove(key);
64+
} else {
65+
const data = aaaa.subarray(0, Math.random() * 1500);
66+
db.put('audit' + auditTime, data);
67+
await db.put(key, data);
68+
}
69+
}
70+
},
71+
}));
72+
});
73+
});

‎benchmark/setup.ts‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import { RocksDatabase, RocksDatabaseOptions } from '../dist/index.mjs';
2-
import { tmpdir } from 'node:os';
32
import { join } from 'node:path';
43
import * as lmdb from 'lmdb';
54
import { randomBytes } from 'node:crypto';
@@ -25,7 +24,8 @@ type BenchmarkOptions<T, U> = {
2524
name?: string,
2625
setup?: (ctx: BenchmarkContext<T>) => void | Promise<void>,
2726
timeout?: number,
28-
teardown?: (ctx: BenchmarkContext<T>) => void | Promise<void>
27+
teardown?: (ctx: BenchmarkContext<T>) => void | Promise<void>,
28+
mode?: 'essential' | 'full'
2929
};
3030

3131
export function benchmark(type: 'rocksdb', options: BenchmarkOptions<RocksDatabase, RocksDatabaseOptions>): void;
@@ -35,12 +35,14 @@ export function benchmark(type: string, options: any): void {
3535
throw new Error(`Unsupported benchmark type: ${type}`);
3636
}
3737

38-
if ((process.env.ROCKSDB_ONLY && type !== 'rocksdb') || (process.env.LMDB_ONLY && type !== 'lmdb')) {
38+
if ((process.env.ROCKSDB_ONLY && type !== 'rocksdb') || (process.env.LMDB_ONLY && type !== 'lmdb') ||
39+
(process.env.BENCHMARK_MODE === 'essential' && options.mode !== 'essential')) {
3940
return;
4041
}
4142

4243
const { bench, setup, teardown, dbOptions, name } = options;
43-
const dbPath = join(tmpdir(), `rocksdb-benchmark-${randomBytes(8).toString('hex')}`);
44+
// it is important to run benchmarks on a real filesystem (not a tempfs)
45+
const dbPath = join('benchmark', 'data', `rocksdb-benchmark-${randomBytes(8).toString('hex')}`);
4446
let ctx: BenchmarkContext<any>;
4547

4648
vitestBench(name || type, () => {
@@ -65,7 +67,7 @@ export function benchmark(type: string, options: any): void {
6567
if (type === 'rocksdb') {
6668
ctx = { db: RocksDatabase.open(dbPath, dbOptions), mode };
6769
} else {
68-
ctx = { db: lmdb.open({ dbPath, compression: true, ...dbOptions }), mode };
70+
ctx = { db: lmdb.open({ path: dbPath, compression: true, ...dbOptions }), mode };
6971
}
7072
}
7173
if (typeof setup === 'function') {
@@ -291,7 +293,8 @@ export function workerBenchmark(type: string, options: any): void {
291293
throw new Error(`Unsupported benchmark type: ${type}`);
292294
}
293295

294-
if ((process.env.ROCKSDB_ONLY && type !== 'rocksdb') || (process.env.LMDB_ONLY && type !== 'lmdb')) {
296+
if ((process.env.ROCKSDB_ONLY && type !== 'rocksdb') || (process.env.LMDB_ONLY && type !== 'lmdb') ||
297+
(process.env.BENCHMARK_MODE === 'essential' && options.mode !== 'essential')) {
295298
return;
296299
}
297300

@@ -335,7 +338,7 @@ export function workerBenchmark(type: string, options: any): void {
335338
throws: true,
336339
async setup(_task, mode) {
337340
if (mode === 'run') return;
338-
const path = join(tmpdir(), `rocksdb-benchmark-${randomBytes(8).toString('hex')}`);
341+
const path = join('benchmark', 'data', `rocksdb-benchmark-${randomBytes(8).toString('hex')}`);
339342

340343
// launch all workers and wait for them to initialize
341344
await Promise.all(Array.from({ length: numWorkers }, (_, i) => {
@@ -378,6 +381,7 @@ export function workerBenchmark(type: string, options: any): void {
378381
});
379382
}));
380383
},
384+
time: 2000,
381385
async teardown(_task, mode) {
382386
if (mode === 'warmup') return;
383387
// tell all workers to teardown and wait
@@ -442,7 +446,7 @@ export async function workerInit() {
442446
if (type === 'rocksdb') {
443447
ctx = { db: RocksDatabase.open(path, dbOptions) };
444448
} else {
445-
ctx = { db: lmdb.open({ path, compression: true, ...dbOptions }) };
449+
ctx = { db: lmdb.open({ path, ...dbOptions }) };
446450
}
447451
if (typeof setup === 'function') {
448452
await setup(ctx);

‎benchmark/transaction-log.bench.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ describe('Transaction log', () => {
3535
});
3636
describe('read 100 iterators while write log with 100 byte records', () => {
3737
benchmark('rocksdb', concurrent({
38+
mode: 'essential',
3839
async setup(ctx: BenchmarkContext<RocksDatabase>) {
3940
const db = ctx.db;
4041
const log = db.useLog('0');
@@ -56,6 +57,7 @@ describe('Transaction log', () => {
5657
}));
5758

5859
benchmark('lmdb', concurrent({
60+
mode: 'essential',
5961
async setup(ctx: BenchmarkContext<LMDBDatabase>) {
6062
let start = Date.now();
6163
ctx.index = start;
@@ -75,6 +77,7 @@ describe('Transaction log', () => {
7577
});
7678
describe('read one entry from random position from log with 1000 100 byte records', () => {
7779
benchmark('rocksdb', {
80+
mode: 'essential',
7881
async setup(ctx: BenchmarkContext<RocksDatabase>) {
7982
const db = ctx.db;
8083
const log = db.useLog('0');
@@ -97,6 +100,7 @@ describe('Transaction log', () => {
97100
});
98101

99102
benchmark('lmdb', {
103+
mode: 'essential',
100104
async setup(ctx: BenchmarkContext<LMDBDatabase>) {
101105
let start = Date.now();
102106
const value = Buffer.alloc(100, 'a');

‎benchmark/worker-put-sync.bench.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ describe('putSync()', () => {
5959

6060
describe('random keys - small key size (100 records, 10 workers)', () => {
6161
benchmark('rocksdb', {
62+
mode: 'essential',
6263
numWorkers: 10,
6364
setup(ctx) {
6465
ctx.data = generateRandomKeys(SMALL_DATASET);
@@ -71,6 +72,7 @@ describe('putSync()', () => {
7172
});
7273

7374
benchmark('lmdb', {
75+
mode: 'essential',
7476
numWorkers: 10,
7577
setup(ctx) {
7678
ctx.data = generateRandomKeys(SMALL_DATASET);

‎benchmark/worker-transaction-log.bench.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ describe('Transaction log with workers', () => {
1212

1313
describe('write log with 100 byte records', () => {
1414
benchmark('rocksdb', concurrent({
15+
mode: 'essential',
1516
numWorkers: 4,
1617
async setup(ctx: BenchmarkContext<RocksDatabase>) {
1718
const db = ctx.db;
@@ -26,6 +27,7 @@ describe('Transaction log with workers', () => {
2627
}));
2728

2829
benchmark('lmdb', concurrent({
30+
mode: 'essential',
2931
numWorkers: 4,
3032
async setup(ctx: BenchmarkContext<LMDBDatabase>) {
3133
let start = Date.now();

0 commit comments

Comments
 (0)