From f931e209ee467b946c2afb3a83148d5678d9b54c Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sun, 2 Feb 2025 23:06:04 +0100 Subject: [PATCH 1/3] Add `getSync()` method Category: addition --- README.md | 10 ++++++ abstract-level.js | 67 +++++++++++++++++++++++++++++++++++++++ lib/abstract-sublevel.js | 4 +++ test/get-sync-test.js | 62 ++++++++++++++++++++++++++++++++++++ test/index.js | 4 +++ test/util.js | 8 ++++- types/abstract-level.d.ts | 10 +++--- 7 files changed, 160 insertions(+), 5 deletions(-) create mode 100644 test/get-sync-test.js diff --git a/README.md b/README.md index d96ab72..cc01659 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,10 @@ Get a value from the database by `key`. The optional `options` object may contai Returns a promise for the value. If the `key` was not found then the value will be `undefined`. +### `db.getSync(key[, options])` + +Synchronously get a value from the database by `key`. This blocks the event loop but can be significantly faster than `db.get()`. Options are the same. Returns the value, or `undefined` if not found. + ### `db.getMany(keys[, options])` Get multiple values from the database by an array of `keys`. The optional `options` object may contain: @@ -1509,6 +1513,12 @@ If the database indicates support of snapshots via `db.supports.implicitSnapshot The default `_get()` returns a promise for an `undefined` value. It must be overridden. +### `db._getSync(key, options)` + +Synchronously get a value by `key`. Receives the same options as `db._get()`. Must return a value, or `undefined` if not found. + +The default `_getSync()` throws a [`LEVEL_NOT_SUPPORTED`](#level_not_supported) error. It should be overridden but support of `_getSync()` is currently opt-in. Set `manifest.getSync` to `true` in order to enable tests. + ### `db._getMany(keys, options)` Get multiple values by an array of `keys`. The `options` object will always have the following properties: `keyEncoding` and `valueEncoding`. Must return a promise. If an error occurs, reject the promise. Otherwise resolve the promise with an array of values. If a key does not exist, set the relevant value to `undefined`. diff --git a/abstract-level.js b/abstract-level.js index d0621e0..da9d255 100644 --- a/abstract-level.js +++ b/abstract-level.js @@ -351,6 +351,73 @@ class AbstractLevel extends EventEmitter { return undefined } + getSync (key, options) { + if (this.status !== 'open') { + throw new ModuleError('Database is not open', { + code: 'LEVEL_DATABASE_NOT_OPEN' + }) + } + + this._assertValidKey(key) + + // Fast-path for default options (known encoding, no cloning, no snapshot) + if (options == null) { + const encodedKey = this.#keyEncoding.encode(key) + const mappedKey = this.prefixKey(encodedKey, this.#keyEncoding.format, true) + const value = this._getSync(mappedKey, this.#defaultOptions.entryFormat) + + try { + return value !== undefined ? this.#valueEncoding.decode(value) : undefined + } catch (err) { + throw new ModuleError('Could not decode value', { + code: 'LEVEL_DECODE_ERROR', + cause: err + }) + } + } + + const snapshot = options.snapshot + const keyEncoding = this.keyEncoding(options.keyEncoding) + const valueEncoding = this.valueEncoding(options.valueEncoding) + const keyFormat = keyEncoding.format + const valueFormat = valueEncoding.format + + // Forward encoding options. Avoid cloning if possible. + if (options.keyEncoding !== keyFormat || options.valueEncoding !== valueFormat) { + options = { ...options, keyEncoding: keyFormat, valueEncoding: valueFormat } + } + + const encodedKey = keyEncoding.encode(key) + const mappedKey = this.prefixKey(encodedKey, keyFormat, true) + + let value + + // Keep snapshot open during operation + snapshot?.ref() + + try { + value = this._getSync(mappedKey, options) + } finally { + // Release snapshot + snapshot?.unref() + } + + try { + return value !== undefined ? valueEncoding.decode(value) : undefined + } catch (err) { + throw new ModuleError('Could not decode value', { + code: 'LEVEL_DECODE_ERROR', + cause: err + }) + } + } + + _getSync (key, options) { + throw new ModuleError('Database does not support getSync()', { + code: 'LEVEL_NOT_SUPPORTED' + }) + } + async getMany (keys, options) { options = getOptions(options, this.#defaultOptions.entry) diff --git a/lib/abstract-sublevel.js b/lib/abstract-sublevel.js index 6b102b1..64b12de 100644 --- a/lib/abstract-sublevel.js +++ b/lib/abstract-sublevel.js @@ -147,6 +147,10 @@ module.exports = function ({ AbstractLevel }) { return this.#parent.get(key, options) } + _getSync (key, options) { + return this.#parent.getSync(key, options) + } + async _getMany (keys, options) { return this.#parent.getMany(keys, options) } diff --git a/test/get-sync-test.js b/test/get-sync-test.js new file mode 100644 index 0000000..d981df2 --- /dev/null +++ b/test/get-sync-test.js @@ -0,0 +1,62 @@ +'use strict' + +const { illegalKeys } = require('./util') +const traits = require('./traits') + +let db + +exports.setUp = function (test, testCommon) { + test('getSync() setup', async function (t) { + db = testCommon.factory() + return db.open() + }) +} + +exports.args = function (test, testCommon) { + test('getSync() with illegal keys', function (t) { + t.plan(illegalKeys.length * 2) + + for (const { name, key } of illegalKeys) { + try { + db.getSync(key) + } catch (err) { + t.ok(err instanceof Error, name + ' - is Error') + t.is(err.code, 'LEVEL_INVALID_KEY', name + ' - correct error code') + } + } + }) +} + +exports.getSync = function (test, testCommon) { + test('simple getSync()', async function (t) { + await db.put('foo', 'bar') + + t.is(db.getSync('foo'), 'bar') + t.is(db.getSync('foo', {}), 'bar') // same but with {} + t.is(db.getSync('foo', { valueEncoding: 'utf8' }), 'bar') + }) + + test('getSync() on non-existent key', async function (t) { + for (const key of ['non-existent', Math.random()]) { + t.is(db.getSync(key), undefined, 'not found') + } + }) + + traits.closed('getSync()', testCommon, async function (t, db) { + db.getSync('foo') + }) +} + +exports.tearDown = function (test, testCommon) { + test('getSync() teardown', async function (t) { + return db.close() + }) +} + +// TODO: test encodings, snapshots +exports.all = function (test, testCommon) { + exports.setUp(test, testCommon) + exports.args(test, testCommon) + exports.getSync(test, testCommon) + exports.tearDown(test, testCommon) +} diff --git a/test/index.js b/test/index.js index 8dd02bf..c7264a7 100644 --- a/test/index.js +++ b/test/index.js @@ -30,6 +30,10 @@ function suite (options) { require('./has-many-test').all(test, testCommon) } + if (testCommon.supports.getSync) { + require('./get-sync-test').all(test, testCommon) + } + require('./batch-test').all(test, testCommon) require('./chained-batch-test').all(test, testCommon) diff --git a/test/util.js b/test/util.js index a78d774..d922831 100644 --- a/test/util.js +++ b/test/util.js @@ -92,7 +92,8 @@ class MinimalLevel extends AbstractLevel { encodings: { utf8: true }, seek: true, has: true, - explicitSnapshots: true + explicitSnapshots: true, + getSync: true }, options) this[kEntries] = new Map() @@ -109,6 +110,11 @@ class MinimalLevel extends AbstractLevel { return entries.get(key) } + _getSync (key, options) { + const entries = (options.snapshot || this)[kEntries] + return entries.get(key) + } + async _getMany (keys, options) { const entries = (options.snapshot || this)[kEntries] return keys.map(k => entries.get(k)) diff --git a/types/abstract-level.d.ts b/types/abstract-level.d.ts index 625d13b..3371f1d 100644 --- a/types/abstract-level.d.ts +++ b/types/abstract-level.d.ts @@ -78,11 +78,13 @@ declare class AbstractLevel * Get a value from the database by {@link key}. */ get (key: KDefault): Promise + get (key: K, options: AbstractGetOptions): Promise - get ( - key: K, - options: AbstractGetOptions - ): Promise + /** + * Synchronously get a value from the database by {@link key}. + */ + getSync (key: KDefault): VDefault | undefined + getSync (key: K, options: AbstractGetOptions): V | undefined /** * Get multiple values from the database by an array of {@link keys}. From 3c54d51626fc17aa03378f48966d1381e49f1ee2 Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sat, 12 Apr 2025 14:42:11 +0200 Subject: [PATCH 2/3] Test encodings and snapshots --- test/encoding-buffer-test.js | 63 +++++++++++++++++++++++-- test/encoding-custom-test.js | 6 +++ test/encoding-decode-error-test.js | 26 ++++++++-- test/encoding-json-test.js | 6 +++ test/encoding-test.js | 11 +++++ test/get-sync-test.js | 1 - test/iterator-explicit-snapshot-test.js | 39 +++++++++++---- test/put-get-del-test.js | 11 +++++ 8 files changed, 147 insertions(+), 16 deletions(-) diff --git a/test/encoding-buffer-test.js b/test/encoding-buffer-test.js index 0ce4b06..0ef5205 100644 --- a/test/encoding-buffer-test.js +++ b/test/encoding-buffer-test.js @@ -218,12 +218,10 @@ exports.all = function (test, testCommon) { const db = testCommon.factory({ keyEncoding }) await db.open() + // These are equal when compared as strings but not when compared as buffers const one = Buffer.from('80', 'hex') const two = Buffer.from('c0', 'hex') - t.ok(two.toString() === one.toString(), 'would be equal when not byte-aware') - t.ok(two.compare(one) > 0, 'but greater when byte-aware') - await db.put(one, 'one') t.is(await db.get(one), 'one', 'value one ok') @@ -232,6 +230,65 @@ exports.all = function (test, testCommon) { return db.close() }) + + if (testCommon.supports.getSync) { + test(`storage is byte-aware (${keyEncoding} encoding) (sync)`, async function (t) { + const db = testCommon.factory({ keyEncoding }) + await db.open() + + // These are equal when compared as strings but not when compared as buffers + const one = Buffer.from('80', 'hex') + const two = Buffer.from('c0', 'hex') + + await db.put(one, 'one') + t.is(db.getSync(one), 'one', 'value one ok') + + await db.put(two, 'two') + t.is(db.getSync(one), 'one', 'value one did not change') + + return db.close() + }) + } + + test(`respects buffer offset and length (${keyEncoding} encoding)`, async function (t) { + const db = testCommon.factory({ keyEncoding }) + await db.open() + + const a = Buffer.from('000102', 'hex') + const b = a.subarray(1) // 0102 + const c = a.subarray(0, 1) // 00 + + await db.put(a, 'a') + await db.put(b, 'b') + await db.put(c, 'c') + + t.is(await db.get(a), 'a', 'value a ok') + t.is(await db.get(b), 'b', 'value b ok') + t.is(await db.get(c), 'c', 'value c ok') + + return db.close() + }) + + if (testCommon.supports.getSync) { + test(`respects buffer offset (${keyEncoding} encoding) (sync)`, async function (t) { + const db = testCommon.factory({ keyEncoding }) + await db.open() + + const a = Buffer.from('000102', 'hex') + const b = a.subarray(1) // 0102 + const c = a.subarray(0, 1) // 00 + + await db.put(a, 'a') + await db.put(b, 'b') + await db.put(c, 'c') + + t.is(db.getSync(a), 'a', 'value a ok') + t.is(db.getSync(b), 'b', 'value b ok') + t.is(db.getSync(c), 'c', 'value c ok') + + return db.close() + }) + } } } diff --git a/test/encoding-custom-test.js b/test/encoding-custom-test.js index fbeec2e..a50641d 100644 --- a/test/encoding-custom-test.js +++ b/test/encoding-custom-test.js @@ -81,6 +81,12 @@ exports.all = function (test, testCommon) { t.same(await db.get(entry.key), entry.value) } + if (testCommon.supports.getSync) { + for (const entry of entries) { + t.same(db.getSync(entry.key), entry.value) + } + } + return db.close() } } diff --git a/test/encoding-decode-error-test.js b/test/encoding-decode-error-test.js index aee8157..83d7561 100644 --- a/test/encoding-decode-error-test.js +++ b/test/encoding-decode-error-test.js @@ -12,8 +12,8 @@ exports.all = function (test, testCommon) { }) // NOTE: adapted from encoding-down - test('decode error is wrapped by get() and getMany()', async function (t) { - t.plan(4) + test('decode error is wrapped by get() and variants', async function (t) { + t.plan(testCommon.supports.getSync ? 6 : 4) const key = testKey() const valueEncoding = { @@ -37,11 +37,20 @@ exports.all = function (test, testCommon) { t.is(err.code, 'LEVEL_DECODE_ERROR') t.is(err.cause.message, 'decode error xyz') } + + if (testCommon.supports.getSync) { + try { + db.getSync(key, { valueEncoding }) + } catch (err) { + t.is(err.code, 'LEVEL_DECODE_ERROR') + t.is(err.cause.message, 'decode error xyz') + } + } }) // NOTE: adapted from encoding-down - test('get() and getMany() yield decode error if stored value is invalid', async function (t) { - t.plan(4) + test('get() and variants yield decode error if stored value is invalid', async function (t) { + t.plan(testCommon.supports.getSync ? 6 : 4) const key = testKey() await db.put(key, 'this {} is [] not : json', { valueEncoding: 'utf8' }) @@ -59,6 +68,15 @@ exports.all = function (test, testCommon) { t.is(err.code, 'LEVEL_DECODE_ERROR') t.is(err.cause.name, 'SyntaxError') // From JSON.parse() } + + if (testCommon.supports.getSync) { + try { + db.getSync(key, { valueEncoding: 'json' }) + } catch (err) { + t.is(err.code, 'LEVEL_DECODE_ERROR') + t.is(err.cause.name, 'SyntaxError') // From JSON.parse() + } + } }) test('decode error teardown', async function (t) { diff --git a/test/encoding-json-test.js b/test/encoding-json-test.js index f68604a..aa710c4 100644 --- a/test/encoding-json-test.js +++ b/test/encoding-json-test.js @@ -55,6 +55,12 @@ exports.all = function (test, testCommon) { await db.batch(operations) await Promise.all([...entries.map(testGet), testIterator()]) + if (testCommon.supports.getSync) { + for (const entry of entries) { + t.same(db.getSync(entry.key), entry.value) + } + } + return db.close() async function testGet (entry) { diff --git a/test/encoding-test.js b/test/encoding-test.js index d9ec58e..f102f59 100644 --- a/test/encoding-test.js +++ b/test/encoding-test.js @@ -59,6 +59,7 @@ exports.all = function (test, testCommon) { if (!deferred) await db.open() await db.put(1, 2) t.is(await db.get(1), '2') + testCommon.supports.getSync && t.is(db.getSync(1), '2') return db.close() }) } @@ -68,7 +69,12 @@ exports.all = function (test, testCommon) { const key = testKey() const data = { thisis: 'json' } await db.put(key, JSON.stringify(data), { valueEncoding: 'utf8' }) + t.same(await db.get(key, { valueEncoding: 'json' }), data, 'got parsed object') + + if (testCommon.supports.getSync) { + t.same(db.getSync(key, { valueEncoding: 'json' }), data, 'got parsed object (sync)') + } }) // NOTE: adapted from encoding-down @@ -76,7 +82,12 @@ exports.all = function (test, testCommon) { const data = { thisis: 'json' } const key = testKey() await db.put(key, data, { valueEncoding: 'json' }) + t.same(await db.get(key, { valueEncoding: 'utf8' }), JSON.stringify(data), 'got unparsed JSON string') + + if (testCommon.supports.getSync) { + t.same(db.getSync(key, { valueEncoding: 'utf8' }), JSON.stringify(data), 'got unparsed JSON string (sync)') + } }) // NOTE: adapted from encoding-down diff --git a/test/get-sync-test.js b/test/get-sync-test.js index d981df2..5d6444e 100644 --- a/test/get-sync-test.js +++ b/test/get-sync-test.js @@ -53,7 +53,6 @@ exports.tearDown = function (test, testCommon) { }) } -// TODO: test encodings, snapshots exports.all = function (test, testCommon) { exports.setUp(test, testCommon) exports.args(test, testCommon) diff --git a/test/iterator-explicit-snapshot-test.js b/test/iterator-explicit-snapshot-test.js index e8a51ea..aa1d393 100644 --- a/test/iterator-explicit-snapshot-test.js +++ b/test/iterator-explicit-snapshot-test.js @@ -18,8 +18,6 @@ exports.get = function (test, testCommon) { const { testFresh, testClose } = testFactory(test, testCommon) testFresh('get() changed entry from snapshot', async function (t, db) { - t.plan(3) - await db.put('abc', 'before') const snapshot = db.snapshot() await db.put('abc', 'after') @@ -28,12 +26,16 @@ exports.get = function (test, testCommon) { t.is(await db.get('abc', { snapshot }), 'before') t.is(await db.get('other', { snapshot }), undefined) + if (testCommon.supports.getSync) { + t.is(db.getSync('abc'), 'after') + t.is(db.getSync('abc', { snapshot }), 'before') + t.is(db.getSync('other', { snapshot }), undefined) + } + return snapshot.close() }) testFresh('get() deleted entry from snapshot', async function (t, db) { - t.plan(3) - await db.put('abc', 'before') const snapshot = db.snapshot() await db.del('abc') @@ -42,18 +44,27 @@ exports.get = function (test, testCommon) { t.is(await db.get('abc', { snapshot }), 'before') t.is(await db.get('other', { snapshot }), undefined) + if (testCommon.supports.getSync) { + t.is(db.getSync('abc'), undefined) + t.is(db.getSync('abc', { snapshot }), 'before') + t.is(db.getSync('other', { snapshot }), undefined) + } + return snapshot.close() }) testFresh('get() non-existent entry from snapshot', async function (t, db) { - t.plan(2) - const snapshot = db.snapshot() await db.put('abc', 'after') t.is(await db.get('abc'), 'after') t.is(await db.get('abc', { snapshot }), undefined) + if (testCommon.supports.getSync) { + t.is(db.getSync('abc'), 'after') + t.is(db.getSync('abc', { snapshot }), undefined) + } + return snapshot.close() }) @@ -61,8 +72,6 @@ exports.get = function (test, testCommon) { const snapshots = [] const iterations = 100 - t.plan(iterations) - for (let i = 0; i < iterations; i++) { await db.put('number', i.toString()) snapshots.push(db.snapshot()) @@ -73,6 +82,10 @@ exports.get = function (test, testCommon) { const value = i.toString() t.is(await db.get('number', { snapshot }), value) + + if (testCommon.supports.getSync) { + t.is(db.getSync('number', { snapshot }), value) + } } return Promise.all(snapshots.map(x => x.close())) @@ -90,12 +103,22 @@ exports.get = function (test, testCommon) { // Closing one snapshot should not affect the other t.is(await db.get('abc', { snapshot: snapshot2 }), 'before') + if (testCommon.supports.getSync) { + t.is(db.getSync('abc', { snapshot: snapshot2 }), 'before') + } + return snapshot2.close() }) testClose('get()', async function (db, snapshot) { return db.get('xyz', { snapshot }) }) + + if (testCommon.supports.getSync) { + testClose('getSync()', async function (db, snapshot) { + return db.getSync('xyz', { snapshot }) + }) + } } exports.getMany = function (test, testCommon) { diff --git a/test/put-get-del-test.js b/test/put-get-del-test.js index d274675..abb09d3 100644 --- a/test/put-get-del-test.js +++ b/test/put-get-del-test.js @@ -9,9 +9,20 @@ function makeTest (test, type, key, value, expectedValue) { test('put(), get(), del() with ' + type, async function (t) { await db.put(key, value) + t.is((await db.get(key)).toString(), stringValue) + + if (db.supports.getSync) { + t.is(db.getSync(key).toString(), stringValue) + } + await db.del(key) + t.is(await db.get(key), undefined, 'not found') + + if (db.supports.getSync) { + t.is(db.getSync(key), undefined, 'not found') + } }) } From 52a078fdd186f13ae79e476f849ceb247d627127 Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sat, 12 Apr 2025 16:59:46 +0200 Subject: [PATCH 3/3] Test buffer values --- test/encoding-buffer-test.js | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/encoding-buffer-test.js b/test/encoding-buffer-test.js index 0ef5205..0bbab47 100644 --- a/test/encoding-buffer-test.js +++ b/test/encoding-buffer-test.js @@ -11,7 +11,13 @@ exports.all = function (test, testCommon) { const db = testCommon.factory() await db.open() await db.put('test', testBuffer(), { valueEncoding: 'buffer' }) + t.same(await db.get('test', { valueEncoding: 'buffer' }), testBuffer()) + + if (testCommon.supports.getSync) { + t.same(db.getSync('test', { valueEncoding: 'buffer' }), testBuffer(), 'sync') + } + return db.close() }) @@ -20,7 +26,13 @@ exports.all = function (test, testCommon) { const db = testCommon.factory({ valueEncoding: 'buffer' }) await db.open() await db.put('test', testBuffer()) + t.same(await db.get('test'), testBuffer()) + + if (testCommon.supports.getSync) { + t.same(db.getSync('test'), testBuffer(), 'sync') + } + return db.close() }) @@ -29,7 +41,13 @@ exports.all = function (test, testCommon) { const db = testCommon.factory() await db.open() await db.put(testBuffer(), 'test', { keyEncoding: 'buffer' }) + t.same(await db.get(testBuffer(), { keyEncoding: 'buffer' }), 'test') + + if (testCommon.supports.getSync) { + t.same(db.getSync(testBuffer(), { keyEncoding: 'buffer' }), 'test', 'sync') + } + return db.close() }) @@ -38,7 +56,13 @@ exports.all = function (test, testCommon) { const db = testCommon.factory() await db.open() await db.put(Buffer.from('foo🐄'), 'test', { keyEncoding: 'utf8' }) + t.same(await db.get(Buffer.from('foo🐄'), { keyEncoding: 'utf8' }), 'test') + + if (testCommon.supports.getSync) { + t.same(db.getSync(Buffer.from('foo🐄'), { keyEncoding: 'utf8' }), 'test', 'sync') + } + return db.close() }) @@ -47,8 +71,15 @@ exports.all = function (test, testCommon) { const db = testCommon.factory() await db.open() await db.put('test', 'foo🐄', { valueEncoding: 'buffer' }) + t.same(await db.get('test', { valueEncoding: 'buffer' }), Buffer.from('foo🐄')) t.same(await db.get('test', { valueEncoding: 'utf8' }), 'foo🐄') + + if (testCommon.supports.getSync) { + t.same(db.getSync('test', { valueEncoding: 'buffer' }), Buffer.from('foo🐄'), 'sync') + t.same(db.getSync('test', { valueEncoding: 'utf8' }), 'foo🐄', 'sync') + } + return db.close() }) @@ -62,11 +93,19 @@ exports.all = function (test, testCommon) { const promise1 = db.put(a, a).then(async () => { const value = await db.get(Buffer.from(a), enc) t.same(value, Buffer.from(a), 'got buffer value') + + if (testCommon.supports.getSync) { + t.same(db.getSync(Buffer.from(a), enc), Buffer.from(a), 'got buffer value (sync)') + } }) const promise2 = db.put(Buffer.from(b), Buffer.from(b), enc).then(async () => { const value = await db.get(b) t.same(value, b, 'got string value') + + if (testCommon.supports.getSync) { + t.same(db.getSync(b), b, 'got string value (sync)') + } }) await Promise.all([promise1, promise2])