Merge pull request #298 from HarperFast/delete-range

Delete range

Author: cb1kenobi

src/binding/database.cpp

@@ -55,15 +55,12 @@ napi_value Database::Constructor(napi_env env, napi_callback_info info) {
 }
 
 /**
- * Removes all entries from the RocksDB database by dropping the column family
- * and recreating it.
+ *  Removes all entries in a RocksDB database column family using an uncapped range.
  *
  * @example
  * ```typescript
  * const db = new NativeDatabase();
- * await db.clear(); // default batch size of 10000
- *
- * await db.clear(1000); // batch size of 1000
+ * await db.clear();
  * ```
  */
 napi_value Database::Clear(napi_env env, napi_callback_info info) {
@@ -72,12 +69,6 @@ napi_value Database::Clear(napi_env env, napi_callback_info info) {
 
 	napi_value resolve = argv[0];
 	napi_value reject = argv[1];
-	uint32_t batchSize = 10000;
-	napi_valuetype batchSizeType;
-	NAPI_STATUS_THROWS(::napi_typeof(env, argv[2], &batchSizeType));
-	if (batchSizeType == napi_number) {
-		NAPI_STATUS_THROWS(::napi_get_value_uint32(env, argv[2], &batchSize));
-	}
 
 	napi_value name;
 	NAPI_STATUS_THROWS(::napi_create_string_utf8(
@@ -87,7 +78,7 @@ napi_value Database::Clear(napi_env env, napi_callback_info info) {
 		&name
 	))
 
-	auto state = new AsyncClearState(env, *dbHandle, batchSize);
+	auto state = new AsyncClearState(env, *dbHandle);
 	NAPI_STATUS_THROWS(::napi_create_reference(env, resolve, 1, &state->resolveRef))
 	NAPI_STATUS_THROWS(::napi_create_reference(env, reject, 1, &state->rejectRef))
 
@@ -101,7 +92,7 @@ napi_value Database::Clear(napi_env env, napi_callback_info info) {
 			if (!state->handle || !state->handle->opened() || state->handle->isCancelled()) {
 				state->status = rocksdb::Status::Aborted("Database closed during clear operation");
 			} else {
-				state->status = state->handle->clear(state->batchSize, state->deleted);
+				state->status = state->handle->clear();
 			}
 			// signal that execute handler is complete
 			state->signalExecuteCompleted();
@@ -116,9 +107,9 @@ napi_value Database::Clear(napi_env env, napi_callback_info info) {
 				NAPI_STATUS_THROWS_VOID(::napi_get_global(env, &global))
 
 				if (state->status.ok()) {
-					napi_value result;
-					NAPI_STATUS_THROWS_VOID(::napi_create_int64(env, state->deleted, &result))
-					state->callResolve(result);
+					napi_value undefined;
+					NAPI_STATUS_THROWS_VOID(::napi_get_undefined(env, &undefined))
+					state->callResolve(undefined);
 				} else {
 					ROCKSDB_STATUS_CREATE_NAPI_ERROR_VOID(state->status, "Clear failed")
 					state->callReject(error);
@@ -140,40 +131,24 @@ napi_value Database::Clear(napi_env env, napi_callback_info info) {
 }
 
 /**
- * Removes all entries from the RocksDB database by dropping the column family
- * and recreating it.
+ *  Removes all entries in a RocksDB database column family using an uncapped range (synchronously).
  *
  * @example
  * ```typescript
  * const db = new NativeDatabase();
- * db.clearSync(); // default batch size of 10000
- *
- * db.clearSync(1000); // batch size of 1000
+ * db.clearSync();
  * ```
  */
 napi_value Database::ClearSync(napi_env env, napi_callback_info info) {
 	NAPI_METHOD_ARGV(1)
 	UNWRAP_DB_HANDLE_AND_OPEN()
-
-	// get the batch size
-	uint32_t batchSize = 10000;
-	napi_valuetype batchSizeType;
-	NAPI_STATUS_THROWS(::napi_typeof(env, argv[0], &batchSizeType));
-	if (batchSizeType == napi_number) {
-		NAPI_STATUS_THROWS(::napi_get_value_uint32(env, argv[0], &batchSize));
-	}
-
-	uint64_t deleted = 0;
-	rocksdb::Status status = (*dbHandle)->clear(batchSize, deleted);
+	rocksdb::Status status = (*dbHandle)->clear();
 	if (!status.ok()) {
 		ROCKSDB_STATUS_CREATE_NAPI_ERROR(status, "Clear failed to write batch")
 		::napi_throw(env, error);
 		return nullptr;
 	}
-
-	napi_value result;
-	NAPI_STATUS_THROWS(::napi_create_int64(env, deleted, &result))
-	return result;
+	NAPI_RETURN_UNDEFINED()
 }
 
 /**
@@ -864,6 +839,7 @@ napi_value Database::PutSync(napi_env env, napi_callback_info info) {
 	NAPI_RETURN_UNDEFINED()
 }
 
+
 /**
  * Removes a key from the RocksDB database.
  */

src/binding/database.h

@@ -71,14 +71,11 @@ struct Database final {
 struct AsyncClearState final : BaseAsyncState<std::shared_ptr<DBHandle>> {
 	AsyncClearState(
 		napi_env env,
-		std::shared_ptr<DBHandle> handle,
-		uint32_t batchSize
+		std::shared_ptr<DBHandle> handle
 	) :
-		BaseAsyncState<std::shared_ptr<DBHandle>>(env, handle),
-		batchSize(batchSize) {}
+		BaseAsyncState<std::shared_ptr<DBHandle>>(env, handle)
+	{}
 
-	uint64_t deleted;
-	uint32_t batchSize;
 };
 
 /**

src/binding/db_handle.cpp

@@ -22,60 +22,20 @@ DBHandle::~DBHandle() {
 /**
  * Clears all data in the database's column family.
  */
-rocksdb::Status DBHandle::clear(uint32_t batchSize, uint64_t& deleted) {
+rocksdb::Status DBHandle::clear() {
 	ASSERT_OPENED_AND_NOT_CANCELLED(this, "clear")
-
-	// create a write batch and iterator
-	rocksdb::WriteBatch batch;
-	std::unique_ptr<rocksdb::Iterator> it = std::unique_ptr<rocksdb::Iterator>(
-		this->descriptor->db->NewIterator(
-			rocksdb::ReadOptions(),
-			this->column.get()
-		)
-	);
-
-	DEBUG_LOG("%p DBHandle::Clear Starting clear with batch size %u\n", this, batchSize)
-
-	// iterate over the database and add each key to the write batch
-	rocksdb::Status status;
-	deleted = 0;
-	it->SeekToFirst();
-	bool valid = it->Valid();
-	while (valid) {
-		ASSERT_OPENED_AND_NOT_CANCELLED(this, "clear")
-
-		batch.Delete(it->key());
-		++deleted;
-		it->Next();
-		valid = it->Valid();
-
-		// if we've reached the end of the iterator or the batch is full, write the batch
-		if (!valid || batch.Count() >= batchSize) {
-			ASSERT_OPENED_AND_NOT_CANCELLED(this, "clear")
-			DEBUG_LOG("%p DBHandle::Clear Writing batch with %zu keys\n", this, batch.Count())
-
-			rocksdb::WriteOptions writeOptions;
-			writeOptions.disableWAL = this->disableWAL;
-
-			status = this->descriptor->db->Write(writeOptions, &batch);
-			if (!status.ok()) {
-				return status;
-			}
-
-			batch.Clear();
-		}
-	}
-
-	// Check one final time before compaction
-	ASSERT_OPENED_AND_NOT_CANCELLED(this, "clear")
-
 	// compact the database to reclaim space
-	return this->descriptor->db->CompactRange(
+	rocksdb::Status status = this->descriptor->db->CompactRange(
 		rocksdb::CompactRangeOptions(),
 		this->column.get(),
 		nullptr,
 		nullptr
 	);
+	if (!status.ok()) {
+		return status;
+	}
+	// it appears we do not need to call WaitForCompact for this to work
+	return rocksdb::DeleteFilesInRange(this->descriptor->db.get(), this->column.get(), nullptr, nullptr);
 }
 
 /**

src/binding/db_handle.h

@@ -65,7 +65,7 @@ struct DBHandle final : Closable, AsyncWorkHandle, public std::enable_shared_fro
 	~DBHandle();
 
 	napi_ref addListener(napi_env env, std::string key, napi_value callback);
-	rocksdb::Status clear(uint32_t batchSize, uint64_t& deleted);
+	rocksdb::Status clear();
 	void close();
 	napi_value get(
 		napi_env env,

src/database.ts

@@ -3,7 +3,6 @@ import { DBI, type DBITransactional } from './dbi.js';
 import { Store, type UserSharedBufferOptions, type ArrayBufferWithNotify, type StoreOptions } from './store.js';
 import { config, type PurgeLogsOptions, type RocksDatabaseConfig, type TransactionOptions } from './load-binding.js';
 import { Encoder as MsgpackEncoder } from 'msgpackr';
-import { withResolvers } from './util.js';
 import * as orderedBinary from 'ordered-binary';
 import type { Encoder, EncoderFunction, Key } from './encoding.js';
 
@@ -49,12 +48,10 @@ export class RocksDatabase extends DBI<DBITransactional> {
 	 * @example
 	 * ```typescript
 	 * const db = RocksDatabase.open('/path/to/database');
-	 * await db.clear(); // default batch size of 10000
-	 *
-	 * await db.clear(1000); // batch size of 1000
+	 * await db.clear();
 	 * ```
 	 */
-	clear(options?: { batchSize?: number }): Promise<number> {
+	clear(): Promise<void> {
 		if (!this.store.db.opened) {
 			return Promise.reject(new Error('Database not open'));
 		}
@@ -63,9 +60,9 @@ export class RocksDatabase extends DBI<DBITransactional> {
 			this.store.encoder.structures = [];
 		}
 
-		const { resolve, reject, promise } = withResolvers<number>();
-		this.store.db.clear(resolve, reject, options?.batchSize);
-		return promise;
+		return new Promise((resolve, reject) => {
+			this.store.db.clear(resolve, reject);
+		});
 	}
 
 	/**
@@ -74,12 +71,10 @@ export class RocksDatabase extends DBI<DBITransactional> {
 	 * @example
 	 * ```typescript
 	 * const db = RocksDatabase.open('/path/to/database');
-	 * db.clearSync(); // default batch size of 10000
-	 *
-	 * db.clearSync(1000); // batch size of 1000
+	 * db.clearSync();
 	 * ```
 	 */
-	clearSync(options?: { batchSize?: number }): number {
+	clearSync(): void {
 		if (!this.store.db.opened) {
 			throw new Error('Database not open');
 		}
@@ -88,7 +83,7 @@ export class RocksDatabase extends DBI<DBITransactional> {
 			this.store.encoder.structures = [];
 		}
 
-		return this.store.db.clearSync(options?.batchSize);
+		return this.store.db.clearSync();
 	}
 
 	/**

src/load-binding.ts

@@ -101,8 +101,8 @@ export type PurgeLogsOptions = {
 export type NativeDatabase = {
 	new(): NativeDatabase;
 	addListener(event: string, callback: (...args: any[]) => void): void;
-	clear(resolve: ResolveCallback<number>, reject: RejectCallback, batchSize?: number): void;
-	clearSync(batchSize?: number): number;
+	clear(resolve: ResolveCallback<void>, reject: RejectCallback): void;
+	clearSync(): void;
 	close(): void;
 	flush(resolve: ResolveCallback<void>, reject: RejectCallback): void;
 	flushSync(): void;