Optimize allocations for large values (#245)

* Update dependencies

* Optimize allocations

* Version bump

Author: arkpar

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "parity-db"
-version = "0.5.0"
+version = "0.5.1"
 authors = ["Parity Technologies <[email protected]>"]
 edition = "2021"
 license = "MIT OR Apache-2.0"
@@ -13,28 +13,28 @@ instrumentation = []
 bench = []
 
 [dependencies]
-blake2 = "0.10.4"
-crc32fast = "1.2.0"
+blake2 = "0.10.6"
+crc32fast = "1.4.2"
 fs2 = "0.4.3"
-hex = "0.4.2"
+hex = "0.4.3"
 libc = "0.2"
-log = "0.4.8"
-lz4 = "1.24.0"
-memmap2 = "0.5"
-parking_lot = "0.12.0"
-rand = "0.8.4"
+log = "0.4.27"
+lz4 = "1.28.1"
+memmap2 = "0.9"
+parking_lot = "0.12.3"
+rand = "0.9.1"
 snap = "1"
-loom = { version = "0.5.1", optional = true }
-siphasher = "0.3.10"
+loom = { version = "0.7.2", optional = true }
+siphasher = "1.0.1"
 
 [target.'cfg(windows)'.dependencies]
 winapi = "0.3.9"
 
 [dev-dependencies]
-env_logger = { version = "0.10.0", default-features = false, features = ["auto-color", "humantime"] }
-fdlimit = "0.2.1"
-rand = { version = "0.8.2", features = ["small_rng"] }
-tempfile = "3.2"
+env_logger = { version = "0.11.8", default-features = false, features = ["auto-color", "humantime"] }
+fdlimit = "0.3.0"
+rand = { version = "0.9.1", features = ["small_rng"] }
+tempfile = "3.19"
 
 [profile.release]
 panic = "abort"

admin/Cargo.toml

@@ -7,14 +7,14 @@ edition = "2018"
 
 [dependencies]
 clap = { version = "4", features = ["derive"] }
-env_logger = { version = "0.10.0", default-features = false, features = ["auto-color", "humantime"] }
-fdlimit = "0.2.1"
-log = "0.4.8"
+env_logger = { version = "0.11.8", default-features = false, features = ["auto-color", "humantime"] }
+fdlimit = "0.3.0"
+log = "0.4.27"
 parity-db = { path = ".." }
-parking_lot = "0.12.0"
-rand = { version = "0.8.5", features = ["small_rng"] }
-blake2 = "0.10.4"
-siphasher = "0.3.10"
+parking_lot = "0.12.3"
+rand = { version = "0.9.1", features = ["small_rng"] }
+blake2 = "0.10.6"
+siphasher = "1.0.1"
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
-jemallocator = "0.5.0"
+jemallocator = "0.5.4"

admin/src/main.rs

@@ -10,7 +10,7 @@
 static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
 fn main() {
-	fdlimit::raise_fd_limit();
+	fdlimit::raise_fd_limit().unwrap();
 
 	parity_db_admin::run().unwrap();
 }

src/db.rs

@@ -275,6 +275,7 @@ impl DbInner {
 				if let Some(v) = overlay.get(col as usize).and_then(|o| o.get(&key)) {
 					return Ok(v.map(|i| i.value().clone()))
 				}
+				std::mem::drop(overlay);
 				// Go into tables and log overlay.
 				let log = self.log.overlays();
 				Ok(column.get(&key, log)?.map(|(v, _rc)| v))
@@ -284,6 +285,7 @@ impl DbInner {
 				if let Some(l) = overlay.get(col as usize).and_then(|o| o.btree_get(key)) {
 					return Ok(l.map(|i| i.value().clone()))
 				}
+				std::mem::drop(overlay);
 				// We lock log, if btree structure changed while reading that would be an issue.
 				let log = self.log.overlays().read();
 				column.with_locked(|btree| BTreeTable::get(key, &*log, btree))
@@ -1814,7 +1816,7 @@ impl Db {
 		if let Err(e) = self.inner.kill_logs(&self.inner) {
 			log::warn!(target: "parity-db", "Shutdown error: {:?}", e);
 		}
-		if let Err(e) = self.inner.lock_file.unlock() {
+		if let Err(e) = fs2::FileExt::unlock(&self.inner.lock_file) {
 			log::debug!(target: "parity-db", "Error removing file lock: {:?}", e);
 		}
 	}
@@ -2853,12 +2855,12 @@ mod tests {
 		let mut iter = db.iter(0).unwrap();
 
 		for _ in 0..100 {
-			let at = rng.gen_range(0u64..=1024);
+			let at = rng.random_range(0u64..=1024);
 			iter.seek(&at.to_be_bytes()).unwrap();
 
-			let mut prev_run: bool = rng.gen();
+			let mut prev_run: bool = rng.random();
 			let at = if prev_run {
-				let take = rng.gen_range(1..100);
+				let take = rng.random_range(1..100);
 				let got = std::iter::from_fn(|| iter.next().unwrap())
 					.map(|(k, _)| u64::from_be_bytes(k.try_into().unwrap()))
 					.take(take)
@@ -2877,7 +2879,7 @@ mod tests {
 			};
 
 			let at = {
-				let take = rng.gen_range(1..100);
+				let take = rng.random_range(1..100);
 				let got = std::iter::from_fn(|| iter.prev().unwrap())
 					.map(|(k, _)| u64::from_be_bytes(k.try_into().unwrap()))
 					.take(take)
@@ -2895,7 +2897,7 @@ mod tests {
 				expected.last().copied().unwrap_or(at)
 			};
 
-			let take = rng.gen_range(1..100);
+			let take = rng.random_range(1..100);
 			let mut got = std::iter::from_fn(|| iter.next().unwrap())
 				.map(|(k, _)| u64::from_be_bytes(k.try_into().unwrap()))
 				.take(take)
@@ -2910,7 +2912,7 @@ mod tests {
 			assert_eq!(got, expected);
 		}
 
-		let take = rng.gen_range(20..100);
+		let take = rng.random_range(20..100);
 		iter.seek_to_last().unwrap();
 		let got = std::iter::from_fn(|| iter.prev().unwrap())
 			.map(|(k, _)| u64::from_be_bytes(k.try_into().unwrap()))
@@ -3007,7 +3009,7 @@ mod tests {
 
 	#[test]
 	fn test_random() {
-		fdlimit::raise_fd_limit();
+		fdlimit::raise_fd_limit().unwrap();
 		for i in 0..100 {
 			test_random_inner(60, 60, i);
 		}

src/index.rs

@@ -681,7 +681,7 @@ mod test {
 		let mut keys = [0u64; CHUNK_ENTRIES];
 		let mut rng = rand::prelude::SmallRng::from_seed(Default::default());
 		for i in 0..CHUNK_ENTRIES {
-			keys[i] = rng.gen();
+			keys[i] = rng.random();
 			let partial_key = Entry::extract_key(keys[i], 18);
 			let e = Entry::new(Address::new(0, 0), partial_key, 18);
 			entries[i] = e;
@@ -763,7 +763,7 @@ mod test {
 		let mut keys = [0u64; CHUNK_ENTRIES];
 		let mut rng = rand::prelude::SmallRng::from_seed(Default::default());
 		for i in 0..CHUNK_ENTRIES {
-			keys[i] = rng.gen();
+			keys[i] = rng.random();
 			let partial_key = Entry::extract_key(keys[i], 18);
 			let e = Entry::new(Address::new(0, 0), partial_key, 18);
 			IndexTable::write_entry(&e, i, &mut chunk);

src/options.rs

@@ -9,9 +9,10 @@ use crate::{
 use rand::Rng;
 use std::{collections::HashMap, path::Path};
 
-pub const CURRENT_VERSION: u32 = 8;
+pub const CURRENT_VERSION: u32 = 9;
 // TODO on last supported 5, remove MULTIHEAD_V4 and MULTIPART_V4
 // TODO on last supported 8, remove XOR with salt in column::hash
+// TODO on last supported 9, remove MULTIHEAD_V8
 const LAST_SUPPORTED_VERSION: u32 = 4;
 
 pub const DEFAULT_COMPRESSION_THRESHOLD: u32 = 4096;
@@ -250,7 +251,7 @@ impl Options {
 			}
 			Ok(meta)
 		} else if create {
-			let s: Salt = self.salt.unwrap_or_else(|| rand::thread_rng().gen());
+			let s: Salt = self.salt.unwrap_or_else(|| rand::rng().random());
 			self.write_metadata(&self.path, &s)?;
 			Ok(Metadata { version: CURRENT_VERSION, columns: self.columns.clone(), salt: s })
 		} else {

src/ref_count.rs

@@ -479,8 +479,8 @@ mod test {
 		let mut addresses = [0u64; CHUNK_ENTRIES];
 		let mut rng = rand::prelude::SmallRng::from_seed(Default::default());
 		for i in 0..CHUNK_ENTRIES {
-			addresses[i] = rng.gen();
-			let ref_count = rng.gen();
+			addresses[i] = rng.random();
+			let ref_count = rng.random();
 			let e = Entry::new(Address::from_u64(addresses[i]), ref_count);
 			entries[i] = e;
 			RefCountTable::write_entry(&e, i, &mut chunk);
@@ -528,8 +528,8 @@ mod test {
 		let mut addresses = [0u64; CHUNK_ENTRIES];
 		let mut rng = rand::prelude::SmallRng::from_seed(Default::default());
 		for i in 0..CHUNK_ENTRIES {
-			addresses[i] = rng.gen();
-			let ref_count = rng.gen();
+			addresses[i] = rng.random();
+			let ref_count = rng.random();
 			let e = Entry::new(Address::from_u64(addresses[i]), ref_count);
 			RefCountTable::write_entry(&e, i, &mut chunk);
 		}