Replace parity-snappy with snap (#11715)

vorot93 · web-flow · commit e68295b8d69a · 2020-05-20T16:03:52.000+03:00
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/ethcore/snapshot/Cargo.toml b/ethcore/snapshot/Cargo.toml
@@ -36,7 +36,7 @@ parking_lot = "0.10.0"
 rlp = "0.4.5"
 rlp-derive = "0.1"
 scopeguard = "1.1.0"
-snappy = { package = "parity-snappy", version ="0.1.0" }
+snap = "1"
 trie-db = "0.20.0"
 triehash = { package = "triehash-ethereum", version = "0.2",  path = "../../util/triehash-ethereum" }
 
diff --git a/ethcore/snapshot/snapshot-tests/Cargo.toml b/ethcore/snapshot/snapshot-tests/Cargo.toml
@@ -31,7 +31,7 @@ parity-crypto = { version = "0.6.1", features = ["publickey"] }
 rand = "0.7.3"
 rand_xorshift = "0.2.0"
 rlp = "0.4.5"
-snappy = { package = "parity-snappy", version ="0.1.0" }
+snap = "1"
 snapshot = { path = "../../snapshot", features = ["test-helpers"] }
 spec = { path = "../../spec" }
 tempfile = "3.1"
diff --git a/ethcore/snapshot/snapshot-tests/src/helpers.rs b/ethcore/snapshot/snapshot-tests/src/helpers.rs
@@ -181,14 +181,22 @@ pub fn restore(
 	for state_chunk_hash in manifest.state_hashes.iter() {
 		trace!(target: "snapshot", "state chunk hash: {}", state_chunk_hash);
 		let chunk = reader.chunk(*state_chunk_hash).unwrap();
-		let len = snappy::decompress_into(&chunk, &mut snappy_buffer).unwrap();
+		let required_len = snap::raw::decompress_len(&chunk)?;
+		if snappy_buffer.len() < required_len {
+			snappy_buffer.resize_with(required_len, Default::default);
+		}
+		let len = snap::raw::Decoder::new().decompress(&chunk, &mut snappy_buffer).unwrap();
 		state.feed(&snappy_buffer[..len], &flag)?;
 	}
 
 	trace!(target: "snapshot", "restoring secondary");
 	for chunk_hash in manifest.block_hashes.iter() {
 		let chunk = reader.chunk(*chunk_hash).unwrap();
-		let len = snappy::decompress_into(&chunk, &mut snappy_buffer).unwrap();
+		let required_len = snap::raw::decompress_len(&chunk)?;
+		if snappy_buffer.len() < required_len {
+			snappy_buffer.resize_with(required_len, Default::default);
+		}
+		let len = snap::raw::Decoder::new().decompress(&chunk, &mut snappy_buffer).unwrap();
 		secondary.feed(&snappy_buffer[..len], engine, &flag)?;
 	}
 
diff --git a/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs b/ethcore/snapshot/snapshot-tests/src/proof_of_work.rs
@@ -32,7 +32,6 @@ use snapshot::{
 	PowSnapshot,
 };
 use parking_lot::{Mutex, RwLock};
-use snappy;
 use keccak_hash::KECCAK_NULL_RLP;
 use kvdb::DBTransaction;
 use ethcore::test_helpers;
@@ -97,7 +96,7 @@ fn chunk_and_restore(amount: u64) {
 	let flag = AtomicBool::new(true);
 	for chunk_hash in &reader.manifest().block_hashes {
 		let compressed = reader.chunk(*chunk_hash).unwrap();
-		let chunk = snappy::decompress(&compressed).unwrap();
+		let chunk = snap::raw::Decoder::new().decompress_vec(&compressed).unwrap();
 		rebuilder.feed(&chunk, engine.as_ref(), &flag).unwrap();
 	}
 
diff --git a/ethcore/snapshot/snapshot-tests/src/state.rs b/ethcore/snapshot/snapshot-tests/src/state.rs
@@ -86,7 +86,7 @@ fn snap_and_restore() {
 
 		for chunk_hash in &reader.manifest().state_hashes {
 			let raw = reader.chunk(*chunk_hash).unwrap();
-			let chunk = snappy::decompress(&raw).unwrap();
+			let chunk = snap::raw::Decoder::new().decompress_vec(&raw).unwrap();
 
 			rebuilder.feed(&chunk, &flag).unwrap();
 		}
@@ -210,7 +210,7 @@ fn checks_flag() {
 
 		for chunk_hash in &reader.manifest().state_hashes {
 			let raw = reader.chunk(*chunk_hash).unwrap();
-			let chunk = snappy::decompress(&raw).unwrap();
+			let chunk = snap::raw::Decoder::new().decompress_vec(&raw).unwrap();
 
 			match rebuilder.feed(&chunk, &flag) {
 				Err(Error::Snapshot(SnapshotError::RestorationAborted)) => {},
diff --git a/ethcore/snapshot/src/lib.rs b/ethcore/snapshot/src/lib.rs
@@ -48,7 +48,6 @@ use log::{debug, info, trace};
 use num_cpus;
 use rand::{Rng, rngs::OsRng};
 use rlp::{RlpStream, Rlp};
-use snappy;
 use trie_db::{Trie, TrieMut};
 
 pub use self::consensus::*;
@@ -202,11 +201,11 @@ pub fn chunk_secondary<'a>(
 	progress: &'a RwLock<Progress>
 ) -> Result<Vec<H256>, Error> {
 	let mut chunk_hashes = Vec::new();
-	let mut snappy_buffer = vec![0; snappy::max_compressed_len(PREFERRED_CHUNK_SIZE)];
+	let mut snappy_buffer = vec![0; snap::raw::max_compress_len(PREFERRED_CHUNK_SIZE)];
 
 	{
 		let mut chunk_sink = |raw_data: &[u8]| {
-			let compressed_size = snappy::compress_into(raw_data, &mut snappy_buffer);
+			let compressed_size = snap::raw::Encoder::new().compress(raw_data, &mut snappy_buffer)?;
 			let compressed = &snappy_buffer[..compressed_size];
 			let hash = keccak(&compressed);
 			let size = compressed.len();
@@ -265,7 +264,11 @@ impl<'a> StateChunker<'a> {
 
 		let raw_data = stream.out();
 
-		let compressed_size = snappy::compress_into(&raw_data, &mut self.snappy_buffer);
+		let required = snap::raw::max_compress_len(raw_data.len());
+		if self.snappy_buffer.len() < required {
+			self.snappy_buffer.resize_with(required, Default::default);
+		}
+		let compressed_size = snap::raw::Encoder::new().compress(&raw_data, &mut self.snappy_buffer)?;
 		let compressed = &self.snappy_buffer[..compressed_size];
 		let hash = keccak(&compressed);
 
@@ -307,7 +310,7 @@ pub fn chunk_state<'a>(
 		hashes: Vec::new(),
 		rlps: Vec::new(),
 		cur_size: 0,
-		snappy_buffer: vec![0; snappy::max_compressed_len(PREFERRED_CHUNK_SIZE)],
+		snappy_buffer: vec![0; snap::raw::max_compress_len(PREFERRED_CHUNK_SIZE)],
 		writer,
 		progress,
 		thread_idx,
diff --git a/ethcore/snapshot/src/service.rs b/ethcore/snapshot/src/service.rs
@@ -41,7 +41,6 @@ use keccak_hash::keccak;
 use kvdb::DBTransaction;
 use log::{debug, error, info, trace, warn};
 use parking_lot::{Mutex, RwLock, RwLockReadGuard};
-use snappy;
 use trie_db::TrieError;
 
 use crate::{SnapshotClient, SnapshotWriter};
@@ -150,12 +149,15 @@ impl Restoration {
 	/// Feeds a chunk of state data to the Restoration. Aborts early if `flag` becomes false.
 	pub fn feed_state(&mut self, hash: H256, chunk: &[u8], flag: &AtomicBool) -> Result<(), Error> {
 		if self.state_chunks_left.contains(&hash) {
-			let expected_len = snappy::decompressed_len(chunk)?;
+			let expected_len = snap::raw::decompress_len(chunk)?;
 			if expected_len > MAX_CHUNK_SIZE {
 				trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE);
 				return Err(SnapshotError::ChunkTooLarge.into());
 			}
-			let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?;
+			if self.snappy_buffer.len() < expected_len {
+				self.snappy_buffer.resize_with(expected_len, Default::default);
+			}
+			let len = snap::raw::Decoder::new().decompress(chunk, &mut self.snappy_buffer)?;
 
 			self.state.feed(&self.snappy_buffer[..len], flag)?;
 
@@ -173,12 +175,15 @@ impl Restoration {
 	/// Feeds a chunk of block data to the `Restoration`. Aborts early if `flag` becomes false.
 	pub fn feed_blocks(&mut self, hash: H256, chunk: &[u8], engine: &dyn Engine, flag: &AtomicBool) -> Result<(), Error> {
 		if self.block_chunks_left.contains(&hash) {
-			let expected_len = snappy::decompressed_len(chunk)?;
+			let expected_len = snap::raw::decompress_len(chunk)?;
 			if expected_len > MAX_CHUNK_SIZE {
 				trace!(target: "snapshot", "Discarding large chunk: {} vs {}", expected_len, MAX_CHUNK_SIZE);
 				return Err(SnapshotError::ChunkTooLarge.into());
 			}
-			let len = snappy::decompress_into(chunk, &mut self.snappy_buffer)?;
+			if self.snappy_buffer.len() < expected_len {
+				self.snappy_buffer.resize_with(expected_len, Default::default);
+			}
+			let len = snap::raw::Decoder::new().decompress(chunk, &mut self.snappy_buffer)?;
 
 			self.secondary.feed(&self.snappy_buffer[..len], engine, flag)?;
 			if let Some(ref mut writer) = self.writer.as_mut() {
diff --git a/ethcore/types/Cargo.toml b/ethcore/types/Cargo.toml
@@ -15,10 +15,10 @@ ethjson = { path = "../../json" }
 hash = { package = "keccak-hash", version = "0.5" }
 parity-crypto = { version = "0.6.1", features = ["publickey"] }
 parity-util-mem = "0.6.0"
-parity-snappy = "0.1"
 ethtrie = { package = "patricia-trie-ethereum", path = "../../util/patricia-trie-ethereum" }
 rlp = "0.4.5"
 rlp-derive = "0.1"
+snap = "1"
 unexpected = { path = "../../util/unexpected" }
 vm = { path = "../vm"}
 
diff --git a/ethcore/types/src/errors/ethcore_error.rs b/ethcore/types/src/errors/ethcore_error.rs
@@ -24,7 +24,6 @@ use std::{error, fmt};
 use derive_more::{Display, From};
 use ethereum_types::{U256, U512};
 use ethtrie::TrieError;
-use parity_snappy::InvalidInput;
 use parity_crypto::publickey::Error as EthPublicKeyCryptoError;
 
 /// Ethcore Result
@@ -59,7 +58,7 @@ pub enum EthcoreError {
 	Transaction(TransactionError),
 	/// Snappy error
 	#[display(fmt = "Snappy error: {}", _0)]
-	Snappy(InvalidInput),
+	Snappy(snap::Error),
 	/// Consensus vote error.
 	#[display(fmt = "Engine error: {}", _0)]
 	Engine(EngineError),
diff --git a/ethcore/types/src/errors/snapshot_error.rs b/ethcore/types/src/errors/snapshot_error.rs
@@ -70,6 +70,8 @@ pub enum SnapshotError {
 	WrongChunkFormat(String),
 	/// Unlinked ancient block chain; includes the parent hash where linkage failed
 	UnlinkedAncientBlockChain(H256),
+	/// Compression error
+	CompressionError(snap::Error),
 }
 
 impl error::Error for SnapshotError {
@@ -87,7 +89,7 @@ impl error::Error for SnapshotError {
 impl fmt::Display for SnapshotError {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 		use self::SnapshotError::*;
-		match *self {
+		match self {
 			InvalidStartingBlock(ref id) => write!(f, "Invalid starting block: {:?}", id),
 			BlockNotFound(ref hash) => write!(f, "Block not found in chain: {}", hash),
 			IncompleteChain => write!(f, "Incomplete blockchain."),
@@ -111,6 +113,7 @@ impl fmt::Display for SnapshotError {
 			BadEpochProof(i) => write!(f, "Bad epoch proof for transition to epoch {}", i),
 			WrongChunkFormat(ref msg) => write!(f, "Wrong chunk format: {}", msg),
 			UnlinkedAncientBlockChain(parent_hash) => write!(f, "Unlinked ancient blocks chain at parent_hash={:#x}", parent_hash),
+			CompressionError(err) => write!(f, "Compression error: {}", err),
 		}
 	}
 }
@@ -133,6 +136,12 @@ impl From<DecoderError> for SnapshotError {
 	}
 }
 
+impl From<snap::Error> for SnapshotError {
+	fn from(err: snap::Error) -> Self {
+		SnapshotError::CompressionError(err)
+	}
+}
+
 impl<E> From<Box<E>> for SnapshotError where SnapshotError: From<E> {
 	fn from(err: Box<E>) -> Self {
 		SnapshotError::from(*err)
diff --git a/util/network-devp2p/Cargo.toml b/util/network-devp2p/Cargo.toml
@@ -26,14 +26,14 @@ network = { package = "ethcore-network", path = "../network" }
 parity-bytes = "0.1"
 parity-crypto = { version = "0.6.1", features = ["publickey"] }
 parity-path = "0.1"
-parity-snappy = "0.1"
 parking_lot = "0.10.0"
 rand = "0.7.3"
 rlp = "0.4.5"
 secp256k1 = "0.17"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 slab = "0.4.2"
+snap = "1"
 tiny-keccak = "2.0.2"
 
 [dev-dependencies]
diff --git a/util/network-devp2p/src/session.rs b/util/network-devp2p/src/session.rs
diff --git a/util/network/Cargo.toml b/util/network/Cargo.toml
diff --git a/util/network/src/error.rs b/util/network/src/error.rs
diff --git a/util/network/src/lib.rs b/util/network/src/lib.rs
