diff --git a/dash-spv-ffi/src/platform_integration.rs b/dash-spv-ffi/src/platform_integration.rs index c4142ad01..183491bbd 100644 --- a/dash-spv-ffi/src/platform_integration.rs +++ b/dash-spv-ffi/src/platform_integration.rs @@ -171,9 +171,9 @@ pub unsafe extern "C" fn ffi_dash_spv_get_quorum_public_key( ); } - // Copy the public key directly from the global index - let pubkey_ptr = public_key as *const _ as *const u8; - std::ptr::copy_nonoverlapping(pubkey_ptr, out_pubkey, QUORUM_PUBKEY_SIZE); + // Get the public key's canonical 48-byte representation safely + let pubkey_bytes: &[u8; 48] = public_key.as_ref(); + std::ptr::copy_nonoverlapping(pubkey_bytes.as_ptr(), out_pubkey, QUORUM_PUBKEY_SIZE); // Return success FFIResult { diff --git a/dash-spv-ffi/tests/test_platform_integration_safety.rs b/dash-spv-ffi/tests/test_platform_integration_safety.rs index 2c6357da3..18a236a87 100644 --- a/dash-spv-ffi/tests/test_platform_integration_safety.rs +++ b/dash-spv-ffi/tests/test_platform_integration_safety.rs @@ -357,10 +357,11 @@ fn test_error_string_lifecycle() { fn test_handle_lifecycle() { unsafe { // Test null handle operations - let null_handle = ptr::null_mut(); + let null_client: *mut FFIDashSpvClient = ptr::null_mut(); + let null_handle: *mut CoreSDKHandle = ptr::null_mut(); // Getting core handle from null client - let handle = ffi_dash_spv_get_core_handle(null_handle); + let handle = ffi_dash_spv_get_core_handle(null_client); assert!(handle.is_null()); // Releasing null handle should be safe diff --git a/dash-spv-ffi/tests/unit/test_type_conversions.rs b/dash-spv-ffi/tests/unit/test_type_conversions.rs index 581f62481..9df36642e 100644 --- a/dash-spv-ffi/tests/unit/test_type_conversions.rs +++ b/dash-spv-ffi/tests/unit/test_type_conversions.rs @@ -172,6 +172,8 @@ mod tests { current_filter_tip: None, masternode_engine: None, last_masternode_diff_height: None, + sync_base_height: 0, + synced_from_checkpoint: false, }; let ffi_state = FFIChainState::from(state); @@ -213,6 +215,10 @@ mod tests { pending_filter_requests: 0, filter_request_timeouts: u64::MAX, filter_requests_retried: u64::MAX, + connected_peers: 0, + total_peers: 0, + header_height: 0, + filter_height: 0, }; let ffi_stats = FFISpvStats::from(stats); diff --git a/dash-spv/src/network/message_handler.rs b/dash-spv/src/network/message_handler.rs index df00c8912..f846ec3b9 100644 --- a/dash-spv/src/network/message_handler.rs +++ b/dash-spv/src/network/message_handler.rs @@ -92,11 +92,11 @@ impl MessageHandler { self.stats.qrinfo_messages += 1; MessageHandleResult::QRInfo(qr_info) } - NetworkMessage::GetQRInfo(_) => { + NetworkMessage::GetQRInfo(req) => { // We don't serve QRInfo requests, only make them tracing::warn!("Received unexpected GetQRInfo request"); self.stats.other_messages += 1; - MessageHandleResult::Unhandled(message) + MessageHandleResult::Unhandled(NetworkMessage::GetQRInfo(req)) } other => { self.stats.other_messages += 1; diff --git a/dash-spv/src/sync/chainlock_validation.rs b/dash-spv/src/sync/chainlock_validation.rs index 10a7bdfb4..ef15112f5 100644 --- a/dash-spv/src/sync/chainlock_validation.rs +++ b/dash-spv/src/sync/chainlock_validation.rs @@ -223,20 +223,38 @@ impl ChainLockValidator { // Get the masternode list at or before the height let mn_list_height = engine.masternode_lists.range(..=height).rev().next().map(|(h, _)| *h); - if let Some(list_height) = mn_list_height { - if let Some(mn_list) = engine.masternode_lists.get(&list_height) { - // Find the chain lock quorum - if let Some(quorums) = mn_list.quorums.get(&self.config.required_llmq_type) { - // Get the most recent quorum - if let Some((quorum_hash, entry)) = quorums.iter().next() { - // Get public key from the quorum entry - return Ok(Some((*quorum_hash, entry.quorum_entry.quorum_public_key))); - } - } - } - } - - Ok(None) + let list_height = mn_list_height.ok_or_else(|| { + SyncError::Validation(format!( + "No masternode list found at or before height {}", + height + )) + })?; + + let mn_list = engine.masternode_lists.get(&list_height).ok_or_else(|| { + SyncError::Validation(format!( + "Masternode list not found at height {}", + list_height + )) + })?; + + // Find the chain lock quorum + let quorums = mn_list.quorums.get(&self.config.required_llmq_type).ok_or_else(|| { + SyncError::Validation(format!( + "No quorums found for LLMQ type {:?} at masternode list height {}", + self.config.required_llmq_type, list_height + )) + })?; + + // Get the most recent quorum + let (quorum_hash, entry) = quorums.iter().next().ok_or_else(|| { + SyncError::Validation(format!( + "No quorum entries found for LLMQ type {:?}", + self.config.required_llmq_type + )) + })?; + + // Get public key from the quorum entry + Ok(Some((*quorum_hash, entry.quorum_entry.quorum_public_key))) } /// Verify chain lock signature using the engine's built-in verification diff --git a/dash-spv/src/sync/sequential/mod.rs b/dash-spv/src/sync/sequential/mod.rs index 3e7bee2dc..d4ac87aff 100644 --- a/dash-spv/src/sync/sequential/mod.rs +++ b/dash-spv/src/sync/sequential/mod.rs @@ -1823,7 +1823,7 @@ impl SequentialSyncManager { // If we have masternodes enabled, request masternode list updates for ChainLock validation if self.config.enable_masternodes { // For ChainLock validation, we need masternode lists at (block_height - CHAINLOCK_VALIDATION_MASTERNODE_OFFSET) - // So we request the masternode diff for this new block to maintain our rolling window + // We request the masternode diff for each new block (not just offset blocks) to maintain a complete rolling window let base_block_hash = if height > 0 { // Get the previous block hash storage diff --git a/dash-spv/src/sync/sequential/phases.rs b/dash-spv/src/sync/sequential/phases.rs index 0454fe3bc..420170740 100644 --- a/dash-spv/src/sync/sequential/phases.rs +++ b/dash-spv/src/sync/sequential/phases.rs @@ -546,17 +546,22 @@ impl SyncPhase { if let SyncPhase::DownloadingMnList { sync_strategy: Some(HybridSyncStrategy::EngineDiscovery { + qr_info_requests, qr_info_completed, .. }), requests_completed, + requests_total, last_progress, .. } = self { - *qr_info_completed += 1; - *requests_completed += 1; - *last_progress = Instant::now(); + // Only increment if we haven't reached the planned total + if *qr_info_completed < *qr_info_requests { + *qr_info_completed += 1; + *requests_completed = (*requests_completed + 1).min(*requests_total); + *last_progress = Instant::now(); + } } } @@ -565,19 +570,24 @@ impl SyncPhase { if let SyncPhase::DownloadingMnList { sync_strategy: Some(HybridSyncStrategy::EngineDiscovery { + mn_diff_requests, mn_diff_completed, .. }), requests_completed, + requests_total, diffs_processed, last_progress, .. } = self { - *mn_diff_completed += 1; - *requests_completed += 1; - *diffs_processed += 1; // Backward compatibility - *last_progress = Instant::now(); + // Only increment if we haven't reached the planned total + if *mn_diff_completed < *mn_diff_requests { + *mn_diff_completed += 1; + *requests_completed = (*requests_completed + 1).min(*requests_total); + *diffs_processed += 1; // Backward compatibility + *last_progress = Instant::now(); + } } } diff --git a/dash-spv/src/sync/validation.rs b/dash-spv/src/sync/validation.rs index 7ca7f764c..5c2961c53 100644 --- a/dash-spv/src/sync/validation.rs +++ b/dash-spv/src/sync/validation.rs @@ -233,14 +233,15 @@ impl ValidationEngine { // Validate masternode list diffs for diff in &qr_info.mn_list_diff_list { + let block_height = engine.block_container.get_height(&diff.block_hash).unwrap_or(0); match self.validate_mn_list_diff(diff, engine) { Ok(true) => items_validated += 1, Ok(false) => errors.push(ValidationError::InvalidMnListDiff( - 0, // We don't have block height in MnListDiff + block_height, "Validation failed".to_string(), )), Err(e) => errors.push(ValidationError::InvalidMnListDiff( - 0, // We don't have block height in MnListDiff + block_height, e.to_string(), )), } @@ -277,7 +278,8 @@ impl ValidationEngine { diff: &MnListDiff, engine: &MasternodeListEngine, ) -> SyncResult { - let cache_key = ValidationCacheKey::MasternodeList(0); // Use 0 as we don't have block height + let block_height = engine.block_container.get_height(&diff.block_hash).unwrap_or(0); + let cache_key = ValidationCacheKey::MasternodeList(block_height); // Check cache if let Some(cached) = self.get_cached_result(&cache_key) { @@ -297,11 +299,11 @@ impl ValidationEngine { fn perform_mn_list_diff_validation( &self, diff: &MnListDiff, - _engine: &MasternodeListEngine, + engine: &MasternodeListEngine, ) -> SyncResult { // Check if we have the base list - // Note: We can't check by height as MnListDiff doesn't contain block height - // We would need to look up the height from the block hash + // We can resolve block height from the diff's block hash using the engine's block container + let block_height = engine.block_container.get_height(&diff.block_hash).unwrap_or(0); // Validate merkle root matches // TODO: Implement merkle root validation diff --git a/dash-spv/src/sync/validation_test.rs b/dash-spv/src/sync/validation_test.rs index 01b3ce044..e3127164a 100644 --- a/dash-spv/src/sync/validation_test.rs +++ b/dash-spv/src/sync/validation_test.rs @@ -3,7 +3,7 @@ #[cfg(test)] mod tests { use crate::client::ClientConfig; - use crate::storage::MemoryStorage; + use crate::storage::MemoryStorageManager; use crate::sync::chainlock_validation::{ChainLockValidationConfig, ChainLockValidator}; use crate::sync::masternodes::MasternodeSyncManager; use crate::sync::validation::{ValidationConfig, ValidationEngine}; @@ -136,7 +136,7 @@ mod tests { async fn test_qr_info_validation() { let config = create_test_config(); let _sync_manager = MasternodeSyncManager::new(&config); - let _storage = MemoryStorage::new(); + let _storage = MemoryStorageManager::new().await.expect("Failed to create MemoryStorageManager"); // Create mock QRInfo let _qr_info = create_mock_qr_info(); diff --git a/dash-spv/tests/chainlock_validation_test.rs b/dash-spv/tests/chainlock_validation_test.rs index f78b2bfc8..e77b64388 100644 --- a/dash-spv/tests/chainlock_validation_test.rs +++ b/dash-spv/tests/chainlock_validation_test.rs @@ -288,7 +288,7 @@ async fn test_chainlock_queue_and_process_flow() { let storage_path = temp_dir.path().to_path_buf(); // Create storage - let storage = Box::new(DiskStorageManager::new(storage_path).await.unwrap()); + let mut storage = Box::new(DiskStorageManager::new(storage_path).await.unwrap()); let network = Box::new(MockNetworkManager::new()); // Create client config @@ -325,7 +325,6 @@ async fn test_chainlock_queue_and_process_flow() { // Process pending (will fail validation but clear the queue) let chain_state = ChainState::new(); - let storage = client.storage(); let _ = chainlock_manager.validate_pending_chainlocks(&chain_state, &mut *storage).await; // Verify queue is cleared @@ -363,13 +362,12 @@ async fn test_chainlock_manager_cache_operations() { // Add test headers let genesis = genesis_block(Network::Dash).header; - let storage = client.storage(); + let mut storage = client.storage(); // storage.store_header(&genesis, 0).await.unwrap(); // Create and process a ChainLock let chain_lock = create_test_chainlock(0, genesis.block_hash()); let chain_state = ChainState::new(); - let storage = client.storage(); let _ = chainlock_manager.process_chain_lock(chain_lock.clone(), &chain_state, &mut *storage).await; diff --git a/dash-spv/tests/error_handling_test.rs b/dash-spv/tests/error_handling_test.rs index fed2c14db..0a7ea7c47 100644 --- a/dash-spv/tests/error_handling_test.rs +++ b/dash-spv/tests/error_handling_test.rs @@ -36,7 +36,7 @@ use dash_spv::network::TcpConnection; use dash_spv::storage::{DiskStorageManager, MemoryStorageManager, StorageManager}; use dash_spv::sync::sequential::phases::SyncPhase; use dash_spv::sync::sequential::recovery::{RecoveryManager, RecoveryStrategy}; -use dash_spv::types::{ChainState, MempoolState}; +use dash_spv::types::{ChainState, MempoolState, UnconfirmedTransaction}; use dash_spv::wallet::Utxo; /// Mock network manager for testing error scenarios @@ -202,7 +202,7 @@ impl StorageManager for MockStorageManager { self } - async fn store_headers(&mut self, headers: &[BlockHeader]) -> StorageResult<()> { + async fn store_headers(&mut self, _headers: &[BlockHeader]) -> StorageResult<()> { if self.lock_poisoned { return Err(StorageError::LockPoisoned("Mock lock poisoned".to_string())); } @@ -218,6 +218,13 @@ impl StorageManager for MockStorageManager { Ok(()) } + async fn load_headers(&self, _range: std::ops::Range) -> StorageResult> { + if self.fail_on_read { + return Err(StorageError::ReadFailed("Mock read failure".to_string())); + } + Ok(vec![]) + } + async fn get_header(&self, _height: u32) -> StorageResult> { if self.lock_poisoned { return Err(StorageError::LockPoisoned("Mock lock poisoned".to_string())); @@ -231,77 +238,110 @@ impl StorageManager for MockStorageManager { Ok(None) } - async fn get_header_by_hash( - &self, - _hash: &BlockHash, - ) -> StorageResult> { + async fn get_tip_height(&self) -> StorageResult> { + if self.fail_on_read { + return Err(StorageError::ReadFailed("Mock read failure".to_string())); + } + Ok(Some(0)) + } + + async fn store_filter_headers(&mut self, _headers: &[FilterHeader]) -> StorageResult<()> { + if self.fail_on_write { + return Err(StorageError::WriteFailed("Mock write failure".to_string())); + } + Ok(()) + } + + async fn load_filter_headers(&self, _range: std::ops::Range) -> StorageResult> { + if self.fail_on_read { + return Err(StorageError::ReadFailed("Mock read failure".to_string())); + } + Ok(vec![]) + } + + async fn get_filter_header(&self, _height: u32) -> StorageResult> { if self.fail_on_read { return Err(StorageError::ReadFailed("Mock read failure".to_string())); } Ok(None) } - async fn get_tip_height(&self) -> StorageResult> { + async fn get_filter_tip_height(&self) -> StorageResult> { if self.fail_on_read { return Err(StorageError::ReadFailed("Mock read failure".to_string())); } Ok(Some(0)) } - async fn get_headers_range( + async fn store_masternode_state( + &mut self, + _state: &dash_spv::storage::MasternodeState, + ) -> StorageResult<()> { + if self.fail_on_write { + return Err(StorageError::WriteFailed("Mock write failure".to_string())); + } + Ok(()) + } + + async fn load_masternode_state( &self, - _range: std::ops::Range, - ) -> StorageResult> { + ) -> StorageResult> { if self.fail_on_read { return Err(StorageError::ReadFailed("Mock read failure".to_string())); } - Ok(vec![]) + Ok(None) } - async fn store_filter_header( - &mut self, - _height: u32, - _filter_header: &FilterHeader, - ) -> StorageResult<()> { + async fn store_chain_state(&mut self, _state: &ChainState) -> StorageResult<()> { if self.fail_on_write { return Err(StorageError::WriteFailed("Mock write failure".to_string())); } Ok(()) } - async fn get_filter_header(&self, _height: u32) -> StorageResult> { + async fn load_chain_state(&self) -> StorageResult> { if self.fail_on_read { return Err(StorageError::ReadFailed("Mock read failure".to_string())); } Ok(None) } - async fn get_filter_tip_height(&self) -> StorageResult> { + async fn store_filter(&mut self, _height: u32, _filter: &[u8]) -> StorageResult<()> { + if self.fail_on_write { + return Err(StorageError::WriteFailed("Mock write failure".to_string())); + } + Ok(()) + } + + async fn load_filter(&self, _height: u32) -> StorageResult>> { if self.fail_on_read { return Err(StorageError::ReadFailed("Mock read failure".to_string())); } - Ok(Some(0)) + Ok(None) } - async fn store_chain_state(&mut self, _state: &ChainState) -> StorageResult<()> { + async fn store_metadata(&mut self, _key: &str, _value: &[u8]) -> StorageResult<()> { if self.fail_on_write { return Err(StorageError::WriteFailed("Mock write failure".to_string())); } Ok(()) } - async fn get_chain_state(&self) -> StorageResult> { + async fn load_metadata(&self, _key: &str) -> StorageResult>> { if self.fail_on_read { return Err(StorageError::ReadFailed("Mock read failure".to_string())); } Ok(None) } - async fn compact_storage(&mut self) -> StorageResult<()> { + async fn clear(&mut self) -> StorageResult<()> { + if self.fail_on_write { + return Err(StorageError::WriteFailed("Mock write failure".to_string())); + } Ok(()) } - async fn get_stats(&self) -> StorageResult { + async fn stats(&self) -> StorageResult { Ok(dash_spv::storage::StorageStats { headers_count: 0, filter_headers_count: 0, @@ -314,10 +354,21 @@ impl StorageManager for MockStorageManager { }) } - async fn get_utxos_by_address( + async fn get_header_height_by_hash( &self, - _address: &Address, - ) -> StorageResult> { + _hash: &dashcore::BlockHash, + ) -> StorageResult> { + if self.fail_on_read { + return Err(StorageError::ReadFailed("Mock read failure".to_string())); + } + Ok(None) + } + + async fn get_headers_batch( + &self, + _start_height: u32, + _end_height: u32, + ) -> StorageResult> { if self.fail_on_read { return Err(StorageError::ReadFailed("Mock read failure".to_string())); } @@ -331,18 +382,18 @@ impl StorageManager for MockStorageManager { Ok(()) } - async fn remove_utxo(&mut self, _outpoint: &OutPoint) -> StorageResult> { + async fn remove_utxo(&mut self, _outpoint: &OutPoint) -> StorageResult<()> { if self.fail_on_write { return Err(StorageError::WriteFailed("Mock write failure".to_string())); } - Ok(None) + Ok(()) } - async fn get_utxo(&self, _outpoint: &OutPoint) -> StorageResult> { + async fn get_utxos_for_address(&self, _address: &Address) -> StorageResult> { if self.fail_on_read { return Err(StorageError::ReadFailed("Mock read failure".to_string())); } - Ok(None) + Ok(vec![]) } async fn get_all_utxos(&self) -> StorageResult> { @@ -352,23 +403,31 @@ impl StorageManager for MockStorageManager { Ok(HashMap::new()) } - async fn store_mempool_state(&mut self, _state: &MempoolState) -> StorageResult<()> { + async fn store_sync_state(&mut self, _state: &dash_spv::storage::PersistentSyncState) -> StorageResult<()> { if self.fail_on_write { return Err(StorageError::WriteFailed("Mock write failure".to_string())); } Ok(()) } - async fn get_mempool_state(&self) -> StorageResult> { + async fn load_sync_state(&self) -> StorageResult> { if self.fail_on_read { return Err(StorageError::ReadFailed("Mock read failure".to_string())); } Ok(None) } - async fn store_masternode_state( + async fn clear_sync_state(&mut self) -> StorageResult<()> { + if self.fail_on_write { + return Err(StorageError::WriteFailed("Mock write failure".to_string())); + } + Ok(()) + } + + async fn store_sync_checkpoint( &mut self, - _state: &dash_spv::storage::MasternodeState, + _height: u32, + _checkpoint: &dash_spv::storage::sync_state::SyncCheckpoint, ) -> StorageResult<()> { if self.fail_on_write { return Err(StorageError::WriteFailed("Mock write failure".to_string())); @@ -376,17 +435,124 @@ impl StorageManager for MockStorageManager { Ok(()) } - async fn get_masternode_state( + async fn get_sync_checkpoints( &self, - ) -> StorageResult> { + _start_height: u32, + _end_height: u32, + ) -> StorageResult> { + if self.fail_on_read { + return Err(StorageError::ReadFailed("Mock read failure".to_string())); + } + Ok(vec![]) + } + + async fn store_chain_lock( + &mut self, + _height: u32, + _chain_lock: &dashcore::ChainLock, + ) -> StorageResult<()> { + if self.fail_on_write { + return Err(StorageError::WriteFailed("Mock write failure".to_string())); + } + Ok(()) + } + + async fn load_chain_lock(&self, _height: u32) -> StorageResult> { + if self.fail_on_read { + return Err(StorageError::ReadFailed("Mock read failure".to_string())); + } + Ok(None) + } + + async fn get_chain_locks( + &self, + _start_height: u32, + _end_height: u32, + ) -> StorageResult> { + if self.fail_on_read { + return Err(StorageError::ReadFailed("Mock read failure".to_string())); + } + Ok(vec![]) + } + + async fn store_instant_lock( + &mut self, + _txid: dashcore::Txid, + _instant_lock: &dashcore::InstantLock, + ) -> StorageResult<()> { + if self.fail_on_write { + return Err(StorageError::WriteFailed("Mock write failure".to_string())); + } + Ok(()) + } + + async fn load_instant_lock( + &self, + _txid: dashcore::Txid, + ) -> StorageResult> { + if self.fail_on_read { + return Err(StorageError::ReadFailed("Mock read failure".to_string())); + } + Ok(None) + } + + async fn store_mempool_transaction( + &mut self, + _txid: &Txid, + _tx: &UnconfirmedTransaction, + ) -> StorageResult<()> { + if self.fail_on_write { + return Err(StorageError::WriteFailed("Mock write failure".to_string())); + } + Ok(()) + } + + async fn remove_mempool_transaction(&mut self, _txid: &Txid) -> StorageResult<()> { + if self.fail_on_write { + return Err(StorageError::WriteFailed("Mock write failure".to_string())); + } + Ok(()) + } + + async fn get_mempool_transaction( + &self, + _txid: &Txid, + ) -> StorageResult> { if self.fail_on_read { return Err(StorageError::ReadFailed("Mock read failure".to_string())); } Ok(None) } - // Terminal block methods removed from StorageManager trait - // These methods are no longer part of the trait + async fn get_all_mempool_transactions( + &self, + ) -> StorageResult> { + if self.fail_on_read { + return Err(StorageError::ReadFailed("Mock read failure".to_string())); + } + Ok(HashMap::new()) + } + + async fn store_mempool_state(&mut self, _state: &MempoolState) -> StorageResult<()> { + if self.fail_on_write { + return Err(StorageError::WriteFailed("Mock write failure".to_string())); + } + Ok(()) + } + + async fn load_mempool_state(&self) -> StorageResult> { + if self.fail_on_read { + return Err(StorageError::ReadFailed("Mock read failure".to_string())); + } + Ok(None) + } + + async fn clear_mempool(&mut self) -> StorageResult<()> { + if self.fail_on_write { + return Err(StorageError::WriteFailed("Mock write failure".to_string())); + } + Ok(()) + } } // ===== Network Error Tests ===== diff --git a/dash-spv/tests/qrinfo_integration_test.rs b/dash-spv/tests/qrinfo_integration_test.rs index 69b8710d7..84673c26f 100644 --- a/dash-spv/tests/qrinfo_integration_test.rs +++ b/dash-spv/tests/qrinfo_integration_test.rs @@ -124,8 +124,8 @@ async fn test_qrinfo_config_defaults() { // Test default configuration values let config = ClientConfig::default(); - // QR info is always enabled in current implementation - assert!(config.qr_info_extra_share); + // QR info extra share is disabled by default + assert!(!config.qr_info_extra_share); assert_eq!(config.qr_info_timeout, Duration::from_secs(30)); } diff --git a/dash-spv/tests/smart_fetch_integration_test.rs b/dash-spv/tests/smart_fetch_integration_test.rs index 48a13f64a..d9a5ea696 100644 --- a/dash-spv/tests/smart_fetch_integration_test.rs +++ b/dash-spv/tests/smart_fetch_integration_test.rs @@ -91,19 +91,23 @@ async fn test_smart_fetch_quorum_discovery() { deleted_masternodes: vec![], new_masternodes: vec![], deleted_quorums: vec![], - new_quorums: vec![QuorumEntry { - version: 1, - llmq_type: LLMQType::Llmqtype50_60, - quorum_hash: dashcore::QuorumHash::all_zeros(), - quorum_index: None, - signers: vec![true; 50], - valid_members: vec![true; 50], - quorum_public_key: dashcore::bls_sig_utils::BLSPublicKey::from([0; 48]), - quorum_vvec_hash: dashcore::hash_types::QuorumVVecHash::all_zeros(), - threshold_sig: dashcore::bls_sig_utils::BLSSignature::from([0; 96]), - all_commitment_aggregated_signature: dashcore::bls_sig_utils::BLSSignature::from( - [0; 96], - ), + new_quorums: vec![{ + let llmq_type = LLMQType::Llmqtype50_60; + let quorum_size = llmq_type.size() as usize; + QuorumEntry { + version: 1, + llmq_type, + quorum_hash: dashcore::QuorumHash::all_zeros(), + quorum_index: None, + signers: vec![true; quorum_size], + valid_members: vec![true; quorum_size], + quorum_public_key: dashcore::bls_sig_utils::BLSPublicKey::from([0; 48]), + quorum_vvec_hash: dashcore::hash_types::QuorumVVecHash::all_zeros(), + threshold_sig: dashcore::bls_sig_utils::BLSSignature::from([0; 96]), + all_commitment_aggregated_signature: dashcore::bls_sig_utils::BLSSignature::from( + [0; 96], + ), + } }], quorums_chainlock_signatures: vec![], }; diff --git a/dash/src/sml/llmq_type/network.rs b/dash/src/sml/llmq_type/network.rs index 1a9800b06..20ecc9b4e 100644 --- a/dash/src/sml/llmq_type/network.rs +++ b/dash/src/sml/llmq_type/network.rs @@ -96,16 +96,6 @@ impl NetworkLLMQExt for Network { ); for llmq_type in self.enabled_llmq_types() { - // Skip platform quorums before activation if needed - if self.should_skip_quorum_type(&llmq_type, start) { - log::trace!( - "Skipping {:?} for height {} (activation threshold not met)", - llmq_type, - start - ); - continue; - } - let type_windows = llmq_type.get_dkg_windows_in_range(start, end); log::debug!( "LLMQ type {:?}: found {} DKG windows in range {}-{}", @@ -116,6 +106,16 @@ impl NetworkLLMQExt for Network { ); for window in type_windows { + // Skip platform quorums before activation if needed + if self.should_skip_quorum_type(&llmq_type, window.mining_start) { + log::trace!( + "Skipping {:?} for height {} (activation threshold not met)", + llmq_type, + window.mining_start + ); + continue; + } + // Group windows by their mining start for efficient fetching windows_by_height.entry(window.mining_start).or_insert_with(Vec::new).push(window); } diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 7a62ced53..63dd1be50 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -11,6 +11,7 @@ cargo-fuzz = true [dependencies] honggfuzz = { version = "0.5", default-features = false } dashcore = { path = "../dash", features = [ "serde" ] } +key-wallet = { path = "../key-wallet" } serde = { version = "1.0.219", features = [ "derive" ] } serde_json = "1.0" diff --git a/fuzz/fuzz_targets/dash/deserialize_psbt.rs b/fuzz/fuzz_targets/dash/deserialize_psbt.rs index c0a0a581d..7159b9982 100644 --- a/fuzz/fuzz_targets/dash/deserialize_psbt.rs +++ b/fuzz/fuzz_targets/dash/deserialize_psbt.rs @@ -1,15 +1,15 @@ use honggfuzz::fuzz; fn do_test(data: &[u8]) { - let psbt: Result = - dashcore::psbt::Psbt::deserialize(data); + let psbt: Result = + key_wallet::psbt::Psbt::deserialize(data); match psbt { Err(_) => {} Ok(psbt) => { - let ser = dashcore::psbt::Psbt::serialize(&psbt); - let deser = dashcore::psbt::Psbt::deserialize(&ser).unwrap(); + let ser = key_wallet::psbt::Psbt::serialize(&psbt); + let deser = key_wallet::psbt::Psbt::deserialize(&ser).unwrap(); // Since the fuzz data could order psbt fields differently, we compare to our deser/ser instead of data - assert_eq!(ser, dashcore::psbt::Psbt::serialize(&deser)); + assert_eq!(ser, key_wallet::psbt::Psbt::serialize(&deser)); } } } diff --git a/key-wallet-ffi/src/lib.rs b/key-wallet-ffi/src/lib.rs index a15cd4e48..7cbced7bd 100644 --- a/key-wallet-ffi/src/lib.rs +++ b/key-wallet-ffi/src/lib.rs @@ -4,8 +4,9 @@ use std::str::FromStr; use std::sync::Arc; use key_wallet::{ - self as kw, address as kw_address, derivation::HDWallet as KwHDWallet, mnemonic as kw_mnemonic, - DerivationPath as KwDerivationPath, ExtendedPrivKey, ExtendedPubKey, Network as KwNetwork, + self as kw, derivation::HDWallet as KwHDWallet, mnemonic as kw_mnemonic, Address as KwAddress, + AddressType as KwAddressType, DerivationPath as KwDerivationPath, ExtendedPrivKey, + ExtendedPubKey, Network as KwNetwork, }; use secp256k1::{PublicKey, Secp256k1}; @@ -75,20 +76,21 @@ pub enum AddressType { P2SH, } -impl From for AddressType { - fn from(t: kw_address::AddressType) -> Self { +impl From for AddressType { + fn from(t: KwAddressType) -> Self { match t { - kw_address::AddressType::P2PKH => AddressType::P2PKH, - kw_address::AddressType::P2SH => AddressType::P2SH, + KwAddressType::P2pkh => AddressType::P2PKH, + KwAddressType::P2sh => AddressType::P2SH, + _ => AddressType::P2PKH, // Default to P2PKH for unknown types } } } -impl From for kw_address::AddressType { +impl From for KwAddressType { fn from(t: AddressType) -> Self { match t { - AddressType::P2PKH => kw_address::AddressType::P2PKH, - AddressType::P2SH => kw_address::AddressType::P2SH, + AddressType::P2PKH => KwAddressType::P2pkh, + AddressType::P2SH => KwAddressType::P2sh, } } } @@ -189,6 +191,15 @@ impl From for KeyWalletError { kw::Error::KeyError(msg) => KeyWalletError::KeyError { message: msg, }, + kw::Error::CoinJoinNotEnabled => KeyWalletError::KeyError { + message: "CoinJoin not enabled".into(), + }, + kw::Error::Serialization(msg) => KeyWalletError::KeyError { + message: format!("Serialization error: {}", msg), + }, + kw::Error::InvalidParameter(msg) => KeyWalletError::KeyError { + message: format!("Invalid parameter: {}", msg), + }, } } } @@ -201,6 +212,14 @@ impl From for KeyWalletError { } } +impl From for KeyWalletError { + fn from(e: kw::dashcore::address::Error) -> Self { + KeyWalletError::AddressError { + message: e.to_string(), + } + } +} + // Validate mnemonic function pub fn validate_mnemonic(phrase: String, language: Language) -> Result { Ok(kw::Mnemonic::validate(&phrase, language.into())) @@ -439,41 +458,20 @@ impl ExtPubKey { // Address wrapper pub struct Address { - inner: kw_address::Address, + inner: KwAddress, } impl Address { pub fn from_string(address: String, network: Network) -> Result { - let inner = kw_address::Address::from_str(&address).map_err(|e| KeyWalletError::from(e))?; + let unchecked_addr = KwAddress::from_str(&address).map_err(|e| KeyWalletError::from(e))?; - // Validate that the parsed network matches the expected network - // Note: Testnet, Devnet, and Regtest all share the same address prefixes (140/19) - // so we need to be flexible when comparing these networks - let parsed_network: KwNetwork = inner.network; + // Convert to expected network and require it let expected_network: KwNetwork = network.into(); - - let networks_compatible = match (parsed_network, expected_network) { - // Exact matches are always OK - (n1, n2) if n1 == n2 => true, - // Testnet addresses can be used on devnet/regtest and vice versa - (KwNetwork::Testnet, KwNetwork::Devnet) - | (KwNetwork::Testnet, KwNetwork::Regtest) - | (KwNetwork::Devnet, KwNetwork::Testnet) - | (KwNetwork::Devnet, KwNetwork::Regtest) - | (KwNetwork::Regtest, KwNetwork::Testnet) - | (KwNetwork::Regtest, KwNetwork::Devnet) => true, - // All other combinations are incompatible - _ => false, - }; - - if !networks_compatible { - return Err(KeyWalletError::AddressError { - message: format!( - "Address is for network {:?}, expected {:?}", - inner.network, network - ), - }); - } + let inner = unchecked_addr.require_network(expected_network).map_err(|e| { + KeyWalletError::AddressError { + message: format!("Address network validation failed: {}", e), + } + })?; Ok(Self { inner, @@ -481,11 +479,12 @@ impl Address { } pub fn from_public_key(public_key: Vec, network: Network) -> Result { - let pubkey = + let secp_pubkey = PublicKey::from_slice(&public_key).map_err(|e| KeyWalletError::Secp256k1Error { message: e.to_string(), })?; - let inner = kw_address::Address::p2pkh(&pubkey, network.into()); + let dashcore_pubkey = kw::dashcore::PublicKey::new(secp_pubkey); + let inner = KwAddress::p2pkh(&dashcore_pubkey, network.into()); Ok(Self { inner, }) @@ -496,11 +495,11 @@ impl Address { } pub fn get_type(&self) -> AddressType { - self.inner.address_type.into() + self.inner.address_type().unwrap_or(KwAddressType::P2pkh).into() } pub fn get_network(&self) -> Network { - match self.inner.network { + match *self.inner.network() { KwNetwork::Dash => Network::Dash, KwNetwork::Testnet => Network::Testnet, KwNetwork::Regtest => Network::Regtest, @@ -510,19 +509,19 @@ impl Address { } pub fn get_script_pubkey(&self) -> Vec { - self.inner.script_pubkey() + self.inner.script_pubkey().into() } } // Address generator wrapper pub struct AddressGenerator { - inner: kw_address::AddressGenerator, + network: Network, } impl AddressGenerator { pub fn new(network: Network) -> Self { Self { - inner: kw_address::AddressGenerator::new(network.into()), + network, } } @@ -538,15 +537,44 @@ impl AddressGenerator { message: e.to_string(), })?; - // Generate addresses for a single index - let addrs = self - .inner - .generate_range(&xpub, external, index, 1) - .map_err(|e| KeyWalletError::from(e))?; + let secp = Secp256k1::new(); - let addr = addrs.into_iter().next().ok_or_else(|| KeyWalletError::KeyError { - message: "Failed to generate address".into(), - })?; + // Derive child key: 0 for external (receiving), 1 for internal (change) + let chain_code = if external { + 0 + } else { + 1 + }; + let child_chain = xpub + .ckd_pub( + &secp, + kw::ChildNumber::from_normal_idx(chain_code).map_err(|e| { + KeyWalletError::InvalidDerivationPath { + message: e.to_string(), + } + })?, + ) + .map_err(|e| KeyWalletError::KeyError { + message: e.to_string(), + })?; + + // Derive specific index + let child = child_chain + .ckd_pub( + &secp, + kw::ChildNumber::from_normal_idx(index).map_err(|e| { + KeyWalletError::InvalidDerivationPath { + message: e.to_string(), + } + })?, + ) + .map_err(|e| KeyWalletError::KeyError { + message: e.to_string(), + })?; + + // Generate P2PKH address from the public key + let dashcore_pubkey = kw::dashcore::PublicKey::new(child.public_key); + let addr = KwAddress::p2pkh(&dashcore_pubkey, self.network.into()); Ok(Arc::new(Address { inner: addr, @@ -560,25 +588,14 @@ impl AddressGenerator { start: u32, count: u32, ) -> Result>, KeyWalletError> { - // Parse the extended public key from string - let xpub = - ExtendedPubKey::from_str(&account_xpub.xpub).map_err(|e| KeyWalletError::KeyError { - message: e.to_string(), - })?; + let mut addresses = Vec::new(); - let addrs = self - .inner - .generate_range(&xpub, external, start, count) - .map_err(|e| KeyWalletError::from(e))?; + for i in 0..count { + let addr = self.generate(account_xpub.clone(), external, start + i)?; + addresses.push(addr); + } - Ok(addrs - .into_iter() - .map(|addr| { - Arc::new(Address { - inner: addr, - }) - }) - .collect()) + Ok(addresses) } } diff --git a/qr_info_spv_plan/PHASE_1.md b/qr_info_spv_plan/PHASE_1.md deleted file mode 100644 index 038d8143d..000000000 --- a/qr_info_spv_plan/PHASE_1.md +++ /dev/null @@ -1,748 +0,0 @@ -# Phase 1: Add QRInfo Support to dash-spv - -## Overview - -This phase adds comprehensive QRInfo message handling to dash-spv, establishing the foundation for efficient batch-based masternode synchronization. We'll implement the network protocol, message processing, and basic integration with the masternode sync manager. - -## Objectives - -1. **Add QRInfo Protocol Support**: Implement network message handling -2. **Basic Integration**: Connect QRInfo to masternode sync manager -3. **Test Infrastructure**: Comprehensive test suite with real data -4. **Fallback Compatibility**: Maintain existing MnListDiff functionality - -## Detailed Implementation Plan - -### 1. Network Layer Implementation - -#### 1.1 Add QRInfo Message Types - -**File**: `dash-spv/src/network/message_handler.rs` - -**Implementation**: -```rust -// Add QRInfo imports -use dashcore::network::message_qrinfo::{QRInfo, GetQRInfo}; - -// Add to NetworkMessage handling -pub async fn handle_message(&mut self, message: NetworkMessage) -> Result<(), NetworkError> { - match message { - // ... existing cases ... - NetworkMessage::QRInfo(qr_info) => { - self.handle_qr_info(qr_info).await?; - } - // Add to request handling - NetworkMessage::GetQRInfo(get_qr_info) => { - // We don't serve QRInfo requests, only make them - tracing::warn!("Received unexpected GetQRInfo request"); - } - _ => {} // existing catch-all - } - Ok(()) -} - -async fn handle_qr_info(&mut self, qr_info: QRInfo) -> Result<(), NetworkError> { - // Route to masternode sync manager - if let Some(sync_sender) = &self.masternode_sync_sender { - sync_sender.send(MasternodeSyncMessage::QRInfo(qr_info)) - .await - .map_err(|e| NetworkError::Internal(format!("Failed to send QRInfo: {}", e)))?; - } - Ok(()) -} -``` - -**Test File**: `tests/network/test_qr_info_message_handling.rs` -```rust -#[tokio::test] -async fn test_qr_info_message_parsing() { - // Load real QRInfo test vector - let qr_info_bytes = load_test_vector("qr_info_mainnet_height_2240504.bin"); - let qr_info: QRInfo = deserialize(&qr_info_bytes).expect("Failed to parse QRInfo"); - - // Verify all components present - assert!(!qr_info.mn_list_diff_list.is_empty()); - assert!(!qr_info.last_commitment_per_index.is_empty()); - assert!(qr_info.quorum_snapshot_and_mn_list_diff_at_h_minus_4c.is_some()); -} - -#[tokio::test] -async fn test_qr_info_network_routing() { - let mut handler = setup_test_message_handler().await; - let test_qr_info = create_test_qr_info(); - - let result = handler.handle_message(NetworkMessage::QRInfo(test_qr_info)).await; - assert!(result.is_ok()); - - // Verify message was routed to sync manager - let received_message = handler.sync_receiver.recv().await.unwrap(); - assert!(matches!(received_message, MasternodeSyncMessage::QRInfo(_))); -} -``` - -#### 1.2 Add QRInfo Request Capability - -**File**: `dash-spv/src/network/network_manager.rs` - -**Implementation**: -```rust -impl NetworkManagerImpl { - /// Request QRInfo from the network - pub async fn request_qr_info( - &mut self, - base_block_hash: BlockHash, - block_hash: BlockHash, - extra_share: bool, - ) -> Result<(), NetworkError> { - let get_qr_info = GetQRInfo { - base_block_hash, - block_hash, - extra_share, - }; - - self.send_message(NetworkMessage::GetQRInfo(get_qr_info)).await?; - - tracing::debug!( - "Requested QRInfo from {} to {}, extra_share={}", - base_block_hash, - block_hash, - extra_share - ); - - Ok(()) - } -} -``` - -**Test File**: `tests/network/test_qr_info_requests.rs` -```rust -#[tokio::test] -async fn test_qr_info_request_construction() { - let mut network = setup_test_network().await; - let base_hash = BlockHash::from_str("000000000000000fcc3b58235989afa1962b6d6f238a2201190452123231a704").unwrap(); - let tip_hash = BlockHash::from_str("000000000000001912a0ac17300c5b7bfd1385a418137c3bc8d273ac3d9f85d7").unwrap(); - - let result = network.request_qr_info(base_hash, tip_hash, true).await; - assert!(result.is_ok()); - - // Verify message was sent - let sent_message = network.get_last_sent_message().unwrap(); - assert!(matches!(sent_message, NetworkMessage::GetQRInfo(_))); -} - -#[tokio::test] -async fn test_qr_info_request_extra_share_flag() { - let mut network = setup_test_network().await; - let base_hash = test_genesis_hash(); - let tip_hash = test_block_hash(100); - - // Test with extra_share = false - network.request_qr_info(base_hash, tip_hash, false).await.unwrap(); - let message = network.get_last_sent_message().unwrap(); - if let NetworkMessage::GetQRInfo(req) = message { - assert!(!req.extra_share); - } else { - panic!("Expected GetQRInfo message"); - } -} -``` - -### 2. Masternode Sync Manager Integration - -#### 2.1 Add QRInfo Message Channel - -**File**: `dash-spv/src/sync/masternodes.rs` - -**Implementation**: -```rust -// Add to MasternodeSyncMessage enum -#[derive(Debug)] -pub enum MasternodeSyncMessage { - MnListDiff(MnListDiff), - QRInfo(QRInfo), // NEW - Reset, - Status, -} - -impl MasternodeSyncManager { - /// Process received QRInfo message - pub async fn handle_qr_info( - &mut self, - qr_info: QRInfo, - storage: &dyn StorageManager, - ) -> SyncResult<()> { - tracing::info!( - "Received QRInfo with {} diffs and {} snapshots", - qr_info.mn_list_diff_list.len(), - qr_info.quorum_snapshot_list.len() - ); - - // Get engine or return early - let engine = self.engine.as_mut().ok_or_else(|| { - SyncError::Configuration("Masternode engine not initialized".to_string()) - })?; - - // Create block height fetcher - let block_height_fetcher = |block_hash: &BlockHash| -> Result { - self.get_block_height_from_storage(block_hash, storage) - }; - - // Process QRInfo through engine - engine.feed_qr_info( - qr_info, - true, // verify_tip_non_rotated_quorums - true, // verify_rotated_quorums - Some(block_height_fetcher) - ).map_err(|e| SyncError::Validation(format!("QRInfo processing failed: {}", e)))?; - - tracing::info!("Successfully processed QRInfo"); - Ok(()) - } - - /// Get block height from storage for QRInfo processing - fn get_block_height_from_storage( - &self, - block_hash: &BlockHash, - storage: &dyn StorageManager, - ) -> Result { - // First check if it's in our block container - if let Some(engine) = &self.engine { - if let Some(height) = engine.block_container.get_height(block_hash) { - return Ok(height); - } - } - - // Fall back to storage lookup (convert storage height to blockchain height) - let sync_base = self.sync_base_height; - - // TODO: Implement efficient block hash -> height lookup in storage - // For now, we'll need to search or maintain an index - for height in 0..=1000 { // Reasonable search range - if let Ok(Some(header)) = futures::executor::block_on(storage.get_header(height)) { - if header.block_hash() == *block_hash { - return Ok(height + sync_base); - } - } - } - - Err(ClientDataRetrievalError::BlockNotFound(*block_hash)) - } -} -``` - -**Test File**: `tests/sync/test_qr_info_processing.rs` -```rust -#[tokio::test] -async fn test_qr_info_basic_processing() { - let mut sync_manager = setup_test_sync_manager().await; - let storage = setup_test_storage().await; - let qr_info = load_test_qr_info("mainnet_2240504"); - - let result = sync_manager.handle_qr_info(qr_info, &storage).await; - assert!(result.is_ok()); - - // Verify engine state was updated - let engine = sync_manager.engine().unwrap(); - assert!(!engine.masternode_lists.is_empty()); - assert!(!engine.known_snapshots.is_empty()); -} - -#[tokio::test] -async fn test_qr_info_block_height_fetching() { - let sync_manager = setup_sync_manager_with_blocks().await; - let storage = setup_storage_with_headers().await; - let test_hash = BlockHash::from_str("000000000000001912a0ac17300c5b7bfd1385a418137c3bc8d273ac3d9f85d7").unwrap(); - - let height = sync_manager.get_block_height_from_storage(&test_hash, &storage) - .expect("Should find block height"); - - assert_eq!(height, 2240504); -} - -#[tokio::test] -async fn test_qr_info_engine_integration() { - let mut sync_manager = setup_test_sync_manager().await; - let storage = setup_test_storage().await; - - // Load real QRInfo with known expected state - let qr_info = load_test_qr_info("mainnet_rotation_cycle"); - let initial_list_count = sync_manager.engine().unwrap().masternode_lists.len(); - - sync_manager.handle_qr_info(qr_info, &storage).await.unwrap(); - - let engine = sync_manager.engine().unwrap(); - assert!(engine.masternode_lists.len() > initial_list_count); - assert!(!engine.rotated_quorums_per_cycle.is_empty()); - - // Verify we can look up quorums now - let quorum_hashes = engine.latest_masternode_list_quorum_hashes(&[]); - assert!(!quorum_hashes.is_empty()); -} -``` - -#### 2.2 Add Storage Block Hash Lookup - -**File**: `dash-spv/src/storage/mod.rs` - -**Implementation** (Using Existing Efficient Storage): -```rust -// NO NEW METHODS NEEDED - Use existing StorageManager interface -// The existing StorageManager already provides efficient O(1) hash-to-height lookups: - -use crate::storage::StorageManager; -use crate::sml::quorum_validation_error::ClientDataRetrievalError; - -/// Create block height fetcher using storage's existing efficient O(1) index -/// -/// Note: dash-spv already implements HashMap for O(1) lookups -/// in disk.rs via header_hash_index. No linear scan required! -pub fn create_block_height_fetcher( - storage: &S, -) -> impl Fn(&BlockHash) -> Result + '_ { - |block_hash: &BlockHash| { - // Use existing efficient storage method - storage.get_header_height(block_hash) - .map_err(|e| ClientDataRetrievalError::StorageError(e.to_string()))? - .ok_or_else(|| ClientDataRetrievalError::BlockNotFound(*block_hash)) - } -} - -/// Async wrapper for block height fetching (if needed) -pub struct AsyncBlockHeightFetcher<'a, S: StorageManager> { - storage: &'a S, -} - -impl<'a, S: StorageManager> AsyncBlockHeightFetcher<'a, S> { - pub fn new(storage: &'a S) -> Self { - Self { storage } - } - - /// Fetch block height using storage's efficient O(1) index - pub async fn fetch_height( - &self, - block_hash: &BlockHash, - ) -> Result { - // Use storage's existing O(1) hash-to-height lookup - self.storage - .get_header_height(block_hash) - .await - .map_err(|e| ClientDataRetrievalError::StorageError(e.to_string()))? - .ok_or_else(|| ClientDataRetrievalError::BlockNotFound(*block_hash)) - } -} -``` - -**Storage Integration** (No changes to existing storage layer needed): -```rust -// The existing storage layer already provides what we need: -impl StorageManager for DiskStorageManager { - // ALREADY EXISTS: O(1) hash-to-height lookup - async fn get_header_height(&self, hash: &BlockHash) -> StorageResult> { - // Uses existing header_hash_index: Arc>> - // This is already implemented and tested! - } -} -``` -``` - -**Test File**: `tests/storage/test_block_hash_lookup.rs` -```rust -#[tokio::test] -async fn test_block_hash_index_creation() { - let storage = setup_test_disk_storage().await; - - // Add some test headers - for i in 0..100 { - let header = create_test_header(i); - storage.store_header(i, &header).await.unwrap(); - } - - // Build index - storage.build_block_hash_index().await.unwrap(); - - // Test lookups - let test_hash = storage.get_header(50).await.unwrap().unwrap().block_hash(); - let found_height = storage.get_block_height(&test_hash).await.unwrap(); - assert_eq!(found_height, Some(50)); -} - -#[tokio::test] -async fn test_block_hash_lookup_performance() { - let storage = setup_large_test_storage(10000).await; // 10k headers - storage.build_block_hash_index().await.unwrap(); - - let start = std::time::Instant::now(); - - // Test 100 random lookups - for _ in 0..100 { - let random_height = rand::random::() % 10000; - let header = storage.get_header(random_height).await.unwrap().unwrap(); - let hash = header.block_hash(); - - let found_height = storage.get_block_height(&hash).await.unwrap(); - assert_eq!(found_height, Some(random_height)); - } - - let elapsed = start.elapsed(); - assert!(elapsed < std::time::Duration::from_millis(100)); // Should be very fast -} -``` - -### 3. Test Infrastructure - -#### 3.1 QRInfo Test Data Generation - -**File**: `tests/fixtures/qr_info_generator.rs` -```rust -/// Generate test QRInfo messages for various scenarios -pub struct QRInfoTestGenerator { - network: Network, - base_height: u32, -} - -impl QRInfoTestGenerator { - pub fn new(network: Network, base_height: u32) -> Self { - Self { network, base_height } - } - - /// Generate QRInfo for normal sync scenario - pub fn generate_normal_sync(&self, tip_height: u32) -> QRInfo { - let base_hash = self.block_hash_at_height(self.base_height); - let tip_hash = self.block_hash_at_height(tip_height); - - QRInfo { - // Generate required snapshots at h-c, h-2c, h-3c - quorum_snapshot_at_h_minus_c: self.generate_snapshot(tip_height - self.cycle_length()), - quorum_snapshot_at_h_minus_2c: self.generate_snapshot(tip_height - 2 * self.cycle_length()), - quorum_snapshot_at_h_minus_3c: self.generate_snapshot(tip_height - 3 * self.cycle_length()), - - // Generate required diffs - mn_list_diff_tip: self.generate_diff(tip_height - 1, tip_height), - mn_list_diff_h: self.generate_diff(tip_height - 8, tip_height), - mn_list_diff_at_h_minus_c: self.generate_diff(tip_height - self.cycle_length() - 8, tip_height - self.cycle_length()), - mn_list_diff_at_h_minus_2c: self.generate_diff(tip_height - 2 * self.cycle_length() - 8, tip_height - 2 * self.cycle_length()), - mn_list_diff_at_h_minus_3c: self.generate_diff(tip_height - 3 * self.cycle_length() - 8, tip_height - 3 * self.cycle_length()), - - // Optional h-4c data for extra validation - quorum_snapshot_and_mn_list_diff_at_h_minus_4c: Some(( - self.generate_snapshot(tip_height - 4 * self.cycle_length()), - self.generate_diff(tip_height - 4 * self.cycle_length() - 8, tip_height - 4 * self.cycle_length()) - )), - - // Last commitment per index for rotating quorums - last_commitment_per_index: self.generate_last_commitments(tip_height), - - // Additional snapshots and diffs - quorum_snapshot_list: vec![], - mn_list_diff_list: vec![], - } - } - - /// Generate QRInfo with rotation cycle - pub fn generate_with_rotation(&self, tip_height: u32) -> QRInfo { - let mut qr_info = self.generate_normal_sync(tip_height); - - // Add rotating quorum data - qr_info.last_commitment_per_index = self.generate_rotating_commitments(tip_height); - - qr_info - } - - fn cycle_length(&self) -> u32 { - match self.network { - Network::Dash => 576, // ~24 hours at 2.5min blocks - Network::Testnet => 24, - _ => 24 - } - } - - fn generate_snapshot(&self, height: u32) -> QuorumSnapshot { - // Generate realistic quorum snapshot - let mut active_quorum_members = Vec::new(); - - // Add some test quorum members - for i in 0..10 { - active_quorum_members.push([i as u8; 32]); // Dummy member IDs - } - - QuorumSnapshot { - active_quorum_members, - } - } - - fn generate_diff(&self, base_height: u32, tip_height: u32) -> MnListDiff { - let base_hash = self.block_hash_at_height(base_height); - let tip_hash = self.block_hash_at_height(tip_height); - - MnListDiff { - base_block_hash: base_hash, - block_hash: tip_hash, - cb_tx_merkle_tree: vec![], - cb_tx: None, - deleted_mns: vec![], - mn_list: vec![], - deleted_quorums: vec![], - new_quorums: self.generate_test_quorums(tip_height), - } - } - - fn generate_last_commitments(&self, tip_height: u32) -> Vec { - // Generate test quorum entries for last commitments - (0..4).map(|i| { - QuorumEntry { - llmq_type: LLMQType::Llmqtype400_60, - quorum_hash: self.block_hash_at_height(tip_height - i * 100), - quorum_index: Some(i as u32), - quorum_public_key: BLSPublicKey::default(), - quorum_vvec_hash: [0u8; 32], - quorum_sig: BLSSignature::default(), - sig: BLSSignature::default(), - } - }).collect() - } - - fn block_hash_at_height(&self, height: u32) -> BlockHash { - // Generate deterministic test block hashes - let mut hasher = Sha256::new(); - hasher.update(b"test_block_"); - hasher.update(&height.to_le_bytes()); - let hash = hasher.finalize(); - BlockHash::from_byte_array(hash.into()) - } -} -``` - -**Test File**: `tests/fixtures/test_qr_info_generation.rs` -```rust -#[test] -fn test_qr_info_generation_normal() { - let generator = QRInfoTestGenerator::new(Network::Testnet, 1000); - let qr_info = generator.generate_normal_sync(2000); - - // Verify structure - assert_eq!(qr_info.mn_list_diff_list.len(), 0); // Normal sync has no extra diffs - assert!(!qr_info.last_commitment_per_index.is_empty()); - assert!(qr_info.quorum_snapshot_and_mn_list_diff_at_h_minus_4c.is_some()); - - // Verify diffs are properly constructed - assert_ne!(qr_info.mn_list_diff_tip.base_block_hash, qr_info.mn_list_diff_tip.block_hash); -} - -#[test] -fn test_qr_info_generation_with_rotation() { - let generator = QRInfoTestGenerator::new(Network::Testnet, 1000); - let qr_info = generator.generate_with_rotation(2000); - - // Should have rotating quorum commitments - assert!(!qr_info.last_commitment_per_index.is_empty()); - - // Verify quorum types are appropriate for rotation - for commitment in &qr_info.last_commitment_per_index { - assert!(commitment.llmq_type.is_rotating_quorum_type()); - } -} -``` - -#### 3.2 Integration Test Suite - -**File**: `tests/integration/test_qr_info_sync_flow.rs` -```rust -use dashcore::Network; -use dash_spv::test_utils::*; - -#[tokio::test] -async fn test_complete_qr_info_sync_flow() { - let config = test_client_config(Network::Testnet); - let mut client = create_test_client(config).await; - - // Set up mock network to provide QRInfo responses - let mut mock_network = MockNetworkManager::new(); - mock_network.expect_request_qr_info() - .returning(|base, tip, extra| { - let generator = QRInfoTestGenerator::new(Network::Testnet, 0); - let qr_info = generator.generate_normal_sync(1000); - // Simulate async network response - tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(100)).await; - client.handle_qr_info(qr_info).await; - }); - Ok(()) - }); - - // Start sync - client.start_sync().await.unwrap(); - - // Wait for completion - let result = tokio::time::timeout( - Duration::from_secs(10), - client.wait_for_masternode_sync() - ).await.unwrap(); - - assert!(result.is_ok()); - - // Verify final state - let engine = client.masternode_list_engine().unwrap(); - assert!(!engine.masternode_lists.is_empty()); - assert!(!engine.known_snapshots.is_empty()); -} - -#[tokio::test] -async fn test_qr_info_error_recovery() { - let mut client = create_test_client_with_error_network().await; - - // First QRInfo request fails - client.network.set_next_qr_info_response(Err(NetworkError::Timeout)); - - let sync_result = tokio::time::timeout( - Duration::from_secs(5), - client.start_sync() - ).await; - - // Should handle error gracefully and retry - assert!(sync_result.is_err() || sync_result.unwrap().is_err()); - - // Second attempt succeeds - let qr_info = QRInfoTestGenerator::new(Network::Testnet, 0).generate_normal_sync(1000); - client.network.set_next_qr_info_response(Ok(qr_info)); - - let retry_result = client.retry_sync().await; - assert!(retry_result.is_ok()); -} - -#[tokio::test] -async fn test_qr_info_vs_mn_diff_compatibility() { - // Test that QRInfo and MnListDiff can work together - let mut client = create_test_client(test_client_config(Network::Testnet)).await; - - // Start with some MnListDiff data - let mn_diff = create_test_mn_list_diff(0, 500); - client.handle_mn_list_diff(mn_diff).await.unwrap(); - - // Then add QRInfo data - let qr_info = QRInfoTestGenerator::new(Network::Testnet, 500).generate_normal_sync(1000); - client.handle_qr_info(qr_info).await.unwrap(); - - // Verify combined state is consistent - let engine = client.masternode_list_engine().unwrap(); - - // Should have masternode lists from both sources - assert!(engine.masternode_lists.contains_key(&500)); // From MnListDiff - assert!(engine.masternode_lists.contains_key(&1000)); // From QRInfo - - // State should be internally consistent - let all_heights: Vec = engine.masternode_lists.keys().cloned().collect(); - assert!(all_heights.windows(2).all(|w| w[0] < w[1])); // Sorted -} -``` - -### 4. Configuration and Feature Flags - -#### 4.1 Add Configuration Options - -**File**: `dash-spv/src/client/config.rs` -```rust -#[derive(Clone, Debug)] -pub struct ClientConfig { - // ... existing fields ... - - /// Enable QRInfo-based masternode sync (default: true) - pub enable_qr_info: bool, - - /// Fall back to MnListDiff if QRInfo fails (default: true) - pub qr_info_fallback: bool, - - /// Request extra share data in QRInfo (default: true for better validation) - pub qr_info_extra_share: bool, - - /// Timeout for QRInfo requests (default: 30 seconds) - pub qr_info_timeout: Duration, -} - -impl Default for ClientConfig { - fn default() -> Self { - Self { - // ... existing defaults ... - enable_qr_info: true, - qr_info_fallback: true, - qr_info_extra_share: true, - qr_info_timeout: Duration::from_secs(30), - } - } -} -``` - -**Test File**: `tests/config/test_qr_info_config.rs` -```rust -#[test] -fn test_qr_info_config_defaults() { - let config = ClientConfig::default(); - assert!(config.enable_qr_info); - assert!(config.qr_info_fallback); - assert!(config.qr_info_extra_share); - assert_eq!(config.qr_info_timeout, Duration::from_secs(30)); -} - -#[test] -fn test_qr_info_config_disabled() { - let mut config = ClientConfig::default(); - config.enable_qr_info = false; - - // When QRInfo is disabled, should fall back to old MnListDiff behavior - let client = create_test_client(config); - // Test that only MnListDiff requests are made -} -``` - -## Success Criteria - -### Functional Requirements -- [ ] QRInfo messages can be parsed and processed successfully -- [ ] QRInfo data integrates properly with masternode list engine -- [ ] Block hash lookups work efficiently with storage layer -- [ ] Existing MnListDiff functionality remains unbroken -- [ ] Configuration options work as expected - -### Performance Requirements -- [ ] QRInfo processing completes within 5 seconds for typical data -- [ ] Block hash lookup performance < 1ms average -- [ ] Memory usage increase < 10% during QRInfo processing -- [ ] No performance regression in existing sync paths - -### Quality Requirements -- [ ] >90% test coverage for all new code -- [ ] All integration tests pass with real QRInfo data -- [ ] Error handling covers all failure scenarios -- [ ] Logging provides sufficient debugging information - -## Risk Mitigation - -### High Risk: QRInfo Protocol Complexity -**Risk**: QRInfo message format is complex with many nested structures -**Mitigation**: -- Use real network test vectors for validation -- Implement comprehensive parsing tests -- Add detailed error messages for parsing failures - -### Medium Risk: Storage Performance Impact -**Risk**: Block hash index might impact storage performance -**Mitigation**: -- Implement lazy index building -- Add configuration option to disable if needed -- Monitor storage performance in tests - -### Low Risk: Configuration Complexity -**Risk**: Too many configuration options might confuse users -**Mitigation**: -- Provide sensible defaults for all options -- Clear documentation for each setting -- Simple enable/disable for main QRInfo functionality - -## Next Steps - -Upon completion of Phase 1: -1. **Validate** all tests pass with real network data -2. **Performance** benchmark QRInfo processing vs MnListDiff -3. **Documentation** update API docs and examples -4. **Phase 2** proceed to engine discovery integration - -The foundation established in Phase 1 enables the more advanced optimizations in subsequent phases while maintaining full backward compatibility and comprehensive test coverage. \ No newline at end of file diff --git a/qr_info_spv_plan/PHASE_2.md b/qr_info_spv_plan/PHASE_2.md deleted file mode 100644 index 7d6da9f44..000000000 --- a/qr_info_spv_plan/PHASE_2.md +++ /dev/null @@ -1,969 +0,0 @@ -# Phase 2: Engine Discovery Integration - -## Overview - -This phase replaces dash-spv's manual height tracking approach with the masternode list engine's intended discovery methods. Instead of dash-spv deciding what to request next, the engine will tell us exactly which masternode lists are missing and needed for validation. - -## Objectives - -1. **Replace Manual Tracking**: Remove hardcoded height progression logic -2. **Engine-Driven Discovery**: Use engine methods to identify missing data -3. **Intelligent Batching**: Group missing data into efficient QRInfo requests -4. **Demand-Driven Sync**: Only request data that's actually needed - -## Current State Analysis - -### Current Inefficient Approach -```rust -// dash-spv currently does this manually: -impl MasternodeSyncManager { - fn determine_next_diff_to_request(&self) -> (u32, u32) { - // Manual calculation of next height range - let base_height = self.last_processed_height; - let target_height = base_height + DIFF_BATCH_SIZE; - (base_height, target_height) - } -} -``` - -### Engine's Intended Approach -```rust -// Engine provides these discovery methods: -engine.latest_masternode_list_non_rotating_quorum_hashes(&[], true) -// Returns: Block hashes where we DON'T have masternode lists - -engine.masternode_list_for_block_hash(&block_hash) -// Returns: Masternode list if we have it, None if missing -``` - -## Detailed Implementation Plan - -### 1. Engine Discovery API Integration - -#### 1.1 Create Discovery Service - -**File**: `dash-spv/src/sync/discovery.rs` - -**Implementation**: -```rust -use dashcore::sml::{ - llmq_type::LLMQType, - masternode_list_engine::MasternodeListEngine, -}; -use std::collections::{BTreeSet, BTreeMap}; - -/// Service for discovering missing masternode data using engine methods -pub struct MasternodeDiscoveryService { - /// LLMQ types to exclude from discovery (configurable) - excluded_quorum_types: Vec, -} - -impl MasternodeDiscoveryService { - pub fn new() -> Self { - Self { - // Exclude types we don't need for SPV - excluded_quorum_types: vec![ - LLMQType::Llmqtype5_60, // Too small for meaningful validation - LLMQType::Llmqtype50_60, // Platform-specific, not needed for SPV - ], - } - } - - /// Discover which masternode lists are missing from the engine - pub fn discover_missing_masternode_lists( - &self, - engine: &MasternodeListEngine, - ) -> DiscoveryResult { - // Use engine's built-in discovery method - let missing_hashes = engine.latest_masternode_list_non_rotating_quorum_hashes( - &self.excluded_quorum_types, - true // only_return_block_hashes_with_missing_masternode_lists_from_engine - ); - - tracing::info!("Discovered {} missing masternode lists", missing_hashes.len()); - - // Convert block hashes to heights using engine's block container - let mut missing_heights = BTreeMap::new(); - for hash in missing_hashes { - if let Some(height) = engine.block_container.get_height(&hash) { - missing_heights.insert(height, hash); - tracing::debug!("Missing masternode list at height {}: {:x}", height, hash); - } else { - tracing::warn!("Found missing hash {:x} but no height mapping", hash); - } - } - - DiscoveryResult { - missing_by_height: missing_heights, - total_discovered: missing_heights.len(), - requires_qr_info: !missing_heights.is_empty(), - } - } - - /// Discover rotating quorums that need validation - pub fn discover_rotating_quorum_needs( - &self, - engine: &MasternodeListEngine, - ) -> RotatingQuorumDiscovery { - let rotating_hashes = engine.latest_masternode_list_rotating_quorum_hashes( - &self.excluded_quorum_types - ); - - let mut needs_validation = Vec::new(); - let mut missing_cycle_data = Vec::new(); - - for hash in rotating_hashes { - if let Some(height) = engine.block_container.get_height(&hash) { - // Check if we have the quorum cycle data - if !engine.rotated_quorums_per_cycle.contains_key(&hash) { - missing_cycle_data.push((height, hash)); - } - - // Check if quorum needs validation - if let Some(list) = engine.masternode_lists.get(&height) { - for (llmq_type, quorums) in &list.quorums { - if llmq_type.is_rotating_quorum_type() { - for (_, quorum_entry) in quorums { - if quorum_entry.verified == LLMQEntryVerificationStatus::Unknown { - needs_validation.push((height, hash, *llmq_type)); - } - } - } - } - } - } - } - - RotatingQuorumDiscovery { - needs_validation, - missing_cycle_data, - } - } - - /// Create optimal QRInfo requests based on discovery results - pub fn plan_qr_info_requests( - &self, - discovery: &DiscoveryResult, - max_request_span: u32, - ) -> Vec { - let mut requests = Vec::new(); - - if discovery.missing_by_height.is_empty() { - return requests; - } - - // Group missing heights into ranges for efficient QRInfo requests - let heights: Vec = discovery.missing_by_height.keys().cloned().collect(); - let mut current_range_start = heights[0]; - let mut current_range_end = heights[0]; - - for &height in &heights[1..] { - if height - current_range_end <= max_request_span && - height - current_range_start <= max_request_span * 3 { - // Extend current range - current_range_end = height; - } else { - // Finalize current range and start new one - requests.push(QRInfoRequest { - base_height: current_range_start.saturating_sub(8), // h-8 for validation - tip_height: current_range_end, - base_hash: discovery.missing_by_height[¤t_range_start], - tip_hash: discovery.missing_by_height[¤t_range_end], - extra_share: true, // Always request extra validation data - priority: self.calculate_priority(current_range_start, current_range_end), - }); - - current_range_start = height; - current_range_end = height; - } - } - - // Add final range - requests.push(QRInfoRequest { - base_height: current_range_start.saturating_sub(8), - tip_height: current_range_end, - base_hash: discovery.missing_by_height[¤t_range_start], - tip_hash: discovery.missing_by_height[¤t_range_end], - extra_share: true, - priority: self.calculate_priority(current_range_start, current_range_end), - }); - - // Sort by priority (most recent first for SPV) - requests.sort_by(|a, b| b.priority.cmp(&a.priority)); - - tracing::info!( - "Planned {} QRInfo requests covering {} heights", - requests.len(), - discovery.total_discovered - ); - - requests - } - - fn calculate_priority(&self, start_height: u32, end_height: u32) -> u32 { - // More recent blocks have higher priority for SPV - end_height - } -} - -#[derive(Debug)] -pub struct DiscoveryResult { - pub missing_by_height: BTreeMap, - pub total_discovered: usize, - pub requires_qr_info: bool, -} - -#[derive(Debug)] -pub struct RotatingQuorumDiscovery { - pub needs_validation: Vec<(u32, BlockHash, LLMQType)>, - pub missing_cycle_data: Vec<(u32, BlockHash)>, -} - -#[derive(Debug, Clone)] -pub struct QRInfoRequest { - pub base_height: u32, - pub tip_height: u32, - pub base_hash: BlockHash, - pub tip_hash: BlockHash, - pub extra_share: bool, - pub priority: u32, -} -``` - -**Test File**: `tests/sync/test_masternode_discovery.rs` -```rust -#[tokio::test] -async fn test_discovery_finds_missing_lists() { - let mut engine = create_test_engine_with_gaps().await; - let discovery_service = MasternodeDiscoveryService::new(); - - // Add some masternode lists but leave gaps - engine.masternode_lists.insert(1000, create_test_masternode_list(1000)); - engine.masternode_lists.insert(1500, create_test_masternode_list(1500)); - // Gap: missing 1200, 1300, 1400 - - // Add block hashes for the missing heights - for height in 1200..=1400 { - let hash = test_block_hash(height); - engine.feed_block_height(height, hash); - - // Add quorum references that point to these missing heights - if let Some(list) = engine.masternode_lists.get_mut(&1500) { - list.add_quorum_reference(height, hash); // This would create the "missing" reference - } - } - - let result = discovery_service.discover_missing_masternode_lists(&engine); - - assert_eq!(result.total_discovered, 3); // 1200, 1300, 1400 - assert!(result.requires_qr_info); - assert!(result.missing_by_height.contains_key(&1200)); - assert!(result.missing_by_height.contains_key(&1300)); - assert!(result.missing_by_height.contains_key(&1400)); -} - -#[tokio::test] -async fn test_discovery_no_missing_lists() { - let engine = create_complete_test_engine().await; - let discovery_service = MasternodeDiscoveryService::new(); - - let result = discovery_service.discover_missing_masternode_lists(&engine); - - assert_eq!(result.total_discovered, 0); - assert!(!result.requires_qr_info); - assert!(result.missing_by_height.is_empty()); -} - -#[tokio::test] -async fn test_qr_info_request_planning() { - let discovery_service = MasternodeDiscoveryService::new(); - - // Create discovery result with scattered missing heights - let mut missing_by_height = BTreeMap::new(); - missing_by_height.insert(1000, test_block_hash(1000)); - missing_by_height.insert(1001, test_block_hash(1001)); - missing_by_height.insert(1002, test_block_hash(1002)); - missing_by_height.insert(1100, test_block_hash(1100)); // Gap - missing_by_height.insert(1200, test_block_hash(1200)); // Another gap - - let discovery = DiscoveryResult { - missing_by_height, - total_discovered: 5, - requires_qr_info: true, - }; - - let requests = discovery_service.plan_qr_info_requests(&discovery, 50); - - // Should group 1000-1002 together, 1100 separate, 1200 separate - assert_eq!(requests.len(), 3); - - // Check first request covers the grouped heights - assert_eq!(requests[0].tip_height, 1002); - assert!(requests[0].base_height <= 1000); - - // Check priorities (higher for more recent) - assert!(requests[0].priority >= requests[1].priority); -} - -#[tokio::test] -async fn test_rotating_quorum_discovery() { - let mut engine = create_test_engine_with_rotating_quorums().await; - let discovery_service = MasternodeDiscoveryService::new(); - - // Add some rotating quorums that need validation - add_unvalidated_rotating_quorum(&mut engine, 2000, LLMQType::Llmqtype400_60); - - let result = discovery_service.discover_rotating_quorum_needs(&engine); - - assert!(!result.needs_validation.is_empty()); - assert_eq!(result.needs_validation[0].2, LLMQType::Llmqtype400_60); -} -``` - -#### 1.2 Integrate Discovery into Sync Manager - -**File**: `dash-spv/src/sync/masternodes.rs` - -**Implementation**: -```rust -impl MasternodeSyncManager { - /// Perform engine-driven discovery of missing data - pub async fn discover_sync_needs(&mut self) -> SyncResult { - let engine = self.engine.as_ref().ok_or_else(|| { - SyncError::Configuration("Masternode engine not initialized".to_string()) - })?; - - let discovery_service = MasternodeDiscoveryService::new(); - - // Discover missing masternode lists - let missing_lists = discovery_service.discover_missing_masternode_lists(engine); - - // Discover rotating quorum needs - let rotating_needs = discovery_service.discover_rotating_quorum_needs(engine); - - // Plan QRInfo requests - let qr_info_requests = discovery_service.plan_qr_info_requests( - &missing_lists, - self.config.qr_info_max_span.unwrap_or(500) // ~20 hours of blocks - ); - - let plan = SyncPlan { - qr_info_requests, - rotating_validation_needed: !rotating_needs.needs_validation.is_empty(), - estimated_completion_time: self.estimate_sync_time(&missing_lists), - fallback_to_mn_diff: missing_lists.total_discovered > 1000, // Large gaps - }; - - tracing::info!( - "Sync plan: {} QRInfo requests, rotating_validation={}, fallback={}", - plan.qr_info_requests.len(), - plan.rotating_validation_needed, - plan.fallback_to_mn_diff - ); - - Ok(plan) - } - - /// Execute the sync plan using engine discovery - pub async fn execute_engine_driven_sync( - &mut self, - network: &mut dyn NetworkManager, - plan: SyncPlan, - ) -> SyncResult<()> { - if plan.qr_info_requests.is_empty() { - tracing::info!("No sync needed - engine has all required data"); - return Ok(()); - } - - // Execute QRInfo requests in priority order - for (i, request) in plan.qr_info_requests.iter().enumerate() { - tracing::info!( - "Executing QRInfo request {}/{}: heights {}-{}", - i + 1, - plan.qr_info_requests.len(), - request.base_height, - request.tip_height - ); - - // Request QRInfo - network.request_qr_info( - request.base_hash, - request.tip_hash, - request.extra_share - ).await.map_err(|e| { - SyncError::Network(format!("Failed to request QRInfo: {}", e)) - })?; - - // Wait for response with timeout - let timeout = tokio::time::timeout( - self.config.qr_info_timeout, - self.wait_for_qr_info_response() - ).await; - - match timeout { - Ok(Ok(qr_info)) => { - self.process_qr_info_response(qr_info).await?; - tracing::info!("Successfully processed QRInfo response {}/{}", i + 1, plan.qr_info_requests.len()); - } - Ok(Err(e)) => { - if plan.fallback_to_mn_diff { - tracing::warn!("QRInfo failed, falling back to MnListDiff: {}", e); - self.fallback_to_mn_diff_sync(request, network).await?; - } else { - return Err(e); - } - } - Err(_) => { - tracing::error!("QRInfo request timed out for heights {}-{}", request.base_height, request.tip_height); - if plan.fallback_to_mn_diff { - self.fallback_to_mn_diff_sync(request, network).await?; - } else { - return Err(SyncError::Network("QRInfo request timeout".to_string())); - } - } - } - - // Brief pause between requests to be network-friendly - tokio::time::sleep(Duration::from_millis(100)).await; - } - - // Perform any additional rotating quorum validation if needed - if plan.rotating_validation_needed { - self.validate_rotating_quorums().await?; - } - - tracing::info!("Engine-driven sync completed successfully"); - Ok(()) - } - - /// Process QRInfo response using engine - async fn process_qr_info_response(&mut self, qr_info: QRInfo) -> SyncResult<()> { - let engine = self.engine.as_mut().ok_or_else(|| { - SyncError::Configuration("Masternode engine not initialized".to_string()) - })?; - - // Create block height fetcher for engine - let block_height_fetcher = |block_hash: &BlockHash| -> Result { - if let Some(height) = engine.block_container.get_height(block_hash) { - Ok(height) - } else { - Err(ClientDataRetrievalError::BlockNotFound(*block_hash)) - } - }; - - // Process through engine - engine.feed_qr_info( - qr_info, - true, // verify_tip_non_rotated_quorums - true, // verify_rotated_quorums - Some(block_height_fetcher) - ).map_err(|e| SyncError::Validation(format!("Engine QRInfo processing failed: {}", e)))?; - - // Update sync progress - self.update_sync_progress_from_engine(); - - Ok(()) - } - - /// Fallback to individual MnListDiff requests if QRInfo fails - async fn fallback_to_mn_diff_sync( - &mut self, - request: &QRInfoRequest, - network: &mut dyn NetworkManager, - ) -> SyncResult<()> { - tracing::info!( - "Falling back to MnListDiff sync for heights {}-{}", - request.base_height, - request.tip_height - ); - - // Request individual diffs for the range - for height in request.base_height..=request.tip_height { - let base_height = height.saturating_sub(1); - self.request_masternode_diff(network, storage, base_height, height).await?; - - // Wait for response - let diff = self.wait_for_mn_diff_response().await?; - self.process_mn_diff(diff).await?; - } - - Ok(()) - } - - /// Update sync progress based on engine state - fn update_sync_progress_from_engine(&mut self) { - if let Some(engine) = &self.engine { - let total_lists = engine.masternode_lists.len(); - let latest_height = engine.masternode_lists.keys().max().copied().unwrap_or(0); - - self.sync_progress = MasternodeSyncProgress { - total_lists, - latest_height, - quorum_validation_complete: self.check_quorum_validation_complete(engine), - estimated_remaining_time: self.estimate_remaining_time(engine), - }; - } - } - - fn estimate_sync_time(&self, discovery: &DiscoveryResult) -> Duration { - // Estimate based on number of QRInfo requests and network latency - let base_time_per_request = Duration::from_secs(2); // Conservative estimate - let total_requests = (discovery.total_discovered / 100).max(1); // ~100 blocks per request - base_time_per_request * total_requests as u32 - } -} - -#[derive(Debug)] -pub struct SyncPlan { - pub qr_info_requests: Vec, - pub rotating_validation_needed: bool, - pub estimated_completion_time: Duration, - pub fallback_to_mn_diff: bool, -} - -#[derive(Debug)] -pub struct MasternodeSyncProgress { - pub total_lists: usize, - pub latest_height: u32, - pub quorum_validation_complete: bool, - pub estimated_remaining_time: Duration, -} -``` - -**Test File**: `tests/sync/test_engine_driven_sync.rs` -```rust -#[tokio::test] -async fn test_engine_driven_discovery() { - let mut sync_manager = setup_sync_manager_with_gaps().await; - - // Engine has some data but is missing critical pieces - add_masternode_list_with_gaps(&mut sync_manager).await; - - let plan = sync_manager.discover_sync_needs().await.unwrap(); - - assert!(!plan.qr_info_requests.is_empty()); - assert!(plan.estimated_completion_time > Duration::ZERO); - - // Verify requests cover the gaps - let covered_heights: Vec = plan.qr_info_requests - .iter() - .flat_map(|req| req.base_height..=req.tip_height) - .collect(); - - assert!(covered_heights.contains(&1200)); // Known gap - assert!(covered_heights.contains(&1300)); // Known gap -} - -#[tokio::test] -async fn test_sync_plan_execution() { - let mut sync_manager = create_test_sync_manager().await; - let mut mock_network = create_mock_network_with_qr_info().await; - - let plan = SyncPlan { - qr_info_requests: vec![create_test_qr_info_request()], - rotating_validation_needed: false, - estimated_completion_time: Duration::from_secs(5), - fallback_to_mn_diff: false, - }; - - let result = sync_manager.execute_engine_driven_sync(&mut mock_network, plan).await; - assert!(result.is_ok()); - - // Verify network requests were made - assert_eq!(mock_network.get_qr_info_request_count(), 1); - - // Verify engine state updated - let engine = sync_manager.engine().unwrap(); - assert!(!engine.masternode_lists.is_empty()); -} - -#[tokio::test] -async fn test_qr_info_timeout_fallback() { - let mut sync_manager = create_test_sync_manager().await; - let mut mock_network = create_mock_network_with_timeout().await; - - let plan = SyncPlan { - qr_info_requests: vec![create_test_qr_info_request()], - rotating_validation_needed: false, - estimated_completion_time: Duration::from_secs(5), - fallback_to_mn_diff: true, // Enable fallback - }; - - // Should succeed despite QRInfo timeout due to fallback - let result = sync_manager.execute_engine_driven_sync(&mut mock_network, plan).await; - assert!(result.is_ok()); - - // Verify fallback was used - assert!(mock_network.get_mn_diff_request_count() > 0); -} - -#[tokio::test] -async fn test_no_sync_needed() { - let sync_manager = create_complete_sync_manager().await; - - let plan = sync_manager.discover_sync_needs().await.unwrap(); - - assert!(plan.qr_info_requests.is_empty()); - assert!(!plan.rotating_validation_needed); - assert_eq!(plan.estimated_completion_time, Duration::ZERO); -} -``` - -### 2. Replace Manual Height Tracking - -#### 2.1 Remove Old Logic - -**File**: `dash-spv/src/sync/masternodes.rs` (modifications) - -**Changes**: -```rust -impl MasternodeSyncManager { - // REMOVE these manual tracking methods: - // fn determine_next_diff_to_request(&self) -> (u32, u32) - // fn calculate_next_height_range(&self) -> Option<(u32, u32)> - // fn update_last_processed_height(&mut self, height: u32) - - // REPLACE with engine-driven approach: - - /// Start masternode sync using engine discovery - pub async fn start_sync( - &mut self, - network: &mut dyn NetworkManager, - storage: &dyn StorageManager, - ) -> SyncResult<()> { - tracing::info!("Starting engine-driven masternode sync"); - - // Initialize engine if needed - if self.engine.is_none() { - self.initialize_engine_from_storage(storage).await?; - } - - // Discover what we need - let plan = self.discover_sync_needs().await?; - - if plan.qr_info_requests.is_empty() { - tracing::info!("Masternode sync already complete"); - return Ok(()); - } - - // Execute the plan - self.execute_engine_driven_sync(network, plan).await?; - - // Perform final validation - self.validate_final_state().await?; - - tracing::info!("Engine-driven masternode sync completed"); - Ok(()) - } - - /// Check if sync is complete based on engine state - pub fn is_sync_complete(&self) -> bool { - if let Some(engine) = &self.engine { - // Check if we have all required masternode lists - let discovery_service = MasternodeDiscoveryService::new(); - let missing = discovery_service.discover_missing_masternode_lists(engine); - - missing.total_discovered == 0 - } else { - false - } - } - - /// Get sync progress based on engine analysis - pub fn get_sync_progress(&self) -> MasternodeSyncProgress { - if let Some(engine) = &self.engine { - let discovery_service = MasternodeDiscoveryService::new(); - let missing = discovery_service.discover_missing_masternode_lists(engine); - - let total_known = engine.masternode_lists.len(); - let total_needed = total_known + missing.total_discovered; - let completion_percentage = if total_needed > 0 { - (total_known as f32 / total_needed as f32) * 100.0 - } else { - 100.0 - }; - - MasternodeSyncProgress { - total_lists: total_known, - latest_height: engine.masternode_lists.keys().max().copied().unwrap_or(0), - quorum_validation_complete: self.check_quorum_validation_complete(engine), - completion_percentage, - estimated_remaining_time: self.estimate_remaining_time_from_missing(&missing), - } - } else { - MasternodeSyncProgress::default() - } - } -} -``` - -**Test File**: `tests/sync/test_engine_driven_replacement.rs` -```rust -#[tokio::test] -async fn test_old_manual_logic_removed() { - let sync_manager = create_test_sync_manager().await; - - // Verify old manual methods are no longer available - // This test will fail to compile if manual methods still exist - // assert!(!has_method(&sync_manager, "determine_next_diff_to_request")); - // assert!(!has_method(&sync_manager, "calculate_next_height_range")); - - // Verify new engine-driven methods work - let is_complete = sync_manager.is_sync_complete(); - assert!(is_complete == false || is_complete == true); // Should not panic - - let progress = sync_manager.get_sync_progress(); - assert!(progress.completion_percentage >= 0.0); - assert!(progress.completion_percentage <= 100.0); -} - -#[tokio::test] -async fn test_sync_completion_detection() { - let mut sync_manager = create_complete_sync_manager().await; - - assert!(sync_manager.is_sync_complete()); - - let progress = sync_manager.get_sync_progress(); - assert_eq!(progress.completion_percentage, 100.0); - assert_eq!(progress.estimated_remaining_time, Duration::ZERO); -} - -#[tokio::test] -async fn test_sync_progress_accuracy() { - let mut sync_manager = create_sync_manager_with_known_gaps().await; - - // We know there are exactly 5 missing lists out of 20 total needed - let progress = sync_manager.get_sync_progress(); - - assert_eq!(progress.total_lists, 15); // 20 - 5 missing - assert!((progress.completion_percentage - 75.0).abs() < 1.0); // 15/20 = 75% - assert!(progress.estimated_remaining_time > Duration::ZERO); -} - -#[tokio::test] -async fn test_engine_driven_vs_manual_compatibility() { - // Test that engine-driven approach produces same results as old manual approach - // but more efficiently - - let manual_result = simulate_old_manual_sync().await; - let engine_result = simulate_engine_driven_sync().await; - - // Same final state - assert_eq!(manual_result.final_height, engine_result.final_height); - assert_eq!(manual_result.total_lists, engine_result.total_lists); - - // But engine-driven should be more efficient - assert!(engine_result.network_requests < manual_result.network_requests); - assert!(engine_result.sync_time < manual_result.sync_time); -} -``` - -### 3. Intelligent Batching Strategies - -#### 3.1 Advanced Request Optimization - -**File**: `dash-spv/src/sync/batching.rs` - -**Implementation**: -```rust -/// Advanced batching strategies for QRInfo requests -pub struct QRInfoBatchingStrategy { - network_latency: Duration, - bandwidth_limit: Option, // bytes per second - max_concurrent_requests: usize, -} - -impl QRInfoBatchingStrategy { - pub fn new() -> Self { - Self { - network_latency: Duration::from_millis(100), // Conservative default - bandwidth_limit: None, - max_concurrent_requests: 3, // Conservative for SPV - } - } - - /// Optimize QRInfo requests based on network conditions - pub fn optimize_requests( - &self, - requests: Vec, - network_conditions: &NetworkConditions, - ) -> Vec { - let mut optimized = Vec::new(); - - // Adjust strategy based on network conditions - let effective_latency = if network_conditions.high_latency { - self.network_latency * 2 - } else { - self.network_latency - }; - - let batch_size = if network_conditions.low_bandwidth { - 2 // Smaller batches for slow connections - } else { - 5 // Larger batches for fast connections - }; - - // Group requests into batches - for chunk in requests.chunks(batch_size) { - let batch = OptimizedQRInfoBatch { - requests: chunk.to_vec(), - priority: chunk.iter().map(|r| r.priority).max().unwrap_or(0), - estimated_response_size: self.estimate_response_size(chunk), - can_execute_parallel: chunk.len() <= self.max_concurrent_requests, - }; - - optimized.push(batch); - } - - // Sort batches by priority and network efficiency - optimized.sort_by(|a, b| { - b.priority.cmp(&a.priority) - .then_with(|| a.estimated_response_size.cmp(&b.estimated_response_size)) - }); - - optimized - } - - fn estimate_response_size(&self, requests: &[QRInfoRequest]) -> usize { - // Rough estimation based on typical QRInfo content - let base_size = 1024; // Base QRInfo overhead - let per_diff_size = 2048; // Average MnListDiff size - let per_snapshot_size = 512; // Average QuorumSnapshot size - - requests.iter().map(|req| { - let height_span = req.tip_height - req.base_height + 1; - let estimated_diffs = height_span / 8; // Diffs every ~8 blocks typically - let estimated_snapshots = 4; // h-c, h-2c, h-3c, h-4c - - base_size + - (estimated_diffs * per_diff_size as u32) as usize + - (estimated_snapshots * per_snapshot_size) - }).sum() - } -} - -#[derive(Debug)] -pub struct OptimizedQRInfoBatch { - pub requests: Vec, - pub priority: u32, - pub estimated_response_size: usize, - pub can_execute_parallel: bool, -} - -#[derive(Debug)] -pub struct NetworkConditions { - pub high_latency: bool, - pub low_bandwidth: bool, - pub unstable_connection: bool, -} -``` - -**Test File**: `tests/sync/test_batching_optimization.rs` -```rust -#[tokio::test] -async fn test_batching_optimization() { - let strategy = QRInfoBatchingStrategy::new(); - let conditions = NetworkConditions { - high_latency: false, - low_bandwidth: false, - unstable_connection: false, - }; - - let requests = create_test_qr_info_requests(10); - let optimized = strategy.optimize_requests(requests, &conditions); - - // Should create efficient batches - assert!(!optimized.is_empty()); - assert!(optimized.len() <= 5); // Should batch multiple requests together - - // Higher priority batches should come first - let priorities: Vec = optimized.iter().map(|b| b.priority).collect(); - assert!(priorities.windows(2).all(|w| w[0] >= w[1])); -} - -#[tokio::test] -async fn test_batching_with_poor_network() { - let strategy = QRInfoBatchingStrategy::new(); - let conditions = NetworkConditions { - high_latency: true, - low_bandwidth: true, - unstable_connection: true, - }; - - let requests = create_test_qr_info_requests(10); - let optimized = strategy.optimize_requests(requests, &conditions); - - // Should create smaller, more conservative batches - let avg_batch_size: f32 = optimized.iter() - .map(|b| b.requests.len()) - .sum::() as f32 / optimized.len() as f32; - - assert!(avg_batch_size <= 3.0); // Smaller batches for poor network -} -``` - -## Success Criteria - -### Functional Requirements -- [ ] Engine discovery methods work correctly to identify missing data -- [ ] QRInfo request planning creates optimal batches -- [ ] Manual height tracking completely replaced -- [ ] Sync completion detection is accurate -- [ ] Fallback to MnListDiff works when QRInfo fails - -### Performance Requirements -- [ ] Discovery phase completes in <1 second for typical engines -- [ ] QRInfo request optimization reduces total requests by >60% -- [ ] Sync progress reporting updates smoothly -- [ ] Memory usage remains stable throughout discovery - -### Quality Requirements -- [ ] >90% test coverage for all discovery logic -- [ ] All edge cases handled (empty engines, complete engines, gaps) -- [ ] Error handling preserves engine state consistency -- [ ] Comprehensive logging for debugging - -## Risk Mitigation - -### High Risk: Engine State Consistency -**Risk**: Discovery methods might return inconsistent results -**Mitigation**: -- Extensive unit tests with various engine states -- Integration tests with real engine data -- State validation checks after each operation - -### Medium Risk: Batching Complexity -**Risk**: Over-optimization might make batching logic fragile -**Mitigation**: -- Keep batching strategies simple and configurable -- Fallback to individual requests if batching fails -- Performance monitoring to detect issues - -### Low Risk: Progress Reporting Accuracy -**Risk**: Progress percentage might be misleading -**Mitigation**: -- Use conservative estimates -- Provide detailed progress breakdown -- Clear documentation about limitations - -## Integration Points - -### Phase 1 Dependencies -- QRInfo message handling must be complete -- Storage block hash lookup must work efficiently -- Engine integration must be stable - -### Phase 3 Preparation -- Discovery results will feed into parallel processing -- Batching strategies will be extended for concurrent execution -- Progress reporting will support multiple concurrent operations - -## Next Steps - -Upon completion of Phase 2: -1. **Validation**: Comprehensive testing with real engine states -2. **Performance**: Benchmark discovery speed and accuracy -3. **Documentation**: Update sync flow diagrams and API docs -4. **Phase 3**: Proceed to network efficiency optimization - -The engine-driven approach established in Phase 2 transforms dash-spv from a blind sequential sync to an intelligent, demand-driven architecture that only requests data it actually needs. \ No newline at end of file diff --git a/qr_info_spv_plan/PHASE_3.md b/qr_info_spv_plan/PHASE_3.md deleted file mode 100644 index be138939d..000000000 --- a/qr_info_spv_plan/PHASE_3.md +++ /dev/null @@ -1,1480 +0,0 @@ -# Phase 3: Network Efficiency Optimization - -## Overview - -This phase optimizes network efficiency by implementing parallel QRInfo processing, intelligent request scheduling, and robust error recovery. Building on the discovery foundation from Phase 2, we'll maximize sync speed while maintaining reliability and network-friendly behavior. - -## Objectives - -1. **Parallel Processing**: Execute multiple QRInfo requests concurrently -2. **Request Scheduling**: Intelligent timing and prioritization of requests -3. **Error Recovery**: Robust handling of network failures and timeouts -4. **Bandwidth Management**: Efficient use of available network capacity -5. **Progress Tracking**: Real-time progress reporting for parallel operations - -## Network Efficiency Analysis - -### Current Sequential Bottlenecks -```rust -// Phase 2 approach (sequential): -for request in qr_info_requests { - network.request_qr_info(request).await?; - let response = wait_for_response().await?; - process_response(response).await?; -} -// Total time: N * (network_latency + processing_time) -``` - -### Target Parallel Efficiency -```rust -// Phase 3 approach (parallel): -let futures = qr_info_requests.map(|req| async { - let response = network.request_qr_info(req).await?; - process_response(response).await -}); -join_all(futures).await; -// Total time: max(network_latency + processing_time) + scheduling_overhead -``` - -## Detailed Implementation Plan - -### 1. Parallel Request Executor - -#### 1.1 Concurrent QRInfo Request Manager - -**File**: `dash-spv/src/sync/parallel.rs` - -**Implementation**: -```rust -use tokio::{sync::Semaphore, task::JoinSet, time::timeout}; -use std::sync::Arc; -use dashcore::BlockHash; - -/// Manages parallel execution of QRInfo requests with concurrency control -pub struct ParallelQRInfoExecutor { - /// Maximum concurrent requests - max_concurrent: usize, - /// Semaphore for controlling concurrency - semaphore: Arc, - /// Network timeout for individual requests - request_timeout: Duration, - /// Progress reporting channel - progress_tx: Option>, -} - -impl ParallelQRInfoExecutor { - pub fn new(max_concurrent: usize, request_timeout: Duration) -> Self { - Self { - max_concurrent, - semaphore: Arc::new(Semaphore::new(max_concurrent)), - request_timeout, - progress_tx: None, - } - } - - pub fn with_progress_reporting(mut self, tx: mpsc::UnboundedSender) -> Self { - self.progress_tx = Some(tx); - self - } - - /// Execute multiple QRInfo requests in parallel with controlled concurrency - pub async fn execute_parallel_requests( - &self, - requests: Vec, - network: Arc>, - processor: Arc>, - ) -> Result, ParallelExecutionError> { - if requests.is_empty() { - return Ok(Vec::new()); - } - - tracing::info!( - "Starting parallel execution of {} QRInfo requests with max_concurrent={}", - requests.len(), - self.max_concurrent - ); - - let mut join_set = JoinSet::new(); - let total_requests = requests.len(); - let completed = Arc::new(AtomicUsize::new(0)); - - // Launch parallel tasks - for (index, request) in requests.into_iter().enumerate() { - let semaphore = self.semaphore.clone(); - let network = network.clone(); - let processor = processor.clone(); - let completed = completed.clone(); - let progress_tx = self.progress_tx.clone(); - let timeout_duration = self.request_timeout; - - let task = async move { - // Acquire semaphore permit for concurrency control - let _permit = semaphore.acquire().await - .map_err(|_| ParallelExecutionError::SemaphoreError)?; - - tracing::debug!("Starting QRInfo request {}/{}: heights {}-{}", - index + 1, total_requests, request.base_height, request.tip_height); - - // Execute request with timeout - let result = timeout(timeout_duration, async { - // Send network request - { - let mut net = network.lock().await; - net.request_qr_info( - request.base_hash, - request.tip_hash, - request.extra_share - ).await?; - } - - // Wait for and process response - let qr_info = Self::wait_for_qr_info_response(&request).await?; - - { - let mut proc = processor.lock().await; - proc.process_qr_info(qr_info).await?; - } - - Ok::(QRInfoResult { - request: request.clone(), - success: true, - processing_time: std::time::Instant::now(), - error: None, - }) - }).await; - - // Update progress - let completed_count = completed.fetch_add(1, Ordering::Relaxed) + 1; - if let Some(ref tx) = progress_tx { - let _ = tx.send(SyncProgress { - completed_requests: completed_count, - total_requests, - current_operation: format!("QRInfo {}/{}", completed_count, total_requests), - estimated_remaining: Self::estimate_remaining_time( - completed_count, total_requests, timeout_duration - ), - }); - } - - match result { - Ok(Ok(success_result)) => { - tracing::debug!("Completed QRInfo request {}/{} successfully", - index + 1, total_requests); - Ok(success_result) - } - Ok(Err(e)) => { - tracing::warn!("QRInfo request {}/{} failed: {}", - index + 1, total_requests, e); - Ok(QRInfoResult { - request, - success: false, - processing_time: std::time::Instant::now(), - error: Some(e), - }) - } - Err(_) => { - tracing::error!("QRInfo request {}/{} timed out after {:?}", - index + 1, total_requests, timeout_duration); - Ok(QRInfoResult { - request, - success: false, - processing_time: std::time::Instant::now(), - error: Some(ParallelExecutionError::Timeout), - }) - } - } - }; - - join_set.spawn(task); - } - - // Collect all results - let mut results = Vec::with_capacity(total_requests); - while let Some(task_result) = join_set.join_next().await { - match task_result { - Ok(qr_info_result) => results.push(qr_info_result?), - Err(e) => { - tracing::error!("Task execution error: {}", e); - return Err(ParallelExecutionError::TaskError(e.to_string())); - } - } - } - - // Sort results back to original request order for consistency - results.sort_by_key(|r| r.request.priority); - - let success_count = results.iter().filter(|r| r.success).count(); - let failure_count = results.len() - success_count; - - tracing::info!( - "Parallel QRInfo execution completed: {}/{} successful, {} failed", - success_count, total_requests, failure_count - ); - - Ok(results) - } - - /// Wait for QRInfo response for a specific request - async fn wait_for_qr_info_response(request: &QRInfoRequest) -> Result { - // Implementation would depend on how network responses are routed - // This is a placeholder - actual implementation would use request correlation - todo!("Implement QRInfo response correlation and waiting") - } - - fn estimate_remaining_time(completed: usize, total: usize, avg_time: Duration) -> Duration { - if completed == 0 || completed >= total { - return Duration::ZERO; - } - - let remaining = total - completed; - let completion_rate = completed as f32 / avg_time.as_secs_f32(); - Duration::from_secs_f32(remaining as f32 / completion_rate) - } -} - -#[derive(Debug, Clone)] -pub struct QRInfoResult { - pub request: QRInfoRequest, - pub success: bool, - pub processing_time: std::time::Instant, - pub error: Option, -} - -#[derive(Debug, thiserror::Error)] -pub enum ParallelExecutionError { - #[error("Network error: {0}")] - Network(String), - #[error("Processing error: {0}")] - Processing(String), - #[error("Request timed out")] - Timeout, - #[error("Semaphore error")] - SemaphoreError, - #[error("Task error: {0}")] - TaskError(String), -} -``` - -**Test File**: `tests/sync/test_parallel_execution.rs` -```rust -#[tokio::test] -async fn test_parallel_qr_info_execution() { - let executor = ParallelQRInfoExecutor::new(3, Duration::from_secs(5)); - - let requests = create_test_qr_info_requests(10); - let mock_network = Arc::new(Mutex::new(create_mock_network())); - let mock_processor = Arc::new(Mutex::new(create_mock_processor())); - - let start_time = Instant::now(); - let results = executor.execute_parallel_requests( - requests, - mock_network, - mock_processor - ).await.unwrap(); - let elapsed = start_time.elapsed(); - - assert_eq!(results.len(), 10); - assert!(results.iter().all(|r| r.success)); - - // Should be much faster than sequential (which would be ~10 * request_time) - // With 3 concurrent, should be roughly ~4 * request_time - assert!(elapsed < Duration::from_secs(8)); // Allow some buffer -} - -#[tokio::test] -async fn test_concurrency_limiting() { - let executor = ParallelQRInfoExecutor::new(2, Duration::from_secs(1)); - - let slow_network = Arc::new(Mutex::new(create_slow_mock_network())); - let mock_processor = Arc::new(Mutex::new(create_mock_processor())); - let requests = create_test_qr_info_requests(5); - - let concurrent_count = Arc::new(AtomicUsize::new(0)); - let max_concurrent = Arc::new(AtomicUsize::new(0)); - - // Track maximum concurrency achieved - let counter_clone = concurrent_count.clone(); - let max_clone = max_concurrent.clone(); - - tokio::spawn(async move { - loop { - let current = counter_clone.load(Ordering::Relaxed); - let max = max_clone.load(Ordering::Relaxed); - if current > max { - max_clone.store(current, Ordering::Relaxed); - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }); - - let results = executor.execute_parallel_requests( - requests, - slow_network, - mock_processor - ).await.unwrap(); - - assert_eq!(results.len(), 5); - - // Should never exceed our concurrency limit - let max_achieved = max_concurrent.load(Ordering::Relaxed); - assert!(max_achieved <= 2, "Max concurrency {} exceeded limit 2", max_achieved); -} - -#[tokio::test] -async fn test_error_handling_and_partial_failure() { - let executor = ParallelQRInfoExecutor::new(3, Duration::from_secs(2)); - - // Network that fails 50% of requests - let flaky_network = Arc::new(Mutex::new(create_flaky_mock_network(0.5))); - let mock_processor = Arc::new(Mutex::new(create_mock_processor())); - let requests = create_test_qr_info_requests(10); - - let results = executor.execute_parallel_requests( - requests, - flaky_network, - mock_processor - ).await.unwrap(); - - assert_eq!(results.len(), 10); // Should get results for all requests - - let success_count = results.iter().filter(|r| r.success).count(); - let failure_count = results.iter().filter(|r| !r.success).count(); - - assert!(success_count > 0, "At least some requests should succeed"); - assert!(failure_count > 0, "Some requests should fail with flaky network"); - assert_eq!(success_count + failure_count, 10); - - // Failed results should have error information - for result in results.iter().filter(|r| !r.success) { - assert!(result.error.is_some()); - } -} - -#[tokio::test] -async fn test_progress_reporting() { - let executor = ParallelQRInfoExecutor::new(2, Duration::from_secs(1)); - let (progress_tx, mut progress_rx) = mpsc::unbounded_channel(); - let executor = executor.with_progress_reporting(progress_tx); - - let requests = create_test_qr_info_requests(5); - let mock_network = Arc::new(Mutex::new(create_mock_network())); - let mock_processor = Arc::new(Mutex::new(create_mock_processor())); - - let execution_handle = tokio::spawn(async move { - executor.execute_parallel_requests(requests, mock_network, mock_processor).await - }); - - let mut progress_updates = Vec::new(); - - // Collect progress updates - while let Some(progress) = progress_rx.recv().await { - progress_updates.push(progress); - if progress.completed_requests >= 5 { - break; - } - } - - let results = execution_handle.await.unwrap().unwrap(); - assert_eq!(results.len(), 5); - - // Should have received progress updates - assert!(!progress_updates.is_empty()); - assert!(progress_updates.len() <= 5); // At most one per request - - // Progress should increase monotonically - let completed_counts: Vec = progress_updates.iter() - .map(|p| p.completed_requests) - .collect(); - - assert!(completed_counts.windows(2).all(|w| w[0] <= w[1])); - assert_eq!(completed_counts.last(), Some(&5)); -} -``` - -#### 1.2 Request Correlation and Response Matching - -**File**: `dash-spv/src/network/correlation.rs` - -**Implementation**: -```rust -use std::collections::HashMap; -use tokio::sync::oneshot; -use dashcore::{BlockHash, network::message_qrinfo::QRInfo}; - -/// Correlates QRInfo requests with responses for parallel processing -pub struct QRInfoCorrelationManager { - /// Pending requests waiting for responses - pending_requests: HashMap, - /// Next request ID - next_request_id: AtomicU64, -} - -impl QRInfoCorrelationManager { - pub fn new() -> Self { - Self { - pending_requests: HashMap::new(), - next_request_id: AtomicU64::new(1), - } - } - - /// Register a QRInfo request and get a channel to wait for the response - pub fn register_request( - &mut self, - base_hash: BlockHash, - tip_hash: BlockHash, - ) -> (RequestId, oneshot::Receiver) { - let request_id = RequestId(self.next_request_id.fetch_add(1, Ordering::Relaxed)); - let (response_tx, response_rx) = oneshot::channel(); - - let pending = PendingQRInfoRequest { - base_hash, - tip_hash, - response_sender: response_tx, - timestamp: Instant::now(), - }; - - self.pending_requests.insert(request_id, pending); - - tracing::debug!( - "Registered QRInfo request {} for range {:x} to {:x}", - request_id.0, base_hash, tip_hash - ); - - (request_id, response_rx) - } - - /// Handle incoming QRInfo response and match it to pending request - pub fn handle_qr_info_response(&mut self, qr_info: QRInfo) -> Result<(), CorrelationError> { - // Find matching request based on QRInfo content - // We need to match based on the diff ranges in the QRInfo - let matching_request_id = self.find_matching_request(&qr_info)?; - - if let Some(pending) = self.pending_requests.remove(&matching_request_id) { - if pending.response_sender.send(qr_info).is_err() { - tracing::warn!( - "Failed to send QRInfo response for request {} - receiver dropped", - matching_request_id.0 - ); - } else { - tracing::debug!("Successfully correlated QRInfo response to request {}", matching_request_id.0); - } - - Ok(()) - } else { - Err(CorrelationError::RequestNotFound(matching_request_id)) - } - } - - /// Clean up expired requests (requests that have been waiting too long) - pub fn cleanup_expired_requests(&mut self, timeout: Duration) { - let now = Instant::now(); - let expired_ids: Vec = self.pending_requests - .iter() - .filter(|(_, pending)| now.duration_since(pending.timestamp) > timeout) - .map(|(id, _)| *id) - .collect(); - - for request_id in expired_ids { - if let Some(pending) = self.pending_requests.remove(&request_id) { - let _ = pending.response_sender.send_error(CorrelationError::Timeout); - tracing::warn!("Cleaned up expired QRInfo request {}", request_id.0); - } - } - } - - /// Find the pending request that matches this QRInfo response - fn find_matching_request(&self, qr_info: &QRInfo) -> Result { - // Strategy: Match based on the block hashes in the QRInfo diffs - // The tip diff should match our request's tip_hash - let tip_hash = qr_info.mn_list_diff_tip.block_hash; - - for (request_id, pending) in &self.pending_requests { - if pending.tip_hash == tip_hash { - return Ok(*request_id); - } - } - - // Fallback: Try to match based on height ranges if we can derive them - // This is more complex but handles edge cases - self.find_matching_request_by_content(qr_info) - } - - fn find_matching_request_by_content(&self, qr_info: &QRInfo) -> Result { - // More sophisticated matching based on analyzing all diffs in QRInfo - // This is a backup strategy if simple tip_hash matching fails - - for (request_id, pending) in &self.pending_requests { - // Check if any of the diffs in QRInfo match our expected range - let diffs = [ - &qr_info.mn_list_diff_tip, - &qr_info.mn_list_diff_h, - &qr_info.mn_list_diff_at_h_minus_c, - &qr_info.mn_list_diff_at_h_minus_2c, - &qr_info.mn_list_diff_at_h_minus_3c, - ]; - - for diff in diffs { - if diff.base_block_hash == pending.base_hash || diff.block_hash == pending.tip_hash { - return Ok(*request_id); - } - } - - // Check additional diffs if present - for diff in &qr_info.mn_list_diff_list { - if diff.base_block_hash == pending.base_hash || diff.block_hash == pending.tip_hash { - return Ok(*request_id); - } - } - } - - Err(CorrelationError::NoMatchFound) - } - - pub fn pending_count(&self) -> usize { - self.pending_requests.len() - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct RequestId(u64); - -#[derive(Debug)] -struct PendingQRInfoRequest { - base_hash: BlockHash, - tip_hash: BlockHash, - response_sender: oneshot::Sender, - timestamp: Instant, -} - -#[derive(Debug, thiserror::Error)] -pub enum CorrelationError { - #[error("Request {0:?} not found")] - RequestNotFound(RequestId), - #[error("No matching request found for QRInfo response")] - NoMatchFound, - #[error("Request timed out")] - Timeout, -} -``` - -**Test File**: `tests/network/test_qr_info_correlation.rs` -```rust -#[tokio::test] -async fn test_request_response_correlation() { - let mut correlator = QRInfoCorrelationManager::new(); - - let base_hash = test_block_hash(1000); - let tip_hash = test_block_hash(1100); - - let (request_id, response_rx) = correlator.register_request(base_hash, tip_hash); - - // Create matching QRInfo response - let mut qr_info = create_test_qr_info(); - qr_info.mn_list_diff_tip.block_hash = tip_hash; - qr_info.mn_list_diff_tip.base_block_hash = base_hash; - - // Handle the response - let result = correlator.handle_qr_info_response(qr_info.clone()); - assert!(result.is_ok()); - - // Should receive the response - let received_qr_info = response_rx.await.unwrap(); - assert_eq!(received_qr_info.mn_list_diff_tip.block_hash, tip_hash); -} - -#[tokio::test] -async fn test_multiple_concurrent_requests() { - let mut correlator = QRInfoCorrelationManager::new(); - - let mut request_receivers = Vec::new(); - let mut expected_responses = Vec::new(); - - // Register multiple requests - for i in 0..5 { - let base_hash = test_block_hash(i * 100); - let tip_hash = test_block_hash(i * 100 + 50); - - let (_, response_rx) = correlator.register_request(base_hash, tip_hash); - request_receivers.push(response_rx); - - let mut qr_info = create_test_qr_info(); - qr_info.mn_list_diff_tip.block_hash = tip_hash; - expected_responses.push(qr_info); - } - - // Send responses in different order - let response_order = [2, 0, 4, 1, 3]; - for &index in &response_order { - let result = correlator.handle_qr_info_response(expected_responses[index].clone()); - assert!(result.is_ok(), "Failed to handle response {}", index); - } - - // All requests should receive their responses - for (i, response_rx) in request_receivers.into_iter().enumerate() { - let received = response_rx.await.unwrap(); - assert_eq!( - received.mn_list_diff_tip.block_hash, - expected_responses[i].mn_list_diff_tip.block_hash - ); - } - - assert_eq!(correlator.pending_count(), 0); -} - -#[tokio::test] -async fn test_expired_request_cleanup() { - let mut correlator = QRInfoCorrelationManager::new(); - - let (_, response_rx) = correlator.register_request( - test_block_hash(1000), - test_block_hash(1100) - ); - - assert_eq!(correlator.pending_count(), 1); - - // Clean up with very short timeout - correlator.cleanup_expired_requests(Duration::from_millis(1)); - - // Wait a bit to ensure expiration - tokio::time::sleep(Duration::from_millis(10)).await; - correlator.cleanup_expired_requests(Duration::from_millis(1)); - - assert_eq!(correlator.pending_count(), 0); - - // Should receive timeout error - let result = response_rx.await; - assert!(result.is_err()); // Channel was closed due to timeout -} -``` - -### 2. Intelligent Request Scheduling - -#### 2.1 Priority-Based Scheduler - -**File**: `dash-spv/src/sync/scheduler.rs` - -**Implementation**: -```rust -use std::cmp::Reverse; -use std::collections::BinaryHeap; -use tokio::time::{interval, Interval}; - -/// Intelligent scheduler for QRInfo requests with priority and rate limiting -pub struct QRInfoScheduler { - /// Priority queue of pending requests - request_queue: BinaryHeap, - /// Rate limiter for network requests - rate_limiter: RateLimiter, - /// Maximum requests per time window - max_requests_per_window: usize, - /// Time window for rate limiting - rate_limit_window: Duration, - /// Network condition monitor - network_monitor: NetworkConditionMonitor, -} - -impl QRInfoScheduler { - pub fn new(max_requests_per_window: usize, rate_limit_window: Duration) -> Self { - Self { - request_queue: BinaryHeap::new(), - rate_limiter: RateLimiter::new(max_requests_per_window, rate_limit_window), - max_requests_per_window, - rate_limit_window, - network_monitor: NetworkConditionMonitor::new(), - } - } - - /// Schedule a QRInfo request with priority and timing - pub fn schedule_request(&mut self, request: QRInfoRequest, priority: SchedulePriority) { - let scheduled = ScheduledRequest { - request, - priority, - scheduled_time: Instant::now(), - retry_count: 0, - max_retries: 3, - }; - - self.request_queue.push(scheduled); - - tracing::debug!( - "Scheduled QRInfo request with priority {:?}, queue size: {}", - priority, self.request_queue.len() - ); - } - - /// Get the next batch of requests ready for execution - pub async fn get_next_batch(&mut self, max_batch_size: usize) -> Vec { - let mut batch = Vec::new(); - let network_conditions = self.network_monitor.get_current_conditions().await; - - // Adjust batch size based on network conditions - let effective_batch_size = self.calculate_effective_batch_size(max_batch_size, &network_conditions); - - while batch.len() < effective_batch_size && !self.request_queue.is_empty() { - // Check rate limiting - if !self.rate_limiter.can_make_request().await { - tracing::debug!("Rate limit reached, deferring requests"); - break; - } - - // Get highest priority request - if let Some(scheduled) = self.request_queue.pop() { - // Check if it's time to execute this request - if self.is_ready_for_execution(&scheduled, &network_conditions) { - batch.push(scheduled.request); - self.rate_limiter.record_request().await; - } else { - // Put it back for later - self.request_queue.push(scheduled); - break; - } - } - } - - if !batch.is_empty() { - tracing::info!( - "Scheduled batch of {} requests (conditions: {:?})", - batch.len(), - network_conditions - ); - } - - batch - } - - /// Handle a failed request - reschedule with backoff if retries available - pub fn handle_request_failure(&mut self, mut request: QRInfoRequest, error: &ParallelExecutionError) { - if let Some(mut scheduled) = self.find_scheduled_request(&request) { - scheduled.retry_count += 1; - - if scheduled.retry_count <= scheduled.max_retries { - // Reschedule with exponential backoff - let backoff_delay = Duration::from_secs(2_u64.pow(scheduled.retry_count as u32)); - scheduled.scheduled_time = Instant::now() + backoff_delay; - scheduled.priority = self.adjust_priority_for_retry(scheduled.priority, &error); - - self.request_queue.push(scheduled); - - tracing::info!( - "Rescheduled failed request (retry {}/{}) with {}s backoff", - scheduled.retry_count, scheduled.max_retries, backoff_delay.as_secs() - ); - } else { - tracing::error!( - "Request failed permanently after {} retries: {:?}", - scheduled.max_retries, error - ); - } - } - } - - /// Calculate effective batch size based on network conditions - fn calculate_effective_batch_size(&self, max_batch_size: usize, conditions: &NetworkConditions) -> usize { - let mut effective_size = max_batch_size; - - if conditions.high_latency { - effective_size = (effective_size * 80 / 100).max(1); // Reduce by 20% - } - - if conditions.low_bandwidth { - effective_size = (effective_size * 50 / 100).max(1); // Reduce by 50% - } - - if conditions.unstable_connection { - effective_size = (effective_size * 60 / 100).max(1); // Reduce by 40% - } - - effective_size - } - - /// Check if a request is ready for execution - fn is_ready_for_execution(&self, scheduled: &ScheduledRequest, conditions: &NetworkConditions) -> bool { - let now = Instant::now(); - - // Basic time check - if now < scheduled.scheduled_time { - return false; - } - - // Network condition checks - match scheduled.priority { - SchedulePriority::Critical => true, // Critical requests always go through - SchedulePriority::High => !conditions.unstable_connection, - SchedulePriority::Normal => !conditions.high_latency && !conditions.unstable_connection, - SchedulePriority::Low => conditions.is_optimal(), - } - } - - fn adjust_priority_for_retry(&self, current: SchedulePriority, error: &ParallelExecutionError) -> SchedulePriority { - match error { - ParallelExecutionError::Timeout => { - // Timeouts might be due to network congestion - lower priority - match current { - SchedulePriority::Critical => SchedulePriority::High, - SchedulePriority::High => SchedulePriority::Normal, - _ => SchedulePriority::Low, - } - } - ParallelExecutionError::Network(_) => { - // Network errors might be temporary - keep same priority - current - } - _ => current, - } - } - - pub fn pending_count(&self) -> usize { - self.request_queue.len() - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct ScheduledRequest { - request: QRInfoRequest, - priority: SchedulePriority, - scheduled_time: Instant, - retry_count: u32, - max_retries: u32, -} - -impl Ord for ScheduledRequest { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - // Higher priority first, then earlier scheduled time - self.priority.cmp(&other.priority) - .then_with(|| other.scheduled_time.cmp(&self.scheduled_time)) - } -} - -impl PartialOrd for ScheduledRequest { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum SchedulePriority { - Low = 0, - Normal = 1, - High = 2, - Critical = 3, -} - -/// Simple rate limiter using token bucket algorithm -struct RateLimiter { - tokens: Arc, - max_tokens: usize, - refill_interval: Interval, - refill_amount: usize, -} - -impl RateLimiter { - fn new(max_requests: usize, window: Duration) -> Self { - let refill_amount = max_requests; - let refill_interval = interval(window); - - Self { - tokens: Arc::new(AtomicUsize::new(max_requests)), - max_tokens: max_requests, - refill_interval, - refill_amount, - } - } - - async fn can_make_request(&self) -> bool { - self.tokens.load(Ordering::Relaxed) > 0 - } - - async fn record_request(&mut self) { - let current = self.tokens.fetch_sub(1, Ordering::Relaxed); - if current == 0 { - // Wait for refill - self.refill_interval.tick().await; - self.tokens.store(self.max_tokens, Ordering::Relaxed); - } - } -} - -/// Monitor network conditions for scheduling decisions -struct NetworkConditionMonitor { - last_measurement: Arc>>, - measurement_interval: Duration, -} - -impl NetworkConditionMonitor { - fn new() -> Self { - Self { - last_measurement: Arc::new(Mutex::new(None)), - measurement_interval: Duration::from_secs(30), - } - } - - async fn get_current_conditions(&mut self) -> NetworkConditions { - let mut measurement = self.last_measurement.lock().await; - - if let Some((conditions, timestamp)) = &*measurement { - if timestamp.elapsed() < self.measurement_interval { - return *conditions; - } - } - - // Perform fresh measurement - let conditions = self.measure_network_conditions().await; - *measurement = Some((conditions, Instant::now())); - conditions - } - - async fn measure_network_conditions(&self) -> NetworkConditions { - // Implementation would measure actual network conditions - // This is simplified for the example - NetworkConditions { - high_latency: false, - low_bandwidth: false, - unstable_connection: false, - } - } -} - -#[derive(Debug, Clone, Copy)] -pub struct NetworkConditions { - pub high_latency: bool, - pub low_bandwidth: bool, - pub unstable_connection: bool, -} - -impl NetworkConditions { - pub fn is_optimal(&self) -> bool { - !self.high_latency && !self.low_bandwidth && !self.unstable_connection - } -} -``` - -**Test File**: `tests/sync/test_request_scheduling.rs` -```rust -#[tokio::test] -async fn test_priority_based_scheduling() { - let mut scheduler = QRInfoScheduler::new(10, Duration::from_secs(60)); - - // Schedule requests with different priorities - let low_priority_req = create_test_qr_info_request(); - let high_priority_req = create_test_qr_info_request(); - let critical_req = create_test_qr_info_request(); - - scheduler.schedule_request(low_priority_req.clone(), SchedulePriority::Low); - scheduler.schedule_request(high_priority_req.clone(), SchedulePriority::High); - scheduler.schedule_request(critical_req.clone(), SchedulePriority::Critical); - - // Get next batch - should return critical first - let batch = scheduler.get_next_batch(3).await; - - assert_eq!(batch.len(), 3); - // First request should be critical (highest priority) - assert_eq!(batch[0].priority, critical_req.priority); -} - -#[tokio::test] -async fn test_rate_limiting() { - let mut scheduler = QRInfoScheduler::new(2, Duration::from_secs(1)); // Only 2 per second - - // Schedule more requests than rate limit allows - for i in 0..5 { - let request = create_test_qr_info_request_with_id(i); - scheduler.schedule_request(request, SchedulePriority::Normal); - } - - // First batch should be limited by rate limit - let batch1 = scheduler.get_next_batch(5).await; - assert_eq!(batch1.len(), 2); // Rate limited to 2 - - // Immediate second batch should be empty (rate limited) - let batch2 = scheduler.get_next_batch(5).await; - assert_eq!(batch2.len(), 0); - - // After rate limit window, should get more - tokio::time::sleep(Duration::from_secs(1)).await; - let batch3 = scheduler.get_next_batch(5).await; - assert_eq!(batch3.len(), 2); // Next 2 requests -} - -#[tokio::test] -async fn test_retry_with_backoff() { - let mut scheduler = QRInfoScheduler::new(10, Duration::from_secs(60)); - let request = create_test_qr_info_request(); - - scheduler.schedule_request(request.clone(), SchedulePriority::High); - - // Get and "fail" the request - let batch = scheduler.get_next_batch(1).await; - assert_eq!(batch.len(), 1); - - // Handle failure - should reschedule with backoff - scheduler.handle_request_failure(request, &ParallelExecutionError::Timeout); - - // Should not be immediately available (backoff delay) - let immediate_batch = scheduler.get_next_batch(1).await; - assert_eq!(immediate_batch.len(), 0); - - // Should still have pending request - assert_eq!(scheduler.pending_count(), 1); -} - -#[tokio::test] -async fn test_network_condition_adaptation() { - let mut scheduler = QRInfoScheduler::new(10, Duration::from_secs(60)); - - // Schedule requests - for i in 0..8 { - let request = create_test_qr_info_request_with_id(i); - scheduler.schedule_request(request, SchedulePriority::Normal); - } - - // Simulate poor network conditions - // This would be done through the NetworkConditionMonitor in real implementation - // For testing, we can simulate by observing batch size changes - - let batch_good_conditions = scheduler.get_next_batch(5).await; - // In good conditions, should get full batch (limited by rate limit) - - assert!(batch_good_conditions.len() <= 5); - assert!(batch_good_conditions.len() > 0); -} -``` - -### 3. Error Recovery and Resilience - -#### 3.1 Comprehensive Error Recovery System - -**File**: `dash-spv/src/sync/recovery.rs` - -**Implementation**: -```rust -/// Comprehensive error recovery system for parallel QRInfo sync -pub struct QRInfoRecoveryManager { - /// Failed requests awaiting retry - failed_requests: VecDeque, - /// Error statistics for adaptive behavior - error_stats: ErrorStatistics, - /// Recovery strategies - recovery_strategies: Vec>, - /// Circuit breaker for catastrophic failures - circuit_breaker: CircuitBreaker, -} - -impl QRInfoRecoveryManager { - pub fn new() -> Self { - let mut recovery_strategies: Vec> = vec![ - Box::new(ExponentialBackoffStrategy::new()), - Box::new(NetworkSwitchStrategy::new()), - Box::new(FallbackToSequentialStrategy::new()), - Box::new(MnListDiffFallbackStrategy::new()), - ]; - - Self { - failed_requests: VecDeque::new(), - error_stats: ErrorStatistics::new(), - recovery_strategies, - circuit_breaker: CircuitBreaker::new(5, Duration::from_secs(300)), // 5 failures in 5 minutes - } - } - - /// Handle a failed QRInfo request and determine recovery action - pub async fn handle_failure( - &mut self, - request: QRInfoRequest, - error: ParallelExecutionError, - attempt_count: u32, - ) -> RecoveryAction { - // Record error statistics - self.error_stats.record_error(&error); - - // Check circuit breaker - if self.circuit_breaker.should_block() { - tracing::error!("Circuit breaker activated - too many failures"); - return RecoveryAction::StopSync; - } - - let failed_request = FailedRequest { - request, - error: error.clone(), - attempt_count, - first_failure_time: Instant::now(), - last_attempt_time: Instant::now(), - }; - - // Try each recovery strategy until one accepts the request - for strategy in &self.recovery_strategies { - if let Some(action) = strategy.handle_failure(&failed_request, &self.error_stats).await { - match &action { - RecoveryAction::Retry { delay, .. } => { - tracing::info!( - "Scheduling retry for request with {}ms delay using strategy: {}", - delay.as_millis(), - strategy.name() - ); - self.failed_requests.push_back(failed_request); - } - RecoveryAction::FallbackStrategy { strategy_name } => { - tracing::warn!( - "Switching to fallback strategy: {} for request", - strategy_name - ); - } - RecoveryAction::SkipRequest => { - tracing::warn!("Permanently skipping failed request after exhausting retries"); - } - RecoveryAction::StopSync => { - tracing::error!("Recovery manager recommends stopping sync due to persistent failures"); - } - } - - return action; - } - } - - // No strategy could handle this - default to skip - tracing::error!("No recovery strategy could handle failed request - skipping"); - RecoveryAction::SkipRequest - } - - /// Get requests that are ready for retry - pub async fn get_retry_requests(&mut self) -> Vec { - let now = Instant::now(); - let mut ready_requests = Vec::new(); - - while let Some(failed_request) = self.failed_requests.front() { - if self.is_ready_for_retry(failed_request, now) { - let failed_request = self.failed_requests.pop_front().unwrap(); - ready_requests.push(RetryRequest { - original_request: failed_request.request, - retry_count: failed_request.attempt_count, - recovery_metadata: RecoveryMetadata::default(), - }); - } else { - break; // Queue is ordered by retry time - } - } - - if !ready_requests.is_empty() { - tracing::info!("Found {} requests ready for retry", ready_requests.len()); - } - - ready_requests - } - - /// Check if sync should continue based on error patterns - pub fn should_continue_sync(&self) -> bool { - !self.circuit_breaker.should_block() && - self.error_stats.success_rate() > 0.1 // At least 10% success rate - } - - /// Get current error statistics - pub fn get_error_statistics(&self) -> &ErrorStatistics { - &self.error_stats - } - - fn is_ready_for_retry(&self, failed_request: &FailedRequest, now: Instant) -> bool { - // Calculate exponential backoff delay - let base_delay = Duration::from_secs(2); - let backoff_delay = base_delay * 2_u32.pow(failed_request.attempt_count.saturating_sub(1)); - let max_delay = Duration::from_secs(300); // Cap at 5 minutes - - let effective_delay = backoff_delay.min(max_delay); - - now >= failed_request.last_attempt_time + effective_delay - } -} - -#[derive(Debug)] -struct FailedRequest { - request: QRInfoRequest, - error: ParallelExecutionError, - attempt_count: u32, - first_failure_time: Instant, - last_attempt_time: Instant, -} - -#[derive(Debug)] -pub struct RetryRequest { - pub original_request: QRInfoRequest, - pub retry_count: u32, - pub recovery_metadata: RecoveryMetadata, -} - -#[derive(Debug, Default)] -pub struct RecoveryMetadata { - pub use_different_peer: bool, - pub reduce_batch_size: bool, - pub fallback_to_mn_diff: bool, -} - -pub enum RecoveryAction { - Retry { - delay: Duration, - metadata: RecoveryMetadata, - }, - FallbackStrategy { - strategy_name: String, - metadata: RecoveryMetadata, - }, - SkipRequest, - StopSync, -} - -/// Statistics tracking for error analysis and adaptive recovery -#[derive(Debug)] -pub struct ErrorStatistics { - total_requests: AtomicU64, - successful_requests: AtomicU64, - timeout_errors: AtomicU64, - network_errors: AtomicU64, - processing_errors: AtomicU64, - recent_errors: Mutex>, -} - -impl ErrorStatistics { - fn new() -> Self { - Self { - total_requests: AtomicU64::new(0), - successful_requests: AtomicU64::new(0), - timeout_errors: AtomicU64::new(0), - network_errors: AtomicU64::new(0), - processing_errors: AtomicU64::new(0), - recent_errors: Mutex::new(VecDeque::new()), - } - } - - pub fn record_success(&self) { - self.total_requests.fetch_add(1, Ordering::Relaxed); - self.successful_requests.fetch_add(1, Ordering::Relaxed); - } - - pub fn record_error(&self, error: &ParallelExecutionError) { - self.total_requests.fetch_add(1, Ordering::Relaxed); - - match error { - ParallelExecutionError::Timeout => { - self.timeout_errors.fetch_add(1, Ordering::Relaxed); - } - ParallelExecutionError::Network(_) => { - self.network_errors.fetch_add(1, Ordering::Relaxed); - } - ParallelExecutionError::Processing(_) => { - self.processing_errors.fetch_add(1, Ordering::Relaxed); - } - _ => {} - } - - // Track recent errors for pattern analysis - if let Ok(mut recent) = self.recent_errors.lock() { - recent.push_back((Instant::now(), error.clone())); - - // Keep only last 100 errors - while recent.len() > 100 { - recent.pop_front(); - } - } - } - - pub fn success_rate(&self) -> f64 { - let total = self.total_requests.load(Ordering::Relaxed); - if total == 0 { - return 1.0; - } - - let successful = self.successful_requests.load(Ordering::Relaxed); - successful as f64 / total as f64 - } - - pub fn timeout_rate(&self) -> f64 { - let total = self.total_requests.load(Ordering::Relaxed); - if total == 0 { - return 0.0; - } - - let timeouts = self.timeout_errors.load(Ordering::Relaxed); - timeouts as f64 / total as f64 - } -} - -/// Circuit breaker to prevent catastrophic failure cascades -struct CircuitBreaker { - failure_threshold: u32, - reset_timeout: Duration, - state: Mutex, -} - -impl CircuitBreaker { - fn new(failure_threshold: u32, reset_timeout: Duration) -> Self { - Self { - failure_threshold, - reset_timeout, - state: Mutex::new(CircuitState::Closed { failure_count: 0 }), - } - } - - fn should_block(&self) -> bool { - if let Ok(state) = self.state.lock() { - matches!(*state, CircuitState::Open { .. }) - } else { - false - } - } - - fn record_failure(&self) { - if let Ok(mut state) = self.state.lock() { - match *state { - CircuitState::Closed { failure_count } => { - if failure_count + 1 >= self.failure_threshold { - *state = CircuitState::Open { - opened_at: Instant::now(), - }; - } else { - *state = CircuitState::Closed { - failure_count: failure_count + 1, - }; - } - } - CircuitState::HalfOpen => { - *state = CircuitState::Open { - opened_at: Instant::now(), - }; - } - _ => {} // Already open - } - } - } - - fn record_success(&self) { - if let Ok(mut state) = self.state.lock() { - match *state { - CircuitState::HalfOpen => { - *state = CircuitState::Closed { failure_count: 0 }; - } - CircuitState::Closed { .. } => { - *state = CircuitState::Closed { failure_count: 0 }; - } - _ => {} - } - } - } -} - -#[derive(Debug)] -enum CircuitState { - Closed { failure_count: u32 }, - Open { opened_at: Instant }, - HalfOpen, -} - -/// Trait for different recovery strategies -#[async_trait::async_trait] -trait RecoveryStrategy: Send + Sync { - fn name(&self) -> &'static str; - - async fn handle_failure( - &self, - failed_request: &FailedRequest, - error_stats: &ErrorStatistics, - ) -> Option; -} - -/// Exponential backoff with jitter -struct ExponentialBackoffStrategy { - max_retries: u32, - base_delay: Duration, - max_delay: Duration, -} - -#[async_trait::async_trait] -impl RecoveryStrategy for ExponentialBackoffStrategy { - fn name(&self) -> &'static str { - "ExponentialBackoff" - } - - async fn handle_failure( - &self, - failed_request: &FailedRequest, - _error_stats: &ErrorStatistics, - ) -> Option { - if failed_request.attempt_count >= self.max_retries { - return None; // Let next strategy handle it - } - - let exponential_delay = self.base_delay * 2_u32.pow(failed_request.attempt_count); - let delay = exponential_delay.min(self.max_delay); - - // Add jitter to prevent thundering herd - let jitter = Duration::from_millis(rand::random::() % 1000); - let final_delay = delay + jitter; - - Some(RecoveryAction::Retry { - delay: final_delay, - metadata: RecoveryMetadata::default(), - }) - } -} -``` - -## Success Criteria - -### Performance Requirements -- [ ] >80% reduction in total sync time compared to sequential approach -- [ ] Maintain <3 concurrent requests per peer to be network-friendly -- [ ] Handle up to 50% network failure rate gracefully -- [ ] Memory usage remains stable during parallel operations - -### Reliability Requirements -- [ ] Error recovery succeeds in >90% of transient failure cases -- [ ] Circuit breaker prevents cascade failures -- [ ] Progress reporting accuracy within 5% of actual completion -- [ ] No data corruption during parallel processing - -### Network Efficiency Requirements -- [ ] Intelligent batching reduces total network requests by >70% -- [ ] Request scheduling adapts to network conditions -- [ ] Rate limiting prevents overwhelming network peers -- [ ] Correlation system handles out-of-order responses correctly - -## Risk Mitigation - -### High Risk: Request/Response Correlation -**Risk**: Responses might be matched to wrong requests in parallel execution -**Mitigation**: -- Comprehensive correlation testing with concurrent requests -- Fallback matching algorithms for edge cases -- Request ID validation and logging - -### Medium Risk: Network Congestion -**Risk**: Too many parallel requests might overwhelm network or peers -**Mitigation**: -- Configurable concurrency limits with conservative defaults -- Network condition monitoring and adaptive batch sizing -- Circuit breaker to stop when network is struggling - -### Low Risk: Memory Usage Growth -**Risk**: Parallel processing might increase memory usage significantly -**Mitigation**: -- Memory profiling throughout development -- Bounded queues and cleanup of completed requests -- Configurable limits on pending requests - -## Integration Points - -### Phase 2 Dependencies -- Discovery results feed into parallel execution planning -- Engine state must remain consistent during parallel updates -- Batching strategies from Phase 2 are extended for concurrency - -### Phase 4 Preparation -- Parallel processing will extend to validation operations -- Error recovery will incorporate validation-specific strategies -- Progress reporting will include validation status - -## Next Steps - -Upon completion of Phase 3: -1. **Performance Testing**: Comprehensive benchmarks vs sequential approach -2. **Network Stress Testing**: Test with various network conditions and failures -3. **Memory Profiling**: Ensure no memory leaks or excessive usage -4. **Phase 4**: Proceed to enhanced validation with parallel support - -The network efficiency optimizations in Phase 3 transform dash-spv into a high-performance, resilient sync system that can handle real-world network conditions while maximizing throughput and maintaining reliability. \ No newline at end of file diff --git a/qr_info_spv_plan/PHASE_4.md b/qr_info_spv_plan/PHASE_4.md deleted file mode 100644 index a8f32d8ed..000000000 --- a/qr_info_spv_plan/PHASE_4.md +++ /dev/null @@ -1,1046 +0,0 @@ -# Phase 4: Enhanced Validation Integration - -## Overview - -This phase implements comprehensive quorum validation leveraging the rich validation context provided by QRInfo messages. Building on the parallel processing foundation from Phase 3, we'll enable full cryptographic validation of rotating quorums, chain locks, and masternode list integrity while maintaining sync performance. - -## Objectives - -1. **Full Quorum Validation**: Enable comprehensive cryptographic validation using QRInfo context -2. **Rotating Quorum Cycles**: Implement complete validation of LLMQ rotation cycles -3. **Chain Lock Verification**: Integrate chain lock validation with historical quorum data -4. **State Consistency**: Ensure engine state integrity during validation failures -5. **Performance Balance**: Maintain sync speed while adding validation overhead -6. **Error Recovery**: Robust validation error handling and recovery mechanisms - -## Validation Architecture - -### Current State (Post Phase 3) -```rust -// Phase 3 result: Efficient parallel sync with basic processing -let qr_info_batch = parallel_executor.fetch_qr_info_batch(requests).await?; -for qr_info in qr_info_batch { - // Basic processing without validation - engine.feed_qr_info(qr_info, false, false, Some(fetch_block_height))?; -} -``` - -### Target State (Phase 4) -```rust -// Phase 4 goal: Full validation with performance optimization -let qr_info_batch = parallel_executor.fetch_qr_info_batch(requests).await?; -let validation_config = ValidationConfig::comprehensive(); - -for qr_info in qr_info_batch { - // Comprehensive validation with context-aware processing - engine.feed_qr_info(qr_info, true, true, Some(fetch_block_height))?; - - // Additional validation layers - validation_engine.validate_quorum_cycles(&qr_info).await?; - chain_lock_validator.verify_historical_locks(&qr_info).await?; -} -``` - -## Detailed Implementation Plan - -### 1. Comprehensive Quorum Validation - -#### 1.1 Enhanced Validation Configuration - -**File**: `dash-spv/src/sync/validation.rs` - -**Implementation**: -```rust -use dashcore::sml::llmq_entry_verification::LLMQEntryVerificationStatus; -use dashcore::sml::llmq_type::LLMQType; -use dashcore::bls_sig_utils::{BLSSignature, BLSPublicKey}; -use std::collections::{BTreeSet, BTreeMap}; - -#[derive(Debug, Clone)] -pub struct ValidationConfig { - /// Enable non-rotating quorum validation at tip - pub verify_tip_non_rotated: bool, - - /// Enable rotating quorum validation - pub verify_rotated_quorums: bool, - - /// Enable chain lock signature verification - pub verify_chain_locks: bool, - - /// Quorum types to exclude from validation - pub exclude_quorum_types: BTreeSet, - - /// Maximum validation failures before sync abort - pub max_validation_failures: u32, - - /// Enable parallel validation of independent quorums - pub parallel_validation: bool, - - /// Validation timeout for individual operations - pub validation_timeout: Duration, -} - -impl ValidationConfig { - pub fn comprehensive() -> Self { - Self { - verify_tip_non_rotated: true, - verify_rotated_quorums: true, - verify_chain_locks: true, - exclude_quorum_types: BTreeSet::new(), - max_validation_failures: 3, - parallel_validation: true, - validation_timeout: Duration::from_secs(30), - } - } - - pub fn minimal() -> Self { - Self { - verify_tip_non_rotated: false, - verify_rotated_quorums: false, - verify_chain_locks: false, - exclude_quorum_types: BTreeSet::new(), - max_validation_failures: 10, - parallel_validation: false, - validation_timeout: Duration::from_secs(5), - } - } -} - -#[derive(Debug, Clone)] -pub struct ValidationResult { - pub success: bool, - pub verified_quorums: u32, - pub failed_quorums: u32, - pub validation_errors: Vec, - pub processing_time: Duration, -} - -#[derive(Debug, Clone)] -pub enum ValidationError { - InvalidQuorumSignature { quorum_hash: QuorumHash, reason: String }, - RotationCycleInconsistent { cycle_hash: BlockHash, reason: String }, - ChainLockVerificationFailed { height: u32, reason: String }, - MissingValidationData { required_data: String }, - ValidationTimeout { operation: String, timeout: Duration }, -} -``` - -**Tests**: -```rust -// File: dash-spv/src/sync/validation.rs -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_validation_config_comprehensive() { - let config = ValidationConfig::comprehensive(); - assert!(config.verify_tip_non_rotated); - assert!(config.verify_rotated_quorums); - assert!(config.verify_chain_locks); - assert!(config.parallel_validation); - assert_eq!(config.max_validation_failures, 3); - } - - #[test] - fn test_validation_config_minimal() { - let config = ValidationConfig::minimal(); - assert!(!config.verify_tip_non_rotated); - assert!(!config.verify_rotated_quorums); - assert!(!config.verify_chain_locks); - assert!(!config.parallel_validation); - assert_eq!(config.max_validation_failures, 10); - } - - #[test] - fn test_validation_result_creation() { - let result = ValidationResult { - success: true, - verified_quorums: 15, - failed_quorums: 0, - validation_errors: vec![], - processing_time: Duration::from_millis(250), - }; - assert!(result.success); - assert_eq!(result.verified_quorums, 15); - assert!(result.validation_errors.is_empty()); - } -} -``` - -#### 1.2 Validation Engine Integration - -**File**: `dash-spv/src/sync/validation.rs` (continued) - -**Implementation**: -```rust -pub struct ValidationEngine { - config: ValidationConfig, - validation_stats: ValidationStats, - failure_tracker: FailureTracker, -} - -impl ValidationEngine { - pub fn new(config: ValidationConfig) -> Self { - Self { - config, - validation_stats: ValidationStats::default(), - failure_tracker: FailureTracker::new(config.max_validation_failures), - } - } - - /// Validate QRInfo with comprehensive checks - pub async fn validate_qr_info( - &mut self, - qr_info: &QRInfo, - engine: &mut MasternodeListEngine, - fetch_block_height: impl Fn(&BlockHash) -> Result, - ) -> Result { - let start_time = Instant::now(); - let mut verified_quorums = 0; - let mut failed_quorums = 0; - let mut errors = Vec::new(); - - // Pre-validation: ensure we have required data - self.validate_qr_info_completeness(qr_info)?; - - // Phase 1: Feed QRInfo to engine with validation enabled - if let Err(e) = engine.feed_qr_info( - qr_info.clone(), - self.config.verify_tip_non_rotated, - self.config.verify_rotated_quorums, - Some(&fetch_block_height), - ) { - errors.push(ValidationError::from(e)); - failed_quorums += qr_info.last_commitment_per_index.len() as u32; - } else { - verified_quorums += qr_info.last_commitment_per_index.len() as u32; - } - - // Phase 2: Additional validation layers - if self.config.verify_rotated_quorums { - match self.validate_rotation_cycles(qr_info, engine).await { - Ok(cycle_results) => { - verified_quorums += cycle_results.verified_cycles; - failed_quorums += cycle_results.failed_cycles; - errors.extend(cycle_results.errors); - }, - Err(e) => errors.push(e), - } - } - - if self.config.verify_chain_locks { - match self.validate_chain_lock_context(qr_info, engine).await { - Ok(lock_results) => { - verified_quorums += lock_results.verified_locks; - failed_quorums += lock_results.failed_locks; - errors.extend(lock_results.errors); - }, - Err(e) => errors.push(e), - } - } - - let processing_time = start_time.elapsed(); - let success = errors.is_empty() || failed_quorums == 0; - - // Update failure tracking - if !success { - self.failure_tracker.record_failure(); - if self.failure_tracker.should_abort() { - return Err(ValidationError::TooManyFailures { - failure_count: self.failure_tracker.failure_count(), - max_failures: self.config.max_validation_failures, - }); - } - } else { - self.failure_tracker.record_success(); - } - - // Update statistics - self.validation_stats.update(verified_quorums, failed_quorums, processing_time); - - Ok(ValidationResult { - success, - verified_quorums, - failed_quorums, - validation_errors: errors, - processing_time, - }) - } - - async fn validate_rotation_cycles( - &self, - qr_info: &QRInfo, - engine: &MasternodeListEngine, - ) -> Result { - // Implementation for validating rotating quorum cycles - // This uses the engine's validation methods for rotation consistency - - let mut verified_cycles = 0; - let mut failed_cycles = 0; - let mut errors = Vec::new(); - - // Validate each rotation cycle in last_commitment_per_index - for (index, quorum_entry) in qr_info.last_commitment_per_index.iter().enumerate() { - if !quorum_entry.llmq_type.is_rotating_quorum_type() { - continue; - } - - match engine.validate_rotation_cycle_quorums(&[quorum_entry]) { - Ok(statuses) => { - if let Some(status) = statuses.get(&quorum_entry.quorum_hash) { - match status { - LLMQEntryVerificationStatus::Verified => verified_cycles += 1, - _ => { - failed_cycles += 1; - errors.push(ValidationError::RotationCycleInconsistent { - cycle_hash: quorum_entry.quorum_hash, - reason: format!("Rotation validation failed with status: {:?}", status), - }); - } - } - } - }, - Err(e) => { - failed_cycles += 1; - errors.push(ValidationError::RotationCycleInconsistent { - cycle_hash: quorum_entry.quorum_hash, - reason: format!("Validation error: {:?}", e), - }); - } - } - } - - Ok(CycleValidationResult { - verified_cycles, - failed_cycles, - errors, - }) - } -} - -#[derive(Debug)] -struct CycleValidationResult { - verified_cycles: u32, - failed_cycles: u32, - errors: Vec, -} - -#[derive(Debug)] -struct ChainLockValidationResult { - verified_locks: u32, - failed_locks: u32, - errors: Vec, -} -``` - -#### 1.3 Integration with Masternode Sync Manager - -**File**: `dash-spv/src/sync/masternodes.rs` - -**Modifications**: -```rust -// Add to MasternodeSyncManager -impl MasternodeSyncManager { - // Add validation engine field - validation_engine: Option, - - pub fn enable_validation(&mut self, config: ValidationConfig) { - self.validation_engine = Some(ValidationEngine::new(config)); - } - - /// Enhanced sync_batch_qr_info with validation - pub async fn sync_batch_qr_info_with_validation( - &mut self, - requests: Vec, - network: &mut dyn NetworkManager, - storage: &dyn StorageManager, - ) -> SyncResult { - let mut validation_summary = ValidationSummary::new(); - - // Use parallel executor from Phase 3 - let qr_info_batch = self.parallel_executor - .fetch_qr_info_batch(requests, network) - .await?; - - // Process each QRInfo with validation - for (qr_info, original_request) in qr_info_batch { - let validation_result = match &mut self.validation_engine { - Some(validator) => { - let fetch_height = |hash: &BlockHash| -> Result { - // Implementation to fetch block height from storage - self.fetch_block_height_from_storage(hash, storage) - .map_err(|e| ClientDataRetrievalError::StorageError(e.to_string())) - }; - - validator.validate_qr_info(&qr_info, &mut self.engine.unwrap(), fetch_height).await - }, - None => { - // Fallback to basic processing without validation - self.engine.as_mut().unwrap().feed_qr_info( - qr_info, - false, - false, - None:: Result>, - )?; - - ValidationResult { - success: true, - verified_quorums: 0, - failed_quorums: 0, - validation_errors: vec![], - processing_time: Duration::from_millis(0), - } - } - }; - - validation_summary.add_result(original_request, validation_result); - } - - Ok(validation_summary) - } -} - -#[derive(Debug)] -pub struct ValidationSummary { - pub total_requests: u32, - pub successful_validations: u32, - pub failed_validations: u32, - pub total_verified_quorums: u32, - pub total_failed_quorums: u32, - pub total_processing_time: Duration, - pub validation_errors: Vec<(QRInfoRequest, ValidationError)>, -} - -impl ValidationSummary { - pub fn new() -> Self { - Self { - total_requests: 0, - successful_validations: 0, - failed_validations: 0, - total_verified_quorums: 0, - total_failed_quorums: 0, - total_processing_time: Duration::from_secs(0), - validation_errors: vec![], - } - } - - pub fn add_result(&mut self, request: QRInfoRequest, result: ValidationResult) { - self.total_requests += 1; - if result.success { - self.successful_validations += 1; - } else { - self.failed_validations += 1; - } - - self.total_verified_quorums += result.verified_quorums; - self.total_failed_quorums += result.failed_quorums; - self.total_processing_time += result.processing_time; - - for error in result.validation_errors { - self.validation_errors.push((request.clone(), error)); - } - } - - pub fn success_rate(&self) -> f64 { - if self.total_requests == 0 { - 1.0 - } else { - self.successful_validations as f64 / self.total_requests as f64 - } - } -} -``` - -### 2. Chain Lock Integration - -#### 2.1 Historical Chain Lock Verification - -**File**: `dash-spv/src/sync/chainlock_validation.rs` - -**Implementation**: -```rust -use dashcore::chain::chainlock::ChainLock; -use dashcore::sml::masternode_list_engine::message_request_verification::ChainLockVerificationExt; - -pub struct ChainLockValidator { - /// Cache of verified chain locks to avoid re-verification - verified_cache: BTreeMap, - /// Maximum cache size to prevent memory bloat - max_cache_size: usize, -} - -impl ChainLockValidator { - pub fn new(max_cache_size: usize) -> Self { - Self { - verified_cache: BTreeMap::new(), - max_cache_size, - } - } - - /// Verify chain locks using QRInfo context data - pub async fn verify_historical_locks( - &mut self, - qr_info: &QRInfo, - engine: &MasternodeListEngine, - ) -> Result { - let mut verified_locks = 0; - let mut failed_locks = 0; - let mut errors = Vec::new(); - - // Extract chain lock data from QRInfo diffs - let chain_lock_candidates = self.extract_chain_locks_from_qr_info(qr_info); - - for (block_hash, chain_lock) in chain_lock_candidates { - // Check cache first - if self.verified_cache.contains_key(&block_hash) { - verified_locks += 1; - continue; - } - - // Perform verification using engine's chain lock verification - match self.verify_chain_lock_with_engine(&chain_lock, engine).await { - Ok(true) => { - verified_locks += 1; - self.cache_verified_lock(block_hash, chain_lock); - }, - Ok(false) => { - failed_locks += 1; - errors.push(ValidationError::ChainLockVerificationFailed { - height: block_hash.height_hint().unwrap_or(0), - reason: "Chain lock signature verification failed".to_string(), - }); - }, - Err(e) => { - failed_locks += 1; - errors.push(ValidationError::ChainLockVerificationFailed { - height: block_hash.height_hint().unwrap_or(0), - reason: format!("Chain lock verification error: {:?}", e), - }); - } - } - } - - Ok(ChainLockValidationResult { - verified_locks, - failed_locks, - errors, - }) - } - - async fn verify_chain_lock_with_engine( - &self, - chain_lock: &ChainLock, - engine: &MasternodeListEngine, - ) -> Result> { - // Use engine's chain lock verification methods - // This leverages the comprehensive quorum validation data - - let block_height = chain_lock.block_height; - - // Get the appropriate quorum for this chain lock - let quorum_under_result = engine.chain_lock_potential_quorum_under( - block_height, - &chain_lock.block_hash, - ); - - let quorum_over_result = engine.chain_lock_potential_quorum_over( - block_height, - &chain_lock.block_hash, - ); - - // Verify against both potential quorums - let verification_results = vec![ - quorum_under_result.and_then(|quorum| { - quorum.verify_chain_lock_signature(&chain_lock.signature, &chain_lock.block_hash) - }), - quorum_over_result.and_then(|quorum| { - quorum.verify_chain_lock_signature(&chain_lock.signature, &chain_lock.block_hash) - }), - ]; - - // Chain lock is valid if any quorum can verify it - Ok(verification_results.into_iter().any(|result| result.unwrap_or(false))) - } - - fn extract_chain_locks_from_qr_info(&self, qr_info: &QRInfo) -> Vec<(BlockHash, ChainLock)> { - let mut chain_locks = Vec::new(); - - // Extract from various QRInfo components that might contain chain lock data - // This includes masternode list diffs and quorum snapshots - - let diffs = vec![ - &qr_info.mn_list_diff_tip, - &qr_info.mn_list_diff_h, - &qr_info.mn_list_diff_at_h_minus_c, - &qr_info.mn_list_diff_at_h_minus_2c, - &qr_info.mn_list_diff_at_h_minus_3c, - ]; - - for diff in diffs { - if let Some(chain_lock) = &diff.chain_lock { - chain_locks.push((diff.block_hash, chain_lock.clone())); - } - } - - // Also check h-4c if available - if let Some((_, diff_4c)) = &qr_info.quorum_snapshot_and_mn_list_diff_at_h_minus_4c { - if let Some(chain_lock) = &diff_4c.chain_lock { - chain_locks.push((diff_4c.block_hash, chain_lock.clone())); - } - } - - chain_locks - } - - fn cache_verified_lock(&mut self, block_hash: BlockHash, chain_lock: ChainLock) { - // Implement LRU cache behavior - if self.verified_cache.len() >= self.max_cache_size { - // Remove oldest entry - if let Some(oldest_key) = self.verified_cache.keys().next().copied() { - self.verified_cache.remove(&oldest_key); - } - } - - self.verified_cache.insert(block_hash, chain_lock); - } -} -``` - -### 3. State Consistency and Error Recovery - -#### 3.1 Validation State Manager - -**File**: `dash-spv/src/sync/validation_state.rs` - -**Implementation**: -```rust -use dashcore::sml::masternode_list_engine::MasternodeListEngine; -use std::collections::BTreeMap; - -/// Manages validation state consistency and recovery -pub struct ValidationStateManager { - /// Snapshots of engine state for rollback - engine_snapshots: BTreeMap, - /// Maximum number of snapshots to maintain - max_snapshots: usize, - /// Validation checkpoints for recovery - validation_checkpoints: BTreeMap, -} - -#[derive(Debug, Clone)] -pub struct ValidationCheckpoint { - pub height: u32, - pub validated_quorums: u32, - pub engine_state_hash: u64, - pub timestamp: Instant, -} - -impl ValidationStateManager { - pub fn new(max_snapshots: usize) -> Self { - Self { - engine_snapshots: BTreeMap::new(), - max_snapshots, - validation_checkpoints: BTreeMap::new(), - } - } - - /// Create a snapshot of engine state before validation - pub fn create_snapshot( - &mut self, - height: u32, - engine: &MasternodeListEngine, - ) -> Result<(), ValidationError> { - // Clean up old snapshots if at limit - if self.engine_snapshots.len() >= self.max_snapshots { - if let Some(oldest_height) = self.engine_snapshots.keys().next().copied() { - self.engine_snapshots.remove(&oldest_height); - self.validation_checkpoints.remove(&oldest_height); - } - } - - // Create deep copy of engine state - let engine_snapshot = engine.clone(); - let checkpoint = ValidationCheckpoint { - height, - validated_quorums: engine.quorum_statuses.values() - .map(|type_map| type_map.len() as u32) - .sum(), - engine_state_hash: self.compute_engine_hash(engine), - timestamp: Instant::now(), - }; - - self.engine_snapshots.insert(height, engine_snapshot); - self.validation_checkpoints.insert(height, checkpoint); - - Ok(()) - } - - /// Restore engine state from snapshot - pub fn restore_snapshot( - &mut self, - height: u32, - engine: &mut MasternodeListEngine, - ) -> Result<(), ValidationError> { - if let Some(snapshot) = self.engine_snapshots.get(&height) { - *engine = snapshot.clone(); - - tracing::info!( - "Restored engine state from snapshot at height {}", - height - ); - - Ok(()) - } else { - Err(ValidationError::MissingValidationData { - required_data: format!("Engine snapshot at height {}", height), - }) - } - } - - /// Validate engine state consistency - pub fn validate_state_consistency( - &self, - engine: &MasternodeListEngine, - ) -> Result<(), ValidationError> { - // Verify internal consistency of engine state - - // 1. Check that all quorum hashes in quorum_statuses have corresponding block heights - for (llmq_type, type_quorums) in &engine.quorum_statuses { - for (quorum_hash, (heights, _public_key, _status)) in type_quorums { - for height in heights { - if !engine.masternode_lists.contains_key(height) { - return Err(ValidationError::EngineStateInconsistent { - reason: format!( - "Quorum {} of type {:?} references height {} but no masternode list exists at that height", - quorum_hash, llmq_type, height - ), - }); - } - } - } - } - - // 2. Check that block_container heights match masternode_lists keys - for height in engine.masternode_lists.keys() { - if let Some(block_hash) = engine.block_container.get_hash(height) { - if engine.block_container.get_height(block_hash) != Some(*height) { - return Err(ValidationError::EngineStateInconsistent { - reason: format!( - "Block container height/hash mapping inconsistent for height {}", - height - ), - }); - } - } - } - - // 3. Validate rotating quorum cycle consistency - for (cycle_hash, quorum_entries) in &engine.rotated_quorums_per_cycle { - for quorum_entry in quorum_entries { - // Ensure all quorums in a cycle are actually rotating types - if !quorum_entry.quorum_entry.llmq_type.is_rotating_quorum_type() { - return Err(ValidationError::EngineStateInconsistent { - reason: format!( - "Non-rotating quorum {:?} found in rotation cycle {}", - quorum_entry.quorum_entry.llmq_type, cycle_hash - ), - }); - } - } - } - - Ok(()) - } - - fn compute_engine_hash(&self, engine: &MasternodeListEngine) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut hasher = DefaultHasher::new(); - - // Hash key components of engine state - engine.masternode_lists.len().hash(&mut hasher); - engine.quorum_statuses.len().hash(&mut hasher); - engine.rotated_quorums_per_cycle.len().hash(&mut hasher); - engine.network.hash(&mut hasher); - - hasher.finish() - } -} - -#[derive(Debug, Clone)] -pub enum ValidationError { - // ... existing variants ... - EngineStateInconsistent { reason: String }, - TooManyFailures { failure_count: u32, max_failures: u32 }, -} -``` - -### 4. Testing Strategy - -#### 4.1 Comprehensive Test Suite - -**File**: `dash-spv/tests/integration/validation_tests.rs` - -**Implementation**: -```rust -use dash_spv::sync::{ValidationConfig, ValidationEngine}; -use dashcore::network::message_qrinfo::QRInfo; -use dashcore::sml::masternode_list_engine::MasternodeListEngine; -use std::time::Duration; -use tokio::test; - -#[tokio::test] -async fn test_comprehensive_validation_flow() { - // Test the complete validation flow with real QRInfo data - - let mut engine = create_test_engine_with_genesis(); - let mut validator = ValidationEngine::new(ValidationConfig::comprehensive()); - - let qr_info = create_test_qr_info_with_rotating_quorums(); - let fetch_height = |hash: &BlockHash| -> Result { - // Mock implementation for test - Ok(test_height_for_hash(hash)) - }; - - let result = validator.validate_qr_info(&qr_info, &mut engine, fetch_height).await; - - assert!(result.is_ok()); - let validation_result = result.unwrap(); - assert!(validation_result.success); - assert!(validation_result.verified_quorums > 0); - assert_eq!(validation_result.failed_quorums, 0); -} - -#[tokio::test] -async fn test_validation_with_invalid_signatures() { - // Test validation behavior with corrupted quorum signatures - - let mut engine = create_test_engine_with_genesis(); - let mut validator = ValidationEngine::new(ValidationConfig::comprehensive()); - - let mut qr_info = create_test_qr_info_with_rotating_quorums(); - // Corrupt some signatures - corrupt_quorum_signatures(&mut qr_info); - - let fetch_height = |hash: &BlockHash| -> Result { - Ok(test_height_for_hash(hash)) - }; - - let result = validator.validate_qr_info(&qr_info, &mut engine, fetch_height).await; - - assert!(result.is_ok()); - let validation_result = result.unwrap(); - assert!(!validation_result.success); - assert!(validation_result.failed_quorums > 0); - assert!(!validation_result.validation_errors.is_empty()); -} - -#[tokio::test] -async fn test_validation_state_rollback() { - // Test state consistency and rollback functionality - - let mut engine = create_test_engine_with_genesis(); - let mut state_manager = ValidationStateManager::new(5); - - // Create snapshot - state_manager.create_snapshot(100, &engine).unwrap(); - - // Modify engine state - modify_engine_state(&mut engine); - - // Verify state changed - assert_ne!(compute_test_engine_hash(&engine), compute_initial_engine_hash()); - - // Restore snapshot - state_manager.restore_snapshot(100, &mut engine).unwrap(); - - // Verify state restored - assert_eq!(compute_test_engine_hash(&engine), compute_initial_engine_hash()); -} - -#[tokio::test] -async fn test_parallel_validation_performance() { - // Test that parallel validation actually improves performance - - let qr_info_batch = create_large_qr_info_batch(20); - - // Sequential validation - let start_sequential = Instant::now(); - let sequential_results = validate_sequential(&qr_info_batch).await; - let sequential_time = start_sequential.elapsed(); - - // Parallel validation - let start_parallel = Instant::now(); - let parallel_results = validate_parallel(&qr_info_batch).await; - let parallel_time = start_parallel.elapsed(); - - // Verify results are equivalent - assert_eq!(sequential_results.len(), parallel_results.len()); - - // Verify parallel is faster (with some tolerance for small batches) - if qr_info_batch.len() > 5 { - assert!(parallel_time < sequential_time * 2 / 3, - "Parallel validation should be significantly faster"); - } -} - -#[tokio::test] -async fn test_chain_lock_historical_verification() { - // Test chain lock verification with historical context - - let mut engine = create_test_engine_with_chain_locks(); - let mut chain_lock_validator = ChainLockValidator::new(100); - - let qr_info = create_qr_info_with_chain_locks(); - - let result = chain_lock_validator - .verify_historical_locks(&qr_info, &engine) - .await - .unwrap(); - - assert!(result.verified_locks > 0); - assert_eq!(result.failed_locks, 0); - assert!(result.errors.is_empty()); -} - -// Helper functions for tests -fn create_test_engine_with_genesis() -> MasternodeListEngine { - // Implementation to create a test engine with genesis state - unimplemented!("Test helper - create engine with known good state") -} - -fn create_test_qr_info_with_rotating_quorums() -> QRInfo { - // Implementation to create QRInfo with valid rotating quorum data - unimplemented!("Test helper - create valid QRInfo for testing") -} - -fn corrupt_quorum_signatures(qr_info: &mut QRInfo) { - // Implementation to corrupt signatures for negative testing - unimplemented!("Test helper - corrupt signatures for failure testing") -} -``` - -#### 4.2 Performance Benchmarks - -**File**: `dash-spv/benches/validation_benchmarks.rs` - -**Implementation**: -```rust -use criterion::{black_box, criterion_group, criterion_main, Criterion, BenchmarkId}; -use dash_spv::sync::{ValidationConfig, ValidationEngine}; -use dashcore::network::message_qrinfo::QRInfo; -use std::time::Duration; - -fn benchmark_validation_modes(c: &mut Criterion) { - let mut group = c.benchmark_group("validation_modes"); - - let qr_info_batch = create_benchmark_qr_info_batch(); - - for batch_size in [1, 5, 10, 20].iter() { - let batch = qr_info_batch.iter().take(*batch_size).collect::>(); - - group.benchmark_with_input( - BenchmarkId::new("minimal_validation", batch_size), - &batch, - |b, batch| { - b.to_async(tokio::runtime::Runtime::new().unwrap()) - .iter(|| async { - let config = ValidationConfig::minimal(); - let mut validator = ValidationEngine::new(config); - validate_batch(black_box(&batch), &mut validator).await - }); - }, - ); - - group.benchmark_with_input( - BenchmarkId::new("comprehensive_validation", batch_size), - &batch, - |b, batch| { - b.to_async(tokio::runtime::Runtime::new().unwrap()) - .iter(|| async { - let config = ValidationConfig::comprehensive(); - let mut validator = ValidationEngine::new(config); - validate_batch(black_box(&batch), &mut validator).await - }); - }, - ); - } - - group.finish(); -} - -fn benchmark_chain_lock_validation(c: &mut Criterion) { - let mut group = c.benchmark_group("chain_lock_validation"); - - for chain_lock_count in [10, 50, 100].iter() { - let qr_info = create_qr_info_with_chain_locks(*chain_lock_count); - - group.benchmark_with_input( - BenchmarkId::new("chain_lock_verification", chain_lock_count), - &qr_info, - |b, qr_info| { - b.to_async(tokio::runtime::Runtime::new().unwrap()) - .iter(|| async { - let mut validator = ChainLockValidator::new(1000); - let engine = create_benchmark_engine(); - validator.verify_historical_locks(black_box(qr_info), &engine).await - }); - }, - ); - } - - group.finish(); -} - -criterion_group!( - validation_benches, - benchmark_validation_modes, - benchmark_chain_lock_validation -); -criterion_main!(validation_benches); -``` - -## Implementation Timeline - -### Week 1: Core Validation Infrastructure -- [ ] Implement `ValidationConfig` and `ValidationEngine` -- [ ] Add comprehensive validation error types -- [ ] Create basic validation integration tests -- [ ] Update `MasternodeSyncManager` for validation support - -### Week 2: Advanced Validation Features -- [ ] Implement rotating quorum cycle validation -- [ ] Add chain lock verification with historical context -- [ ] Create validation state management and rollback -- [ ] Add performance benchmarks - -### Week 3: Integration and Testing -- [ ] Integrate validation with Phase 3 parallel processing -- [ ] Add comprehensive error recovery mechanisms -- [ ] Create extensive integration test suite -- [ ] Performance optimization and tuning - -### Week 4: Production Readiness -- [ ] Add validation metrics and monitoring -- [ ] Create configuration management for different validation levels -- [ ] Documentation and usage examples -- [ ] Final integration testing with real network data - -## Success Criteria - -1. **Validation Coverage**: 100% of quorum types can be validated with appropriate context -2. **Performance Impact**: Validation adds <50% overhead to sync time -3. **Error Recovery**: Robust handling of validation failures with state consistency -4. **Chain Lock Integration**: Historical chain lock verification with >99% accuracy -5. **State Consistency**: Engine state remains consistent across validation failures -6. **Test Coverage**: >95% test coverage for all validation components - -## Migration Strategy - -1. **Gradual Rollout**: Start with minimal validation, gradually enable comprehensive validation -2. **Configuration-Driven**: Allow runtime configuration of validation levels -3. **Fallback Mechanisms**: Graceful degradation when validation fails -4. **Monitoring Integration**: Comprehensive metrics for validation performance and accuracy -5. **Backward Compatibility**: Maintain compatibility with non-validating sync modes \ No newline at end of file diff --git a/test-utils/src/fixtures.rs b/test-utils/src/fixtures.rs index bb225c44d..9e6c4214e 100644 --- a/test-utils/src/fixtures.rs +++ b/test-utils/src/fixtures.rs @@ -99,8 +99,26 @@ mod tests { let testnet = testnet_genesis_hash(); assert_ne!(mainnet, testnet); - assert_eq!(mainnet.to_string(), MAINNET_GENESIS_HASH); - assert_eq!(testnet.to_string(), TESTNET_GENESIS_HASH); + + // Create expected BlockHash instances from the constants for proper comparison + let expected_mainnet = { + let bytes = decode(MAINNET_GENESIS_HASH).unwrap(); + let mut reversed = [0u8; 32]; + reversed.copy_from_slice(&bytes); + reversed.reverse(); + BlockHash::from_slice(&reversed).unwrap() + }; + + let expected_testnet = { + let bytes = decode(TESTNET_GENESIS_HASH).unwrap(); + let mut reversed = [0u8; 32]; + reversed.copy_from_slice(&bytes); + reversed.reverse(); + BlockHash::from_slice(&reversed).unwrap() + }; + + assert_eq!(mainnet, expected_mainnet); + assert_eq!(testnet, expected_testnet); } #[test] diff --git a/test_smart_algo.sh b/test_smart_algo.sh index 8a5ab2042..7f5f8d27d 100644 --- a/test_smart_algo.sh +++ b/test_smart_algo.sh @@ -4,12 +4,11 @@ # Enable debug logging for the relevant modules export RUST_LOG=dash_spv::sync::masternodes=debug,dash_spv::sync::sequential=debug -# Run with checkpoint at 1100000 to trigger the smart algorithm for the range 1260302-1290302 +# Run with start height at 1100000 to trigger the smart algorithm for the range 1260302-1290302 ./target/debug/dash-spv \ --network testnet \ --data-dir ./test-smart-algo \ - --checkpoint 1100000 \ - --checkpoint-hash 00000bafbc94add76cb75e2ec92894837288a481e5c005f6563d91623bf8bc2c \ + --start-height 1100000 \ 2>&1 | tee smart_algo_debug.log echo "Debug log saved to smart_algo_debug.log" \ No newline at end of file