Reduce number of blocks (duration)

Author: akonring

.config/nextest.toml

@@ -1,2 +1,7 @@
 [profile.default]
 slow-timeout = { period = "60s", terminate-after = 2 }
+
+[[profile.default.overrides]]
+filter = 'test(/^tests::timeboost::timeboost_handover::/)'
+threads-required = 2
+slow-timeout = { period = "300s", terminate-after = 1 }

tests/src/tests/timeboost.rs

@@ -23,6 +23,7 @@ use timeboost::crypto::prelude::DkgDecKey;
 use timeboost::sequencer::{Sequencer, SequencerConfig};
 use timeboost::types::{BlockNumber, BundleVariant, DecryptionKeyCell, KeyStore, Transaction};
 use timeboost_utils::load_generation::make_bundle;
+use timeboost_utils::with_retry;
 use tokio::sync::broadcast;
 use tokio::time::{Duration, sleep};
 use tracing::warn;
@@ -194,27 +195,6 @@ impl Round2Block {
     }
 }
 
-async fn with_retry<T, E, F, Fut>(operation: F, error_message: impl Fn(&E) -> String) -> T
-where
-    E: std::fmt::Display,
-    F: Fn() -> Fut,
-    Fut: std::future::Future<Output = Result<T, E>>,
-{
-    let mut delay = Duration::from_millis(100);
-    const MAX_DELAY: Duration = Duration::from_secs(5);
-
-    loop {
-        match operation().await {
-            Ok(result) => return result,
-            Err(e) => {
-                warn!("{}: {e}, retrying in {:?}...", error_message(&e), delay);
-                sleep(delay).await;
-                delay = (delay * 2).min(MAX_DELAY);
-            }
-        }
-    }
-}
-
 async fn start_sequencer_with_retry(seq_conf: SequencerConfig) -> Sequencer {
     with_retry(
         || async { Sequencer::new(seq_conf.clone(), &NoMetrics).await },

tests/src/tests/timeboost/block_order.rs

@@ -1,12 +1,10 @@
-use metrics::NoMetrics;
 use multisig::Certificate;
 use std::collections::HashMap;
 use std::iter::once;
 use std::num::NonZeroUsize;
 use std::sync::Arc;
 use std::time::Duration;
-use timeboost::builder::Certifier;
-use timeboost::sequencer::{Output, Sequencer};
+use timeboost::sequencer::Output;
 use timeboost::types::{Block, BlockInfo};
 use timeboost_utils::types::logging::init_logging;
 use tokio::select;
@@ -17,7 +15,9 @@ use tokio_util::sync::CancellationToken;
 use tokio_util::task::TaskTracker;
 use tracing::{debug, error, info};
 
-use crate::tests::timeboost::{Round2Block, hash};
+use crate::tests::timeboost::{
+    Round2Block, hash, start_certifier_with_retry, start_sequencer_with_retry,
+};
 
 use super::{gen_bundles, make_configs};
 
@@ -49,8 +49,8 @@ async fn block_order() {
                 // delay start of a recovering node:
                 sleep(Duration::from_secs(5)).await
             }
-            let mut s = Sequencer::new(c, &NoMetrics).await.unwrap();
-            let mut p = Certifier::new(b, &NoMetrics).await.unwrap();
+            let mut s = start_sequencer_with_retry(c).await;
+            let mut p = start_certifier_with_retry(b).await;
             let mut r = None;
             let handle = p.handle();
             loop {

tests/src/tests/timeboost/timeboost_handover.rs

@@ -36,12 +36,12 @@ async fn run_handover(
     curr: Vec<(DecryptionKeyCell, SequencerConfig, CertifierConfig)>,
     next: Vec<(DecryptionKeyCell, SequencerConfig, CertifierConfig)>,
 ) {
-    const NEXT_COMMITTEE_DELAY: u64 = 10;
+    const NEXT_COMMITTEE_DELAY: u64 = 15;
     const NUM_OF_BLOCKS: usize = 50;
 
     let mut out1 = Vec::new();
     let tasks = TaskTracker::new();
-    let (bcast, _) = broadcast::channel(200);
+    let (bcast, _) = broadcast::channel(500);
     let finish = CancellationToken::new();
     let round2block = Arc::new(Round2Block::new());
 
@@ -130,6 +130,8 @@ async fn run_handover(
         key.read().await;
     }
 
+    tokio::time::sleep(Duration::from_secs(5)).await;
+
     tasks.spawn({
         let (key, _, _) = &curr[0];
         let key = key.clone();

tests/src/tests/timeboost/transaction_order.rs

@@ -4,9 +4,8 @@ use std::num::NonZeroUsize;
 use std::time::Duration;
 
 use alloy::primitives::B256;
-use metrics::NoMetrics;
 use sailfish_types::RoundNumber;
-use timeboost::sequencer::{Output, Sequencer};
+use timeboost::sequencer::Output;
 use timeboost_utils::types::logging::init_logging;
 use tokio::select;
 use tokio::sync::broadcast::error::RecvError;
@@ -16,6 +15,8 @@ use tokio_util::sync::CancellationToken;
 use tokio_util::task::TaskTracker;
 use tracing::{debug, info};
 
+use crate::tests::timeboost::start_sequencer_with_retry;
+
 use super::{gen_bundles, make_configs};
 
 const NUM_OF_TRANSACTIONS: usize = 500;
@@ -52,7 +53,7 @@ async fn transaction_order() {
                 // delay start of a recovering node:
                 sleep(Duration::from_secs(5)).await
             }
-            let mut s = Sequencer::new(c, &NoMetrics).await.unwrap();
+            let mut s = start_sequencer_with_retry(c).await;
             loop {
                 select! {
                     trx = brx.recv() => match trx {

timeboost-sequencer/src/decrypt.rs

@@ -1408,7 +1408,7 @@ mod tests {
         time::Instant,
     };
 
-    use timeboost_utils::types::logging;
+    use timeboost_utils::{types::logging, with_retry};
 
     use cliquenet::AddressableCommittee;
     use multisig::{Committee, KeyId, Keypair, SecretKey, Signed, VoteAccumulator, x25519};
@@ -2373,13 +2373,7 @@ mod tests {
                 .threshold_dec_key(encryption_key_cell.clone())
                 .build();
 
-            let decrypter = Decrypter::new(
-                decrypter_config,
-                &NoMetrics,
-                Arc::new(SequencerMetrics::default()),
-            )
-            .await
-            .expect("Decrypter creation should succeed");
+            let decrypter = start_decrypter_with_retry(decrypter_config).await;
             decrypters.push(decrypter);
             encryption_key_cells.push(encryption_key_cell);
         }
@@ -2394,4 +2388,19 @@ mod tests {
             ),
         )
     }
+
+    async fn start_decrypter_with_retry(dec_config: DecrypterConfig) -> Decrypter {
+        with_retry(
+            || async {
+                Decrypter::new(
+                    dec_config.clone(),
+                    &NoMetrics,
+                    Arc::new(SequencerMetrics::default()),
+                )
+                .await
+            },
+            |e| format!("failed to create decrypter: {e}"),
+        )
+        .await
+    }
 }

timeboost-utils/src/lib.rs

@@ -7,7 +7,7 @@ use std::time::Duration;
 use cliquenet::Address;
 use multisig::x25519;
 use tokio::time::sleep;
-use tracing::{error, info};
+use tracing::{error, info, warn};
 
 pub fn unsafe_zero_keypair<N: Into<u64>>(i: N) -> multisig::Keypair {
     sig_keypair_from_seed_indexed([0u8; 32], i.into())
@@ -68,3 +68,24 @@ pub async fn wait_for_live_peer(host: &Address) -> anyhow::Result<()> {
         sleep(Duration::from_secs(3)).await;
     }
 }
+
+pub async fn with_retry<T, E, F, Fut>(operation: F, error_message: impl Fn(&E) -> String) -> T
+where
+    E: std::fmt::Display,
+    F: Fn() -> Fut,
+    Fut: std::future::Future<Output = Result<T, E>>,
+{
+    let mut delay = Duration::from_millis(100);
+    const MAX_DELAY: Duration = Duration::from_secs(5);
+
+    loop {
+        match operation().await {
+            Ok(result) => return result,
+            Err(e) => {
+                warn!("{}: {e}, retrying in {:?}...", error_message(&e), delay);
+                sleep(delay).await;
+                delay = (delay * 2).min(MAX_DELAY);
+            }
+        }
+    }
+}