Skip to content

Commit c03dacb

Browse files
GTC6244Copilot
GTC6244
and
Copilot
authored
Chain fault testing framework + Enable auto-rejoin (#51)
* Port code from the prod release PR. * c&p * Update rust/lit-node/lit-node/tests/common/faults.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update rust/lit-node/lit-node/tests/toxiproxy/chain_faults.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * remove comment * Update rust/lit-node/lit-node-testnet/src/testnet/actions.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update rust/lit-node/lit-node/tests/toxiproxy/chain_faults.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * formatting * cargo fmt * ack. * ack. * fix fault test setup * ignore redundant chain fault test - but keep as model. * re-add >= to epoch/previous epoch check --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent e6f4cbd commit c03dacb

File tree

14 files changed

+378
-50
lines changed

14 files changed

+378
-50
lines changed

‎rust/lit-core/lit-blockchain/Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ edition.workspace = true
88
[features]
99
default = []
1010
env-override = []
11+
testing = []
12+
proxy_chatter = []
1113

1214
[dependencies]
1315
alloy.workspace = true

‎rust/lit-core/lit-blockchain/src/contracts/mod.rs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,8 @@ impl
145145
> {
146146
let chain = cfg.blockchain_chain_name()?;
147147
let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
148+
#[cfg(feature = "proxy_chatter")]
149+
let provider = change_port(provider, cfg)?;
148150
let meta_signer = meta_signer_key.into();
149151
let meta_wallet = LocalWallet::from(meta_signer.clone());
150152
let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -207,6 +209,8 @@ impl
207209
> {
208210
let chain = cfg.blockchain_chain_name()?;
209211
let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
212+
#[cfg(feature = "proxy_chatter")]
213+
let provider = change_port(provider, cfg)?;
210214
let meta_signer = meta_signer_key.into();
211215
let meta_wallet = LocalWallet::from(meta_signer.clone());
212216
let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -318,6 +322,8 @@ impl
318322
> {
319323
let chain = cfg.blockchain_chain_name()?;
320324
let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
325+
#[cfg(feature = "proxy_chatter")]
326+
let provider = change_port(provider, cfg)?;
321327
let meta_signer = meta_signer_key.into();
322328
let meta_wallet = LocalWallet::from(meta_signer.clone());
323329
let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -363,6 +369,8 @@ impl
363369
> {
364370
let chain = cfg.blockchain_chain_name()?;
365371
let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
372+
#[cfg(feature = "proxy_chatter")]
373+
let provider = change_port(provider, cfg)?;
366374
let meta_signer = meta_signer_key.into();
367375
let meta_wallet = LocalWallet::from(meta_signer.clone());
368376
let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -428,6 +436,8 @@ impl
428436
> {
429437
let chain = cfg.blockchain_chain_name()?;
430438
let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
439+
#[cfg(feature = "proxy_chatter")]
440+
let provider = change_port(provider, cfg)?;
431441
let meta_signer = meta_signer_key.into();
432442
let meta_wallet = LocalWallet::from(meta_signer.clone());
433443
let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -525,6 +535,8 @@ impl
525535
> {
526536
let chain = cfg.blockchain_chain_name()?;
527537
let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
538+
#[cfg(feature = "proxy_chatter")]
539+
let provider = change_port(provider, cfg)?;
528540
let meta_signer = meta_signer_key.into();
529541
let meta_wallet = LocalWallet::from(meta_signer.clone());
530542
let meta_client = SignerMiddleware::new(provider, meta_wallet);
@@ -571,13 +583,41 @@ pub fn default_local_client(
571583
let chain = cfg.blockchain_chain_name()?;
572584
let wallet = load_wallet(cfg, wallet_key)?;
573585
let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
586+
#[cfg(feature = "proxy_chatter")]
587+
let provider = change_port(provider, cfg)?;
574588

575589
Ok(Arc::new(SignerMiddleware::new(provider, wallet)))
576590
}
577591

578592
pub fn default_local_client_no_wallet(cfg: &LitConfig) -> Result<Arc<Provider<Http>>> {
579593
let chain = cfg.blockchain_chain_name()?;
580594
let provider = ENDPOINT_MANAGER.get_provider(chain.as_str())?;
595+
#[cfg(feature = "proxy_chatter")]
596+
let provider = change_port(provider, cfg)?;
581597

582598
Ok(provider)
583599
}
600+
601+
#[cfg(feature = "proxy_chatter")]
602+
pub fn change_port(provider: Arc<Provider<Http>>, cfg: &LitConfig) -> Result<Arc<Provider<Http>>> {
603+
let chain = cfg.blockchain_chain_name()?;
604+
if chain.to_lowercase() == "localchain" {
605+
let cfg2 = cfg.config();
606+
let port = cfg2
607+
.get_int("node.http.port")
608+
.map_err(|e| crate::error::unexpected_err(e.to_string(), None))?;
609+
let port = port as u16;
610+
let port = 11075 + port; // 10000 + 8545 ( anvil default port ) - 7470 ( lit-node default starting port ) + actual port value
611+
tracing::trace!(
612+
"Changing port for proxy provider {} to {}.",
613+
provider.url().as_str(),
614+
port
615+
);
616+
let mut proxy_provider = (*provider).clone();
617+
proxy_provider.url_mut().set_port(Some(port)).map_err(|_e| {
618+
crate::error::unexpected_err("Could not set port for proxy provider", None)
619+
})?;
620+
return Ok(Arc::new(proxy_provider));
621+
}
622+
Ok(provider)
623+
}

‎rust/lit-node/lit-node-testnet/Cargo.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ edition.workspace = true
88
# this needs to happen at runtime, so is just a place holder for now.
99
sign_test = ["lit-actions"] #enables unsafe endpoint allowing a test to use a PK directly .
1010
lit-actions = []
11-
proxy-chatter = []
1211
proxy_chatter = []
1312
lit-actions-server = ["lit-actions"] # start internal lit_actions server for testing
1413
testing = [] # enables testing features

‎rust/lit-node/lit-node-testnet/src/lib.rs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ pub struct TestSetupBuilder {
2929
register_inactive_validators: bool,
3030
enable_payment: Option<String>,
3131
chain_polling_interval: Option<String>,
32+
signing_round_timeout: Option<String>,
3233
epoch_length: Option<U256>,
3334
max_presign_count: Option<u64>,
3435
min_presign_count: Option<u64>,
@@ -51,6 +52,7 @@ impl Default for TestSetupBuilder {
5152
register_inactive_validators: false,
5253
enable_payment: Some("true".to_string()),
5354
chain_polling_interval: None,
55+
signing_round_timeout: None,
5456
epoch_length: None,
5557
max_presign_count: Some(0),
5658
min_presign_count: Some(0),
@@ -114,6 +116,11 @@ impl TestSetupBuilder {
114116
self
115117
}
116118

119+
pub fn signing_round_timeout_ms(mut self, signing_round_timeout: Option<String>) -> Self {
120+
self.signing_round_timeout = signing_round_timeout;
121+
self
122+
}
123+
117124
pub fn epoch_length(mut self, epoch_length: usize) -> Self {
118125
self.epoch_length = Some(U256::from(epoch_length));
119126
self
@@ -161,10 +168,18 @@ impl TestSetupBuilder {
161168
}
162169
fs::create_dir_all(node_keys_path).unwrap();
163170

171+
let signing_round_timeout_ms = if self.signing_round_timeout.is_some() {
172+
self.signing_round_timeout
173+
} else {
174+
// if not in CI, set a default signing round timeout of 8000ms
175+
Some("8000".to_string())
176+
};
177+
164178
let custom_node_runtime_config = CustomNodeRuntimeConfig::builder()
165179
.enable_payment(self.enable_payment)
166180
.payment_interval_ms(self.payment_interval_ms)
167181
.chain_polling_interval(self.chain_polling_interval)
182+
.signing_round_timeout_ms(signing_round_timeout_ms)
168183
.build();
169184

170185
let mut testnet = Testnet::builder()

‎rust/lit-node/lit-node-testnet/src/testnet/actions.rs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1269,4 +1269,75 @@ impl Actions {
12691269
}
12701270
Ok(())
12711271
}
1272+
1273+
// shortcut function to update all complaint configs to the same interval and tolerance for testing
1274+
pub async fn update_all_complaint_configs(
1275+
&self,
1276+
interval_secs: Option<u64>,
1277+
tolerance: Option<u64>,
1278+
kick_penalty_percent: Option<u64>,
1279+
kick_penalty_demerits: Option<u64>,
1280+
) -> Result<()> {
1281+
info!(
1282+
"Updating all complaint reason configs interval_secs to {:?} and tolerance to {:?}",
1283+
interval_secs, tolerance
1284+
);
1285+
1286+
let interval_secs = if interval_secs.is_some() {
1287+
Some(U256::from(interval_secs.unwrap()))
1288+
} else {
1289+
None
1290+
};
1291+
let tolerance = if tolerance.is_some() {
1292+
Some(U256::from(tolerance.unwrap()))
1293+
} else {
1294+
None
1295+
};
1296+
let kick_penalty_percent = if kick_penalty_percent.is_some() {
1297+
Some(U256::from(kick_penalty_percent.unwrap()))
1298+
} else {
1299+
None
1300+
};
1301+
let kick_penalty_demerits = if kick_penalty_demerits.is_some() {
1302+
Some(U256::from(kick_penalty_demerits.unwrap()))
1303+
} else {
1304+
None
1305+
};
1306+
for i in 0..=MAX_COMPLAINT_REASON_VALUE {
1307+
let reason = U256::from(i);
1308+
// First, get current chain config for this reason.
1309+
let current_config: lit_blockchain::contracts::staking::ComplaintConfig = self
1310+
.contracts
1311+
.staking
1312+
.complaint_config(reason)
1313+
.call()
1314+
.await
1315+
.map_err(|e| anyhow::anyhow!("unable to get complaint config: {:?}", e))?;
1316+
1317+
// Then, set the config with any new values.
1318+
let cc = self.contracts.staking.set_complaint_config(
1319+
reason,
1320+
lit_blockchain::contracts::staking::ComplaintConfig {
1321+
tolerance: tolerance.unwrap_or(current_config.tolerance),
1322+
interval_secs: interval_secs.unwrap_or(current_config.interval_secs),
1323+
kick_penalty_percent: kick_penalty_percent
1324+
.unwrap_or(current_config.kick_penalty_percent),
1325+
kick_penalty_demerits: kick_penalty_demerits
1326+
.unwrap_or(current_config.kick_penalty_demerits),
1327+
},
1328+
);
1329+
if !Contracts::process_contract_call(
1330+
cc,
1331+
format!("updating staking complaint config for reason {:?}", reason).as_str(),
1332+
)
1333+
.await
1334+
{
1335+
return Err(anyhow::anyhow!(
1336+
"Error updating complaint config for reason {:?}",
1337+
reason.as_u64()
1338+
));
1339+
}
1340+
}
1341+
Ok(())
1342+
}
12721343
}

‎rust/lit-node/lit-node-testnet/src/testnet/node_config.rs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,9 @@ use lit_node_common::config::config_names::{
99
CFG_KEY_CHAIN_POLLING_INTERVAL_MS, CFG_KEY_CHATTER_CLIENT_TIMEOUT,
1010
CFG_KEY_ENABLE_EPOCH_TRANSITIONS, CFG_KEY_ENABLE_PAYMENT,
1111
CFG_KEY_ENABLE_PROXIED_CHATTER_CLIENT, CFG_KEY_PAYMENT_INTERVAL_MS,
12+
CFG_KEY_SIGNING_ROUND_TIMEOUT,
1213
};
1314

14-
#[cfg(all(feature = "proxy_chatter", feature = "testing"))]
15-
use lit_node_common::config::config_names::CFG_KEY_SIGNING_ROUND_TIMEOUT;
16-
1715
use tracing::trace;
1816

1917
pub const CUSTOM_NODE_RUNTIME_CONFIG_PATH: &str = "config/test/custom_node_runtime_config.toml";
@@ -36,6 +34,7 @@ pub struct CustomNodeRuntimeConfigBuilder {
3634
enable_payment: Option<String>,
3735
chain_polling_interval: Option<String>,
3836
payment_interval_ms: Option<String>,
37+
signing_round_timeout_ms: Option<String>,
3938
}
4039

4140
impl Default for CustomNodeRuntimeConfigBuilder {
@@ -50,6 +49,7 @@ impl CustomNodeRuntimeConfigBuilder {
5049
enable_payment: Some("true".to_string()),
5150
chain_polling_interval: None,
5251
payment_interval_ms: None,
52+
signing_round_timeout_ms: None,
5353
}
5454
}
5555

@@ -68,11 +68,17 @@ impl CustomNodeRuntimeConfigBuilder {
6868
self
6969
}
7070

71+
pub fn signing_round_timeout_ms(mut self, signing_round_timeout_ms: Option<String>) -> Self {
72+
self.signing_round_timeout_ms = signing_round_timeout_ms;
73+
self
74+
}
75+
7176
pub fn build(self) -> CustomNodeRuntimeConfig {
7277
CustomNodeRuntimeConfig {
7378
enable_payment: self.enable_payment,
7479
chain_polling_interval: self.chain_polling_interval,
7580
payment_interval_ms: self.payment_interval_ms,
81+
signing_round_timeout_ms: self.signing_round_timeout_ms,
7682
}
7783
}
7884
}
@@ -82,6 +88,7 @@ pub struct CustomNodeRuntimeConfig {
8288
enable_payment: Option<String>,
8389
chain_polling_interval: Option<String>,
8490
payment_interval_ms: Option<String>,
91+
signing_round_timeout_ms: Option<String>,
8592
}
8693

8794
impl CustomNodeRuntimeConfig {
@@ -171,6 +178,13 @@ pub fn generate_custom_node_runtime_config(
171178
.unwrap_or("1000".into()),
172179
);
173180

181+
if let Some(signing_round_timeout_ms) = custom_config.signing_round_timeout_ms.clone() {
182+
cfg.insertstr(
183+
section,
184+
CFG_KEY_SIGNING_ROUND_TIMEOUT,
185+
&signing_round_timeout_ms,
186+
);
187+
}
174188
match node_config_path {
175189
Some(path) => {
176190
cfg.write_file(Path::new(&path))

‎rust/lit-node/lit-node/Cargo.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,12 @@ lit-actions-server = [
1717
testing = [
1818
"lit-actions-server",
1919
"lit-observability/testing",
20+
"lit-blockchain/testing",
2021
] # enables testing features
2122
proxy_chatter = [
2223
"lit-node-testnet/proxy_chatter",
23-
"lit-node-testnet/testing",
2424
"lit-node-testnet/lit-actions",
25+
"lit-blockchain/proxy_chatter",
2526
] # enables proxy http for testing
2627

2728
[build-dependencies]

‎rust/lit-node/lit-node/src/git_info.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
pub const GIT_COMMIT_HASH: &str = "b04446f9037be567924971aad3cb78a1f17b7414";
1+
pub const GIT_COMMIT_HASH: &str = "50083ccf8061c4bfe17761be8b496bdab82851a6";

‎rust/lit-node/lit-node/src/peers/peer_state/chain_update.rs‎

Lines changed: 2 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,7 @@
11
use super::super::PeerState;
22
use crate::error::unexpected_err;
3-
use crate::{
4-
error::{EC, Result, blockchain_err, blockchain_err_code},
5-
utils::eth::EthereumAddress,
6-
};
7-
use ethers::{
8-
providers::Middleware,
9-
types::{U64, U256},
10-
};
3+
use crate::error::{EC, Result, blockchain_err, blockchain_err_code};
4+
use ethers::types::{U64, U256};
115
use lit_blockchain::util::decode_revert;
126
use std::time::Duration;
137
use tracing::{Instrument, debug_span, instrument, trace};
@@ -202,40 +196,6 @@ impl PeerState {
202196
return Err(unexpected_err("No realm id set", None));
203197
};
204198

205-
let provider = self.staking_contract.client().provider().clone();
206-
let wallet_address = self
207-
.wallet_keys
208-
.verifying_key()
209-
.to_eth_address()
210-
.map_err(|e| {
211-
blockchain_err(
212-
e,
213-
Some(
214-
"Failed to convert verifying key to eth address during request to join."
215-
.to_string(),
216-
),
217-
)
218-
})?;
219-
let balance = provider
220-
.get_balance(wallet_address, None)
221-
.await
222-
.map_err(|e| {
223-
blockchain_err(
224-
e,
225-
Some(
226-
"Failed to get balance of attested node wallet during request to join."
227-
.to_string(),
228-
),
229-
)
230-
})?;
231-
232-
if balance.is_zero() {
233-
return Err(blockchain_err(
234-
"Aborting request to join as attested node wallet balance is 0.",
235-
None,
236-
));
237-
}
238-
239199
let func = self
240200
.staking_contract
241201
.request_to_join_as_node(realm_id, self.staker_address);

‎rust/lit-node/lit-node/src/tasks/fsm/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ pub async fn node_fsm_worker(
161161
}
162162

163163
// if the epoch seems to have jumped, we need to figure out why and handle it.
164-
if epoch_number > previous_included_epoch_number {
164+
if epoch_number >= previous_included_epoch_number {
165165
// this could be the state if we haven't checked the chain - check it and continue.
166166
if network_state == NetworkState::NextValidatorSetLocked {
167167
wait_on_next_validator_set_locked(

0 commit comments

Comments
 (0)