diff --git a/apps/fortuna/Cargo.lock b/apps/fortuna/Cargo.lock index 012a660b9a..eb343e9009 100644 --- a/apps/fortuna/Cargo.lock +++ b/apps/fortuna/Cargo.lock @@ -1503,7 +1503,7 @@ dependencies = [ [[package]] name = "fortuna" -version = "6.7.2" +version = "6.8.0" dependencies = [ "anyhow", "axum", diff --git a/apps/fortuna/Cargo.toml b/apps/fortuna/Cargo.toml index ee9fad45c7..cd63aed9f3 100644 --- a/apps/fortuna/Cargo.toml +++ b/apps/fortuna/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fortuna" -version = "6.7.2" +version = "6.8.0" edition = "2021" [dependencies] diff --git a/apps/fortuna/config.sample.yaml b/apps/fortuna/config.sample.yaml index 73ffe3da5a..3d5911d5ff 100644 --- a/apps/fortuna/config.sample.yaml +++ b/apps/fortuna/config.sample.yaml @@ -6,6 +6,9 @@ chains: # Keeper configuration for the chain reveal_delay_blocks: 0 gas_limit: 500000 + # Increase the transaction gas limit by 10% each time the callback fails + # defaults to 100 (i.e., don't change the gas limit) if not specified. + backoff_gas_multiplier_pct: 110 min_keeper_balance: 100000000000000000 # Provider configuration diff --git a/apps/fortuna/src/config.rs b/apps/fortuna/src/config.rs index 45bf8569a0..1ffd47a602 100644 --- a/apps/fortuna/src/config.rs +++ b/apps/fortuna/src/config.rs @@ -134,6 +134,10 @@ pub struct EthereumConfig { /// The gas limit to use for entropy callback transactions. pub gas_limit: u64, + /// The percentage multiplier to apply to the gas limit for each backoff. + #[serde(default = "default_backoff_gas_multiplier_pct")] + pub backoff_gas_multiplier_pct: u64, + /// The minimum percentage profit to earn as a function of the callback cost. /// For example, 20 means a profit of 20% over the cost of the callback. /// The fee will be raised if the profit is less than this number. @@ -172,6 +176,10 @@ pub struct EthereumConfig { pub priority_fee_multiplier_pct: u64, } +fn default_backoff_gas_multiplier_pct() -> u64 { + 100 +} + /// A commitment that the provider used to generate random numbers at some point in the past. /// These historical commitments need to be stored in the configuration to support transition points where /// the commitment changes. In theory, this information is stored on the blockchain, but unfortunately it diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index 38da57e2df..b5c9e67f2b 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -55,6 +55,8 @@ const UPDATE_COMMITMENTS_INTERVAL: Duration = Duration::from_secs(30); const UPDATE_COMMITMENTS_THRESHOLD_FACTOR: f64 = 0.95; /// Rety last N blocks const RETRY_PREVIOUS_BLOCKS: u64 = 100; +/// By default, we scale the gas estimate by 25% when submitting the tx. +const DEFAULT_GAS_ESTIMATE_MULTIPLIER_PCT: u64 = 125; #[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] pub struct AccountLabel { @@ -254,6 +256,7 @@ pub async fn run_keeper_threads( }, contract.clone(), gas_limit, + chain_eth_config.backoff_gas_multiplier_pct, chain_state.clone(), metrics.clone(), fulfilled_requests_cache.clone(), @@ -279,6 +282,7 @@ pub async fn run_keeper_threads( rx, Arc::clone(&contract), gas_limit, + chain_eth_config.backoff_gas_multiplier_pct, metrics.clone(), fulfilled_requests_cache.clone(), ) @@ -303,7 +307,9 @@ pub async fn run_keeper_threads( chain_state.provider_address, ADJUST_FEE_INTERVAL, chain_eth_config.legacy_tx, - chain_eth_config.gas_limit, + // NOTE: we adjust fees based on the maximum gas that the keeper will submit a callback with. + // This number is *larger* than the configured gas limit, as we pad gas on transaction submission for reliability. + (chain_eth_config.gas_limit * DEFAULT_GAS_ESTIMATE_MULTIPLIER_PCT) / 100, chain_eth_config.min_profit_pct, chain_eth_config.target_profit_pct, chain_eth_config.max_profit_pct, @@ -372,6 +378,7 @@ pub async fn process_event_with_backoff( chain_state: BlockchainState, contract: Arc, gas_limit: U256, + backoff_gas_multiplier_pct: u64, metrics: Arc, ) { let start_time = std::time::Instant::now(); @@ -388,13 +395,35 @@ pub async fn process_event_with_backoff( max_elapsed_time: Some(Duration::from_secs(300)), // retry for 5 minutes ..Default::default() }; + + let current_multiplier = Arc::new(AtomicU64::new(DEFAULT_GAS_ESTIMATE_MULTIPLIER_PCT)); + match backoff::future::retry_notify( backoff, || async { - process_event(&event, &chain_state, &contract, gas_limit, metrics.clone()).await + let multiplier = current_multiplier.load(std::sync::atomic::Ordering::Relaxed); + process_event( + &event, + &chain_state, + &contract, + gas_limit, + multiplier, + metrics.clone(), + ) + .await }, |e, dur| { - tracing::error!("Error happened at {:?}: {}", dur, e); + let multiplier = current_multiplier.load(std::sync::atomic::Ordering::Relaxed); + tracing::error!( + "Error at duration {:?} with gas multiplier {}: {}", + dur, + multiplier, + e + ); + current_multiplier.store( + multiplier.saturating_mul(backoff_gas_multiplier_pct) / 100, + std::sync::atomic::Ordering::Relaxed, + ); }, ) .await @@ -436,6 +465,8 @@ pub async fn process_event( chain_config: &BlockchainState, contract: &InstrumentedSignablePythContract, gas_limit: U256, + // A value of 100 submits the tx with the same gas as the estimate. + gas_estimate_multiplier_pct: u64, metrics: Arc, ) -> Result<(), backoff::Error> { // ignore requests that are not for the configured provider @@ -466,6 +497,8 @@ pub async fn process_event( backoff::Error::transient(anyhow!("Error estimating gas for reveal: {:?}", e)) })?; + // The gas limit on the simulated transaction is the configured gas limit on the chain, + // but we are willing to pad the gas a bit to ensure reliable submission. if gas_estimate > gas_limit { return Err(backoff::Error::permanent(anyhow!( "Gas estimate for reveal with callback is higher than the gas limit {} > {}", @@ -474,8 +507,10 @@ pub async fn process_event( ))); } - // Pad the gas estimate by 25% after checking it against the gas limit - let gas_estimate = gas_estimate.saturating_mul(5.into()) / 4; + // Pad the gas estimate after checking it against the simulation gas limit, ensuring that + // the padded gas estimate doesn't exceed the maximum amount of gas we are willing to use. + let gas_estimate = gas_estimate.saturating_mul(gas_estimate_multiplier_pct.into()) / 100; + let gas_estimate = gas_estimate.min((gas_limit * DEFAULT_GAS_ESTIMATE_MULTIPLIER_PCT) / 100); let contract_call = contract .reveal_with_callback( @@ -589,6 +624,7 @@ pub async fn process_block_range( block_range: BlockRange, contract: Arc, gas_limit: U256, + backoff_gas_multiplier_pct: u64, chain_state: api::BlockchainState, metrics: Arc, fulfilled_requests_cache: Arc>>, @@ -612,6 +648,7 @@ pub async fn process_block_range( }, contract.clone(), gas_limit, + backoff_gas_multiplier_pct, chain_state.clone(), metrics.clone(), fulfilled_requests_cache.clone(), @@ -634,6 +671,7 @@ pub async fn process_single_block_batch( block_range: BlockRange, contract: Arc, gas_limit: U256, + backoff_gas_multiplier_pct: u64, chain_state: api::BlockchainState, metrics: Arc, fulfilled_requests_cache: Arc>>, @@ -660,6 +698,7 @@ pub async fn process_single_block_batch( chain_state.clone(), contract.clone(), gas_limit, + backoff_gas_multiplier_pct, metrics.clone(), ) .in_current_span(), @@ -806,6 +845,7 @@ pub async fn process_new_blocks( mut rx: mpsc::Receiver, contract: Arc, gas_limit: U256, + backoff_gas_multiplier_pct: u64, metrics: Arc, fulfilled_requests_cache: Arc>>, ) { @@ -816,6 +856,7 @@ pub async fn process_new_blocks( block_range, Arc::clone(&contract), gas_limit, + backoff_gas_multiplier_pct, chain_state.clone(), metrics.clone(), fulfilled_requests_cache.clone(), @@ -832,6 +873,7 @@ pub async fn process_backlog( backlog_range: BlockRange, contract: Arc, gas_limit: U256, + backoff_gas_multiplier_pct: u64, chain_state: BlockchainState, metrics: Arc, fulfilled_requests_cache: Arc>>, @@ -841,6 +883,7 @@ pub async fn process_backlog( backlog_range, contract, gas_limit, + backoff_gas_multiplier_pct, chain_state, metrics, fulfilled_requests_cache,