From 535e8c3d319584d7f44001326f6aeff42097b65b Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Thu, 26 Jun 2025 11:17:49 -0700 Subject: [PATCH] feat(fortuna): add option to disable adjustment threads --- apps/fortuna/README.md | 44 ++++++++++++++--- apps/fortuna/config.sample.yaml | 7 +++ apps/fortuna/src/command/run.rs | 10 +++- apps/fortuna/src/config.rs | 15 ++++++ apps/fortuna/src/keeper.rs | 84 ++++++++++++++++++--------------- 5 files changed, 115 insertions(+), 45 deletions(-) diff --git a/apps/fortuna/README.md b/apps/fortuna/README.md index 961b1e4ec0..ce431508ed 100644 --- a/apps/fortuna/README.md +++ b/apps/fortuna/README.md @@ -56,41 +56,70 @@ Fortuna supports running multiple replica instances for high availability and re - Each replica primarily handles requests assigned to its ID - After a configurable delay, replicas will process requests from other replicas as backup (failover) +### Fee Management with Multiple Instances + +When running multiple Fortuna instances with different keeper wallets but a single provider, only one instance should handle fee management. This instance needs to run using the same private key as the fee manager, because only the registerd fee manager wallet can adjust fees and withdraw funds. + ### Example Configurations -**Two Replica Setup (Blue/Green):** +**Two Replica Setup with Fee Management:** ```yaml -# Replica 0 (Blue) - handles even sequence numbers (0, 2, 4, ...) +# Replica 0 (fee manager wallet) - handles even sequence numbers + fee management keeper: + private_key: + value: 0x replica_config: replica_id: 0 total_replicas: 2 backup_delay_seconds: 30 + run_config: + disable_fee_adjustment: false # Enable fee management (default) + disable_fee_withdrawal: false -# Replica 1 (Green) - handles odd sequence numbers (1, 3, 5, ...) +# Replica 1 (non-fee-manager wallet) - handles odd sequence numbers only keeper: + private_key: + value: 0x replica_config: replica_id: 1 total_replicas: 2 backup_delay_seconds: 30 + run_config: + disable_fee_adjustment: true # Disable fee management + disable_fee_withdrawal: true ``` **Three Replica Setup:** ```yaml -# Replica 0 - handles sequence numbers 0, 3, 6, 9, ... +# Replica 0 (fee manager wallet) - handles sequence numbers 0, 3, 6, 9, ... + fee management keeper: replica_config: replica_id: 0 total_replicas: 3 backup_delay_seconds: 30 + run_config: + disable_fee_adjustment: false + disable_fee_withdrawal: false + +# Replicas 1 & 2 (non-fee-manager wallets) - request processing only +keeper: + replica_config: + replica_id: 1 # or 2 + total_replicas: 3 + backup_delay_seconds: 30 + run_config: + disable_fee_adjustment: true + disable_fee_withdrawal: true ``` ### Deployment Considerations 1. **Separate Wallets**: Each replica MUST use a different private key to avoid nonce conflicts -2. **Backup Delay**: Set `backup_delay_seconds` long enough to allow primary replica to process requests, but short enough for acceptable failover time (recommended: 30-60 seconds) -3. **Monitoring**: Monitor each replica's processing metrics to ensure proper load distribution -4. **Gas Management**: Each replica needs sufficient ETH balance for gas fees +2. **Fee Manager Assignment**: Set the provider's `fee_manager` address to match the primary instance's keeper wallet +3. **Thread Configuration**: Only enable fee management threads on the instance using the fee manager wallet +4. **Backup Delay**: Set `backup_delay_seconds` long enough to allow primary replica to process requests, but short enough for acceptable failover time (recommended: 30-60 seconds) +5. **Monitoring**: Monitor each replica's processing metrics to ensure proper load distribution +6. **Gas Management**: Each replica needs sufficient ETH balance for gas fees ### Failover Behavior @@ -98,6 +127,7 @@ keeper: - Backup replicas wait for `backup_delay_seconds` before checking if request is still unfulfilled - If request is already fulfilled during the delay, backup replica skips processing - This prevents duplicate transactions and wasted gas while ensuring reliability +- Fee management operations (adjustment/withdrawal) only occur on an instance where the keeper wallet is the fee manager wallet. ## Local Development diff --git a/apps/fortuna/config.sample.yaml b/apps/fortuna/config.sample.yaml index d7a3bef205..a2b8fa310f 100644 --- a/apps/fortuna/config.sample.yaml +++ b/apps/fortuna/config.sample.yaml @@ -87,6 +87,13 @@ keeper: # For production, you can store the private key in a file. # file: keeper-key.txt + # Runtime configuration for the keeper service + # Optional: Configure which keeper threads to disable. If running multiple replicas, + # only a single replica should have the fee adjustment and withdrawal threads enabled. + # run_config: + # disable_fee_adjustment: false # Set to true to disable automatic fee adjustment + # disable_fee_withdrawal: false # Set to true to disable automatic fee withdrawal + # Multi-replica configuration # Optional: Multi-replica configuration for high availability and load distribution # Uncomment and configure for production deployments with multiple Fortuna instances diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index e1933db21f..e8c8309ea2 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -3,7 +3,10 @@ use { api::{self, ApiBlockChainState, BlockchainState, ChainId}, chain::ethereum::InstrumentedPythContract, command::register_provider::CommitmentMetadata, - config::{Commitment, Config, EthereumConfig, ProviderConfig, ReplicaConfig, RunOptions}, + config::{ + Commitment, Config, EthereumConfig, ProviderConfig, ReplicaConfig, RunConfig, + RunOptions, + }, eth_utils::traced_client::RpcMetrics, history::History, keeper::{self, keeper_metrics::KeeperMetrics}, @@ -101,6 +104,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> { } let keeper_replica_config = config.keeper.replica_config.clone(); + let keeper_run_config = config.keeper.run_config.clone(); let chains: Arc>> = Arc::new(RwLock::new( config @@ -115,6 +119,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> { let keeper_metrics = keeper_metrics.clone(); let keeper_private_key_option = keeper_private_key_option.clone(); let keeper_replica_config = keeper_replica_config.clone(); + let keeper_run_config = keeper_run_config.clone(); let chains = chains.clone(); let secret_copy = secret.clone(); let rpc_metrics = rpc_metrics.clone(); @@ -129,6 +134,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> { keeper_metrics.clone(), keeper_private_key_option.clone(), keeper_replica_config.clone(), + keeper_run_config.clone(), chains.clone(), &secret_copy, history.clone(), @@ -180,6 +186,7 @@ async fn setup_chain_and_run_keeper( keeper_metrics: Arc, keeper_private_key_option: Option, keeper_replica_config: Option, + keeper_run_config: RunConfig, chains: Arc>>, secret_copy: &str, history: Arc, @@ -203,6 +210,7 @@ async fn setup_chain_and_run_keeper( keeper::run_keeper_threads( keeper_private_key, keeper_replica_config, + keeper_run_config, chain_config, state, keeper_metrics.clone(), diff --git a/apps/fortuna/src/config.rs b/apps/fortuna/src/config.rs index 84471ef8fc..0d0ae8b2d2 100644 --- a/apps/fortuna/src/config.rs +++ b/apps/fortuna/src/config.rs @@ -350,6 +350,17 @@ fn default_chain_sample_interval() -> u64 { 1 } +#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] +pub struct RunConfig { + /// Disable automatic fee adjustment threads + #[serde(default)] + pub disable_fee_adjustment: bool, + + /// Disable automatic fee withdrawal threads + #[serde(default)] + pub disable_fee_withdrawal: bool, +} + #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct ReplicaConfig { pub replica_id: u64, @@ -374,6 +385,10 @@ pub struct KeeperConfig { #[serde(default)] pub replica_config: Option, + + /// Runtime configuration for the keeper service + #[serde(default)] + pub run_config: RunConfig, } // A secret is a string that can be provided either as a literal in the config, diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index e65bc2aaf1..184c6821d4 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -2,7 +2,7 @@ use { crate::{ api::{BlockchainState, ChainId}, chain::ethereum::{InstrumentedPythContract, InstrumentedSignablePythContract}, - config::{EthereumConfig, ReplicaConfig}, + config::{EthereumConfig, ReplicaConfig, RunConfig}, eth_utils::traced_client::RpcMetrics, history::History, keeper::{ @@ -54,10 +54,12 @@ pub enum RequestState { /// Run threads to handle events for the last `BACKLOG_RANGE` blocks, watch for new blocks and /// handle any events for the new blocks. +#[allow(clippy::too_many_arguments)] // Top level orchestration function that needs to configure several threads #[tracing::instrument(name = "keeper", skip_all, fields(chain_id = chain_state.id))] pub async fn run_keeper_threads( keeper_private_key: String, keeper_replica_config: Option, + keeper_run_config: RunConfig, chain_eth_config: EthereumConfig, chain_state: BlockchainState, metrics: Arc, @@ -118,44 +120,52 @@ pub async fn run_keeper_threads( ); // Spawn a thread that watches the keeper wallet balance and submits withdrawal transactions as needed to top-up the balance. - spawn( - withdraw_fees_wrapper( - contract.clone(), - chain_state.provider_address, - WITHDRAW_INTERVAL, - U256::from(chain_eth_config.min_keeper_balance), - ) - .in_current_span(), - ); + if !keeper_run_config.disable_fee_withdrawal { + spawn( + withdraw_fees_wrapper( + contract.clone(), + chain_state.provider_address, + WITHDRAW_INTERVAL, + U256::from(chain_eth_config.min_keeper_balance), + ) + .in_current_span(), + ); + } else { + tracing::info!("Fee withdrawal thread disabled by configuration"); + } // Spawn a thread that periodically adjusts the provider fee. - spawn( - adjust_fee_wrapper( - contract.clone(), - chain_state.clone(), - chain_state.provider_address, - ADJUST_FEE_INTERVAL, - chain_eth_config.legacy_tx, - // NOTE: we are adjusting the fees based on the maximum configured gas for user transactions. - // However, the keeper will pad the gas limit for transactions (per the escalation policy) to ensure reliable submission. - // Consequently, fees can be adjusted such that transactions are still unprofitable. - // While we could scale up this value based on the padding, that ends up overcharging users as most transactions cost nowhere - // near the maximum gas limit. - // In the unlikely event that the keeper fees aren't sufficient, the solution to this is to configure the target - // fee percentage to be higher on that specific chain. - chain_eth_config.gas_limit, - // NOTE: unwrap() here so we panic early if someone configures these values below -100. - u64::try_from(100 + chain_eth_config.min_profit_pct) - .expect("min_profit_pct must be >= -100"), - u64::try_from(100 + chain_eth_config.target_profit_pct) - .expect("target_profit_pct must be >= -100"), - u64::try_from(100 + chain_eth_config.max_profit_pct) - .expect("max_profit_pct must be >= -100"), - chain_eth_config.fee, - metrics.clone(), - ) - .in_current_span(), - ); + if !keeper_run_config.disable_fee_adjustment { + spawn( + adjust_fee_wrapper( + contract.clone(), + chain_state.clone(), + chain_state.provider_address, + ADJUST_FEE_INTERVAL, + chain_eth_config.legacy_tx, + // NOTE: we are adjusting the fees based on the maximum configured gas for user transactions. + // However, the keeper will pad the gas limit for transactions (per the escalation policy) to ensure reliable submission. + // Consequently, fees can be adjusted such that transactions are still unprofitable. + // While we could scale up this value based on the padding, that ends up overcharging users as most transactions cost nowhere + // near the maximum gas limit. + // In the unlikely event that the keeper fees aren't sufficient, the solution to this is to configure the target + // fee percentage to be higher on that specific chain. + chain_eth_config.gas_limit, + // NOTE: unwrap() here so we panic early if someone configures these values below -100. + u64::try_from(100 + chain_eth_config.min_profit_pct) + .expect("min_profit_pct must be >= -100"), + u64::try_from(100 + chain_eth_config.target_profit_pct) + .expect("target_profit_pct must be >= -100"), + u64::try_from(100 + chain_eth_config.max_profit_pct) + .expect("max_profit_pct must be >= -100"), + chain_eth_config.fee, + metrics.clone(), + ) + .in_current_span(), + ); + } else { + tracing::info!("Fee adjustment thread disabled by configuration"); + } spawn(update_commitments_loop(contract.clone(), chain_state.clone()).in_current_span());