From dbbda0b145950885c6d5ad8d832ee3782997057b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 00:24:33 +0000 Subject: [PATCH 01/12] feat(fortuna): separate fee manager and keeper wallets for multi-replica support - Add fee_manager_private_key to KeeperConfig for fee manager operations - Add known_keeper_addresses for balance comparison - Modify withdrawal logic to use fee manager key for fee manager calls - Only withdraw fees if current keeper has lowest balance among known keepers - Maintain backward compatibility with existing single-key setup Co-Authored-By: Tejas Badadare --- apps/fortuna/config.sample.yaml | 9 ++++ apps/fortuna/src/command/run.rs | 35 +++++++++------ apps/fortuna/src/config.rs | 9 ++++ apps/fortuna/src/keeper.rs | 23 +++++++--- apps/fortuna/src/keeper/fee.rs | 79 +++++++++++++++++++++++++++++---- 5 files changed, 125 insertions(+), 30 deletions(-) diff --git a/apps/fortuna/config.sample.yaml b/apps/fortuna/config.sample.yaml index a2b8fa310f..0a64229e49 100644 --- a/apps/fortuna/config.sample.yaml +++ b/apps/fortuna/config.sample.yaml @@ -87,6 +87,15 @@ keeper: # For production, you can store the private key in a file. # file: keeper-key.txt + # Fee manager private key for fee manager operations (optional) + # fee_manager_private_key: + # value: 0xabcd + # # file: fee-manager-key.txt + + # List of known keeper wallet addresses for balance comparison (optional) + # The keeper will only withdraw fees if its balance is the lowest among these addresses. + + # Runtime configuration for the keeper service # Optional: Configure which keeper threads to disable. If running multiple replicas, # only a single replica should have the fee adjustment and withdrawal threads enabled. diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index e8c8309ea2..f9e0b3b6d8 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -3,10 +3,7 @@ use { api::{self, ApiBlockChainState, BlockchainState, ChainId}, chain::ethereum::InstrumentedPythContract, command::register_provider::CommitmentMetadata, - config::{ - Commitment, Config, EthereumConfig, ProviderConfig, ReplicaConfig, RunConfig, - RunOptions, - }, + config::{Commitment, Config, EthereumConfig, ProviderConfig, RunOptions}, eth_utils::traced_client::RpcMetrics, history::History, keeper::{self, keeper_metrics::KeeperMetrics}, @@ -125,16 +122,30 @@ pub async fn run(opts: &RunOptions) -> Result<()> { let rpc_metrics = rpc_metrics.clone(); let provider_config = config.provider.clone(); let history = history.clone(); + let fee_manager_private_key = config.keeper.fee_manager_private_key.clone(); + let known_keeper_addresses = config.keeper.known_keeper_addresses.clone(); spawn(async move { loop { + let keeper_config = if keeper_private_key_option.is_some() { + Some(crate::config::KeeperConfig { + private_key: crate::config::SecretString { + value: keeper_private_key_option.clone(), + file: None, + }, + fee_manager_private_key: fee_manager_private_key.clone(), + known_keeper_addresses: known_keeper_addresses.clone(), + replica_config: keeper_replica_config.clone(), + run_config: keeper_run_config.clone(), + }) + } else { + None + }; let setup_result = setup_chain_and_run_keeper( provider_config.clone(), &chain_id, chain_config.clone(), keeper_metrics.clone(), - keeper_private_key_option.clone(), - keeper_replica_config.clone(), - keeper_run_config.clone(), + keeper_config, chains.clone(), &secret_copy, history.clone(), @@ -184,9 +195,7 @@ async fn setup_chain_and_run_keeper( chain_id: &ChainId, chain_config: EthereumConfig, keeper_metrics: Arc, - keeper_private_key_option: Option, - keeper_replica_config: Option, - keeper_run_config: RunConfig, + keeper_config: Option, chains: Arc>>, secret_copy: &str, history: Arc, @@ -206,11 +215,9 @@ async fn setup_chain_and_run_keeper( chain_id.clone(), ApiBlockChainState::Initialized(state.clone()), ); - if let Some(keeper_private_key) = keeper_private_key_option { + if let Some(keeper_config) = keeper_config { keeper::run_keeper_threads( - keeper_private_key, - keeper_replica_config, - keeper_run_config, + keeper_config, chain_config, state, keeper_metrics.clone(), diff --git a/apps/fortuna/src/config.rs b/apps/fortuna/src/config.rs index 0d0ae8b2d2..aa45d2c3c6 100644 --- a/apps/fortuna/src/config.rs +++ b/apps/fortuna/src/config.rs @@ -383,6 +383,15 @@ pub struct KeeperConfig { /// should ensure this is a different key in order to reduce the severity of security breaches. pub private_key: SecretString, + /// The fee manager's private key for fee manager operations. + /// This key is used to withdraw fees from the contract as the fee manager. + /// Multiple replicas can share the same fee manager private key. + #[serde(default)] + pub fee_manager_private_key: Option, + + #[serde(default)] + pub known_keeper_addresses: Vec
, + #[serde(default)] pub replica_config: Option, diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index 184c6821d4..e11b4c0dbe 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -2,7 +2,7 @@ use { crate::{ api::{BlockchainState, ChainId}, chain::ethereum::{InstrumentedPythContract, InstrumentedSignablePythContract}, - config::{EthereumConfig, ReplicaConfig, RunConfig}, + config::EthereumConfig, eth_utils::traced_client::RpcMetrics, history::History, keeper::{ @@ -57,9 +57,7 @@ pub enum RequestState { #[allow(clippy::too_many_arguments)] // Top level orchestration function that needs to configure several threads #[tracing::instrument(name = "keeper", skip_all, fields(chain_id = chain_state.id))] pub async fn run_keeper_threads( - keeper_private_key: String, - keeper_replica_config: Option, - keeper_run_config: RunConfig, + keeper_config: crate::config::KeeperConfig, chain_eth_config: EthereumConfig, chain_state: BlockchainState, metrics: Arc, @@ -70,6 +68,10 @@ pub async fn run_keeper_threads( let latest_safe_block = get_latest_safe_block(&chain_state).in_current_span().await; tracing::info!("Latest safe block: {}", &latest_safe_block); + let keeper_private_key = keeper_config.private_key.load()?.ok_or_else(|| { + anyhow::anyhow!("Keeper private key is required but not provided in config") + })?; + let contract = Arc::new(InstrumentedSignablePythContract::from_config( &chain_eth_config, &keeper_private_key, @@ -88,7 +90,7 @@ pub async fn run_keeper_threads( contract: contract.clone(), gas_limit, escalation_policy: chain_eth_config.escalation_policy.to_policy(), - replica_config: keeper_replica_config, + replica_config: keeper_config.replica_config.clone(), metrics: metrics.clone(), fulfilled_requests_cache, history, @@ -120,13 +122,20 @@ pub async fn run_keeper_threads( ); // Spawn a thread that watches the keeper wallet balance and submits withdrawal transactions as needed to top-up the balance. - if !keeper_run_config.disable_fee_withdrawal { + if !keeper_config.run_config.disable_fee_withdrawal { + let fee_manager_private_key = keeper_config + .fee_manager_private_key + .as_ref() + .and_then(|key| key.load().ok()) + .flatten(); spawn( withdraw_fees_wrapper( contract.clone(), chain_state.provider_address, WITHDRAW_INTERVAL, U256::from(chain_eth_config.min_keeper_balance), + fee_manager_private_key, + keeper_config.known_keeper_addresses.clone(), ) .in_current_span(), ); @@ -135,7 +144,7 @@ pub async fn run_keeper_threads( } // Spawn a thread that periodically adjusts the provider fee. - if !keeper_run_config.disable_fee_adjustment { + if !keeper_config.run_config.disable_fee_adjustment { spawn( adjust_fee_wrapper( contract.clone(), diff --git a/apps/fortuna/src/keeper/fee.rs b/apps/fortuna/src/keeper/fee.rs index bd9aedeb0a..a8a945244f 100644 --- a/apps/fortuna/src/keeper/fee.rs +++ b/apps/fortuna/src/keeper/fee.rs @@ -16,17 +16,60 @@ use { tracing::{self, Instrument}, }; +async fn should_withdraw_fees( + provider: Arc, + current_keeper_address: Address, + known_keeper_addresses: &[Address], +) -> Result { + if known_keeper_addresses.is_empty() { + return Ok(true); + } + + let current_balance = provider + .get_balance(current_keeper_address, None) + .await + .map_err(|e| anyhow!("Error while getting current keeper balance. error: {:?}", e))?; + + for &address in known_keeper_addresses { + let balance = provider.get_balance(address, None).await.map_err(|e| { + anyhow!( + "Error while getting keeper balance for {:?}. error: {:?}", + address, + e + ) + })?; + + if balance < current_balance { + tracing::info!( + "Skipping fee withdrawal: keeper {:?} has lower balance ({:?}) than current keeper {:?} ({:?})", + address, balance, current_keeper_address, current_balance + ); + return Ok(false); + } + } + + Ok(true) +} + #[tracing::instrument(name = "withdraw_fees", skip_all, fields())] pub async fn withdraw_fees_wrapper( contract: Arc, provider_address: Address, poll_interval: Duration, min_balance: U256, + fee_manager_private_key: Option, + known_keeper_addresses: Vec
, ) { loop { - if let Err(e) = withdraw_fees_if_necessary(contract.clone(), provider_address, min_balance) - .in_current_span() - .await + if let Err(e) = withdraw_fees_if_necessary( + contract.clone(), + provider_address, + min_balance, + fee_manager_private_key.clone(), + known_keeper_addresses.clone(), + ) + .in_current_span() + .await { tracing::error!("Withdrawing fees. error: {:?}", e); } @@ -39,6 +82,8 @@ pub async fn withdraw_fees_if_necessary( contract: Arc, provider_address: Address, min_balance: U256, + fee_manager_private_key: Option, + known_keeper_addresses: Vec
, ) -> Result<()> { let provider = contract.provider(); let wallet = contract.wallet(); @@ -48,22 +93,38 @@ pub async fn withdraw_fees_if_necessary( .await .map_err(|e| anyhow!("Error while getting balance. error: {:?}", e))?; + if !should_withdraw_fees( + Arc::new(provider.clone()), + wallet.address(), + &known_keeper_addresses, + ) + .await? + { + return Ok(()); + } + let provider_info = contract .get_provider_info(provider_address) .call() .await .map_err(|e| anyhow!("Error while getting provider info. error: {:?}", e))?; - if provider_info.fee_manager != wallet.address() { - return Err(anyhow!("Fee manager for provider {:?} is not the keeper wallet. Fee manager: {:?} Keeper: {:?}", provider, provider_info.fee_manager, wallet.address())); - } - let fees = provider_info.accrued_fees_in_wei; if keeper_balance < min_balance && U256::from(fees) > min_balance { tracing::info!("Claiming accrued fees..."); - let contract_call = contract.withdraw_as_fee_manager(provider_address, fees); - send_and_confirm(contract_call).await?; + + if let Some(_fee_manager_key) = fee_manager_private_key { + let contract_call = contract.withdraw_as_fee_manager(provider_address, fees); + send_and_confirm(contract_call).await?; + } else { + if provider_info.fee_manager != wallet.address() { + return Err(anyhow!("Fee manager for provider {:?} is not the keeper wallet and no fee manager private key is configured. Fee manager: {:?} Keeper: {:?}", provider_address, provider_info.fee_manager, wallet.address())); + } + + let contract_call = contract.withdraw_as_fee_manager(provider_address, fees); + send_and_confirm(contract_call).await?; + } } else if keeper_balance < min_balance { // NOTE: This log message triggers a grafana alert. If you want to change the text, please change the alert also. tracing::warn!("Keeper balance {:?} is too low (< {:?}) but provider fees are not sufficient to top-up.", keeper_balance, min_balance) From 68fd3f8c26fe3646de7c33f8d44291f9f569fcd0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 23:07:40 +0000 Subject: [PATCH 02/12] fix: address PR feedback for fee manager/keeper separation Co-Authored-By: Tejas Badadare --- apps/fortuna/src/command/run.rs | 23 ++-------- apps/fortuna/src/config.rs | 2 +- apps/fortuna/src/keeper.rs | 13 +++++- apps/fortuna/src/keeper/fee.rs | 81 +++++++++++++++++++++++++-------- 4 files changed, 78 insertions(+), 41 deletions(-) diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index f9e0b3b6d8..cab4678f2c 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -3,7 +3,7 @@ use { api::{self, ApiBlockChainState, BlockchainState, ChainId}, chain::ethereum::InstrumentedPythContract, command::register_provider::CommitmentMetadata, - config::{Commitment, Config, EthereumConfig, ProviderConfig, RunOptions}, + config::{Commitment, Config, EthereumConfig, KeeperConfig, ProviderConfig, RunOptions}, eth_utils::traced_client::RpcMetrics, history::History, keeper::{self, keeper_metrics::KeeperMetrics}, @@ -100,9 +100,6 @@ pub async fn run(opts: &RunOptions) -> Result<()> { tracing::info!("Not starting keeper service: no keeper private key specified. Please add one to the config if you would like to run the keeper service.") } - let keeper_replica_config = config.keeper.replica_config.clone(); - let keeper_run_config = config.keeper.run_config.clone(); - let chains: Arc>> = Arc::new(RwLock::new( config .chains @@ -115,28 +112,16 @@ pub async fn run(opts: &RunOptions) -> Result<()> { keeper_metrics.add_chain(chain_id.clone(), config.provider.address); let keeper_metrics = keeper_metrics.clone(); let keeper_private_key_option = keeper_private_key_option.clone(); - let keeper_replica_config = keeper_replica_config.clone(); - let keeper_run_config = keeper_run_config.clone(); let chains = chains.clone(); let secret_copy = secret.clone(); let rpc_metrics = rpc_metrics.clone(); let provider_config = config.provider.clone(); let history = history.clone(); - let fee_manager_private_key = config.keeper.fee_manager_private_key.clone(); - let known_keeper_addresses = config.keeper.known_keeper_addresses.clone(); + let keeper_config_base = config.keeper.clone(); spawn(async move { loop { let keeper_config = if keeper_private_key_option.is_some() { - Some(crate::config::KeeperConfig { - private_key: crate::config::SecretString { - value: keeper_private_key_option.clone(), - file: None, - }, - fee_manager_private_key: fee_manager_private_key.clone(), - known_keeper_addresses: known_keeper_addresses.clone(), - replica_config: keeper_replica_config.clone(), - run_config: keeper_run_config.clone(), - }) + Some(keeper_config_base.clone()) } else { None }; @@ -195,7 +180,7 @@ async fn setup_chain_and_run_keeper( chain_id: &ChainId, chain_config: EthereumConfig, keeper_metrics: Arc, - keeper_config: Option, + keeper_config: Option, chains: Arc>>, secret_copy: &str, history: Arc, diff --git a/apps/fortuna/src/config.rs b/apps/fortuna/src/config.rs index aa45d2c3c6..ad7735ebe6 100644 --- a/apps/fortuna/src/config.rs +++ b/apps/fortuna/src/config.rs @@ -385,7 +385,7 @@ pub struct KeeperConfig { /// The fee manager's private key for fee manager operations. /// This key is used to withdraw fees from the contract as the fee manager. - /// Multiple replicas can share the same fee manager private key. + /// Multiple replicas can share the same fee manager private key but different keeper keys (`private_key`). #[serde(default)] pub fee_manager_private_key: Option, diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index e11b4c0dbe..ac91ed8274 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -126,8 +126,17 @@ pub async fn run_keeper_threads( let fee_manager_private_key = keeper_config .fee_manager_private_key .as_ref() - .and_then(|key| key.load().ok()) - .flatten(); + .ok_or_else(|| { + anyhow::anyhow!( + "Fee manager private key is required when fee withdrawal is enabled" + ) + })? + .load()? + .ok_or_else(|| { + anyhow::anyhow!( + "Fee manager private key value is required when fee withdrawal is enabled" + ) + })?; spawn( withdraw_fees_wrapper( contract.clone(), diff --git a/apps/fortuna/src/keeper/fee.rs b/apps/fortuna/src/keeper/fee.rs index a8a945244f..9bc29b7963 100644 --- a/apps/fortuna/src/keeper/fee.rs +++ b/apps/fortuna/src/keeper/fee.rs @@ -8,10 +8,10 @@ use { anyhow::{anyhow, Result}, ethers::{ middleware::Middleware, - signers::Signer, - types::{Address, U256}, + signers::{LocalWallet, Signer}, + types::{Address, TransactionRequest, U256}, }, - std::sync::Arc, + std::{str::FromStr, sync::Arc}, tokio::time::{self, Duration}, tracing::{self, Instrument}, }; @@ -57,7 +57,7 @@ pub async fn withdraw_fees_wrapper( provider_address: Address, poll_interval: Duration, min_balance: U256, - fee_manager_private_key: Option, + fee_manager_private_key: String, known_keeper_addresses: Vec
, ) { loop { @@ -82,17 +82,12 @@ pub async fn withdraw_fees_if_necessary( contract: Arc, provider_address: Address, min_balance: U256, - fee_manager_private_key: Option, + fee_manager_private_key: String, known_keeper_addresses: Vec
, ) -> Result<()> { let provider = contract.provider(); let wallet = contract.wallet(); - let keeper_balance = provider - .get_balance(wallet.address(), None) - .await - .map_err(|e| anyhow!("Error while getting balance. error: {:?}", e))?; - if !should_withdraw_fees( Arc::new(provider.clone()), wallet.address(), @@ -103,6 +98,11 @@ pub async fn withdraw_fees_if_necessary( return Ok(()); } + let keeper_balance = provider + .get_balance(wallet.address(), None) + .await + .map_err(|e| anyhow!("Error while getting balance. error: {:?}", e))?; + let provider_info = contract .get_provider_info(provider_address) .call() @@ -114,16 +114,59 @@ pub async fn withdraw_fees_if_necessary( if keeper_balance < min_balance && U256::from(fees) > min_balance { tracing::info!("Claiming accrued fees..."); - if let Some(_fee_manager_key) = fee_manager_private_key { - let contract_call = contract.withdraw_as_fee_manager(provider_address, fees); - send_and_confirm(contract_call).await?; - } else { - if provider_info.fee_manager != wallet.address() { - return Err(anyhow!("Fee manager for provider {:?} is not the keeper wallet and no fee manager private key is configured. Fee manager: {:?} Keeper: {:?}", provider_address, provider_info.fee_manager, wallet.address())); - } + let contract_call = contract.withdraw_as_fee_manager(provider_address, fees); + send_and_confirm(contract_call).await?; + + let fee_manager_wallet = LocalWallet::from_str(&fee_manager_private_key) + .map_err(|e| anyhow!("Invalid fee manager private key: {:?}", e))?; + let fee_manager_address = fee_manager_wallet.address(); + let keeper_address = wallet.address(); + + if fee_manager_address != keeper_address { + tracing::info!( + "Transferring withdrawn fees from fee manager {:?} to keeper {:?}", + fee_manager_address, + keeper_address + ); + + let transfer_amount = U256::from(fees); + let chain_id = provider + .get_chainid() + .await + .map_err(|e| anyhow!("Failed to get chain ID: {:?}", e))?; + + let gas_price = provider + .get_gas_price() + .await + .map_err(|e| anyhow!("Failed to get gas price: {:?}", e))?; + + let nonce = provider + .get_transaction_count(fee_manager_address, None) + .await + .map_err(|e| anyhow!("Failed to get nonce: {:?}", e))?; + + let tx = TransactionRequest::new() + .to(keeper_address) + .value(transfer_amount) + .from(fee_manager_address) + .gas(21000) // Standard ETH transfer gas limit + .gas_price(gas_price) + .nonce(nonce) + .chain_id(chain_id.as_u64()); + + let typed_tx = tx.into(); + let signature = fee_manager_wallet + .sign_transaction(&typed_tx) + .await + .map_err(|e| anyhow!("Failed to sign transfer transaction: {:?}", e))?; + + let signed_tx = typed_tx.rlp_signed(&signature); + let tx_hash = provider + .send_raw_transaction(signed_tx) + .await + .map_err(|e| anyhow!("Failed to send transfer transaction: {:?}", e))?; - let contract_call = contract.withdraw_as_fee_manager(provider_address, fees); - send_and_confirm(contract_call).await?; + tracing::info!("Transfer transaction sent: {:?}", tx_hash); } } else if keeper_balance < min_balance { // NOTE: This log message triggers a grafana alert. If you want to change the text, please change the alert also. From 0f1b6e99b1d31f92f599b251b11d55d26772fd60 Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Thu, 3 Jul 2025 12:24:38 -0700 Subject: [PATCH 03/12] remove disable_withdrawal flag, improve control flow, update docs --- apps/fortuna/README.md | 61 +++---- apps/fortuna/config.sample.yaml | 15 +- apps/fortuna/src/config.rs | 4 - apps/fortuna/src/eth_utils/utils.rs | 80 +++++++++ apps/fortuna/src/keeper.rs | 28 ++-- apps/fortuna/src/keeper/fee.rs | 241 ++++++++++++++++++---------- 6 files changed, 281 insertions(+), 148 deletions(-) diff --git a/apps/fortuna/README.md b/apps/fortuna/README.md index ce431508ed..7e8974afbe 100644 --- a/apps/fortuna/README.md +++ b/apps/fortuna/README.md @@ -58,58 +58,48 @@ Fortuna supports running multiple replica instances for high availability and re ### Fee Management with Multiple Instances -When running multiple Fortuna instances with different keeper wallets but a single provider, only one instance should handle fee management. This instance needs to run using the same private key as the fee manager, because only the registerd fee manager wallet can adjust fees and withdraw funds. +When running multiple Fortuna instances with different keeper wallets, the system uses a fair fee distribution strategy. Each keeper will withdraw fees from the contract to maintain a balanced distribution across all known keeper addresses. + +The fee manager (configured in the provider section) can be a separate wallet from the keeper wallets. When fees are withdrawn from the contract, they go to the fee manager wallet first, then are automatically transferred to the requesting keeper wallet. + +**Key Configuration:** +- Only one instance should have fee adjustment enabled to avoid multiple keepers racing to adjust the fee (`disable_fee_adjustment: false`) +- All instances should have `fee_manager_private_key` provided so that each keeper can top itself up from contract fees. ### Example Configurations -**Two Replica Setup with Fee Management:** ```yaml -# Replica 0 (fee manager wallet) - handles even sequence numbers + fee management +# Replica 0 (with fee management) - handles even sequence numbers + fee management keeper: private_key: + value: 0x + fee_manager_private_key: value: 0x + known_keeper_addresses: + - 0x # This replica's address + - 0x # Other replica's address replica_config: replica_id: 0 total_replicas: 2 - backup_delay_seconds: 30 + backup_delay_seconds: 15 run_config: - disable_fee_adjustment: false # Enable fee management (default) - disable_fee_withdrawal: false + disable_fee_adjustment: false # Enable fee adjustment (default) -# Replica 1 (non-fee-manager wallet) - handles odd sequence numbers only +# Replica 1 (request processing only) - handles odd sequence numbers keeper: private_key: - value: 0x + value: 0x + fee_manager_private_key: + value: 0x + known_keeper_addresses: + - 0x # Other replica's address + - 0x # This replica's address replica_config: replica_id: 1 total_replicas: 2 - backup_delay_seconds: 30 - run_config: - disable_fee_adjustment: true # Disable fee management - disable_fee_withdrawal: true -``` - -**Three Replica Setup:** -```yaml -# Replica 0 (fee manager wallet) - handles sequence numbers 0, 3, 6, 9, ... + fee management -keeper: - replica_config: - replica_id: 0 - total_replicas: 3 - backup_delay_seconds: 30 - run_config: - disable_fee_adjustment: false - disable_fee_withdrawal: false - -# Replicas 1 & 2 (non-fee-manager wallets) - request processing only -keeper: - replica_config: - replica_id: 1 # or 2 - total_replicas: 3 - backup_delay_seconds: 30 + backup_delay_seconds: 15 run_config: - disable_fee_adjustment: true - disable_fee_withdrawal: true + disable_fee_adjustment: true # Disable fee adjustment ``` ### Deployment Considerations @@ -117,7 +107,7 @@ keeper: 1. **Separate Wallets**: Each replica MUST use a different private key to avoid nonce conflicts 2. **Fee Manager Assignment**: Set the provider's `fee_manager` address to match the primary instance's keeper wallet 3. **Thread Configuration**: Only enable fee management threads on the instance using the fee manager wallet -4. **Backup Delay**: Set `backup_delay_seconds` long enough to allow primary replica to process requests, but short enough for acceptable failover time (recommended: 30-60 seconds) +4. **Backup Delay**: Set `backup_delay_seconds` long enough to allow primary replica to process requests, but short enough for acceptable failover time (recommended: 10-30 seconds) 5. **Monitoring**: Monitor each replica's processing metrics to ensure proper load distribution 6. **Gas Management**: Each replica needs sufficient ETH balance for gas fees @@ -127,7 +117,6 @@ keeper: - Backup replicas wait for `backup_delay_seconds` before checking if request is still unfulfilled - If request is already fulfilled during the delay, backup replica skips processing - This prevents duplicate transactions and wasted gas while ensuring reliability -- Fee management operations (adjustment/withdrawal) only occur on an instance where the keeper wallet is the fee manager wallet. ## Local Development diff --git a/apps/fortuna/config.sample.yaml b/apps/fortuna/config.sample.yaml index 0a64229e49..a20a47c1ba 100644 --- a/apps/fortuna/config.sample.yaml +++ b/apps/fortuna/config.sample.yaml @@ -87,21 +87,22 @@ keeper: # For production, you can store the private key in a file. # file: keeper-key.txt - # Fee manager private key for fee manager operations (optional) - # fee_manager_private_key: - # value: 0xabcd - # # file: fee-manager-key.txt + # Fee manager private key for fee manager operations (if not provided, fee withdrawals won't happen) + fee_manager_private_key: + value: 0xabcd + # file: fee-manager-key.txt - # List of known keeper wallet addresses for balance comparison (optional) + # List of known keeper wallet addresses for balance comparison # The keeper will only withdraw fees if its balance is the lowest among these addresses. - + known_keeper_addresses: + - 0x1234 + - 0x5678 # Runtime configuration for the keeper service # Optional: Configure which keeper threads to disable. If running multiple replicas, # only a single replica should have the fee adjustment and withdrawal threads enabled. # run_config: # disable_fee_adjustment: false # Set to true to disable automatic fee adjustment - # disable_fee_withdrawal: false # Set to true to disable automatic fee withdrawal # Multi-replica configuration # Optional: Multi-replica configuration for high availability and load distribution diff --git a/apps/fortuna/src/config.rs b/apps/fortuna/src/config.rs index ad7735ebe6..46453205aa 100644 --- a/apps/fortuna/src/config.rs +++ b/apps/fortuna/src/config.rs @@ -355,10 +355,6 @@ pub struct RunConfig { /// Disable automatic fee adjustment threads #[serde(default)] pub disable_fee_adjustment: bool, - - /// Disable automatic fee withdrawal threads - #[serde(default)] - pub disable_fee_withdrawal: bool, } #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] diff --git a/apps/fortuna/src/eth_utils/utils.rs b/apps/fortuna/src/eth_utils/utils.rs index ca256dc5e4..c9195576b0 100644 --- a/apps/fortuna/src/eth_utils/utils.rs +++ b/apps/fortuna/src/eth_utils/utils.rs @@ -5,8 +5,10 @@ use { ethabi::ethereum_types::U64, ethers::{ contract::{ContractCall, ContractError}, + core::k256::ecdsa::SigningKey, middleware::Middleware, providers::ProviderError, + signers::Wallet, types::{transaction::eip2718::TypedTransaction, TransactionReceipt, U256}, }, std::{ @@ -18,6 +20,7 @@ use { }; const TX_CONFIRMATION_TIMEOUT_SECS: u64 = 30; +pub const ETH_TRANSFER_GAS_LIMIT: u64 = 21_000; #[derive(Debug)] pub struct SubmitTxResult { @@ -367,3 +370,80 @@ pub async fn submit_tx( Ok(receipt) } + +/// Transfer funds from fee manager wallet to keeper wallet. +/// This is used when the fee manager and keeper are different addresses. +pub async fn submit_transfer_tx( + provider: Arc, + source_wallet: &Wallet, + destination_address: ethers::types::Address, + gas_price: U256, + transfer_amount: U256, +) -> Result { + use ethers::{signers::Signer, types::TransactionRequest}; + + let source_wallet_address = source_wallet.address(); + + tracing::info!( + "Transferring {:?} from {:?} to {:?}", + transfer_amount, + source_wallet_address, + destination_address + ); + + // Get transaction parameters + let chain_id = provider + .get_chainid() + .await + .map_err(|e| anyhow!("Failed to get chain ID: {:?}", e))?; + + let nonce = provider + .get_transaction_count(source_wallet_address, None) + .await + .map_err(|e| anyhow!("Failed to get nonce: {:?}", e))?; + + let tx = TransactionRequest::new() + .to(destination_address) + .value(transfer_amount) + .from(source_wallet_address) + .gas(ETH_TRANSFER_GAS_LIMIT) + .gas_price(gas_price) + .nonce(nonce) + .chain_id(chain_id.as_u64()); + + let typed_tx = tx.into(); + let signature = source_wallet + .sign_transaction(&typed_tx) + .await + .map_err(|e| anyhow!("Failed to sign transfer transaction: {:?}", e))?; + + let signed_tx = typed_tx.rlp_signed(&signature); + + // Send transaction and get pending transaction + let pending_tx = provider + .send_raw_transaction(signed_tx) + .await + .map_err(|e| anyhow!("Failed to send transfer transaction: {:?}", e))?; + + // Wait for confirmation with timeout + let tx_receipt = timeout( + Duration::from_secs(TX_CONFIRMATION_TIMEOUT_SECS), + pending_tx, + ) + .await + .map_err(|_| anyhow!("Transfer transaction confirmation timeout"))? + .map_err(|e| anyhow!("Transfer transaction confirmation error: {:?}", e))? + .ok_or_else(|| anyhow!("Transfer transaction dropped from mempool"))?; + + // Check if transaction was successful + if tx_receipt.status == Some(ethabi::ethereum_types::U64::from(0)) { + return Err(anyhow!( + "Transfer transaction failed on-chain. Receipt: {:?}", + tx_receipt + )); + } + + let tx_hash = tx_receipt.transaction_hash; + tracing::info!("Transfer transaction confirmed: {:?}", tx_hash); + Ok(tx_hash) +} diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index ac91ed8274..9ffbc35b00 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -121,22 +121,14 @@ pub async fn run_keeper_threads( .in_current_span(), ); - // Spawn a thread that watches the keeper wallet balance and submits withdrawal transactions as needed to top-up the balance. - if !keeper_config.run_config.disable_fee_withdrawal { - let fee_manager_private_key = keeper_config - .fee_manager_private_key - .as_ref() - .ok_or_else(|| { - anyhow::anyhow!( - "Fee manager private key is required when fee withdrawal is enabled" - ) - })? - .load()? - .ok_or_else(|| { - anyhow::anyhow!( - "Fee manager private key value is required when fee withdrawal is enabled" - ) - })?; + // Spawn a thread that watches all known keeper wallet balances and submits withdrawal transactions as needed to top-up this keeper's balance. + let fee_manager_private_key = if let Some(ref secret) = keeper_config.fee_manager_private_key { + secret.load()? + } else { + None + }; + + if let Some(fee_manager_private_key) = fee_manager_private_key { spawn( withdraw_fees_wrapper( contract.clone(), @@ -149,7 +141,9 @@ pub async fn run_keeper_threads( .in_current_span(), ); } else { - tracing::info!("Fee withdrawal thread disabled by configuration"); + tracing::warn!( + "Fee manager private key not provided - fee withdrawal thread will not run." + ); } // Spawn a thread that periodically adjusts the provider fee. diff --git a/apps/fortuna/src/keeper/fee.rs b/apps/fortuna/src/keeper/fee.rs index 9bc29b7963..0946a35abf 100644 --- a/apps/fortuna/src/keeper/fee.rs +++ b/apps/fortuna/src/keeper/fee.rs @@ -2,27 +2,41 @@ use { crate::{ api::BlockchainState, chain::ethereum::InstrumentedSignablePythContract, - eth_utils::utils::{estimate_tx_cost, send_and_confirm}, + eth_utils::utils::{ + estimate_tx_cost, send_and_confirm, submit_transfer_tx, ETH_TRANSFER_GAS_LIMIT, + }, keeper::{AccountLabel, ChainId, KeeperMetrics}, }, anyhow::{anyhow, Result}, ethers::{ + core::k256::ecdsa::SigningKey, middleware::Middleware, - signers::{LocalWallet, Signer}, - types::{Address, TransactionRequest, U256}, + signers::{LocalWallet, Signer, Wallet}, + types::{Address, U256}, }, std::{str::FromStr, sync::Arc}, tokio::time::{self, Duration}, tracing::{self, Instrument}, }; -async fn should_withdraw_fees( +/// Determines the amount of fees to withdraw based on fair distribution. +/// Each keeper will try to withdraw up to their fair share of the fees (T/N) +/// where T is the total fees across all keepers and the contract, and N is the +/// number of keepers. +async fn calculate_fair_fee_withdrawal_amount( provider: Arc, current_keeper_address: Address, known_keeper_addresses: &[Address], -) -> Result { + available_fees: U256, +) -> Result { + // Early return if no fees available + if available_fees.is_zero() { + return Ok(U256::zero()); + } + + // If no other keepers, withdraw all available fees if known_keeper_addresses.is_empty() { - return Ok(true); + return Ok(available_fees); } let current_balance = provider @@ -30,6 +44,9 @@ async fn should_withdraw_fees( .await .map_err(|e| anyhow!("Error while getting current keeper balance. error: {:?}", e))?; + // Calculate total funds across all keepers + available fees + let mut total_funds = current_balance + available_fees; + for &address in known_keeper_addresses { let balance = provider.get_balance(address, None).await.map_err(|e| { anyhow!( @@ -38,17 +55,29 @@ async fn should_withdraw_fees( e ) })?; - - if balance < current_balance { - tracing::info!( - "Skipping fee withdrawal: keeper {:?} has lower balance ({:?}) than current keeper {:?} ({:?})", - address, balance, current_keeper_address, current_balance - ); - return Ok(false); - } + total_funds += balance; } - Ok(true) + // Calculate fair share per keeper + let fair_share = total_funds / (known_keeper_addresses.len() + 1); // +1 for current keeper + + // Calculate how much current keeper should withdraw to reach fair share + let withdrawal_amount = if current_balance < fair_share { + let deficit = fair_share - current_balance; + std::cmp::min(deficit, available_fees) + } else { + U256::zero() + }; + + tracing::info!( + "Fair share calculation: total_funds={:?}, fair_share={:?}, current_balance={:?}, withdrawal_amount={:?}", + total_funds, + fair_share, + current_balance, + withdrawal_amount + ); + + Ok(withdrawal_amount) } #[tracing::instrument(name = "withdraw_fees", skip_all, fields())] @@ -86,20 +115,13 @@ pub async fn withdraw_fees_if_necessary( known_keeper_addresses: Vec
, ) -> Result<()> { let provider = contract.provider(); - let wallet = contract.wallet(); - if !should_withdraw_fees( - Arc::new(provider.clone()), - wallet.address(), - &known_keeper_addresses, - ) - .await? - { - return Ok(()); - } + let keeper_wallet = contract.wallet(); + let fee_manager_wallet = LocalWallet::from_str(&fee_manager_private_key) + .map_err(|e| anyhow!("Invalid fee manager private key: {:?}", e))?; let keeper_balance = provider - .get_balance(wallet.address(), None) + .get_balance(keeper_wallet.address(), None) .await .map_err(|e| anyhow!("Error while getting balance. error: {:?}", e))?; @@ -109,70 +131,121 @@ pub async fn withdraw_fees_if_necessary( .await .map_err(|e| anyhow!("Error while getting provider info. error: {:?}", e))?; - let fees = provider_info.accrued_fees_in_wei; + let available_fees = U256::from(provider_info.accrued_fees_in_wei); - if keeper_balance < min_balance && U256::from(fees) > min_balance { - tracing::info!("Claiming accrued fees..."); + // Only withdraw if the keeper balance is below the minimum threshold + if keeper_balance >= min_balance { + return Ok(()); + } - let contract_call = contract.withdraw_as_fee_manager(provider_address, fees); - send_and_confirm(contract_call).await?; + // Determine how much we can fairly withdraw from the contract + let withdrawal_amount = calculate_fair_fee_withdrawal_amount( + Arc::new(provider.clone()), + keeper_wallet.address(), + &known_keeper_addresses, + available_fees, + ) + .await?; - let fee_manager_wallet = LocalWallet::from_str(&fee_manager_private_key) - .map_err(|e| anyhow!("Invalid fee manager private key: {:?}", e))?; - let fee_manager_address = fee_manager_wallet.address(); - let keeper_address = wallet.address(); - - if fee_manager_address != keeper_address { - tracing::info!( - "Transferring withdrawn fees from fee manager {:?} to keeper {:?}", - fee_manager_address, - keeper_address - ); - - let transfer_amount = U256::from(fees); - let chain_id = provider - .get_chainid() - .await - .map_err(|e| anyhow!("Failed to get chain ID: {:?}", e))?; - - let gas_price = provider - .get_gas_price() - .await - .map_err(|e| anyhow!("Failed to get gas price: {:?}", e))?; - - let nonce = provider - .get_transaction_count(fee_manager_address, None) - .await - .map_err(|e| anyhow!("Failed to get nonce: {:?}", e))?; - - let tx = TransactionRequest::new() - .to(keeper_address) - .value(transfer_amount) - .from(fee_manager_address) - .gas(21000) // Standard ETH transfer gas limit - .gas_price(gas_price) - .nonce(nonce) - .chain_id(chain_id.as_u64()); - - let typed_tx = tx.into(); - let signature = fee_manager_wallet - .sign_transaction(&typed_tx) - .await - .map_err(|e| anyhow!("Failed to sign transfer transaction: {:?}", e))?; - - let signed_tx = typed_tx.rlp_signed(&signature); - let tx_hash = provider - .send_raw_transaction(signed_tx) - .await - .map_err(|e| anyhow!("Failed to send transfer transaction: {:?}", e))?; - - tracing::info!("Transfer transaction sent: {:?}", tx_hash); - } - } else if keeper_balance < min_balance { + // Only withdraw more than `min_balance` accrued fees to avoid tiny txs. + // TODO: make this independently configurable + let min_withdrawal_amount = min_balance; + if withdrawal_amount < min_withdrawal_amount { + // We don't have enough to meaningfully top up the balance. // NOTE: This log message triggers a grafana alert. If you want to change the text, please change the alert also. - tracing::warn!("Keeper balance {:?} is too low (< {:?}) but provider fees are not sufficient to top-up.", keeper_balance, min_balance) + tracing::warn!("Keeper balance {:?} is too low (< {:?}) but provider fees are not sufficient to top-up.", keeper_balance, min_balance); + return Ok(()); + } + + tracing::info!( + "Keeper balance {:?} below minimum {:?}, claiming {:?}", + keeper_balance, + min_balance, + withdrawal_amount + ); + + // Proceed with withdrawal + let contract_call = + contract.withdraw_as_fee_manager(provider_address, withdrawal_amount.as_u128()); + send_and_confirm(contract_call).await?; + + // Transfer fees from fee manager to keeper if fee manager is different from keeper + transfer_from_fee_manager_to_keeper( + Arc::new(provider.clone()), + fee_manager_wallet, + keeper_wallet.address(), + ) + .await + .map_err(|e| { + // If we fail to transfer funds from fee manager to keeper, we need to manually intervene + // NOTE: This log message triggers a grafana alert to allow for this. If you want to change the text, please change the alert also. + // TODO: Add retries/run a thread to periodically flush fee manager balance to the keeper. + anyhow!( + "Failed to transfer funds from fee manager to keeper: {:?}", + e + ) + })?; + + Ok(()) +} + +/// Transfer fees from fee manager to keeper if they are different addresses. +/// Transfers the maximum amount possible while preserving gas reserve for the transaction. +async fn transfer_from_fee_manager_to_keeper( + provider: Arc, + fee_manager_wallet: Wallet, + keeper_address: Address, +) -> Result<()> { + let fee_manager_address = fee_manager_wallet.address(); + + if fee_manager_address == keeper_address { + // Same address, no transfer needed + return Ok(()); } + // Calculate gas cost for the transfer transaction + let gas_price = provider.get_gas_price().await.map_err(|e| { + anyhow!( + "Failed to get gas price for transfer cost calculation: {:?}", + e + ) + })?; + + let transfer_gas_cost = U256::from(ETH_TRANSFER_GAS_LIMIT) * gas_price; + + // Add 50% buffer for gas price fluctuations and safety margin + let gas_reserve_required = transfer_gas_cost * 150 / 100; + + let fee_manager_balance = provider + .get_balance(fee_manager_address, None) + .await + .map_err(|e| anyhow!("Failed to get fee manager balance: {:?}", e))?; + + let adjusted_withdrawal_amount = if fee_manager_balance > gas_reserve_required { + fee_manager_balance - gas_reserve_required + } else { + // Fee manager doesn't have enough balance to cover gas costs. Shouldn't be the case after withdrawing from the contract, + // manual intervention likely required. + tracing::error!( + "Fee manager balance {:?} is insufficient to cover gas reserve {:?} for transfer. Skipping transfer.", + fee_manager_balance, + gas_reserve_required + ); + return Ok(()); + }; + + submit_transfer_tx( + provider, + &fee_manager_wallet, + keeper_address, + gas_price, + adjusted_withdrawal_amount, + ) + .await + .map_err(|e| { + anyhow!("Failed to transfer fees from fee manager to keeper! Manual intervention required. error: {:?}", e) + })?; + Ok(()) } From e50c1a4a1546824296a310979f86dcf772bbc51f Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Thu, 3 Jul 2025 12:29:26 -0700 Subject: [PATCH 04/12] update config sample --- apps/fortuna/config.sample.yaml | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/apps/fortuna/config.sample.yaml b/apps/fortuna/config.sample.yaml index a20a47c1ba..7d7d149c17 100644 --- a/apps/fortuna/config.sample.yaml +++ b/apps/fortuna/config.sample.yaml @@ -92,30 +92,24 @@ keeper: value: 0xabcd # file: fee-manager-key.txt - # List of known keeper wallet addresses for balance comparison - # The keeper will only withdraw fees if its balance is the lowest among these addresses. + # List of known keeper wallet addresses for balance comparison and fair fee withdrawals known_keeper_addresses: - 0x1234 - 0x5678 # Runtime configuration for the keeper service # Optional: Configure which keeper threads to disable. If running multiple replicas, - # only a single replica should have the fee adjustment and withdrawal threads enabled. - # run_config: - # disable_fee_adjustment: false # Set to true to disable automatic fee adjustment + # only a single replica should have the fee adjustment thread enabled. + run_config: + disable_fee_adjustment: false # Multi-replica configuration # Optional: Multi-replica configuration for high availability and load distribution # Uncomment and configure for production deployments with multiple Fortuna instances - # replica_config: - # replica_id: 0 # Unique identifier for this replica (0, 1, 2, ...) - # total_replicas: 2 # Total number of replica instances running - # backup_delay_seconds: 30 # Seconds to wait before processing other replicas' requests - # - # Example configurations: - # - # Two-replica setup (Blue/Green): - # - Replica 0: handles even sequence numbers (0, 2, 4, ...) - # - Replica 1: handles odd sequence numbers (1, 3, 5, ...) - # + # See the README for more details. + replica_config: + replica_id: 0 # Unique identifier for this replica (0, 1, 2, ...) + total_replicas: 2 # Total number of replica instances running + backup_delay_seconds: 30 # Seconds to wait before processing other replicas' requests + # IMPORTANT: Each replica must use a different private_key to avoid nonce conflicts! From f78576e3aa062c1fca3ca9a72f0448b5e5457fdc Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Thu, 3 Jul 2025 12:32:19 -0700 Subject: [PATCH 05/12] update docs --- apps/fortuna/config.sample.yaml | 4 ++-- apps/fortuna/src/config.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/fortuna/config.sample.yaml b/apps/fortuna/config.sample.yaml index 7d7d149c17..3453297b4c 100644 --- a/apps/fortuna/config.sample.yaml +++ b/apps/fortuna/config.sample.yaml @@ -73,8 +73,8 @@ provider: # For production, you can store the private key in a file. # file: secret.txt - # Set this to the address of your keeper wallet if you would like the keeper wallet to - # be able to withdraw fees from the contract. + # The address of the fee manager for the provider. Only used for syncing the fee manager address to the contract. + # Fee withdrawals are handled by the fee manager private key defined in the keeper config. fee_manager: 0xADDRESS keeper: # An ethereum wallet address and private key for running the keeper service. diff --git a/apps/fortuna/src/config.rs b/apps/fortuna/src/config.rs index 46453205aa..f558d195d6 100644 --- a/apps/fortuna/src/config.rs +++ b/apps/fortuna/src/config.rs @@ -341,8 +341,8 @@ pub struct ProviderConfig { #[serde(default = "default_chain_sample_interval")] pub chain_sample_interval: u64, - /// The address of the fee manager for the provider. Set this value to the keeper wallet address to - /// enable keeper balance top-ups. + /// The address of the fee manager for the provider. Only used for syncing the fee manager address to the contract. + /// Fee withdrawals are handled by the fee manager private key defined in the keeper config. pub fee_manager: Option
, } From f09b53a8455a8bf8be5bbf454e1d9f72f9e79871 Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Thu, 3 Jul 2025 12:34:07 -0700 Subject: [PATCH 06/12] update docs --- apps/fortuna/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/fortuna/README.md b/apps/fortuna/README.md index 7e8974afbe..e8c56c6f96 100644 --- a/apps/fortuna/README.md +++ b/apps/fortuna/README.md @@ -64,7 +64,7 @@ The fee manager (configured in the provider section) can be a separate wallet fr **Key Configuration:** - Only one instance should have fee adjustment enabled to avoid multiple keepers racing to adjust the fee (`disable_fee_adjustment: false`) -- All instances should have `fee_manager_private_key` provided so that each keeper can top itself up from contract fees. +- All instances should have `keeper.private_key` and `keeper.fee_manager_private_key` provided so that each keeper can top itself up as fee manager from contract fees. ### Example Configurations From 867cee751a00adc66ac4adddc5feb573babd1a78 Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Thu, 3 Jul 2025 14:59:41 -0700 Subject: [PATCH 07/12] docs --- apps/fortuna/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/fortuna/README.md b/apps/fortuna/README.md index e8c56c6f96..9e1fe41596 100644 --- a/apps/fortuna/README.md +++ b/apps/fortuna/README.md @@ -69,7 +69,7 @@ The fee manager (configured in the provider section) can be a separate wallet fr ### Example Configurations ```yaml -# Replica 0 (with fee management) - handles even sequence numbers + fee management +# Replica 0 - handles even sequence numbers + fee management keeper: private_key: value: 0x @@ -85,7 +85,7 @@ keeper: run_config: disable_fee_adjustment: false # Enable fee adjustment (default) -# Replica 1 (request processing only) - handles odd sequence numbers +# Replica 1 - handles odd sequence numbers keeper: private_key: value: 0x From fcab1ec844ca9e15126ec582dc2686ed0de4929e Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Thu, 3 Jul 2025 17:07:20 -0700 Subject: [PATCH 08/12] address PR feedback --- apps/fortuna/README.md | 2 +- apps/fortuna/config.sample.yaml | 5 +- apps/fortuna/src/config.rs | 2 + apps/fortuna/src/eth_utils/utils.rs | 61 +++------ apps/fortuna/src/keeper.rs | 31 +++-- apps/fortuna/src/keeper/fee.rs | 201 +++++++++++----------------- 6 files changed, 122 insertions(+), 180 deletions(-) diff --git a/apps/fortuna/README.md b/apps/fortuna/README.md index 9e1fe41596..801fff065f 100644 --- a/apps/fortuna/README.md +++ b/apps/fortuna/README.md @@ -58,7 +58,7 @@ Fortuna supports running multiple replica instances for high availability and re ### Fee Management with Multiple Instances -When running multiple Fortuna instances with different keeper wallets, the system uses a fair fee distribution strategy. Each keeper will withdraw fees from the contract to maintain a balanced distribution across all known keeper addresses. +When running multiple Fortuna instances with different keeper wallets, the system uses a fair fee distribution strategy. Each keeper will withdraw fees from the contract to maintain a balanced distribution across all known keeper addresses and the fee manager address. The fee manager (configured in the provider section) can be a separate wallet from the keeper wallets. When fees are withdrawn from the contract, they go to the fee manager wallet first, then are automatically transferred to the requesting keeper wallet. diff --git a/apps/fortuna/config.sample.yaml b/apps/fortuna/config.sample.yaml index 3453297b4c..cb914db4dc 100644 --- a/apps/fortuna/config.sample.yaml +++ b/apps/fortuna/config.sample.yaml @@ -75,7 +75,7 @@ provider: # The address of the fee manager for the provider. Only used for syncing the fee manager address to the contract. # Fee withdrawals are handled by the fee manager private key defined in the keeper config. - fee_manager: 0xADDRESS + fee_manager: 0xfee keeper: # An ethereum wallet address and private key for running the keeper service. # This does not have to be the same key as the provider's key above. @@ -92,7 +92,8 @@ keeper: value: 0xabcd # file: fee-manager-key.txt - # List of known keeper wallet addresses for balance comparison and fair fee withdrawals + # List of other known keeper wallet addresses for balance comparison and fair fee withdrawals. + # Do not include this keeper's address. known_keeper_addresses: - 0x1234 - 0x5678 diff --git a/apps/fortuna/src/config.rs b/apps/fortuna/src/config.rs index f558d195d6..07097b5c62 100644 --- a/apps/fortuna/src/config.rs +++ b/apps/fortuna/src/config.rs @@ -385,6 +385,8 @@ pub struct KeeperConfig { #[serde(default)] pub fee_manager_private_key: Option, + /// The addresses of other keepers in the replica set (excluding the current keeper). + /// This is used to distribute fees fairly across all keepers. #[serde(default)] pub known_keeper_addresses: Vec
, diff --git a/apps/fortuna/src/eth_utils/utils.rs b/apps/fortuna/src/eth_utils/utils.rs index c9195576b0..fb57fd86af 100644 --- a/apps/fortuna/src/eth_utils/utils.rs +++ b/apps/fortuna/src/eth_utils/utils.rs @@ -1,15 +1,18 @@ use { - crate::eth_utils::nonce_manager::NonceManaged, + crate::{ + chain::ethereum::InstrumentedSignablePythContract, eth_utils::nonce_manager::NonceManaged, + }, anyhow::{anyhow, Result}, backoff::ExponentialBackoff, ethabi::ethereum_types::U64, ethers::{ contract::{ContractCall, ContractError}, - core::k256::ecdsa::SigningKey, middleware::Middleware, providers::ProviderError, - signers::Wallet, - types::{transaction::eip2718::TypedTransaction, TransactionReceipt, U256}, + signers::Signer, + types::{ + transaction::eip2718::TypedTransaction, TransactionReceipt, TransactionRequest, U256, + }, }, std::{ fmt::Display, @@ -20,7 +23,6 @@ use { }; const TX_CONFIRMATION_TIMEOUT_SECS: u64 = 30; -pub const ETH_TRANSFER_GAS_LIMIT: u64 = 21_000; #[derive(Debug)] pub struct SubmitTxResult { @@ -371,18 +373,13 @@ pub async fn submit_tx( Ok(receipt) } -/// Transfer funds from fee manager wallet to keeper wallet. -/// This is used when the fee manager and keeper are different addresses. -pub async fn submit_transfer_tx( - provider: Arc, - source_wallet: &Wallet, +/// Transfer funds from the signing wallet to the destination address. +pub async fn submit_transfer_tx( + contract: Arc, destination_address: ethers::types::Address, - gas_price: U256, transfer_amount: U256, ) -> Result { - use ethers::{signers::Signer, types::TransactionRequest}; - - let source_wallet_address = source_wallet.address(); + let source_wallet_address = contract.wallet().address(); tracing::info!( "Transferring {:?} from {:?} to {:?}", @@ -391,39 +388,13 @@ pub async fn submit_transfer_tx( destination_address ); - // Get transaction parameters - let chain_id = provider - .get_chainid() - .await - .map_err(|e| anyhow!("Failed to get chain ID: {:?}", e))?; - - let nonce = provider - .get_transaction_count(source_wallet_address, None) - .await - .map_err(|e| anyhow!("Failed to get nonce: {:?}", e))?; - let tx = TransactionRequest::new() .to(destination_address) .value(transfer_amount) - .from(source_wallet_address) - .gas(ETH_TRANSFER_GAS_LIMIT) - .gas_price(gas_price) - .nonce(nonce) - .chain_id(chain_id.as_u64()); - - let typed_tx = tx.into(); - let signature = source_wallet - .sign_transaction(&typed_tx) - .await - .map_err(|e| anyhow!("Failed to sign transfer transaction: {:?}", e))?; + .from(source_wallet_address); - let signed_tx = typed_tx.rlp_signed(&signature); - - // Send transaction and get pending transaction - let pending_tx = provider - .send_raw_transaction(signed_tx) - .await - .map_err(|e| anyhow!("Failed to send transfer transaction: {:?}", e))?; + let client = contract.client(); + let pending_tx = client.send_transaction(tx, None).await?; // Wait for confirmation with timeout let tx_receipt = timeout( @@ -433,10 +404,10 @@ pub async fn submit_transfer_tx( .await .map_err(|_| anyhow!("Transfer transaction confirmation timeout"))? .map_err(|e| anyhow!("Transfer transaction confirmation error: {:?}", e))? - .ok_or_else(|| anyhow!("Transfer transaction dropped from mempool"))?; + .ok_or_else(|| anyhow!("Transfer transaction, probably dropped from mempool"))?; // Check if transaction was successful - if tx_receipt.status == Some(ethabi::ethereum_types::U64::from(0)) { + if tx_receipt.status == Some(U64::from(0)) { return Err(anyhow!( "Transfer transaction failed on-chain. Receipt: {:?}", tx_receipt diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index 9ffbc35b00..58245a58d1 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -17,6 +17,7 @@ use { }, }, }, + anyhow, ethers::{signers::Signer, types::U256}, keeper_metrics::{AccountLabel, KeeperMetrics}, std::{collections::HashSet, sync::Arc}, @@ -72,6 +73,7 @@ pub async fn run_keeper_threads( anyhow::anyhow!("Keeper private key is required but not provided in config") })?; + // Contract that uses the keeper wallet to send transactions let contract = Arc::new(InstrumentedSignablePythContract::from_config( &chain_eth_config, &keeper_private_key, @@ -121,7 +123,7 @@ pub async fn run_keeper_threads( .in_current_span(), ); - // Spawn a thread that watches all known keeper wallet balances and submits withdrawal transactions as needed to top-up this keeper's balance. + // If fee manager private key is provided, spawn fee withdrawal and adjustmnet threads let fee_manager_private_key = if let Some(ref secret) = keeper_config.fee_manager_private_key { secret.load()? } else { @@ -129,28 +131,31 @@ pub async fn run_keeper_threads( }; if let Some(fee_manager_private_key) = fee_manager_private_key { + let contract_as_fee_mgr = Arc::new(InstrumentedSignablePythContract::from_config( + &chain_eth_config, + &fee_manager_private_key, + chain_state.id.clone(), + rpc_metrics.clone(), + chain_state.network_id, + )?); + + // Spawn a thread that periodically withdraws fees to the fee manager and keeper. spawn( withdraw_fees_wrapper( - contract.clone(), + contract_as_fee_mgr.clone(), chain_state.provider_address, WITHDRAW_INTERVAL, U256::from(chain_eth_config.min_keeper_balance), - fee_manager_private_key, + keeper_address, keeper_config.known_keeper_addresses.clone(), ) .in_current_span(), ); - } else { - tracing::warn!( - "Fee manager private key not provided - fee withdrawal thread will not run." - ); - } - // Spawn a thread that periodically adjusts the provider fee. - if !keeper_config.run_config.disable_fee_adjustment { + // Spawn a thread that periodically adjusts the provider fee. spawn( adjust_fee_wrapper( - contract.clone(), + contract_as_fee_mgr.clone(), chain_state.clone(), chain_state.provider_address, ADJUST_FEE_INTERVAL, @@ -176,7 +181,9 @@ pub async fn run_keeper_threads( .in_current_span(), ); } else { - tracing::info!("Fee adjustment thread disabled by configuration"); + tracing::warn!( + "Fee manager private key not provided - fee withdrawal and adjustment threads will not run." + ); } spawn(update_commitments_loop(contract.clone(), chain_state.clone()).in_current_span()); diff --git a/apps/fortuna/src/keeper/fee.rs b/apps/fortuna/src/keeper/fee.rs index 0946a35abf..d2fbcc7c8c 100644 --- a/apps/fortuna/src/keeper/fee.rs +++ b/apps/fortuna/src/keeper/fee.rs @@ -2,31 +2,31 @@ use { crate::{ api::BlockchainState, chain::ethereum::InstrumentedSignablePythContract, - eth_utils::utils::{ - estimate_tx_cost, send_and_confirm, submit_transfer_tx, ETH_TRANSFER_GAS_LIMIT, - }, + eth_utils::utils::{estimate_tx_cost, send_and_confirm, submit_transfer_tx}, keeper::{AccountLabel, ChainId, KeeperMetrics}, }, anyhow::{anyhow, Result}, ethers::{ - core::k256::ecdsa::SigningKey, middleware::Middleware, - signers::{LocalWallet, Signer, Wallet}, + signers::Signer, types::{Address, U256}, }, - std::{str::FromStr, sync::Arc}, + std::sync::Arc, tokio::time::{self, Duration}, tracing::{self, Instrument}, }; /// Determines the amount of fees to withdraw based on fair distribution. /// Each keeper will try to withdraw up to their fair share of the fees (T/N) -/// where T is the total fees across all keepers and the contract, and N is the -/// number of keepers. -async fn calculate_fair_fee_withdrawal_amount( +/// where T is the total fees across all known keepers and the contract, and N is the +/// number of known keepers. +/// +/// `other_keeper_addresses` is expected to not include the `keeper_address`, and should +/// include the fee manager so that the fee manager wallet stays funded. +async fn calculate_fair_fee_withdrawal_amount( provider: Arc, - current_keeper_address: Address, - known_keeper_addresses: &[Address], + keeper_address: Address, + other_keeper_addresses: &[Address], available_fees: U256, ) -> Result { // Early return if no fees available @@ -35,19 +35,19 @@ async fn calculate_fair_fee_withdrawal_amount( } // If no other keepers, withdraw all available fees - if known_keeper_addresses.is_empty() { + if other_keeper_addresses.is_empty() { return Ok(available_fees); } let current_balance = provider - .get_balance(current_keeper_address, None) + .get_balance(keeper_address, None) .await .map_err(|e| anyhow!("Error while getting current keeper balance. error: {:?}", e))?; // Calculate total funds across all keepers + available fees let mut total_funds = current_balance + available_fees; - for &address in known_keeper_addresses { + for &address in other_keeper_addresses { let balance = provider.get_balance(address, None).await.map_err(|e| { anyhow!( "Error while getting keeper balance for {:?}. error: {:?}", @@ -59,7 +59,7 @@ async fn calculate_fair_fee_withdrawal_amount( } // Calculate fair share per keeper - let fair_share = total_funds / (known_keeper_addresses.len() + 1); // +1 for current keeper + let fair_share = total_funds / (other_keeper_addresses.len() + 1); // +1 for current keeper // Calculate how much current keeper should withdraw to reach fair share let withdrawal_amount = if current_balance < fair_share { @@ -82,50 +82,77 @@ async fn calculate_fair_fee_withdrawal_amount( #[tracing::instrument(name = "withdraw_fees", skip_all, fields())] pub async fn withdraw_fees_wrapper( - contract: Arc, + contract_as_fee_manager: Arc, provider_address: Address, poll_interval: Duration, min_balance: U256, - fee_manager_private_key: String, - known_keeper_addresses: Vec
, + keeper_address: Address, + other_keeper_addresses: Vec
, ) { + let fee_manager_wallet = contract_as_fee_manager.wallet().address(); + + // Add the fee manager to the list of other keepers so that we can fairly distribute the fees + // across the fee manager and all the keepers. + let mut other_keepers_and_fee_mgr = other_keeper_addresses.clone(); + other_keepers_and_fee_mgr.push(contract_as_fee_manager.wallet().address()); + loop { + // Top up the fee manager balance + // Do this before attempting to top up the keeper balance, since we need a funded + // fee manager to be able to withdraw & transfer funds to the keeper. if let Err(e) = withdraw_fees_if_necessary( - contract.clone(), + contract_as_fee_manager.clone(), + provider_address, + fee_manager_wallet, + other_keepers_and_fee_mgr.clone(), + min_balance, + ) + .in_current_span() + .await + { + tracing::error!("Withdrawing fees to fee manager. error: {:?}", e); + } + + // Top up the keeper balance + if let Err(e) = withdraw_fees_if_necessary( + contract_as_fee_manager.clone(), provider_address, + keeper_address, + other_keepers_and_fee_mgr.clone(), min_balance, - fee_manager_private_key.clone(), - known_keeper_addresses.clone(), ) .in_current_span() .await { - tracing::error!("Withdrawing fees. error: {:?}", e); + tracing::error!("Withdrawing fees to keeper. error: {:?}", e); } + time::sleep(poll_interval).await; } } /// Withdraws accumulated fees in the contract as needed to maintain the balance of the keeper wallet. pub async fn withdraw_fees_if_necessary( - contract: Arc, + contract_as_fee_manager: Arc, provider_address: Address, + keeper_address: Address, + other_keeper_addresses: Vec
, min_balance: U256, - fee_manager_private_key: String, - known_keeper_addresses: Vec
, ) -> Result<()> { - let provider = contract.provider(); - - let keeper_wallet = contract.wallet(); - let fee_manager_wallet = LocalWallet::from_str(&fee_manager_private_key) - .map_err(|e| anyhow!("Invalid fee manager private key: {:?}", e))?; + let provider = contract_as_fee_manager.provider(); + let fee_manager_wallet = contract_as_fee_manager.wallet(); let keeper_balance = provider - .get_balance(keeper_wallet.address(), None) + .get_balance(keeper_address, None) .await .map_err(|e| anyhow!("Error while getting balance. error: {:?}", e))?; - let provider_info = contract + // Only withdraw if our balance is below the minimum threshold + if keeper_balance >= min_balance { + return Ok(()); + } + + let provider_info = contract_as_fee_manager .get_provider_info(provider_address) .call() .await @@ -133,23 +160,17 @@ pub async fn withdraw_fees_if_necessary( let available_fees = U256::from(provider_info.accrued_fees_in_wei); - // Only withdraw if the keeper balance is below the minimum threshold - if keeper_balance >= min_balance { - return Ok(()); - } - // Determine how much we can fairly withdraw from the contract let withdrawal_amount = calculate_fair_fee_withdrawal_amount( Arc::new(provider.clone()), - keeper_wallet.address(), - &known_keeper_addresses, + keeper_address, + &other_keeper_addresses, available_fees, ) .await?; - // Only withdraw more than `min_balance` accrued fees to avoid tiny txs. - // TODO: make this independently configurable - let min_withdrawal_amount = min_balance; + // Only withdraw as long as we are at least doubling our keeper balance (avoids repeated withdrawals of tiny amounts) + let min_withdrawal_amount = keeper_balance; if withdrawal_amount < min_withdrawal_amount { // We don't have enough to meaningfully top up the balance. // NOTE: This log message triggers a grafana alert. If you want to change the text, please change the alert also. @@ -158,93 +179,33 @@ pub async fn withdraw_fees_if_necessary( } tracing::info!( - "Keeper balance {:?} below minimum {:?}, claiming {:?}", + "Keeper balance {:?} below minimum {:?}, claiming {:?} out of available {:?}", keeper_balance, min_balance, - withdrawal_amount + withdrawal_amount, + available_fees ); // Proceed with withdrawal - let contract_call = - contract.withdraw_as_fee_manager(provider_address, withdrawal_amount.as_u128()); + let contract_call = contract_as_fee_manager + .withdraw_as_fee_manager(provider_address, withdrawal_amount.as_u128()); send_and_confirm(contract_call).await?; - // Transfer fees from fee manager to keeper if fee manager is different from keeper - transfer_from_fee_manager_to_keeper( - Arc::new(provider.clone()), - fee_manager_wallet, - keeper_wallet.address(), - ) - .await - .map_err(|e| { - // If we fail to transfer funds from fee manager to keeper, we need to manually intervene - // NOTE: This log message triggers a grafana alert to allow for this. If you want to change the text, please change the alert also. - // TODO: Add retries/run a thread to periodically flush fee manager balance to the keeper. - anyhow!( - "Failed to transfer funds from fee manager to keeper: {:?}", - e + // Transfer the withdrawn funds from fee manager to keeper if fee manager is different from keeper + if fee_manager_wallet.address() != keeper_address { + submit_transfer_tx( + contract_as_fee_manager.clone(), + keeper_address, + withdrawal_amount, ) - })?; - - Ok(()) -} - -/// Transfer fees from fee manager to keeper if they are different addresses. -/// Transfers the maximum amount possible while preserving gas reserve for the transaction. -async fn transfer_from_fee_manager_to_keeper( - provider: Arc, - fee_manager_wallet: Wallet, - keeper_address: Address, -) -> Result<()> { - let fee_manager_address = fee_manager_wallet.address(); - - if fee_manager_address == keeper_address { - // Same address, no transfer needed - return Ok(()); - } - - // Calculate gas cost for the transfer transaction - let gas_price = provider.get_gas_price().await.map_err(|e| { - anyhow!( - "Failed to get gas price for transfer cost calculation: {:?}", - e - ) - })?; - - let transfer_gas_cost = U256::from(ETH_TRANSFER_GAS_LIMIT) * gas_price; - - // Add 50% buffer for gas price fluctuations and safety margin - let gas_reserve_required = transfer_gas_cost * 150 / 100; - - let fee_manager_balance = provider - .get_balance(fee_manager_address, None) .await - .map_err(|e| anyhow!("Failed to get fee manager balance: {:?}", e))?; - - let adjusted_withdrawal_amount = if fee_manager_balance > gas_reserve_required { - fee_manager_balance - gas_reserve_required - } else { - // Fee manager doesn't have enough balance to cover gas costs. Shouldn't be the case after withdrawing from the contract, - // manual intervention likely required. - tracing::error!( - "Fee manager balance {:?} is insufficient to cover gas reserve {:?} for transfer. Skipping transfer.", - fee_manager_balance, - gas_reserve_required - ); - return Ok(()); - }; - - submit_transfer_tx( - provider, - &fee_manager_wallet, - keeper_address, - gas_price, - adjusted_withdrawal_amount, - ) - .await - .map_err(|e| { - anyhow!("Failed to transfer fees from fee manager to keeper! Manual intervention required. error: {:?}", e) - })?; + .map_err(|e| { + anyhow!( + "Failed to transfer fees from fee manager to keeper. error: {:?}", + e + ) + })?; + } Ok(()) } From e869caab140b4143d3c5395b062e1aa8466a8818 Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Thu, 3 Jul 2025 17:18:08 -0700 Subject: [PATCH 09/12] naming --- apps/fortuna/README.md | 4 ++-- apps/fortuna/config.sample.yaml | 2 +- apps/fortuna/src/config.rs | 2 +- apps/fortuna/src/keeper.rs | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/fortuna/README.md b/apps/fortuna/README.md index 801fff065f..0f33edfab2 100644 --- a/apps/fortuna/README.md +++ b/apps/fortuna/README.md @@ -75,7 +75,7 @@ keeper: value: 0x fee_manager_private_key: value: 0x - known_keeper_addresses: + other_keeper_addresses: - 0x # This replica's address - 0x # Other replica's address replica_config: @@ -91,7 +91,7 @@ keeper: value: 0x fee_manager_private_key: value: 0x - known_keeper_addresses: + other_keeper_addresses: - 0x # Other replica's address - 0x # This replica's address replica_config: diff --git a/apps/fortuna/config.sample.yaml b/apps/fortuna/config.sample.yaml index cb914db4dc..177de21858 100644 --- a/apps/fortuna/config.sample.yaml +++ b/apps/fortuna/config.sample.yaml @@ -94,7 +94,7 @@ keeper: # List of other known keeper wallet addresses for balance comparison and fair fee withdrawals. # Do not include this keeper's address. - known_keeper_addresses: + other_keeper_addresses: - 0x1234 - 0x5678 diff --git a/apps/fortuna/src/config.rs b/apps/fortuna/src/config.rs index 07097b5c62..9ba0ad1736 100644 --- a/apps/fortuna/src/config.rs +++ b/apps/fortuna/src/config.rs @@ -388,7 +388,7 @@ pub struct KeeperConfig { /// The addresses of other keepers in the replica set (excluding the current keeper). /// This is used to distribute fees fairly across all keepers. #[serde(default)] - pub known_keeper_addresses: Vec
, + pub other_keeper_addresses: Vec
, #[serde(default)] pub replica_config: Option, diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index 58245a58d1..c53c95dedd 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -2,7 +2,7 @@ use { crate::{ api::{BlockchainState, ChainId}, chain::ethereum::{InstrumentedPythContract, InstrumentedSignablePythContract}, - config::EthereumConfig, + config::{EthereumConfig, KeeperConfig}, eth_utils::traced_client::RpcMetrics, history::History, keeper::{ @@ -58,7 +58,7 @@ pub enum RequestState { #[allow(clippy::too_many_arguments)] // Top level orchestration function that needs to configure several threads #[tracing::instrument(name = "keeper", skip_all, fields(chain_id = chain_state.id))] pub async fn run_keeper_threads( - keeper_config: crate::config::KeeperConfig, + keeper_config: KeeperConfig, chain_eth_config: EthereumConfig, chain_state: BlockchainState, metrics: Arc, @@ -131,7 +131,7 @@ pub async fn run_keeper_threads( }; if let Some(fee_manager_private_key) = fee_manager_private_key { - let contract_as_fee_mgr = Arc::new(InstrumentedSignablePythContract::from_config( + let contract_as_fee_manager = Arc::new(InstrumentedSignablePythContract::from_config( &chain_eth_config, &fee_manager_private_key, chain_state.id.clone(), @@ -142,12 +142,12 @@ pub async fn run_keeper_threads( // Spawn a thread that periodically withdraws fees to the fee manager and keeper. spawn( withdraw_fees_wrapper( - contract_as_fee_mgr.clone(), + contract_as_fee_manager.clone(), chain_state.provider_address, WITHDRAW_INTERVAL, U256::from(chain_eth_config.min_keeper_balance), keeper_address, - keeper_config.known_keeper_addresses.clone(), + keeper_config.other_keeper_addresses.clone(), ) .in_current_span(), ); @@ -155,7 +155,7 @@ pub async fn run_keeper_threads( // Spawn a thread that periodically adjusts the provider fee. spawn( adjust_fee_wrapper( - contract_as_fee_mgr.clone(), + contract_as_fee_manager.clone(), chain_state.clone(), chain_state.provider_address, ADJUST_FEE_INTERVAL, From 17932faca4d53f6a2dfcb589971a63d0e43a6eb4 Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Thu, 3 Jul 2025 17:19:08 -0700 Subject: [PATCH 10/12] comment --- apps/fortuna/src/keeper.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index c53c95dedd..ec7379a550 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -123,7 +123,7 @@ pub async fn run_keeper_threads( .in_current_span(), ); - // If fee manager private key is provided, spawn fee withdrawal and adjustmnet threads + // If fee manager private key is provided, spawn fee withdrawal and adjustment threads let fee_manager_private_key = if let Some(ref secret) = keeper_config.fee_manager_private_key { secret.load()? } else { From e6fc9ea5291a502947876788c38bacc3522bf423 Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Thu, 3 Jul 2025 17:20:49 -0700 Subject: [PATCH 11/12] remove run config --- apps/fortuna/README.md | 7 ++----- apps/fortuna/config.sample.yaml | 6 ------ apps/fortuna/src/config.rs | 4 ---- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/apps/fortuna/README.md b/apps/fortuna/README.md index 0f33edfab2..274f3eaf31 100644 --- a/apps/fortuna/README.md +++ b/apps/fortuna/README.md @@ -63,7 +63,6 @@ When running multiple Fortuna instances with different keeper wallets, the syste The fee manager (configured in the provider section) can be a separate wallet from the keeper wallets. When fees are withdrawn from the contract, they go to the fee manager wallet first, then are automatically transferred to the requesting keeper wallet. **Key Configuration:** -- Only one instance should have fee adjustment enabled to avoid multiple keepers racing to adjust the fee (`disable_fee_adjustment: false`) - All instances should have `keeper.private_key` and `keeper.fee_manager_private_key` provided so that each keeper can top itself up as fee manager from contract fees. ### Example Configurations @@ -82,8 +81,7 @@ keeper: replica_id: 0 total_replicas: 2 backup_delay_seconds: 15 - run_config: - disable_fee_adjustment: false # Enable fee adjustment (default) + # Replica 1 - handles odd sequence numbers keeper: @@ -98,8 +96,7 @@ keeper: replica_id: 1 total_replicas: 2 backup_delay_seconds: 15 - run_config: - disable_fee_adjustment: true # Disable fee adjustment + ``` ### Deployment Considerations diff --git a/apps/fortuna/config.sample.yaml b/apps/fortuna/config.sample.yaml index 177de21858..2360dde3ea 100644 --- a/apps/fortuna/config.sample.yaml +++ b/apps/fortuna/config.sample.yaml @@ -98,12 +98,6 @@ keeper: - 0x1234 - 0x5678 - # Runtime configuration for the keeper service - # Optional: Configure which keeper threads to disable. If running multiple replicas, - # only a single replica should have the fee adjustment thread enabled. - run_config: - disable_fee_adjustment: false - # Multi-replica configuration # Optional: Multi-replica configuration for high availability and load distribution # Uncomment and configure for production deployments with multiple Fortuna instances diff --git a/apps/fortuna/src/config.rs b/apps/fortuna/src/config.rs index 9ba0ad1736..4f62f767e6 100644 --- a/apps/fortuna/src/config.rs +++ b/apps/fortuna/src/config.rs @@ -392,10 +392,6 @@ pub struct KeeperConfig { #[serde(default)] pub replica_config: Option, - - /// Runtime configuration for the keeper service - #[serde(default)] - pub run_config: RunConfig, } // A secret is a string that can be provided either as a literal in the config, From 035ed53673041360e22c8d96c37f4ab78528e41c Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Mon, 7 Jul 2025 16:18:55 -0700 Subject: [PATCH 12/12] chore(fortuna): bump version --- Cargo.lock | 2 +- apps/fortuna/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca85b2f628..6ce85720ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3046,7 +3046,7 @@ dependencies = [ [[package]] name = "fortuna" -version = "8.0.0" +version = "8.1.0" dependencies = [ "anyhow", "axum 0.6.20", diff --git a/apps/fortuna/Cargo.toml b/apps/fortuna/Cargo.toml index 94d1e7a302..80d5895dc6 100644 --- a/apps/fortuna/Cargo.toml +++ b/apps/fortuna/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fortuna" -version = "8.0.0" +version = "8.1.0" edition = "2021" [lib]