feat(fortuna): Automated fair fee withdrawals with multiple keepers

apps/fortuna/README.md

@@ -58,66 +58,56 @@ Fortuna supports running multiple replica instances for high availability and re
 
 ### Fee Management with Multiple Instances
 
-When running multiple Fortuna instances with different keeper wallets but a single provider, only one instance should handle fee management. This instance needs to run using the same private key as the fee manager, because only the registerd fee manager wallet can adjust fees and withdraw funds.
+When running multiple Fortuna instances with different keeper wallets, the system uses a fair fee distribution strategy. Each keeper will withdraw fees from the contract to maintain a balanced distribution across all known keeper addresses.
+
+The fee manager (configured in the provider section) can be a separate wallet from the keeper wallets. When fees are withdrawn from the contract, they go to the fee manager wallet first, then are automatically transferred to the requesting keeper wallet.
+
+**Key Configuration:**
+- Only one instance should have fee adjustment enabled to avoid multiple keepers racing to adjust the fee (`disable_fee_adjustment: false`)
+- All instances should have `fee_manager_private_key` provided so that each keeper can top itself up from contract fees.
 
 ### Example Configurations
 
-**Two Replica Setup with Fee Management:**
 ```yaml
-# Replica 0 (fee manager wallet) - handles even sequence numbers + fee management
+# Replica 0 (with fee management) - handles even sequence numbers + fee management
 keeper:
   private_key:
+    value: 0x<keeper_0_private_key>
+  fee_manager_private_key:
     value: 0x<fee_manager_private_key>
+  known_keeper_addresses:
+    - 0x<keeper_0_address>  # This replica's address
+    - 0x<keeper_1_address>  # Other replica's address
   replica_config:
     replica_id: 0
     total_replicas: 2
-    backup_delay_seconds: 30
+    backup_delay_seconds: 15
   run_config:
-    disable_fee_adjustment: false  # Enable fee management (default)
-    disable_fee_withdrawal: false
+    disable_fee_adjustment: false  # Enable fee adjustment (default)
 
-# Replica 1 (non-fee-manager wallet) - handles odd sequence numbers only
+# Replica 1 (request processing only) - handles odd sequence numbers
 keeper:
   private_key:
-    value: 0x<other_keeper_private_key>
+    value: 0x<keeper_1_private_key>
+  fee_manager_private_key:
+    value: 0x<fee_manager_private_key>
+  known_keeper_addresses:
+    - 0x<keeper_0_address>  # Other replica's address
+    - 0x<keeper_1_address>  # This replica's address
   replica_config:
     replica_id: 1
     total_replicas: 2
-    backup_delay_seconds: 30
-  run_config:
-    disable_fee_adjustment: true   # Disable fee management
-    disable_fee_withdrawal: true
-```
-
-**Three Replica Setup:**
-```yaml
-# Replica 0 (fee manager wallet) - handles sequence numbers 0, 3, 6, 9, ... + fee management
-keeper:
-  replica_config:
-    replica_id: 0
-    total_replicas: 3
-    backup_delay_seconds: 30
-  run_config:
-    disable_fee_adjustment: false
-    disable_fee_withdrawal: false
-
-# Replicas 1 & 2 (non-fee-manager wallets) - request processing only
-keeper:
-  replica_config:
-    replica_id: 1  # or 2
-    total_replicas: 3
-    backup_delay_seconds: 30
+    backup_delay_seconds: 15
   run_config:
-    disable_fee_adjustment: true
-    disable_fee_withdrawal: true
+    disable_fee_adjustment: true   # Disable fee adjustment
 ```
 
 ### Deployment Considerations
 
 1. **Separate Wallets**: Each replica MUST use a different private key to avoid nonce conflicts
 2. **Fee Manager Assignment**: Set the provider's `fee_manager` address to match the primary instance's keeper wallet
 3. **Thread Configuration**: Only enable fee management threads on the instance using the fee manager wallet
-4. **Backup Delay**: Set `backup_delay_seconds` long enough to allow primary replica to process requests, but short enough for acceptable failover time (recommended: 30-60 seconds)
+4. **Backup Delay**: Set `backup_delay_seconds` long enough to allow primary replica to process requests, but short enough for acceptable failover time (recommended: 10-30 seconds)
 5. **Monitoring**: Monitor each replica's processing metrics to ensure proper load distribution
 6. **Gas Management**: Each replica needs sufficient ETH balance for gas fees
 
@@ -127,7 +117,6 @@ keeper:
 - Backup replicas wait for `backup_delay_seconds` before checking if request is still unfulfilled
 - If request is already fulfilled during the delay, backup replica skips processing
 - This prevents duplicate transactions and wasted gas while ensuring reliability
-- Fee management operations (adjustment/withdrawal) only occur on an instance where the keeper wallet is the fee manager wallet.
 
 ## Local Development
 

apps/fortuna/config.sample.yaml

@@ -87,12 +87,22 @@ keeper:
     # For production, you can store the private key in a file.
     # file: keeper-key.txt
 
+  # Fee manager private key for fee manager operations (if not provided, fee withdrawals won't happen)
+  fee_manager_private_key:
+    value: 0xabcd
+    # file: fee-manager-key.txt
+
+  # List of known keeper wallet addresses for balance comparison
+  # The keeper will only withdraw fees if its balance is the lowest among these addresses.
+  known_keeper_addresses:
+    - 0x1234
+    - 0x5678
+
   # Runtime configuration for the keeper service
   # Optional: Configure which keeper threads to disable. If running multiple replicas,
   # only a single replica should have the fee adjustment and withdrawal threads enabled.
   # run_config:
   #   disable_fee_adjustment: false    # Set to true to disable automatic fee adjustment
-  #   disable_fee_withdrawal: false    # Set to true to disable automatic fee withdrawal
 
   # Multi-replica configuration
   # Optional: Multi-replica configuration for high availability and load distribution

apps/fortuna/src/command/run.rs

@@ -3,10 +3,7 @@ use {
         api::{self, ApiBlockChainState, BlockchainState, ChainId},
         chain::ethereum::InstrumentedPythContract,
         command::register_provider::CommitmentMetadata,
-        config::{
-            Commitment, Config, EthereumConfig, ProviderConfig, ReplicaConfig, RunConfig,
-            RunOptions,
-        },
+        config::{Commitment, Config, EthereumConfig, KeeperConfig, ProviderConfig, RunOptions},
         eth_utils::traced_client::RpcMetrics,
         history::History,
         keeper::{self, keeper_metrics::KeeperMetrics},
@@ -103,9 +100,6 @@ pub async fn run(opts: &RunOptions) -> Result<()> {
         tracing::info!("Not starting keeper service: no keeper private key specified. Please add one to the config if you would like to run the keeper service.")
     }
 
-    let keeper_replica_config = config.keeper.replica_config.clone();
-    let keeper_run_config = config.keeper.run_config.clone();
-
     let chains: Arc<RwLock<HashMap<ChainId, ApiBlockChainState>>> = Arc::new(RwLock::new(
         config
             .chains
@@ -118,23 +112,25 @@ pub async fn run(opts: &RunOptions) -> Result<()> {
         keeper_metrics.add_chain(chain_id.clone(), config.provider.address);
         let keeper_metrics = keeper_metrics.clone();
         let keeper_private_key_option = keeper_private_key_option.clone();
-        let keeper_replica_config = keeper_replica_config.clone();
-        let keeper_run_config = keeper_run_config.clone();
         let chains = chains.clone();
         let secret_copy = secret.clone();
         let rpc_metrics = rpc_metrics.clone();
         let provider_config = config.provider.clone();
         let history = history.clone();
+        let keeper_config_base = config.keeper.clone();
         spawn(async move {
             loop {
+                let keeper_config = if keeper_private_key_option.is_some() {
+                    Some(keeper_config_base.clone())
+                } else {
+                    None
+                };
                 let setup_result = setup_chain_and_run_keeper(
                     provider_config.clone(),
                     &chain_id,
                     chain_config.clone(),
                     keeper_metrics.clone(),
-                    keeper_private_key_option.clone(),
-                    keeper_replica_config.clone(),
-                    keeper_run_config.clone(),
+                    keeper_config,
                     chains.clone(),
                     &secret_copy,
                     history.clone(),
@@ -184,9 +180,7 @@ async fn setup_chain_and_run_keeper(
     chain_id: &ChainId,
     chain_config: EthereumConfig,
     keeper_metrics: Arc<KeeperMetrics>,
-    keeper_private_key_option: Option<String>,
-    keeper_replica_config: Option<ReplicaConfig>,
-    keeper_run_config: RunConfig,
+    keeper_config: Option<KeeperConfig>,
     chains: Arc<RwLock<HashMap<ChainId, ApiBlockChainState>>>,
     secret_copy: &str,
     history: Arc<History>,
@@ -206,11 +200,9 @@ async fn setup_chain_and_run_keeper(
         chain_id.clone(),
         ApiBlockChainState::Initialized(state.clone()),
     );
-    if let Some(keeper_private_key) = keeper_private_key_option {
+    if let Some(keeper_config) = keeper_config {
         keeper::run_keeper_threads(
-            keeper_private_key,
-            keeper_replica_config,
-            keeper_run_config,
+            keeper_config,
             chain_config,
             state,
             keeper_metrics.clone(),

apps/fortuna/src/config.rs

@@ -355,10 +355,6 @@ pub struct RunConfig {
     /// Disable automatic fee adjustment threads
     #[serde(default)]
     pub disable_fee_adjustment: bool,
-
-    /// Disable automatic fee withdrawal threads
-    #[serde(default)]
-    pub disable_fee_withdrawal: bool,
 }
 
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
@@ -383,6 +379,15 @@ pub struct KeeperConfig {
     /// should ensure this is a different key in order to reduce the severity of security breaches.
     pub private_key: SecretString,
 
+    /// The fee manager's private key for fee manager operations.
+    /// This key is used to withdraw fees from the contract as the fee manager.
+    /// Multiple replicas can share the same fee manager private key but different keeper keys (`private_key`).
+    #[serde(default)]
+    pub fee_manager_private_key: Option<SecretString>,
+
+    #[serde(default)]
+    pub known_keeper_addresses: Vec<Address>,
+
     #[serde(default)]
     pub replica_config: Option<ReplicaConfig>,
 

apps/fortuna/src/eth_utils/utils.rs

@@ -5,8 +5,10 @@ use {
     ethabi::ethereum_types::U64,
     ethers::{
         contract::{ContractCall, ContractError},
+        core::k256::ecdsa::SigningKey,
         middleware::Middleware,
         providers::ProviderError,
+        signers::Wallet,
         types::{transaction::eip2718::TypedTransaction, TransactionReceipt, U256},
     },
     std::{
@@ -18,6 +20,7 @@ use {
 };
 
 const TX_CONFIRMATION_TIMEOUT_SECS: u64 = 30;
+pub const ETH_TRANSFER_GAS_LIMIT: u64 = 21_000;
 
 #[derive(Debug)]
 pub struct SubmitTxResult {
@@ -367,3 +370,80 @@ pub async fn submit_tx<T: Middleware + NonceManaged + 'static>(
 
     Ok(receipt)
 }
+
+/// Transfer funds from fee manager wallet to keeper wallet.
+/// This is used when the fee manager and keeper are different addresses.
+pub async fn submit_transfer_tx<M: Middleware>(
+    provider: Arc<M>,
+    source_wallet: &Wallet<SigningKey>,
+    destination_address: ethers::types::Address,
+    gas_price: U256,
+    transfer_amount: U256,
+) -> Result<ethers::types::H256> {
+    use ethers::{signers::Signer, types::TransactionRequest};
+
+    let source_wallet_address = source_wallet.address();
+
+    tracing::info!(
+        "Transferring {:?} from {:?} to {:?}",
+        transfer_amount,
+        source_wallet_address,
+        destination_address
+    );
+
+    // Get transaction parameters
+    let chain_id = provider
+        .get_chainid()
+        .await
+        .map_err(|e| anyhow!("Failed to get chain ID: {:?}", e))?;
+
+    let nonce = provider
+        .get_transaction_count(source_wallet_address, None)
+        .await
+        .map_err(|e| anyhow!("Failed to get nonce: {:?}", e))?;
+
+    let tx = TransactionRequest::new()
+        .to(destination_address)
+        .value(transfer_amount)
+        .from(source_wallet_address)
+        .gas(ETH_TRANSFER_GAS_LIMIT)
+        .gas_price(gas_price)
+        .nonce(nonce)
+        .chain_id(chain_id.as_u64());
+
+    let typed_tx = tx.into();
+    let signature = source_wallet
+        .sign_transaction(&typed_tx)
+        .await
+        .map_err(|e| anyhow!("Failed to sign transfer transaction: {:?}", e))?;
+
+    let signed_tx = typed_tx.rlp_signed(&signature);
+
+    // Send transaction and get pending transaction
+    let pending_tx = provider
+        .send_raw_transaction(signed_tx)
+        .await
+        .map_err(|e| anyhow!("Failed to send transfer transaction: {:?}", e))?;
+
+    // Wait for confirmation with timeout
+    let tx_receipt = timeout(
+        Duration::from_secs(TX_CONFIRMATION_TIMEOUT_SECS),
+        pending_tx,
+    )
+    .await
+    .map_err(|_| anyhow!("Transfer transaction confirmation timeout"))?
+    .map_err(|e| anyhow!("Transfer transaction confirmation error: {:?}", e))?
+    .ok_or_else(|| anyhow!("Transfer transaction dropped from mempool"))?;
+
+    // Check if transaction was successful
+    if tx_receipt.status == Some(ethabi::ethereum_types::U64::from(0)) {
+        return Err(anyhow!(
+            "Transfer transaction failed on-chain. Receipt: {:?}",
+            tx_receipt
+        ));
+    }
+
+    let tx_hash = tx_receipt.transaction_hash;
+    tracing::info!("Transfer transaction confirmed: {:?}", tx_hash);
+    Ok(tx_hash)
+}