Merge remote-tracking branch 'refs/remotes/origin/main' into feat/pyth-lazer/agent/jrpc-impl-1

Author: bplatak

.github/CODEOWNERS

@@ -3,6 +3,7 @@ apps/entropy-debugger @pyth-network/web-team
 apps/insights @pyth-network/web-team
 apps/staking @pyth-network/web-team
 apps/pyth-lazer-agent @merolish
+apps/fortuna @tejasbadadare
 packages/component-library @pyth-network/web-team
 packages/known-publishers @pyth-network/web-team
 Dockerfile.node @pyth-network/web-team

Cargo.lock

@@ -3,4 +3,4 @@
 *secret*
 *private-key*
 .envrc
-fortuna.db
+fortuna.db*

apps/fortuna/.gitignore

@@ -1,6 +1,6 @@
 [package]
 name = "fortuna"
-version = "7.7.0"
+version = "8.1.0"
 edition = "2021"
 
 [lib]
@@ -47,6 +47,7 @@ backoff = { version = "0.4.0", features = ["futures", "tokio"] }
 thiserror = "1.0.61"
 futures-locks = "0.7.1"
 sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite", "chrono"] }
+num-traits = "0.2.19"
 
 [dev-dependencies]
 axum-test = "13.1.1"

apps/fortuna/Cargo.toml

@@ -58,66 +58,53 @@ Fortuna supports running multiple replica instances for high availability and re
 
 ### Fee Management with Multiple Instances
 
-When running multiple Fortuna instances with different keeper wallets but a single provider, only one instance should handle fee management. This instance needs to run using the same private key as the fee manager, because only the registerd fee manager wallet can adjust fees and withdraw funds.
+When running multiple Fortuna instances with different keeper wallets, the system uses a fair fee distribution strategy. Each keeper will withdraw fees from the contract to maintain a balanced distribution across all known keeper addresses and the fee manager address.
+
+The fee manager (configured in the provider section) can be a separate wallet from the keeper wallets. When fees are withdrawn from the contract, they go to the fee manager wallet first, then are automatically transferred to the requesting keeper wallet.
+
+**Key Configuration:**
+- All instances should have `keeper.private_key` and `keeper.fee_manager_private_key` provided so that each keeper can top itself up as fee manager from contract fees.
 
 ### Example Configurations
 
-**Two Replica Setup with Fee Management:**
 ```yaml
-# Replica 0 (fee manager wallet) - handles even sequence numbers + fee management
+# Replica 0 - handles even sequence numbers + fee management
 keeper:
   private_key:
+    value: 0x<keeper_0_private_key>
+  fee_manager_private_key:
     value: 0x<fee_manager_private_key>
+  other_keeper_addresses:
+    - 0x<keeper_0_address>  # This replica's address
+    - 0x<keeper_1_address>  # Other replica's address
   replica_config:
     replica_id: 0
     total_replicas: 2
-    backup_delay_seconds: 30
-  run_config:
-    disable_fee_adjustment: false  # Enable fee management (default)
-    disable_fee_withdrawal: false
+    backup_delay_seconds: 15
+
 
-# Replica 1 (non-fee-manager wallet) - handles odd sequence numbers only
+# Replica 1 - handles odd sequence numbers
 keeper:
   private_key:
-    value: 0x<other_keeper_private_key>
+    value: 0x<keeper_1_private_key>
+  fee_manager_private_key:
+    value: 0x<fee_manager_private_key>
+  other_keeper_addresses:
+    - 0x<keeper_0_address>  # Other replica's address
+    - 0x<keeper_1_address>  # This replica's address
   replica_config:
     replica_id: 1
     total_replicas: 2
-    backup_delay_seconds: 30
-  run_config:
-    disable_fee_adjustment: true   # Disable fee management
-    disable_fee_withdrawal: true
-```
-
-**Three Replica Setup:**
-```yaml
-# Replica 0 (fee manager wallet) - handles sequence numbers 0, 3, 6, 9, ... + fee management
-keeper:
-  replica_config:
-    replica_id: 0
-    total_replicas: 3
-    backup_delay_seconds: 30
-  run_config:
-    disable_fee_adjustment: false
-    disable_fee_withdrawal: false
+    backup_delay_seconds: 15
 
-# Replicas 1 & 2 (non-fee-manager wallets) - request processing only
-keeper:
-  replica_config:
-    replica_id: 1  # or 2
-    total_replicas: 3
-    backup_delay_seconds: 30
-  run_config:
-    disable_fee_adjustment: true
-    disable_fee_withdrawal: true
 ```
 
 ### Deployment Considerations
 
 1. **Separate Wallets**: Each replica MUST use a different private key to avoid nonce conflicts
 2. **Fee Manager Assignment**: Set the provider's `fee_manager` address to match the primary instance's keeper wallet
 3. **Thread Configuration**: Only enable fee management threads on the instance using the fee manager wallet
-4. **Backup Delay**: Set `backup_delay_seconds` long enough to allow primary replica to process requests, but short enough for acceptable failover time (recommended: 30-60 seconds)
+4. **Backup Delay**: Set `backup_delay_seconds` long enough to allow primary replica to process requests, but short enough for acceptable failover time (recommended: 10-30 seconds)
 5. **Monitoring**: Monitor each replica's processing metrics to ensure proper load distribution
 6. **Gas Management**: Each replica needs sufficient ETH balance for gas fees
 
@@ -127,7 +114,6 @@ keeper:
 - Backup replicas wait for `backup_delay_seconds` before checking if request is still unfulfilled
 - If request is already fulfilled during the delay, backup replica skips processing
 - This prevents duplicate transactions and wasted gas while ensuring reliability
-- Fee management operations (adjustment/withdrawal) only occur on an instance where the keeper wallet is the fee manager wallet.
 
 ## Local Development
 

apps/fortuna/README.md

@@ -73,9 +73,9 @@ provider:
     # For production, you can store the private key in a file.
     # file: secret.txt
 
-  # Set this to the address of your keeper wallet if you would like the keeper wallet to
-  # be able to withdraw fees from the contract.
-  fee_manager: 0xADDRESS
+  # The address of the fee manager for the provider. Only used for syncing the fee manager address to the contract.
+  # Fee withdrawals are handled by the fee manager private key defined in the keeper config.
+  fee_manager: 0xfee
 keeper:
   # An ethereum wallet address and private key for running the keeper service.
   # This does not have to be the same key as the provider's key above.
@@ -87,25 +87,24 @@ keeper:
     # For production, you can store the private key in a file.
     # file: keeper-key.txt
 
-  # Runtime configuration for the keeper service
-  # Optional: Configure which keeper threads to disable. If running multiple replicas,
-  # only a single replica should have the fee adjustment and withdrawal threads enabled.
-  # run_config:
-  #   disable_fee_adjustment: false    # Set to true to disable automatic fee adjustment
-  #   disable_fee_withdrawal: false    # Set to true to disable automatic fee withdrawal
+  # Fee manager private key for fee manager operations (if not provided, fee withdrawals won't happen)
+  fee_manager_private_key:
+    value: 0xabcd
+    # file: fee-manager-key.txt
+
+  # List of other known keeper wallet addresses for balance comparison and fair fee withdrawals.
+  # Do not include this keeper's address.
+  other_keeper_addresses:
+    - 0x1234
+    - 0x5678
 
   # Multi-replica configuration
   # Optional: Multi-replica configuration for high availability and load distribution
   # Uncomment and configure for production deployments with multiple Fortuna instances
-  # replica_config:
-  #   replica_id: 0              # Unique identifier for this replica (0, 1, 2, ...)
-  #   total_replicas: 2          # Total number of replica instances running
-  #   backup_delay_seconds: 30   # Seconds to wait before processing other replicas' requests
-  #
-  # Example configurations:
-  #
-  # Two-replica setup (Blue/Green):
-  # - Replica 0: handles even sequence numbers (0, 2, 4, ...)
-  # - Replica 1: handles odd sequence numbers (1, 3, 5, ...)
-  #
+  # See the README for more details.
+  replica_config:
+    replica_id: 0              # Unique identifier for this replica (0, 1, 2, ...)
+    total_replicas: 2          # Total number of replica instances running
+    backup_delay_seconds: 30   # Seconds to wait before processing other replicas' requests
+
   # IMPORTANT: Each replica must use a different private_key to avoid nonce conflicts!

apps/fortuna/config.sample.yaml

@@ -89,7 +89,9 @@ pub async fn revelation(
                 .ok_or(RestError::NoPendingRequest)?;
         }
         None => {
-            let maybe_request_fut = state.contract.get_request(state.provider_address, sequence);
+            let maybe_request_fut = state
+                .contract
+                .get_request_v2(state.provider_address, sequence);
             let (maybe_request, current_block_number) =
                 try_join!(maybe_request_fut, current_block_number_fut).map_err(|e| {
                     tracing::error!(chain_id = chain_id, "RPC request failed {}", e);

apps/fortuna/src/api/revelation.rs

@@ -265,29 +265,22 @@ impl<T: JsonRpcClient + 'static> PythRandom<Provider<T>> {
 
 #[async_trait]
 impl<T: JsonRpcClient + 'static> EntropyReader for PythRandom<Provider<T>> {
-    async fn get_request(
+    async fn get_request_v2(
         &self,
         provider_address: Address,
         sequence_number: u64,
     ) -> Result<Option<reader::Request>> {
-        let r = self
-            .get_request(provider_address, sequence_number)
-            // TODO: This doesn't work for lighlink right now. Figure out how to do this in lightlink
-            // .block(ethers::core::types::BlockNumber::Finalized)
+        let request = self
+            .get_request_v2(provider_address, sequence_number)
             .call()
             .await?;
-
-        // sequence_number == 0 means the request does not exist.
-        if r.sequence_number != 0 {
-            Ok(Some(reader::Request {
-                provider: r.provider,
-                sequence_number: r.sequence_number,
-                block_number: r.block_number,
-                use_blockhash: r.use_blockhash,
-            }))
-        } else {
-            Ok(None)
-        }
+        Ok(Some(reader::Request {
+            provider: request.provider,
+            sequence_number: request.sequence_number,
+            block_number: request.block_number,
+            use_blockhash: request.use_blockhash,
+            callback_status: reader::RequestCallbackStatus::try_from(request.callback_status)?,
+        }))
     }
 
     async fn get_block_number(&self, confirmed_block_status: BlockStatus) -> Result<BlockNumber> {

apps/fortuna/src/chain/ethereum.rs

@@ -60,8 +60,11 @@ pub trait EntropyReader: Send + Sync {
     /// Get an in-flight request (if it exists)
     /// Note that if we support additional blockchains in the future, the type of `provider` may
     /// need to become more generic.
-    async fn get_request(&self, provider: Address, sequence_number: u64)
-        -> Result<Option<Request>>;
+    async fn get_request_v2(
+        &self,
+        provider: Address,
+        sequence_number: u64,
+    ) -> Result<Option<Request>>;
 
     async fn get_block_number(&self, confirmed_block_status: BlockStatus) -> Result<BlockNumber>;
 
@@ -93,12 +96,48 @@ pub struct Request {
     // The block number where this request was created
     pub block_number: BlockNumber,
     pub use_blockhash: bool,
+    pub callback_status: RequestCallbackStatus,
+}
+
+/// Status values for Request.callback_status
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum RequestCallbackStatus {
+    /// Not a request with callback
+    CallbackNotNecessary = 0,
+    /// A request with callback where the callback hasn't been invoked yet
+    CallbackNotStarted = 1,
+    /// A request with callback where the callback is currently in flight (this state is a reentry guard)
+    CallbackInProgress = 2,
+    /// A request with callback where the callback has been invoked and failed
+    CallbackFailed = 3,
+}
+
+impl TryFrom<u8> for RequestCallbackStatus {
+    type Error = anyhow::Error;
+
+    fn try_from(value: u8) -> Result<Self> {
+        match value {
+            0 => Ok(RequestCallbackStatus::CallbackNotNecessary),
+            1 => Ok(RequestCallbackStatus::CallbackNotStarted),
+            2 => Ok(RequestCallbackStatus::CallbackInProgress),
+            3 => Ok(RequestCallbackStatus::CallbackFailed),
+            _ => Err(anyhow::anyhow!("Invalid callback status value: {}", value)),
+        }
+    }
+}
+
+impl From<RequestCallbackStatus> for u8 {
+    fn from(status: RequestCallbackStatus) -> Self {
+        status as u8
+    }
 }
 
 #[cfg(test)]
 pub mod mock {
     use {
-        crate::chain::reader::{BlockNumber, BlockStatus, EntropyReader, Request},
+        crate::chain::reader::{
+            BlockNumber, BlockStatus, EntropyReader, Request, RequestCallbackStatus,
+        },
         anyhow::Result,
         axum::async_trait,
         ethers::types::{Address, U256},
@@ -129,6 +168,7 @@ pub mod mock {
                             sequence_number: s,
                             block_number: b,
                             use_blockhash: u,
+                            callback_status: RequestCallbackStatus::CallbackNotNecessary,
                         })
                         .collect(),
                 ),
@@ -148,6 +188,7 @@ pub mod mock {
                 sequence_number: sequence,
                 block_number,
                 use_blockhash,
+                callback_status: RequestCallbackStatus::CallbackNotNecessary,
             });
             self
         }
@@ -160,7 +201,7 @@ pub mod mock {
 
     #[async_trait]
     impl EntropyReader for MockEntropyReader {
-        async fn get_request(
+        async fn get_request_v2(
             &self,
             provider: Address,
             sequence_number: u64,

apps/fortuna/src/chain/reader.rs

@@ -14,12 +14,12 @@ pub async fn get_request(opts: &GetRequestOptions) -> Result<()> {
         &Config::load(&opts.config.config)?.get_chain_config(&opts.chain_id)?,
     )?);
 
-    let p = contract.get_provider_info(opts.provider).call().await?;
+    let p = contract.get_provider_info_v2(opts.provider).call().await?;
 
     tracing::info!("Found provider: {:?}", p);
 
     let r = contract
-        .get_request(opts.provider, opts.sequence)
+        .get_request_v2(opts.provider, opts.sequence)
         .call()
         .await?;
     tracing::info!("Found request: {:?}", r);