feat(fortuna): add delayed processing for backup replicas

- Add configurable time delay before backup replicas check request status
- Backup replicas now wait backup_delay_seconds before attempting fulfillment
- Add backup_delay_seconds field to ReplicaConfig with default of 30 seconds
- Improves reliability by reducing race conditions between replicas

Co-Authored-By: Tejas Badadare <[email protected]>

Author: devin-ai-integration[bot]

apps/fortuna/src/config.rs

@@ -346,6 +346,12 @@ fn default_chain_sample_interval() -> u64 {
 pub struct ReplicaConfig {
     pub replica_id: u64,
     pub total_replicas: u64,
+    #[serde(default = "default_backup_delay_seconds")]
+    pub backup_delay_seconds: u64,
+}
+
+fn default_backup_delay_seconds() -> u64 {
+    30
 }
 
 /// Configuration values for the keeper service that are shared across chains.

apps/fortuna/src/keeper/process_event.rs

@@ -54,6 +54,18 @@ pub async fn process_event_with_backoff(
         };
 
     if !is_primary_replica {
+        if let Some(replica_config) = &process_param.keeper_config.replica_config {
+            tracing::info!(
+                sequence_number = event.sequence_number,
+                delay_seconds = replica_config.backup_delay_seconds,
+                "Waiting before processing as backup replica"
+            );
+            tokio::time::sleep(tokio::time::Duration::from_secs(
+                replica_config.backup_delay_seconds,
+            ))
+            .await;
+        }
+
         match chain_state
             .contract
             .get_request(event.provider_address, event.sequence_number)
@@ -62,21 +74,21 @@ pub async fn process_event_with_backoff(
             Ok(Some(_)) => {
                 tracing::info!(
                     sequence_number = event.sequence_number,
-                    "Request still open, processing as backup replica"
+                    "Request still open after delay, processing as backup replica"
                 );
             }
             Ok(None) => {
                 tracing::debug!(
                     sequence_number = event.sequence_number,
-                    "Request already fulfilled by primary replica, skipping"
+                    "Request already fulfilled by primary replica during delay, skipping"
                 );
                 return Ok(());
             }
             Err(e) => {
                 tracing::warn!(
                     sequence_number = event.sequence_number,
                     error = ?e,
-                    "Error checking request status, skipping"
+                    "Error checking request status after delay, skipping"
                 );
                 return Ok(());
             }