feat(fortuna): Improve startup flow (#2638)

* fix(fortuna): Do not wait for all chains setup in startup

* Use blocking thread for calculating hashchain

* Put block timestamp lag next to other tracking tasks

* Retry startup

Author: m30m

apps/fortuna/Cargo.lock

@@ -43,7 +43,7 @@ pub struct ApiMetrics {
 
 #[derive(Clone)]
 pub struct ApiState {
-    pub chains: Arc<HashMap<ChainId, BlockchainState>>,
+    pub chains: Arc<RwLock<HashMap<ChainId, ApiBlockChainState>>>,
 
     pub metrics_registry: Arc<RwLock<Registry>>,
 
@@ -53,7 +53,7 @@ pub struct ApiState {
 
 impl ApiState {
     pub async fn new(
-        chains: HashMap<ChainId, BlockchainState>,
+        chains: Arc<RwLock<HashMap<ChainId, ApiBlockChainState>>>,
         metrics_registry: Arc<RwLock<Registry>>,
     ) -> ApiState {
         let metrics = ApiMetrics {
@@ -68,7 +68,7 @@ impl ApiState {
         );
 
         ApiState {
-            chains: Arc::new(chains),
+            chains,
             metrics: Arc::new(metrics),
             metrics_registry,
         }
@@ -94,6 +94,12 @@ pub struct BlockchainState {
     pub confirmed_block_status: BlockStatus,
 }
 
+#[derive(Clone)]
+pub enum ApiBlockChainState {
+    Uninitialized,
+    Initialized(BlockchainState),
+}
+
 pub enum RestError {
     /// The caller passed a sequence number that isn't within the supported range
     InvalidSequenceNumber,
@@ -108,6 +114,9 @@ pub enum RestError {
     /// The server cannot currently communicate with the blockchain, so is not able to verify
     /// which random values have been requested.
     TemporarilyUnavailable,
+    /// The server is not able to process the request because the blockchain initialization
+    /// has not been completed yet.
+    Uninitialized,
     /// A catch-all error for all other types of errors that could occur during processing.
     Unknown,
 }
@@ -137,6 +146,11 @@ impl IntoResponse for RestError {
                 "This service is temporarily unavailable",
             )
                 .into_response(),
+            RestError::Uninitialized => (
+                StatusCode::SERVICE_UNAVAILABLE,
+                "The service is not yet initialized for this chain, please try again in a few minutes",
+            )
+                .into_response(),
             RestError::Unknown => (
                 StatusCode::INTERNAL_SERVER_ERROR,
                 "An unknown error occurred processing the request",
@@ -172,6 +186,7 @@ pub fn get_register_uri(base_uri: &str, chain_id: &str) -> Result<String> {
 
 #[cfg(test)]
 mod test {
+    use crate::api::ApiBlockChainState;
     use {
         crate::{
             api::{self, ApiState, BinaryEncoding, Blob, BlockchainState, GetRandomValueResponse},
@@ -228,10 +243,16 @@ mod test {
         };
 
         let mut chains = HashMap::new();
-        chains.insert("ethereum".into(), eth_state);
-        chains.insert("avalanche".into(), avax_state);
+        chains.insert(
+            "ethereum".into(),
+            ApiBlockChainState::Initialized(eth_state),
+        );
+        chains.insert(
+            "avalanche".into(),
+            ApiBlockChainState::Initialized(avax_state),
+        );
 
-        let api_state = ApiState::new(chains, metrics_registry).await;
+        let api_state = ApiState::new(Arc::new(RwLock::new(chains)), metrics_registry).await;
 
         let app = api::routes(api_state);
         (TestServer::new(app).unwrap(), eth_read, avax_read)

apps/fortuna/src/api.rs

@@ -15,6 +15,12 @@ responses(
 pub async fn chain_ids(
     State(state): State<crate::api::ApiState>,
 ) -> Result<Json<Vec<ChainId>>, RestError> {
-    let chain_ids = state.chains.iter().map(|(id, _)| id.clone()).collect();
+    let chain_ids = state
+        .chains
+        .read()
+        .await
+        .iter()
+        .map(|(id, _)| id.clone())
+        .collect();
     Ok(Json(chain_ids))
 }

apps/fortuna/src/api/chain_ids.rs

@@ -1,3 +1,4 @@
+use crate::api::ApiBlockChainState;
 use crate::chain::reader::BlockNumber;
 use {
     crate::api::{ChainId, RequestLabel, RestError},
@@ -46,8 +47,18 @@ pub async fn revelation(
 
     let state = state
         .chains
+        .read()
+        .await
         .get(&chain_id)
-        .ok_or(RestError::InvalidChainId)?;
+        .ok_or(RestError::InvalidChainId)?
+        .clone();
+
+    let state = match state {
+        ApiBlockChainState::Initialized(state) => state,
+        ApiBlockChainState::Uninitialized => {
+            return Err(RestError::Uninitialized);
+        }
+    };
 
     let current_block_number_fut = state
         .contract

apps/fortuna/src/api/revelation.rs

@@ -52,15 +52,16 @@ pub async fn register_provider_from_config(
 
     let commitment_length = provider_config.chain_length;
     tracing::info!("Generating hash chain");
-    let chain = PebbleHashChain::from_config(
+    let chain = PebbleHashChain::from_config_async(
         &secret,
         chain_id,
         &private_key_string.parse::<LocalWallet>()?.address(),
         &chain_config.contract_addr,
         &random,
         commitment_length,
         provider_config.chain_sample_interval,
-    )?;
+    )
+    .await?;
     tracing::info!("Done generating hash chain");
 
     // Arguments to the contract to register our new provider.