Retry certificate reads for up to a second before erroring. (#4903)

## Motivation

In Testnet deployment we see that block processor fails to find the
certificate in the DB.

## Proposal

Scylla is eventually consistent database and maybe processor tries to
read the cert before Scylla fully commits.

Retry certificate read for up to a second since the first try/failure.

## Test Plan

CI/manual.

## Release Plan

- These changes should be released to testnet deployment of exporter,
and
- If successful backported to `main`.
- 
## Links

- [reviewer
checklist](https://github.com/linera-io/linera-protocol/blob/main/CONTRIBUTING.md#reviewer-checklist)

---------

Signed-off-by: deuszx <[email protected]>
Co-authored-by: Andreas Fackler <[email protected]>
Co-authored-by: Andre da Silva <[email protected]>

Author: 3 people

linera-service/src/exporter/runloops/block_processor/mod.rs

@@ -2,10 +2,12 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::{
+    collections::HashMap,
     future::{Future, IntoFuture},
-    time::Duration,
+    time::{Duration, Instant},
 };
 
+use linera_base::crypto::CryptoHash;
 use linera_execution::committee::Committee;
 use linera_service::config::DestinationId;
 use linera_storage::Storage;
@@ -27,6 +29,10 @@ where
     storage: BlockProcessorStorage<T>,
     new_block_queue: NewBlockQueue,
     committee_destination_update: bool,
+    // Temporary solution.
+    // Tracks certificates that failed to be read from storage
+    // along with the time of the failure to avoid retrying for too long.
+    retried_certs: HashMap<CryptoHash, (u8, Instant)>,
 }
 
 impl<S, T> BlockProcessor<S, T>
@@ -46,6 +52,7 @@ where
             exporters_tracker,
             committee_destination_update,
             new_block_queue,
+            retried_certs: HashMap::new(),
         }
     }
 
@@ -120,6 +127,30 @@ where
                             self.new_block_queue.push_back(next_block_notification);
                         },
 
+                        Err(ExporterError::ReadCertificateError(hash)) => {
+                            match self.retried_certs.remove(&hash) {
+                                // We retry only if the time elapsed since the first attempt is
+                                // less than 1 second. The assumption is that Scylla cannot
+                                // be inconsistent for too long.
+                                Some((retries, first_attempt)) => {
+                                    let elapsed = Instant::now().duration_since(first_attempt);
+                                    if retries < 3 || elapsed < Duration::from_secs(1) {
+                                        tracing::warn!(?hash, retry=retries+1, "retrying to read certificate");
+                                        self.retried_certs.insert(hash, (retries + 1, first_attempt));
+                                        self.new_block_queue.push_back(next_block_notification);
+                                    } else {
+                                        tracing::error!(?hash, "certificate is missing from the database");
+                                        return Err(ExporterError::ReadCertificateError(hash));
+                                    }
+                                },
+                                None => {
+                                    tracing::warn!(?hash, retry=1, "retrying to read certificate");
+                                    self.retried_certs.insert(hash, (1, Instant::now()));
+                                    self.new_block_queue.push_back(next_block_notification);
+                                }
+                            }
+                        },
+
                         Err(e @ (ExporterError::UnprocessedChain
                                 | ExporterError::BadInitialization
                                 | ExporterError::ChainAlreadyExists(_))