[Backport] Add a limit for the number of received_log entries in a chain info response (#4643)

## Motivation

The huge amount of entries in the `received_log` seems to be making it
impossible to sync the old GoL chain on Testnet Conway.

## Proposal

Limit the number of returned entries in a single response. Client should
request more entries with an increased offset until the number of
returned entries is less than the limit (indicating that there are no
more entries).

## Test Plan

CI; Manual testing on the testnet will show whether this fixes the
syncing issue

## Release Plan

- These changes should be:
    - released in a new SDK,
    - released in a validator hotfix.

## Links

- [reviewer
checklist](https://github.com/linera-io/linera-protocol/blob/main/CONTRIBUTING.md#reviewer-checklist)

---------

Co-authored-by: Mathieu Baudet <1105398+ma2bd@users.noreply.github.com>

Author: bart-linera

linera-core/src/chain_worker/config.rs

@@ -7,8 +7,10 @@ use std::sync::Arc;
 
 use linera_base::{crypto::ValidatorSecretKey, time::Duration};
 
+use crate::CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES;
+
 /// Configuration parameters for the [`ChainWorkerState`][`super::state::ChainWorkerState`].
-#[derive(Clone, Default)]
+#[derive(Clone)]
 pub struct ChainWorkerConfig {
     /// The signature key pair of the validator. The key may be missing for replicas
     /// without voting rights (possibly with a partial view of chains).
@@ -27,6 +29,8 @@ pub struct ChainWorkerConfig {
     /// TTL for sender chains.
     // We don't want them to keep in memory forever since usually they're short-lived.
     pub sender_chain_ttl: Duration,
+    /// The size to truncate receive log entries in chain info responses.
+    pub chain_info_max_received_log_entries: usize,
 }
 
 impl ChainWorkerConfig {
@@ -48,3 +52,18 @@ impl ChainWorkerConfig {
         self.key_pair.as_ref().map(Arc::as_ref)
     }
 }
+
+impl Default for ChainWorkerConfig {
+    fn default() -> Self {
+        Self {
+            key_pair: None,
+            allow_inactive_chains: false,
+            allow_messages_from_deprecated_epochs: false,
+            long_lived_services: false,
+            grace_period: Default::default(),
+            ttl: Default::default(),
+            sender_chain_ttl: Default::default(),
+            chain_info_max_received_log_entries: CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES,
+        }
+    }
+}

linera-core/src/chain_worker/state.rs

@@ -1407,7 +1407,11 @@ where
         info.requested_sent_certificate_hashes = hashes;
         if let Some(start) = query.request_received_log_excluding_first_n {
             let start = usize::try_from(start).map_err(|_| ArithmeticError::Overflow)?;
-            info.requested_received_log = chain.received_log.read(start..).await?;
+            let max_received_log_entries = self.config.chain_info_max_received_log_entries;
+            let end = start
+                .saturating_add(max_received_log_entries)
+                .min(chain.received_log.count());
+            info.requested_received_log = chain.received_log.read(start..end).await?;
         }
         if query.request_manager_values {
             info.manager.add_values(&chain.manager);

linera-core/src/client/mod.rs

@@ -75,6 +75,7 @@ use crate::{
     remote_node::RemoteNode,
     updater::{communicate_with_quorum, CommunicateAction, CommunicationError, ValidatorUpdater},
     worker::{Notification, ProcessableCertificate, Reason, WorkerError, WorkerState},
+    CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES,
 };
 
 mod chain_client_state;
@@ -760,9 +761,18 @@ impl<Env: Environment> Client<Env> {
         let (max_epoch, committees) = self.admin_committees().await?;
 
         // Retrieve the list of newly received certificates from this validator.
-        let query = ChainInfoQuery::new(chain_id).with_received_log_excluding_first_n(tracker);
-        let info = remote_node.handle_chain_info_query(query).await?;
-        let remote_log = info.requested_received_log;
+        let mut remote_log = Vec::new();
+        let mut offset = tracker;
+        loop {
+            let query = ChainInfoQuery::new(chain_id).with_received_log_excluding_first_n(offset);
+            let info = remote_node.handle_chain_info_query(query).await?;
+            let received_entries = info.requested_received_log.len();
+            offset += received_entries as u64;
+            remote_log.extend(info.requested_received_log);
+            if received_entries < CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES {
+                break;
+            }
+        }
         let remote_heights = Self::heights_per_chain(&remote_log);
 
         // Obtain the next block height we need in the local node, for each chain.

linera-core/src/lib.rs

@@ -29,3 +29,7 @@ pub use crate::join_set_ext::{JoinSetExt, TaskHandle};
 
 pub mod environment;
 pub use environment::Environment;
+
+/// The maximum number of entries in a `received_log` included in a `ChainInfo` response.
+// TODO(#4638): Revisit the number.
+pub const CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES: usize = 20_000;

linera-core/src/worker.rs

@@ -43,6 +43,7 @@ use crate::{
     join_set_ext::{JoinSet, JoinSetExt},
     notifier::Notifier,
     value_cache::ValueCache,
+    CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES,
 };
 
 const BLOCK_CACHE_SIZE: usize = 5_000;
@@ -421,6 +422,26 @@ where
         self
     }
 
+    /// Returns an instance with the specified maximum size for received_log entries.
+    ///
+    /// Sizes below `CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES` should be avoided.
+    #[instrument(level = "trace", skip(self))]
+    pub fn with_chain_info_max_received_log_entries(
+        mut self,
+        chain_info_max_received_log_entries: usize,
+    ) -> Self {
+        if chain_info_max_received_log_entries < CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES {
+            warn!(
+                "The value set for the maximum size of received_log entries \
+                   may not be compatible with the latest clients: {} instead of {}",
+                chain_info_max_received_log_entries, CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES
+            );
+        }
+        self.chain_worker_config.chain_info_max_received_log_entries =
+            chain_info_max_received_log_entries;
+        self
+    }
+
     #[instrument(level = "trace", skip(self))]
     pub fn nickname(&self) -> &str {
         &self.nickname

linera-service/src/server.rs

@@ -37,7 +37,7 @@ use linera_base::{
     listen_for_shutdown_signals,
 };
 use linera_client::config::{CommitteeConfig, ValidatorConfig, ValidatorServerConfig};
-use linera_core::{worker::WorkerState, JoinSetExt as _};
+use linera_core::{worker::WorkerState, JoinSetExt as _, CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES};
 use linera_execution::{WasmRuntime, WithWasmDefault};
 #[cfg(with_metrics)]
 use linera_metrics::monitoring_server;
@@ -68,6 +68,7 @@ struct ServerContext {
     shard: Option<usize>,
     grace_period: Duration,
     chain_worker_ttl: Duration,
+    chain_info_max_received_log_entries: usize,
 }
 
 impl ServerContext {
@@ -94,7 +95,8 @@ impl ServerContext {
         .with_allow_inactive_chains(false)
         .with_allow_messages_from_deprecated_epochs(false)
         .with_grace_period(self.grace_period)
-        .with_chain_worker_ttl(self.chain_worker_ttl);
+        .with_chain_worker_ttl(self.chain_worker_ttl)
+        .with_chain_info_max_received_log_entries(self.chain_info_max_received_log_entries);
         (state, shard_id, shard.clone())
     }
 
@@ -380,6 +382,15 @@ enum ServerCommand {
             value_parser = util::parse_millis
         )]
         chain_worker_ttl: Duration,
+
+        /// Maximum size for received_log entries in chain info responses. This should
+        /// generally only be increased from the default value.
+        #[arg(
+            long,
+            default_value_t = CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES,
+            env = "LINERA_SERVER_CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES",
+        )]
+        chain_info_max_received_log_entries: usize,
     },
 
     /// Act as a trusted third-party and generate all server configurations
@@ -492,6 +503,7 @@ async fn run(options: ServerOptions) {
             grace_period,
             wasm_runtime,
             chain_worker_ttl,
+            chain_info_max_received_log_entries,
         } => {
             linera_version::VERSION_INFO.log();
 
@@ -505,6 +517,7 @@ async fn run(options: ServerOptions) {
                 shard,
                 grace_period,
                 chain_worker_ttl,
+                chain_info_max_received_log_entries,
             };
             let wasm_runtime = wasm_runtime.with_wasm_default();
             let store_config = storage_config