Validator destination (#3896)

## Motivation

Essential work to close #3666.
Also closes #3667.

## Proposal
- Caching for the recently used blocks and other things from the
storage.
- Dependency resolution with a topological sort for the whole state.
- Periodic routine persistence to the shared storage.
- Parallelization of the exporters across threads.
- Refactoring of the binary crate.
- Validator destination.


## Test Plan
- Unit tests for the block_processor.rs module.
- For the validator destination, tests will be included in a separate
PR, as part of the integration tests.

## Release Plan

- Nothing to do / These changes follow the usual release cycle.

Author: usagi32

Cargo.lock

@@ -148,6 +148,7 @@ kube = "0.88.1"
 linked-hash-map = "0.5.6"
 log = "0.4.21"
 lru = "0.12.3"
+mini-moka = "0.10.3"
 nonzero_lit = "0.1.2"
 num-bigint = "0.4.3"
 num-format = "0.4.4"
@@ -163,6 +164,7 @@ prometheus = "0.13.3"
 prometheus-parse = "0.2.5"
 proptest = { version = "1.6.0", default-features = false, features = ["alloc"] }
 prost = "0.13.2"
+quick_cache = "0.6.13"
 quote = "1.0"
 rand = { version = "0.8.5", default-features = false }
 rand_chacha = { version = "0.3.1", default-features = false }

Cargo.toml

@@ -19,7 +19,7 @@ use linera_execution::{
 };
 use linera_persistent as persistent;
 use linera_rpc::config::{
-    ExporterServiceConfig, ValidatorInternalNetworkConfig, ValidatorPublicNetworkConfig,
+    ExporterServiceConfig, TlsConfig, ValidatorInternalNetworkConfig, ValidatorPublicNetworkConfig,
 };
 use linera_storage::Storage;
 use serde::{Deserialize, Serialize};
@@ -208,15 +208,20 @@ impl GenesisConfig {
 /// The configuration file for the linera-exporter.
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct BlockExporterConfig {
+    /// Identity for the block exporter state.
+    pub id: u32,
+
     /// The server configuration for the linera-exporter.
     pub service_config: ExporterServiceConfig,
 
     /// The configuration file for the export destinations.
     #[serde(default)]
     pub destination_config: DestinationConfig,
 
-    /// Identity for the block exporter state.
-    pub id: u32,
+    /// The configuration file to impose various limits
+    /// on the resources used by the linera-exporter.
+    #[serde(default)]
+    pub limits: LimitsConfig,
 }
 
 /// Configuration file for the exports.
@@ -230,11 +235,60 @@ pub struct DestinationConfig {
 pub type DestinationId = u16;
 
 /// The uri to provide export services to.
-#[allow(dead_code)]
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Destination {
+    /// The gRPC network protocol.
+    pub tls: TlsConfig,
     /// The host name of the target destination (IP or hostname).
     pub endpoint: String,
     /// The port number of the target destination.
     pub port: u16,
 }
+
+impl Destination {
+    pub fn address(&self) -> String {
+        let tls = match self.tls {
+            TlsConfig::ClearText => "http",
+            TlsConfig::Tls => "https",
+        };
+
+        format!("{}://{}:{}", tls, self.endpoint, self.port)
+    }
+}
+
+/// The configuration file to impose various limits
+/// on the resources used by the linera-exporter.
+#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
+pub struct LimitsConfig {
+    /// Time period in milliseconds between periodic persistence
+    /// to the shared storage.
+    pub persistence_period_ms: u32,
+    /// Maximum size of the work queue i.e. maximum number
+    /// of blocks queued up for exports per destination.
+    pub work_queue_size: u16,
+    /// Maximum weight of the blob cache in megabytes.
+    pub blob_cache_weight_mb: u16,
+    /// Estimated number of elements for the blob cache.
+    pub blob_cache_items_capacity: u16,
+    /// Maximum weight of the block cache in megabytes.
+    pub block_cache_weight_mb: u16,
+    /// Estimated number of elements for the block cache.
+    pub block_cache_items_capacity: u16,
+    /// Maximum weight in megabytes for the combined
+    /// cache, consisting of small miscellaneous items.
+    pub auxiliary_cache_size_mb: u16,
+}
+
+impl Default for LimitsConfig {
+    fn default() -> Self {
+        Self {
+            persistence_period_ms: 299 * 1000,
+            work_queue_size: 256,
+            blob_cache_weight_mb: 1024,
+            blob_cache_items_capacity: 8192,
+            block_cache_weight_mb: 1024,
+            block_cache_items_capacity: 8192,
+            auxiliary_cache_size_mb: 1024,
+        }
+    }
+}

linera-client/src/config.rs

@@ -72,6 +72,8 @@ colored.workspace = true
 comfy-table.workspace = true
 convert_case.workspace = true
 current_platform = "0.2.0"
+custom_debug_derive.workspace = true
+dashmap.workspace = true
 dirs.workspace = true
 fs-err = { workspace = true, features = ["tokio"] }
 fs_extra = { workspace = true, optional = true }
@@ -94,10 +96,12 @@ linera-storage.workspace = true
 linera-storage-service = { workspace = true, optional = true }
 linera-version.workspace = true
 linera-views.workspace = true
+mini-moka.workspace = true
 pathdiff = { workspace = true, optional = true }
 port-selector.workspace = true
 prometheus = { workspace = true, optional = true }
 prost = { workspace = true }
+quick_cache.workspace = true
 rand.workspace = true
 reqwest = { workspace = true, features = ["json"] }
 serde.workspace = true
@@ -177,7 +181,7 @@ required-features = ["benchmark"]
 
 [[bin]]
 name = "linera-exporter"
-path = "src/linera-exporter/main.rs"
+path = "src/exporter/main.rs"
 
 [[bench]]
 name = "transfers"

linera-service/Cargo.toml

@@ -622,7 +622,7 @@ impl LocalNet {
         let child = self
             .command_for_binary("linera-exporter")
             .await?
-            .arg(config_path)
+            .args(["--config_path", &config_path])
             .args(["--storage", &storage.to_string()])
             .spawn_into()?;