Implement the caching of the find_{keys,key_value}_by_prefix operations. (#4576)

## Motivation

The `find_keys_by_prefix` has emerged as a choke point for some
benchmarks.
Considering that it is the only part that is not cached, it is
interesting to consider this.

## Proposal

The implementation is done in the following way:
* The `find_{keys,key_values}_by_prefix` are cached.
* The `delete_prefix` leads to a `find_key_values_by_prefix` entry in
the cache.
* All the information about the operation is used to avoid making some
calls to the storage.

## Test Plan

The CI.
Some unit tests have been added to the `lru_caching.rs` file.

## Release Plan

The input file would change because some additional entries have been
added.
However, this can still be ported to existing DevNet / TestNet.

## Links

None.

Author: MathieuDutSik

CLI.md

@@ -188,12 +188,27 @@ Client implementation and command-line tool for the Linera blockchain
 * `--storage-max-cache-size <STORAGE_MAX_CACHE_SIZE>` — The maximal memory used in the storage cache
 
   Default value: `10000000`
-* `--storage-max-entry-size <STORAGE_MAX_ENTRY_SIZE>` — The maximal size of an entry in the storage cache
+* `--storage-max-value-entry-size <STORAGE_MAX_VALUE_ENTRY_SIZE>` — The maximal size of a value entry in the storage cache
+
+  Default value: `1000000`
+* `--storage-max-find-keys-entry-size <STORAGE_MAX_FIND_KEYS_ENTRY_SIZE>` — The maximal size of a find-keys entry in the storage cache
+
+  Default value: `1000000`
+* `--storage-max-find-key-values-entry-size <STORAGE_MAX_FIND_KEY_VALUES_ENTRY_SIZE>` — The maximal size of a find-key-values entry in the storage cache
 
   Default value: `1000000`
 * `--storage-max-cache-entries <STORAGE_MAX_CACHE_ENTRIES>` — The maximal number of entries in the storage cache
 
   Default value: `1000`
+* `--storage-max-cache-value-size <STORAGE_MAX_CACHE_VALUE_SIZE>` — The maximal memory used in the value cache
+
+  Default value: `10000000`
+* `--storage-max-cache-find-keys-size <STORAGE_MAX_CACHE_FIND_KEYS_SIZE>` — The maximal memory used in the find_keys_by_prefix cache
+
+  Default value: `10000000`
+* `--storage-max-cache-find-key-values-size <STORAGE_MAX_CACHE_FIND_KEY_VALUES_SIZE>` — The maximal memory used in the find_key_values_by_prefix cache
+
+  Default value: `10000000`
 * `--storage-replication-factor <STORAGE_REPLICATION_FACTOR>` — The replication factor for the keyspace
 
   Default value: `1`

linera-indexer/lib/src/rocks_db.rs

@@ -5,7 +5,7 @@ use std::path::PathBuf;
 
 use clap::Parser as _;
 use linera_views::{
-    lru_caching::StorageCacheConfig,
+    lru_prefix_cache::StorageCacheConfig,
     rocks_db::{
         PathWithGuard, RocksDbDatabase, RocksDbSpawnMode, RocksDbStoreConfig,
         RocksDbStoreInternalConfig,
@@ -36,17 +36,37 @@ pub struct RocksDbConfig {
     #[arg(long, default_value = "10")]
     pub max_stream_queries: usize,
 
-    /// The maximal memory used in the storage cache.
+    /// The maximal memory used in the storage cache in bytes.
     #[arg(long, default_value = "10000000")]
     pub max_cache_size: usize,
 
-    /// The maximal size of an entry in the storage cache.
+    /// The maximal size of a value entry in the storage cache in bytes.
     #[arg(long, default_value = "1000000")]
-    pub max_entry_size: usize,
+    pub max_value_entry_size: usize,
+
+    /// The maximal size of a find-keys entry in the storage cache in bytes.
+    #[arg(long, default_value = "1000000")]
+    pub max_find_keys_entry_size: usize,
+
+    /// The maximal size of a find-key-values entry in the storage cache in bytes.
+    #[arg(long, default_value = "1000000")]
+    pub max_find_key_values_entry_size: usize,
 
     /// The maximal number of entries in the storage cache.
     #[arg(long, default_value = "1000")]
     pub max_cache_entries: usize,
+
+    /// The maximal memory used in the value cache in bytes.
+    #[arg(long, default_value = "10000000")]
+    pub max_cache_value_size: usize,
+
+    /// The maximal memory used in the find_keys_by_prefix cache in bytes.
+    #[arg(long, default_value = "10000000")]
+    pub max_cache_find_keys_size: usize,
+
+    /// The maximal memory used in the find_key_values_by_prefix cache in bytes.
+    #[arg(long, default_value = "10000000")]
+    pub max_cache_find_key_values_size: usize,
 }
 
 pub type RocksDbRunner = Runner<RocksDbDatabase, RocksDbConfig>;
@@ -56,8 +76,13 @@ impl RocksDbRunner {
         let config = IndexerConfig::<RocksDbConfig>::parse();
         let storage_cache_config = StorageCacheConfig {
             max_cache_size: config.client.max_cache_size,
-            max_entry_size: config.client.max_entry_size,
+            max_value_entry_size: config.client.max_value_entry_size,
+            max_find_keys_entry_size: config.client.max_find_keys_entry_size,
+            max_find_key_values_entry_size: config.client.max_find_key_values_entry_size,
             max_cache_entries: config.client.max_cache_entries,
+            max_cache_value_size: config.client.max_cache_value_size,
+            max_cache_find_keys_size: config.client.max_cache_find_keys_size,
+            max_cache_find_key_values_size: config.client.max_cache_find_key_values_size,
         };
         let path_buf = config.client.storage.as_path().to_path_buf();
         let path_with_guard = PathWithGuard::new(path_buf);

linera-indexer/lib/src/scylla_db.rs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use linera_views::{
-    lru_caching::StorageCacheConfig,
+    lru_prefix_cache::StorageCacheConfig,
     scylla_db::{ScyllaDbDatabase, ScyllaDbStoreConfig, ScyllaDbStoreInternalConfig},
     store::KeyValueDatabase,
 };
@@ -30,18 +30,38 @@ pub struct ScyllaDbConfig {
     #[arg(long, default_value = "10")]
     pub max_stream_queries: usize,
 
-    /// The maximal memory used in the storage cache.
+    /// The maximal memory used in the storage cache in bytes.
     #[arg(long, default_value = "10000000")]
     pub max_cache_size: usize,
 
-    /// The maximal size of an entry in the storage cache.
+    /// The maximal size of a value entry in the storage cache in bytes.
     #[arg(long, default_value = "1000000")]
-    pub max_entry_size: usize,
+    pub max_value_entry_size: usize,
+
+    /// The maximal size of a find-keys entry in the storage cache in bytes.
+    #[arg(long, default_value = "1000000")]
+    pub max_find_keys_entry_size: usize,
+
+    /// The maximal size of a find-key-values entry in the storage cache in bytes.
+    #[arg(long, default_value = "1000000")]
+    pub max_find_key_values_entry_size: usize,
 
     /// The maximal number of entries in the storage cache.
     #[arg(long, default_value = "1000")]
     pub max_cache_entries: usize,
 
+    /// The maximal memory used in the value cache in bytes.
+    #[arg(long, default_value = "10000000")]
+    pub max_cache_value_size: usize,
+
+    /// The maximal memory used in the find_keys_by_prefix cache in bytes.
+    #[arg(long, default_value = "10000000")]
+    pub max_cache_find_keys_size: usize,
+
+    /// The maximal memory used in the find_key_values_by_prefix cache in bytes.
+    #[arg(long, default_value = "10000000")]
+    pub max_cache_find_key_values_size: usize,
+
     /// The replication factor for the keyspace
     #[arg(long, default_value = "1")]
     pub replication_factor: u32,
@@ -54,8 +74,13 @@ impl ScyllaDbRunner {
         let config = <IndexerConfig<ScyllaDbConfig> as clap::Parser>::parse();
         let storage_cache_config = StorageCacheConfig {
             max_cache_size: config.client.max_cache_size,
-            max_entry_size: config.client.max_entry_size,
+            max_value_entry_size: config.client.max_value_entry_size,
+            max_find_keys_entry_size: config.client.max_find_keys_entry_size,
+            max_find_key_values_entry_size: config.client.max_find_key_values_entry_size,
             max_cache_entries: config.client.max_cache_entries,
+            max_cache_value_size: config.client.max_cache_value_size,
+            max_cache_find_keys_size: config.client.max_cache_find_keys_size,
+            max_cache_find_key_values_size: config.client.max_cache_find_key_values_size,
         };
         let inner_config = ScyllaDbStoreInternalConfig {
             uri: config.client.uri.clone(),

linera-service/src/storage.rs

@@ -21,7 +21,7 @@ use linera_views::rocks_db::{
     RocksDbStoreInternalConfig,
 };
 use linera_views::{
-    lru_caching::StorageCacheConfig,
+    lru_prefix_cache::StorageCacheConfig,
     memory::{MemoryDatabase, MemoryStoreConfig},
     store::{KeyValueDatabase, KeyValueStore},
 };
@@ -54,14 +54,34 @@ pub struct CommonStorageOptions {
     #[arg(long, default_value = "10000000", global = true)]
     pub storage_max_cache_size: usize,
 
-    /// The maximal size of an entry in the storage cache.
+    /// The maximal size of a value entry in the storage cache.
     #[arg(long, default_value = "1000000", global = true)]
-    pub storage_max_entry_size: usize,
+    pub storage_max_value_entry_size: usize,
+
+    /// The maximal size of a find-keys entry in the storage cache.
+    #[arg(long, default_value = "1000000", global = true)]
+    pub storage_max_find_keys_entry_size: usize,
+
+    /// The maximal size of a find-key-values entry in the storage cache.
+    #[arg(long, default_value = "1000000", global = true)]
+    pub storage_max_find_key_values_entry_size: usize,
 
     /// The maximal number of entries in the storage cache.
     #[arg(long, default_value = "1000", global = true)]
     pub storage_max_cache_entries: usize,
 
+    /// The maximal memory used in the value cache.
+    #[arg(long, default_value = "10000000", global = true)]
+    pub storage_max_cache_value_size: usize,
+
+    /// The maximal memory used in the find_keys_by_prefix cache.
+    #[arg(long, default_value = "10000000", global = true)]
+    pub storage_max_cache_find_keys_size: usize,
+
+    /// The maximal memory used in the find_key_values_by_prefix cache.
+    #[arg(long, default_value = "10000000", global = true)]
+    pub storage_max_cache_find_key_values_size: usize,
+
     /// The replication factor for the keyspace
     #[arg(long, default_value = "1", global = true)]
     pub storage_replication_factor: u32,
@@ -71,8 +91,13 @@ impl CommonStorageOptions {
     pub fn storage_cache_config(&self) -> StorageCacheConfig {
         StorageCacheConfig {
             max_cache_size: self.storage_max_cache_size,
-            max_entry_size: self.storage_max_entry_size,
+            max_value_entry_size: self.storage_max_value_entry_size,
+            max_find_keys_entry_size: self.storage_max_find_keys_entry_size,
+            max_find_key_values_entry_size: self.storage_max_find_key_values_entry_size,
             max_cache_entries: self.storage_max_cache_entries,
+            max_cache_value_size: self.storage_max_cache_value_size,
+            max_cache_find_keys_size: self.storage_max_cache_find_keys_size,
+            max_cache_find_key_values_size: self.storage_max_cache_find_key_values_size,
         }
     }
 }

linera-storage-service/src/server.rs

@@ -12,7 +12,7 @@ use linera_views::{
 };
 #[cfg(with_rocksdb)]
 use linera_views::{
-    lru_caching::StorageCacheConfig,
+    lru_prefix_cache::StorageCacheConfig,
     rocks_db::{
         PathWithGuard, RocksDbDatabase, RocksDbSpawnMode, RocksDbStoreConfig,
         RocksDbStoreInternalConfig,
@@ -282,12 +282,27 @@ enum StorageServerOptions {
         /// The maximum size of the cache, in bytes (keys size + value sizes)
         #[arg(long, default_value = "10000000")]
         max_cache_size: usize,
-        /// The maximum size of an entry size, in bytes
+        /// The maximum size of a value entry, in bytes
         #[arg(long, default_value = "1000000")]
-        max_entry_size: usize,
+        max_value_entry_size: usize,
+        /// The maximum size of a find-keys entry, in bytes
+        #[arg(long, default_value = "1000000")]
+        max_find_keys_entry_size: usize,
+        /// The maximum size of a find-key-values entry, in bytes
+        #[arg(long, default_value = "1000000")]
+        max_find_key_values_entry_size: usize,
         /// The maximum number of entries in the cache.
         #[arg(long, default_value = "1000")]
         max_cache_entries: usize,
+        /// The maximum value size of the cache, in bytes
+        #[arg(long, default_value = "10000000")]
+        max_cache_value_size: usize,
+        /// The maximum find_keys_by_prefix size of the cache, in bytes
+        #[arg(long, default_value = "10000000")]
+        max_cache_find_keys_size: usize,
+        /// The maximum find_key_values_by_prefix size of the cache, in bytes
+        #[arg(long, default_value = "10000000")]
+        max_cache_find_key_values_size: usize,
     },
 }
 
@@ -647,8 +662,13 @@ async fn main() {
             path,
             max_stream_queries,
             max_cache_size,
-            max_entry_size,
+            max_value_entry_size,
+            max_find_keys_entry_size,
+            max_find_key_values_entry_size,
             max_cache_entries,
+            max_cache_value_size,
+            max_cache_find_keys_size,
+            max_cache_find_key_values_size,
         } => {
             let path_buf = path.into();
             let path_with_guard = PathWithGuard::new(path_buf);
@@ -660,8 +680,13 @@ async fn main() {
             };
             let storage_cache_config = StorageCacheConfig {
                 max_cache_size,
-                max_entry_size,
+                max_value_entry_size,
+                max_find_keys_entry_size,
+                max_find_key_values_entry_size,
                 max_cache_entries,
+                max_cache_value_size,
+                max_cache_find_keys_size,
+                max_cache_find_key_values_size,
             };
             let config = RocksDbStoreConfig {
                 inner_config,