feat(dursto): transactions from serialisation

Changes the database benchmark to choose the transactions for the
benchmark according to a file parsed at build-time.

Author: kurtisc

Cargo.lock

@@ -17,6 +17,8 @@ hex.workspace = true
 octez-riscv-data.workspace = true
 rand.workspace = true
 rocksdb.workspace = true
+serde = { workspace = true, features = ["derive"] }
+serde_json.workspace = true
 tempfile.workspace = true
 thiserror.workspace = true
 tokio.workspace = true

durable-storage/Cargo.toml

@@ -15,6 +15,7 @@ use octez_riscv_durable_storage::random::generate_random_bytes;
 use octez_riscv_durable_storage::repo::DirectoryManager;
 use rand::prelude::*;
 use rand::rng;
+use serde::Deserialize;
 use tokio::runtime::Handle;
 
 #[cfg_attr(
@@ -28,7 +29,7 @@ struct BenchmarkState<'a> {
     database: Database,
     operations: Vec<Operation>,
     prepopulated_node_keys: Vec<Key>,
-    prepopulated_values: Vec<Bytes>,
+    random_data: Vec<u8>,
     read_buffer: Vec<u8>,
     handle: &'a Handle,
     repo: &'a DirectoryManager,
@@ -44,42 +45,95 @@ impl<'a> BenchmarkState<'a> {
             operations: self.operations.clone(),
             read_buffer: vec![0u8; LARGE_READ_MAX_SIZE],
             prepopulated_node_keys: self.prepopulated_node_keys.clone(),
-            prepopulated_values: self.prepopulated_values.clone(),
+            random_data: self.random_data.clone(),
             handle,
             repo,
         }
     }
 }
 
 #[derive(Clone)]
-#[expect(dead_code, reason = "Some operations aren't benchmarked yet")]
 enum Operation {
     Clone,
-    Delete { key_index: usize },
-    Exists { key_index: usize },
+    Delete { key: Key },
+    Exists { key: Key },
     Hash,
-    Read { key_index: usize, size: usize },
-    ValueLength { key_index: usize },
-    Write { key_index: usize, data_index: usize },
+    Read { key: Key, size: usize },
+    ValueLength { key: Key },
+    Write { key: Key, size: usize },
+}
+
+impl From<SerialisedOperation> for Operation {
+    fn from(serialised_operation: SerialisedOperation) -> Self {
+        match serialised_operation {
+            SerialisedOperation::Copy { .. } => Operation::Clone,
+            SerialisedOperation::Delete { path } => Operation::Delete {
+                key: Key::new(path.as_bytes()).expect("The path should be a valid key"),
+            },
+            SerialisedOperation::Has { path } => Operation::Exists {
+                key: Key::new(path.as_bytes()).expect("The path should be a valid key"),
+            },
+            SerialisedOperation::Read { path, size }
+            | SerialisedOperation::ReadAll { path, size }
+            | SerialisedOperation::ReadSlice { path, size } => Operation::Read {
+                key: Key::new(path.as_bytes()).expect("The path should be a valid key"),
+                size,
+            },
+            SerialisedOperation::ValueSize { path } => Operation::ValueLength {
+                key: Key::new(path.as_bytes()).expect("The path should be a valid key"),
+            },
+            SerialisedOperation::Write { path, size }
+            | SerialisedOperation::WriteAll { path, size } => Operation::Write {
+                key: Key::new(path.as_bytes()).expect("The path should be a valid key"),
+                size,
+            },
+        }
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(tag = "operation", rename_all = "snake_case")]
+enum SerialisedOperation {
+    #[serde(rename = "store_copy")]
+    #[expect(dead_code, reason = "The result of the copy is ignored")]
+    Copy { from_path: String, to_path: String },
+    #[serde(rename = "store_delete")]
+    Delete { path: String },
+    #[serde(rename = "store_has")]
+    Has { path: String },
+    #[serde(rename = "store_read")]
+    Read { path: String, size: usize },
+    #[serde(rename = "store_read_all")]
+    ReadAll { path: String, size: usize },
+    #[serde(rename = "store_read_slice")]
+    ReadSlice { path: String, size: usize },
+    #[serde(rename = "store_value_size")]
+    ValueSize { path: String },
+    #[serde(rename = "store_write")]
+    Write { path: String, size: usize },
+    #[serde(rename = "store_write_all")]
+    WriteAll { path: String, size: usize },
+}
+
+#[derive(Deserialize)]
+struct StoreAccesses {
+    setup: Vec<SerialisedOperation>,
+    transaction: Vec<SerialisedOperation>,
+    block_creation: Vec<SerialisedOperation>,
 }
 
 const BLOCK_FREQUENCY: usize = 5_000;
 const ERC_20_TRANSACTIONS: usize = 10_000;
 const LARGE_READ_MIN_SIZE: usize = 96000;
 const LARGE_READ_MAX_SIZE: usize = LARGE_READ_MIN_SIZE * 2;
-const PREPOPULATED_LARGE_NODE_KEYS_COUNT: usize = 100;
-const PREPOPULATED_NODE_KEYS_COUNT: usize = 10_000_000 - PREPOPULATED_LARGE_NODE_KEYS_COUNT;
-const SMALL_READS: usize = 13;
-const SMALL_READ_SIZE: usize = 32;
-const SMALL_WRITES: usize = 7;
+const PREPOPULATED_NODE_KEYS_COUNT: usize = 10_000_000;
 
 fn setup_benchmark_state<'a>(handle: &'a Handle, repo: &'a DirectoryManager) -> BenchmarkState<'a> {
     let mut database = Database::try_new(handle, repo).expect("Creating a database should succeed");
     let mut rng = rng();
 
     const VALUES_COUNT: usize = 100;
     let mut prepopulated_node_keys = Vec::with_capacity(PREPOPULATED_NODE_KEYS_COUNT);
-    let mut prepopulated_large_node_keys = Vec::with_capacity(PREPOPULATED_LARGE_NODE_KEYS_COUNT);
     let mut prepopulated_values = Vec::with_capacity(VALUES_COUNT);
 
     // Pre-populate a small number of values of random small sizes
@@ -98,73 +152,57 @@ fn setup_benchmark_state<'a>(handle: &'a Handle, repo: &'a DirectoryManager) ->
         prepopulated_node_keys.push(key);
     }
 
-    // Pre-populate the database with a small number of random keys with large values
-    let keys = generate_keys(&mut rng, PREPOPULATED_LARGE_NODE_KEYS_COUNT);
-    for key in keys {
-        let value = Bytes::from(generate_random_bytes(
-            &mut rng,
-            LARGE_READ_MIN_SIZE..LARGE_READ_MAX_SIZE,
-        ));
-        database.write(key.clone(), 0, value).ok();
-        prepopulated_large_node_keys.push(key);
-    }
+    let store_accesses_data = include_str!("store_accesses.json");
+    let store_accesses: StoreAccesses = serde_json::from_str(store_accesses_data).expect(
+        "The benchmark data should be a valid serialisation of the store accesses representation",
+    );
 
-    let mut operations =
-        Vec::<Operation>::with_capacity(ERC_20_TRANSACTIONS * (1 + SMALL_READS + SMALL_WRITES));
+    // Convert the serialised form into a sequence of transactions
+    let erc_20_transaction: Vec<Operation> = store_accesses
+        .setup
+        .into_iter()
+        .chain(
+            store_accesses
+                .transaction
+                .into_iter()
+                .chain(store_accesses.block_creation),
+        )
+        .map(Operation::from)
+        .collect();
 
-    for i in 0..ERC_20_TRANSACTIONS {
-        let key_index = rng.random_range(0..prepopulated_large_node_keys.len());
-        let size = rng.random_range(LARGE_READ_MIN_SIZE..LARGE_READ_MAX_SIZE);
-        let operation = Operation::Read { key_index, size };
-
-        let mut vec = vec![0u8; size];
-        rng.fill(vec.as_mut_slice());
-
-        // Write to the node so there's something meaningful and large to read
-        database
-            .write(
-                prepopulated_large_node_keys[key_index].clone(),
-                0,
-                Bytes::from(vec),
-            )
-            .expect("The write should succeed");
-        operations.push(operation);
-
-        for _ in 0..SMALL_READS {
-            let key_index = rng.random_range(0..prepopulated_large_node_keys.len());
-            let size = SMALL_READ_SIZE;
-            operations.push(Operation::Read { key_index, size });
-
-            let mut vec = vec![0u8; size];
-            rng.fill(vec.as_mut_slice());
-
-            // Write to the node so there's something meaningful to read
+    // Populate nodes for read operations where keys are expected to exist
+    for operation in &erc_20_transaction {
+        if let Operation::Read { key, size: _ } = operation {
             database
                 .write(
-                    prepopulated_node_keys[key_index].clone(),
+                    key.clone(),
                     0,
-                    Bytes::from(vec),
+                    Bytes::from(generate_random_bytes(&mut rng, 1..32)),
                 )
                 .expect("The write should succeed");
         }
+    }
 
-        for _ in 0..SMALL_WRITES {
-            operations.push(Operation::Write {
-                key_index: rng.random_range(0..prepopulated_node_keys.len()),
-                data_index: rng.random_range(0..prepopulated_values.len()),
-            });
-        }
-
+    // Repeat the operations in a transaction for the number of transactions that are being
+    // benchmarked
+    let mut operations = Vec::with_capacity(
+        erc_20_transaction.len() * ERC_20_TRANSACTIONS + ERC_20_TRANSACTIONS / BLOCK_FREQUENCY,
+    );
+    for i in 0..ERC_20_TRANSACTIONS {
+        operations.extend_from_slice(&erc_20_transaction);
         if i % BLOCK_FREQUENCY == 0 {
             operations.push(Operation::Hash);
         }
     }
 
+    let mut random_data = vec![0; LARGE_READ_MAX_SIZE];
+    rng.fill(random_data.as_mut_slice());
+
     BenchmarkState {
         database,
         operations,
         prepopulated_node_keys,
-        prepopulated_values,
+        random_data,
         read_buffer: vec![0u8; LARGE_READ_MAX_SIZE],
         handle,
         repo,
@@ -173,8 +211,8 @@ fn setup_benchmark_state<'a>(handle: &'a Handle, repo: &'a DirectoryManager) ->
 
 #[inline(never)]
 fn bench_run(mut state: BenchmarkState) {
-    // `KeyNotFound` errors are allowed, since the (conceptual, not std::hint) black box that is the transaction inputs may delete keys
-    // which are later used.
+    // `KeyNotFound` errors are allowed, since arbitrary operations may delete keys which are later
+    // used.
     for operation in state.operations {
         match operation {
             Operation::Clone => {
@@ -188,50 +226,40 @@ fn bench_run(mut state: BenchmarkState) {
                         .expect("The clone should succeed"),
                 );
             }
-            Operation::Delete { key_index } => match state
-                .database
-                .delete(state.prepopulated_node_keys[key_index].clone())
-            {
+            Operation::Delete { key } => match state.database.delete(key) {
                 Ok(_)
                 | Err(DatabaseError::PersistenceLayer(PersistenceLayerError::KeyNotFound)) => {}
                 Err(e) => panic!("The deletion should succeed: {e:?}"),
             },
-            Operation::Exists { key_index } => {
+            Operation::Exists { key } => {
                 state
                     .database
-                    .exists(&state.prepopulated_node_keys[key_index])
+                    .exists(&key)
                     .expect("The existence check should succeed");
             }
             Operation::Hash => {
                 black_box(state.database.hash());
             }
-            Operation::Read { key_index, size } => {
-                match state.database.read(
-                    &state.prepopulated_node_keys[key_index],
-                    0,
-                    &mut state.read_buffer[0..size],
-                ) {
+            Operation::Read { key, size } => {
+                match state
+                    .database
+                    .read(&key, 0, &mut state.read_buffer[0..size])
+                {
                     Ok(_)
                     | Err(DatabaseError::PersistenceLayer(PersistenceLayerError::KeyNotFound)) => {}
                     Err(e) => panic!("The read should succeed: {e:?}"),
                 }
             }
-            Operation::ValueLength { key_index } => match state
-                .database
-                .value_length(&state.prepopulated_node_keys[key_index])
-            {
+            Operation::ValueLength { key } => match state.database.value_length(&key) {
                 Ok(_)
                 | Err(DatabaseError::PersistenceLayer(PersistenceLayerError::KeyNotFound)) => {}
                 Err(e) => panic!("The value length calculation should succeed: {e:?}"),
             },
-            Operation::Write {
-                key_index,
-                data_index,
-            } => {
+            Operation::Write { key, size } => {
                 match state.database.write(
-                    state.prepopulated_node_keys[key_index].clone(),
+                    key,
                     0,
-                    state.prepopulated_values[data_index].clone(),
+                    Bytes::copy_from_slice(&state.random_data[0..size]),
                 ) {
                     Ok(_)
                     | Err(DatabaseError::PersistenceLayer(PersistenceLayerError::KeyNotFound)) => {}