add stable benchmark (dfinity#92)

* add stable benchmark

* fix

* adjust to 800k

* fix

* 600k

* fix

* fix

* vec_stable

* heap_stable

* readme

Author: chenyan-dfinity

Cargo.lock

@@ -3,9 +3,12 @@ members = [
     "utils/rust",
     "collections/rust/src/hashmap",
     "collections/rust/src/btreemap",
+    "collections/rust/src/btreemap_stable",
     "collections/rust/src/heap",
+    "collections/rust/src/heap_stable",
     "collections/rust/src/imrc_hashmap",
     "collections/rust/src/vector",
+    "collections/rust/src/vector_stable",
     "crypto/rust/src/sha",
     "crypto/rust/src/certified_map",
     "dapps/rust/dip721-nft",

Cargo.toml

@@ -2,6 +2,7 @@
 
 Measure different collection libraries written in both Motoko and Rust. 
 The library names with `_rs` suffix are written in Rust; the rest are written in Motoko.
+The `_stable` and `_stable_rs` suffix represents that the library directly writes the state to stable memory using `Region` in Motoko and `ic-stable-stuctures` in Rust.
 
 We use the same random number generator with fixed seed to ensure that all collections contain
 the same elements, and the queries are exactly the same. Below we explain the measurements of each column in the table:
@@ -11,7 +12,7 @@ the same elements, and the queries are exactly the same. Below we explain the me
 * batch_get 50. Find 50 elements from the collection.
 * batch_put 50. Insert 50 elements to the collection.
 * batch_remove 50. Remove 50 elements from the collection.
-* upgrade. Upgrade the canister with the same Wasm module. The map state is persisted by serializing and deserializing states into stable memory.
+* upgrade. Upgrade the canister with the same Wasm module. For non-stable benchmarks, the map state is persisted by serializing and deserializing states into stable memory. For stable benchmarks, the upgrade takes no cycles, as the state is already in the stable memory.
 
 ## **💎 Takeaways**
 
@@ -29,6 +30,7 @@ the same elements, and the queries are exactly the same. Below we explain the me
 >   + Use stable variable directly in Motoko: `zhenya_hashmap`, `btree`, `vector`
 >   + Expose and serialize external state (`share/unshare` in Motoko, `candid::Encode` in Rust): `rbtree`, `heap`, `btreemap_rs`, `hashmap_rs`, `heap_rs`, `vector_rs`
 >   + Use pre/post-upgrade hooks to convert data into an array: `hashmap`, `splay`, `triemap`, `buffer`, `imrc_hashmap_rs`
+> * The stable benchmarks are much more expensive than their non-stable counterpart, because the stable memory API is much more expensive. The benefit is that they get zero cost upgrade.
 > * `hashmap` uses amortized data structure. When the initial capacity is reached, it has to copy the whole array, thus the cost of `batch_put 50` is much higher than other data structures.
 > * `btree` comes from [mops.one/stableheapbtreemap](https://mops.one/stableheapbtreemap).
 > * `zhenya_hashmap` comes from [mops.one/map](https://mops.one/map).

collections/README.md

@@ -13,15 +13,17 @@ let vector = wasm_profiling("motoko/.dfx/local/canisters/vector/vector.wasm", re
 
 let hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/hashmap/hashmap.wasm", record { start_page = 1 });
 let btreemap_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap/btreemap.wasm", record { start_page = 1 });
+let btreemap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/btreemap_stable/btreemap_stable.wasm", record { start_page = 1 });
 let heap_rs = wasm_profiling("rust/.dfx/local/canisters/heap/heap.wasm", record { start_page = 1 });
+let heap_stable_rs = wasm_profiling("rust/.dfx/local/canisters/heap_stable/heap_stable.wasm", record { start_page = 1 });
 let imrc_hashmap_rs = wasm_profiling("rust/.dfx/local/canisters/imrc_hashmap/imrc_hashmap.wasm", record { start_page = 1 });
 let vector_rs = wasm_profiling("rust/.dfx/local/canisters/vector/vector.wasm", record { start_page = 1 });
+let vector_stable_rs = wasm_profiling("rust/.dfx/local/canisters/vector_stable/vector_stable.wasm", record { start_page = 1 });
 
 //let movm_rs = wasm_profiling("rust/.dfx/local/canisters/movm/movm.wasm");
 //let movm_dynamic_rs = wasm_profiling("rust/.dfx/local/canisters/movm_dynamic/movm_dynamic.wasm");
 
 let file = "README.md";
-output(file, "\n## Map\n\n| |binary_size|generate 1m|max mem|batch_get 50|batch_put 50|batch_remove 50|upgrade|\n|--:|--:|--:|--:|--:|--:|--:|--:|\n");
 
 function perf(wasm, title, init, batch) {
   let cid = install(wasm, encode (), null);
@@ -60,7 +62,7 @@ function perf(wasm, title, init, batch) {
 
 let init_size = 1_000_000;
 let batch_size = 50;
-
+output(file, "\n## Map\n\n| |binary_size|generate 1m|max mem|batch_get 50|batch_put 50|batch_remove 50|upgrade|\n|--:|--:|--:|--:|--:|--:|--:|--:|\n");
 perf(hashmap, "hashmap", init_size, batch_size);
 perf(triemap, "triemap", init_size, batch_size);
 perf(rbtree, "rbtree", init_size, batch_size);
@@ -82,6 +84,15 @@ perf(buffer, "buffer", init_size, batch_size);
 perf(vector, "vector", init_size, batch_size);
 perf(vector_rs, "vec_rs", init_size, batch_size);
 
+let init_size = 50_000;
+let batch_size = 50;
+output(file, "\n## Stable structures\n\n| |binary_size|generate 50k|max mem|batch_get 50|batch_put 50|batch_remove 50|upgrade|\n|--:|--:|--:|--:|--:|--:|--:|--:|\n");
+perf(btreemap_rs, "btreemap_rs", init_size, batch_size);
+perf(btreemap_stable_rs, "btreemap_stable_rs", init_size, batch_size);
+perf(heap_rs, "heap_rs", init_size, batch_size);
+perf(heap_stable_rs, "heap_stable_rs", init_size, batch_size);
+perf(vector_rs, "vec_rs", init_size, batch_size);
+perf(vector_stable_rs, "vec_stable_rs", init_size, batch_size);
 
 /*
 let movm_size = 10000;

collections/perf.sh

@@ -11,11 +11,21 @@
       "candid": "collection.did",
       "package": "btreemap"
     },
+    "btreemap_stable": {
+      "type": "rust",
+      "candid": "collection.did",
+      "package": "btreemap_stable"
+    },
     "heap": {
       "type": "rust",
       "candid": "collection.did",
       "package": "heap"
     },
+    "heap_stable": {
+      "type": "rust",
+      "candid": "collection.did",
+      "package": "heap_stable"
+    },
     "imrc_hashmap": {
       "type": "rust",
       "candid": "collection.did",
@@ -25,6 +35,11 @@
       "type": "rust",
       "candid": "collection.did",
       "package": "vector"
+    },
+    "vector_stable": {
+      "type": "rust",
+      "candid": "collection.did",
+      "package": "vector_stable"
     }
   },
   "defaults": {

collections/rust/dfx.json

@@ -0,0 +1,15 @@
+[package]
+name = "btreemap_stable"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+candid.workspace = true
+ic-cdk.workspace = true
+ic-stable-structures.workspace = true
+utils = { path = "../../../../utils/rust" }

collections/rust/src/btreemap_stable/Cargo.toml

@@ -0,0 +1,70 @@
+use ic_stable_structures::StableBTreeMap;
+use std::cell::RefCell;
+use utils::{Memory, Random};
+
+thread_local! {
+    static MAP: RefCell<StableBTreeMap<u64, u64, Memory>> = RefCell::new(StableBTreeMap::init(utils::get_upgrade_memory()));
+    static RAND: RefCell<Random> = RefCell::new(Random::new(None, 42));
+}
+
+#[ic_cdk::init]
+fn init() {
+    utils::profiling_init();
+}
+
+#[ic_cdk::update]
+fn generate(size: u32) {
+    let rand = Random::new(Some(size), 1);
+    let iter = rand.map(|x| (x, x));
+    MAP.with(|map| {
+        let mut map = map.borrow_mut();
+        for (k, v) in iter {
+            map.insert(k, v);
+        }
+    });
+}
+
+#[ic_cdk::query]
+fn get_mem() -> (u128, u128, u128) {
+    utils::get_upgrade_mem_size()
+}
+
+#[ic_cdk::update]
+fn batch_get(n: u32) {
+    MAP.with(|map| {
+        let map = map.borrow();
+        RAND.with(|rand| {
+            let mut rand = rand.borrow_mut();
+            for _ in 0..n {
+                let k = rand.next().unwrap();
+                map.get(&k);
+            }
+        })
+    })
+}
+
+#[ic_cdk::update]
+fn batch_put(n: u32) {
+    MAP.with(|map| {
+        let mut map = map.borrow_mut();
+        RAND.with(|rand| {
+            let mut rand = rand.borrow_mut();
+            for _ in 0..n {
+                let k = rand.next().unwrap();
+                map.insert(k, k);
+            }
+        })
+    })
+}
+
+#[ic_cdk::update]
+fn batch_remove(n: u32) {
+    let mut rand = Random::new(None, 1);
+    MAP.with(|map| {
+        let mut map = map.borrow_mut();
+        for _ in 0..n {
+            let k = rand.next().unwrap();
+            map.remove(&k);
+        }
+    })
+}

collections/rust/src/btreemap_stable/src/lib.rs

@@ -11,5 +11,4 @@ crate-type = ["cdylib"]
 [dependencies]
 candid.workspace = true
 ic-cdk.workspace = true
-serde.workspace = true
 utils = { path = "../../../../utils/rust" }

collections/rust/src/heap/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "heap_stable"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+candid.workspace = true
+ic-cdk.workspace = true
+ic-stable-structures.workspace = true
+utils = { path = "../../../../utils/rust" }

collections/rust/src/heap_stable/Cargo.toml

@@ -0,0 +1,65 @@
+use ic_stable_structures::StableMinHeap;
+use std::cell::RefCell;
+use std::cmp::Reverse;
+use utils::{Memory, Random};
+
+thread_local! {
+    static MAP: RefCell<StableMinHeap<Reverse<u64>, Memory>> = RefCell::new(StableMinHeap::init(utils::get_upgrade_memory()).unwrap());
+    static RAND: RefCell<Random> = RefCell::new(Random::new(None, 42));
+}
+
+#[ic_cdk::init]
+fn init() {
+    utils::profiling_init();
+}
+
+#[ic_cdk::update]
+fn generate(size: u32) {
+    let rand = Random::new(Some(size), 1);
+    let iter = rand.map(Reverse);
+    MAP.with(|map| {
+        let mut map = map.borrow_mut();
+        for x in iter {
+            map.push(&x).unwrap();
+        }
+    });
+}
+
+#[ic_cdk::query]
+fn get_mem() -> (u128, u128, u128) {
+    utils::get_upgrade_mem_size()
+}
+
+#[ic_cdk::update]
+fn batch_get(n: u32) {
+    MAP.with(|map| {
+        let mut map = map.borrow_mut();
+        for _ in 0..n {
+            map.pop();
+        }
+    })
+}
+
+#[ic_cdk::update]
+fn batch_put(n: u32) {
+    MAP.with(|map| {
+        let mut map = map.borrow_mut();
+        RAND.with(|rand| {
+            let mut rand = rand.borrow_mut();
+            for _ in 0..n {
+                let k = rand.next().unwrap();
+                map.push(&Reverse(k)).unwrap();
+            }
+        })
+    })
+}
+
+#[ic_cdk::update]
+fn batch_remove(n: u32) {
+    MAP.with(|map| {
+        let mut map = map.borrow_mut();
+        for _ in 0..n {
+            map.pop();
+        }
+    })
+}