Merge pull request #2380 from CosmWasm/co/metering-benchmark

Add loop benchmark

Author: chipshort

contracts/cyberpunk/src/contract.rs

@@ -65,13 +65,8 @@ fn execute_argon2(mem_cost: u32, time_cost: u32) -> Result<Response, ContractErr
 }
 
 fn execute_cpu_loop() -> Result<Response, ContractError> {
-    let mut counter = 0u64;
-    loop {
-        counter += 1;
-        if counter >= 9_000_000_000 {
-            counter = 0;
-        }
-    }
+    #[allow(clippy::empty_loop)]
+    loop {}
 }
 
 fn execute_storage_loop(deps: DepsMut) -> Result<Response, ContractError> {

packages/vm/benches/main.rs

@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion};
 
 use rand::Rng;
 use std::sync::Arc;
@@ -12,7 +12,7 @@ use cosmwasm_vm::testing::{
 };
 use cosmwasm_vm::{
     call_execute, call_instantiate, capabilities_from_csv, Cache, CacheOptions, Instance,
-    InstanceOptions, Size,
+    InstanceOptions, Size, VmError,
 };
 
 // Instance
@@ -22,6 +22,7 @@ const DEFAULT_INSTANCE_OPTIONS: InstanceOptions = InstanceOptions {
     gas_limit: DEFAULT_GAS_LIMIT,
 };
 const HIGH_GAS_LIMIT: u64 = 20_000_000_000_000; // ~20s, allows many calls on one instance
+const MEDIUM_GAS_LIMIT: u64 = 1_000_000_000_000; // ~1s
 
 // Cache
 const MEMORY_CACHE_SIZE: Size = Size::mebi(200);
@@ -133,6 +134,48 @@ fn bench_instance(c: &mut Criterion) {
         println!("Gas used: {gas_used}");
     });
 
+    group.bench_function("execute execute (infinite loop)", |b| {
+        let backend = mock_backend(&[]);
+        let medium_gas: InstanceOptions = InstanceOptions {
+            gas_limit: MEDIUM_GAS_LIMIT,
+        };
+        let mut instance =
+            Instance::from_code(CYBERPUNK, backend, medium_gas, Some(DEFAULT_MEMORY_LIMIT))
+                .unwrap();
+
+        let info = mock_info("creator", &coins(1000, "earth"));
+        let contract_result =
+            call_instantiate::<_, _, _, Empty>(&mut instance, &mock_env(), &info, b"{}").unwrap();
+        assert!(contract_result.into_result().is_ok());
+
+        let mut gas_used = 0;
+        b.iter_batched(
+            || {
+                // setup new instance for each iteration because cpu loop will consume all gas
+                Instance::from_code(
+                    CYBERPUNK,
+                    mock_backend(&[]),
+                    medium_gas,
+                    Some(DEFAULT_MEMORY_LIMIT),
+                )
+                .unwrap()
+            },
+            |mut instance| {
+                let gas_before = instance.get_gas_left();
+                let info = mock_info("hasher", &[]);
+                let msg = br#"{"cpu_loop":{}}"#;
+
+                let vm_result =
+                    call_execute::<_, _, _, Empty>(&mut instance, &mock_env(), &info, msg);
+
+                assert!(matches!(vm_result, Err(VmError::GasDepletion { .. })));
+                gas_used = gas_before - instance.get_gas_left();
+            },
+            BatchSize::SmallInput,
+        );
+        println!("Gas used: {gas_used}");
+    });
+
     group.finish();
 }
 

packages/vm/src/wasm_backend/engine.rs

@@ -17,6 +17,8 @@ use super::limiting_tunables::LimitingTunables;
 /// https://github.com/WebAssembly/memory64/blob/master/proposals/memory64/Overview.md
 const MAX_WASM_PAGES: u32 = 65536;
 
+// This function is hashed and put into the `module_version_discriminator` because it is used as
+// part of the compilation process. If it changes, modules need to be recompiled.
 #[hash_function(const_name = "COST_FUNCTION_HASH")]
 fn cost(operator: &Operator) -> u64 {
     // A flat fee for each operation
@@ -25,13 +27,18 @@ fn cost(operator: &Operator) -> u64 {
     // In https://github.com/CosmWasm/cosmwasm/pull/1042 a profiler is developed to
     // identify runtime differences between different Wasm operation, but this is not yet
     // precise enough to derive insights from it.
-    //
-    // Please note that any changes to this function need to be accompanied by a bump of
-    // `MODULE_SERIALIZATION_VERSION` to avoid cached modules from using different amounts of gas
-    // compared to newly compiled ones.
     const GAS_PER_OPERATION: u64 = 115;
 
     if is_accounting(operator) {
+        // Accounting operators are operators where the `Metering` middleware injects instructions
+        // to count the gas usage and check for gas exhaustion. Therefore they are more expensive.
+        //
+        // Benchmarks show that the overhead is about 14 times the cost of a normal operation.
+        // To benchmark this, set `GAS_PER_OPERATION = 100` and run the "infinite loop" and
+        // "argon2" benchmarks. From the "Gas used" output, you can calculate the number of
+        // operations and from that together with the run time the expected gas value per operation:
+        // GAS_PER_OP = GAS_TARGET_PER_SEC / (NUM_OPS / RUNTIME_IN_SECS)
+        // This is repeated with different multipliers to bring the two benchmarks closer together.
         GAS_PER_OPERATION * 14
     } else {
         GAS_PER_OPERATION