Gas metering (#2414)

* WIP: gas

* runtime: Add gas metering

* gas: Fix Gas::add_assign, adjust  size_of for BigInt/Decimal

* gas: Adjust costs, log gas used

* runtime: Test gas usage

* gas: Justify the value of HOST_EXPORT_GAS

* gas: Adjust cost of Ethereum calls

* gas: Remove dead trait ConstGasSizeOf

* gas: Move gas module from runtime to graph crate

This will probably be necessary for multiblockchain.

* gas: Fix `GasSizeOf` impl for `Entity`

* gas: Make MAX_GAS_PER_HANDLER configurable for debugging

* runtime: Fix `ens_name_by_hash` error message

* gas: Cost `log.log` per byte

* gas: Reduce limit to 1000 seconds

* gas: Fix rebase artifact

* gas: Update tests for gas limit change

* runtime: Fix rebase artifacts in tests

* runtime: Fix tests gas costs based on api version

* gas: gas costing for ethereum calls

* gas: Remove ambiguous impls

* instance manager: Print backtrace of start subgraph error

* runtime: Support sign extension instructions

Co-authored-by: Zac Burns <[email protected]>
Co-authored-by: Leo Yvens <[email protected]>

Author: 3 people

Cargo.lock

@@ -7,6 +7,7 @@ use crate::{
 };
 use anyhow::{Context, Error};
 use blockchain::HostFn;
+use graph::runtime::gas::Gas;
 use graph::runtime::{AscIndexId, IndexForAscTypeId};
 use graph::{
     blockchain::{self, BlockPtr, HostFnCtx},
@@ -23,6 +24,15 @@ use graph_runtime_wasm::asc_abi::class::{AscEnumArray, EthereumValueKind};
 
 use super::abi::{AscUnresolvedContractCall, AscUnresolvedContractCall_0_0_4};
 
+// Allow up to 1,000 ethereum calls. The justification is that we don't know how much Ethereum gas a
+// call takes, but we limit the maximum to 25 million. One unit of Ethereum gas is at least 100ns
+// according to these benchmarks [1], so 1000 of our gas. Assuming the worst case, an Ethereum call
+// should therefore consume 25 billion gas. This allows for 400 calls per handler with the current
+// limits.
+//
+// [1] - https://www.sciencedirect.com/science/article/abs/pii/S0166531620300900
+pub const ETHEREUM_CALL: Gas = Gas::new(25_000_000_000);
+
 pub struct RuntimeAdapter {
     pub(crate) eth_adapters: Arc<EthereumNetworkAdapters>,
     pub(crate) call_cache: Arc<dyn EthereumCallCache>,
@@ -60,6 +70,8 @@ fn ethereum_call(
     wasm_ptr: u32,
     abis: &[Arc<MappingABI>],
 ) -> Result<AscEnumArray<EthereumValueKind>, HostExportError> {
+    ctx.gas.consume_host_fn(ETHEREUM_CALL)?;
+
     // For apiVersion >= 0.0.4 the call passed from the mapping includes the
     // function signature; subgraphs using an apiVersion < 0.0.4 don't pass
     // the signature along with the call.

chain/ethereum/src/runtime/runtime_adapter.rs

@@ -223,7 +223,7 @@ where
                 Err(err) => error!(
                     err_logger,
                     "Failed to start subgraph";
-                    "error" => format!("{}", err),
+                    "error" => format!("{:#}", err),
                     "code" => LogCode::SubgraphStartFailure
                 ),
             }
@@ -879,7 +879,7 @@ async fn process_block<T: RuntimeHostBuilder<C>, C: Blockchain>(
     )
     .await
     {
-        // Triggers processed with no errors or with only determinstic errors.
+        // Triggers processed with no errors or with only deterministic errors.
         Ok(block_state) => block_state,
 
         // Some form of unknown or non-deterministic error ocurred.

core/src/subgraph/instance_manager.rs

@@ -18,7 +18,7 @@ use crate::{
     },
     data::subgraph::UnifiedMappingApiVersion,
     prelude::DataSourceContext,
-    runtime::{AscHeap, AscPtr, DeterministicHostError, HostExportError},
+    runtime::{gas::GasCounter, AscHeap, AscPtr, DeterministicHostError, HostExportError},
 };
 use crate::{
     components::{
@@ -300,6 +300,7 @@ pub struct HostFnCtx<'a> {
     pub logger: Logger,
     pub block_ptr: BlockPtr,
     pub heap: &'a mut dyn AscHeap,
+    pub gas: GasCounter,
 }
 
 /// Host fn that receives one u32 argument and returns an u32.

graph/src/blockchain/mod.rs

@@ -1,6 +1,7 @@
 use crate::{
     components::store::{DeploymentLocator, EntityType},
     prelude::{q, r, s, CacheWeight, EntityKey, QueryExecutionError},
+    runtime::gas::{Gas, GasSizeOf},
 };
 use crate::{data::subgraph::DeploymentHash, prelude::EntityChange};
 use anyhow::{anyhow, Error};
@@ -622,6 +623,12 @@ impl CacheWeight for Entity {
     }
 }
 
+impl GasSizeOf for Entity {
+    fn gas_size_of(&self) -> Gas {
+        self.0.gas_size_of()
+    }
+}
+
 /// A value that can (maybe) be converted to an `Entity`.
 pub trait TryIntoEntity {
     fn try_into_entity(self) -> Result<Entity, Error>;

graph/src/data/store/mod.rs

@@ -60,7 +60,7 @@ impl BigDecimal {
         self.0.as_bigint_and_exponent()
     }
 
-    pub(crate) fn digits(&self) -> u64 {
+    pub fn digits(&self) -> u64 {
         self.0.digits()
     }
 

graph/src/data/store/scalar.rs

@@ -0,0 +1,136 @@
+use super::{Gas, GasSizeOf};
+use std::cmp::{max, min};
+
+pub mod complexity {
+    use super::*;
+
+    // Args have additive linear complexity
+    // Eg: O(N₁+N₂)
+    pub struct Linear;
+    // Args have multiplicative complexity
+    // Eg: O(N₁*N₂)
+    pub struct Mul;
+
+    // Exponential complexity.
+    // Eg: O(N₁^N₂)
+    pub struct Exponential;
+
+    // There is only one arg and it scales linearly with it's size
+    pub struct Size;
+
+    // Complexity is captured by the lesser complexity of the two args
+    // Eg: O(min(N₁, N₂))
+    pub struct Min;
+
+    // Complexity is captured by the greater complexity of the two args
+    // Eg: O(max(N₁, N₂))
+    pub struct Max;
+
+    impl GasCombinator for Linear {
+        #[inline(always)]
+        fn combine(lhs: Gas, rhs: Gas) -> Gas {
+            lhs + rhs
+        }
+    }
+
+    impl GasCombinator for Mul {
+        #[inline(always)]
+        fn combine(lhs: Gas, rhs: Gas) -> Gas {
+            Gas(lhs.0.saturating_mul(rhs.0))
+        }
+    }
+
+    impl GasCombinator for Min {
+        #[inline(always)]
+        fn combine(lhs: Gas, rhs: Gas) -> Gas {
+            min(lhs, rhs)
+        }
+    }
+
+    impl GasCombinator for Max {
+        #[inline(always)]
+        fn combine(lhs: Gas, rhs: Gas) -> Gas {
+            max(lhs, rhs)
+        }
+    }
+
+    impl<T> GasSizeOf for Combine<T, Size>
+    where
+        T: GasSizeOf,
+    {
+        fn gas_size_of(&self) -> Gas {
+            self.0.gas_size_of()
+        }
+    }
+}
+
+pub struct Combine<Tuple, Combinator>(pub Tuple, pub Combinator);
+
+pub trait GasCombinator {
+    fn combine(lhs: Gas, rhs: Gas) -> Gas;
+}
+
+impl<T0, T1, C> GasSizeOf for Combine<(T0, T1), C>
+where
+    T0: GasSizeOf,
+    T1: GasSizeOf,
+    C: GasCombinator,
+{
+    fn gas_size_of(&self) -> Gas {
+        let (a, b) = &self.0;
+        C::combine(a.gas_size_of(), b.gas_size_of())
+    }
+
+    #[inline]
+    fn const_gas_size_of() -> Option<Gas> {
+        if let Some(t0) = T0::const_gas_size_of() {
+            if let Some(t1) = T1::const_gas_size_of() {
+                return Some(C::combine(t0, t1));
+            }
+        }
+        None
+    }
+}
+
+impl<T0, T1, T2, C> GasSizeOf for Combine<(T0, T1, T2), C>
+where
+    T0: GasSizeOf,
+    T1: GasSizeOf,
+    T2: GasSizeOf,
+    C: GasCombinator,
+{
+    fn gas_size_of(&self) -> Gas {
+        let (a, b, c) = &self.0;
+        C::combine(
+            C::combine(a.gas_size_of(), b.gas_size_of()),
+            c.gas_size_of(),
+        )
+    }
+
+    #[inline] // Const propagation to the rescue. I hope.
+    fn const_gas_size_of() -> Option<Gas> {
+        if let Some(t0) = T0::const_gas_size_of() {
+            if let Some(t1) = T1::const_gas_size_of() {
+                if let Some(t2) = T2::const_gas_size_of() {
+                    return Some(C::combine(C::combine(t0, t1), t2));
+                }
+            }
+        }
+        None
+    }
+}
+
+impl<T0> GasSizeOf for Combine<(T0, u8), complexity::Exponential>
+where
+    T0: GasSizeOf,
+{
+    fn gas_size_of(&self) -> Gas {
+        let (a, b) = &self.0;
+        Gas(a.gas_size_of().0.saturating_pow(*b as u32))
+    }
+
+    #[inline]
+    fn const_gas_size_of() -> Option<Gas> {
+        None
+    }
+}

graph/src/runtime/gas/combinators.rs

@@ -0,0 +1,91 @@
+//! Stores all the gas costs is one place so they can be compared easily.
+//! Determinism: Once deployed, none of these values can be changed without a version upgrade.
+
+use super::*;
+use lazy_static::lazy_static;
+use std::str::FromStr;
+
+/// Using 10 gas = ~1ns for WASM instructions.
+const GAS_PER_SECOND: u64 = 10_000_000_000;
+
+/// Set max gas to 1000 seconds worth of gas per handler. The intent here is to have the determinism
+/// cutoff be very high, while still allowing more reasonable timer based cutoffs. Having a unit
+/// like 10 gas for ~1ns allows us to be granular in instructions which are aggregated into metered
+/// blocks via https://docs.rs/pwasm-utils/0.16.0/pwasm_utils/fn.inject_gas_counter.html But we can
+/// still charge very high numbers for other things.
+const CONST_MAX_GAS_PER_HANDLER: u64 = 1000 * GAS_PER_SECOND;
+
+lazy_static! {
+    /// This is configurable only for debugging purposes. This value is set by the protocol,
+    /// so indexers running in the network should never set this config.
+    pub static ref MAX_GAS_PER_HANDLER: u64 = std::env::var("GRAPH_MAX_GAS_PER_HANDLER")
+        .ok()
+        .map(|s| {
+            u64::from_str(&s.replace("_", "")).unwrap_or_else(|_| {
+                panic!("GRAPH_LOAD_WINDOW_SIZE must be a number, but is `{}`", s)
+            })
+        })
+        .unwrap_or(CONST_MAX_GAS_PER_HANDLER);
+}
+
+/// Gas for instructions are aggregated into blocks, so hopefully gas calls each have relatively
+/// large gas. But in the case they don't, we don't want the overhead of calling out into a host
+/// export to be the dominant cost that causes unexpectedly high execution times.
+///
+/// This value is based on the benchmark of an empty infinite loop, which does basically nothing
+/// other than call the gas function. The benchmark result was closer to 5000 gas but use 10_000 to
+/// be conservative.
+pub const HOST_EXPORT_GAS: Gas = Gas(10_000);
+
+/// As a heuristic for the cost of host fns it makes sense to reason in terms of bandwidth and
+/// calculate the cost from there. Because we don't have benchmarks for each host fn, we go with
+/// pessimistic assumption of performance of 10 MB/s, which nonetheless allows for 10 GB to be
+/// processed through host exports by a single handler at a 1000 seconds budget.
+const DEFAULT_BYTE_PER_SECOND: u64 = 10_000_000;
+
+/// With the current parameters DEFAULT_GAS_PER_BYTE = 1_000.
+const DEFAULT_GAS_PER_BYTE: u64 = GAS_PER_SECOND / DEFAULT_BYTE_PER_SECOND;
+
+/// Base gas cost for calling any host export.
+/// Security: This must be non-zero.
+pub const DEFAULT_BASE_COST: u64 = 100_000;
+
+pub const DEFAULT_GAS_OP: GasOp = GasOp {
+    base_cost: DEFAULT_BASE_COST,
+    size_mult: DEFAULT_GAS_PER_BYTE,
+};
+
+/// Because big math has a multiplicative complexity, that can result in high sizes, so assume a
+/// bandwidth of 100 MB/s, faster than the default.
+const BIG_MATH_BYTE_PER_SECOND: u64 = 100_000_000;
+const BIG_MATH_GAS_PER_BYTE: u64 = GAS_PER_SECOND / BIG_MATH_BYTE_PER_SECOND;
+
+pub const BIG_MATH_GAS_OP: GasOp = GasOp {
+    base_cost: DEFAULT_BASE_COST,
+    size_mult: BIG_MATH_GAS_PER_BYTE,
+};
+
+// Allow up to 100,000 data sources to be created
+pub const CREATE_DATA_SOURCE: Gas = Gas(CONST_MAX_GAS_PER_HANDLER / 100_000);
+
+pub const LOG_OP: GasOp = GasOp {
+    // Allow up to 100,000 logs
+    base_cost: CONST_MAX_GAS_PER_HANDLER / 100_000,
+    size_mult: DEFAULT_GAS_PER_BYTE,
+};
+
+// Saving to the store is one of the most expensive operations.
+pub const STORE_SET: GasOp = GasOp {
+    // Allow up to 250k entities saved.
+    base_cost: CONST_MAX_GAS_PER_HANDLER / 250_000,
+    // If the size roughly corresponds to bytes, allow 1GB to be saved.
+    size_mult: CONST_MAX_GAS_PER_HANDLER / 1_000_000_000,
+};
+
+// Reading from the store is much cheaper than writing.
+pub const STORE_GET: GasOp = GasOp {
+    base_cost: CONST_MAX_GAS_PER_HANDLER / 10_000_000,
+    size_mult: CONST_MAX_GAS_PER_HANDLER / 10_000_000_000,
+};
+
+pub const STORE_REMOVE: GasOp = STORE_SET;