Implement overcollateralization (#419)

* sim-rs: add config/output for overcollateralization

* sim-rs: respect overcollateralization when building IBs

Author: SupernaviX

data/simulation/config.d.ts

@@ -75,6 +75,11 @@ export interface Config {
   "tx-generation-distribution": Distribution;
   /** Only supported by Rust simulation. */
   "tx-size-bytes-distribution": Distribution;
+  /**
+   * Distribution used to choose the "over-collateralization factor" for a transaction.
+   * 0 means the transaction is not over-collateralized, n means it has enough extra collateral to be included in n shards.
+   * Only supported by Rust simulation.  */
+  "tx-overcollateralization-factor-distribution": Distribution;
   /** Only supported by Rust simulation. */
   "tx-validation-cpu-time-ms": number;
   /** Only supported by Rust simulation. */

data/simulation/config.default.yaml

@@ -47,6 +47,9 @@ tx-size-bytes-distribution:
   distribution: log-normal
   mu: 6.833
   sigma: 1.127
+tx-overcollateralization-factor-distribution:
+  distribution: constant
+  value: 0
 tx-validation-cpu-time-ms: 1.5
 tx-max-size-bytes: 16384
 tx-conflict-fraction: 0

data/simulation/config.schema.json

@@ -273,6 +273,10 @@
       "description": "Extends Leios so that EB producers include IBs directly from previous pipelines\nwhere no certified EB was observed.\n\nOnly supported by Rust simulation.",
       "type": "boolean"
     },
+    "leios-mempool-aggressive-pruning": {
+      "description": "If true, transactions will be removed from the Leios mempool if they conflict with in-flight IBs.",
+      "type": "boolean"
+    },
     "leios-mempool-sampling-strategy": {
       "$ref": "#/definitions/MempoolSamplingStrategy",
       "description": "The strategy to use when selecting TXs from the Leios mempool."
@@ -369,6 +373,10 @@
       "properties": {},
       "type": "number"
     },
+    "tx-overcollateralization-factor-distribution": {
+      "$ref": "#/definitions/Distribution",
+      "description": "Distribution used to choose the \"over-collateralization factor\" for a transaction.\n0 means the transaction is not over-collateralized, n means it has enough extra collateral to be included in n shards.\nOnly supported by Rust simulation."
+    },
     "tx-size-bytes-distribution": {
       "$ref": "#/definitions/Distribution",
       "description": "Only supported by Rust simulation."

data/simulation/example.rust.jsonl

@@ -7,7 +7,7 @@
 {"time_s":0.0,"message":{"type":"CpuTaskScheduled","task":{"node":"node-0","index":1},"task_type":"GenIB","subtasks":1}}
 {"time_s":0.0,"message":{"type":"Cpu","task":{"node":"node-0","index":1},"node":"node-0","cpu_time_s":0.13,"task_label":"GenIB: node-0-1-0","task_type":"GenIB","id":"node-0-1-0"}}
 {"time_s":0.0,"message":{"type":"VTLotteryWon","id":"0-node-1","slot":0,"pipeline":0,"producer":"node-1"}}
-{"time_s":0.0,"message":{"type":"TXGenerated","id":"0","publisher":"node-11","size_bytes":156,"input_id":0}}
+{"time_s":0.0,"message":{"type":"TXGenerated","id":"0","publisher":"node-11","size_bytes":156,"input_id":0,"shard":0,"overcollateralization_factor":0}}
 {"time_s":0.0,"message":{"type":"VTLotteryWon","id":"0-node-3","slot":0,"pipeline":0,"producer":"node-3"}}
 {"time_s":0.0,"message":{"type":"VTLotteryWon","id":"0-node-4","slot":0,"pipeline":0,"producer":"node-4"}}
 {"time_s":0.0,"message":{"type":"EBLotteryWon","id":"0-node-4","slot":0,"pipeline":1,"producer":"node-4"}}

data/simulation/trace.rust.d.ts

@@ -57,6 +57,8 @@ interface GeneratedTransaction {
     publisher: string;
     size_bytes: number;
     input_id: number;
+    shard: number;
+    overcollateralization_factor: number;
 }
 
 interface LostTransaction {

data/simulation/trace.rust.schema.json

@@ -242,9 +242,15 @@
         "input_id": {
           "type": "number"
         },
+        "overcollateralization_factor": {
+          "type": "number"
+        },
         "publisher": {
           "type": "string"
         },
+        "shard": {
+          "type": "number"
+        },
         "size_bytes": {
           "type": "number"
         },
@@ -253,7 +259,15 @@
           "type": "string"
         }
       },
-      "required": ["id", "input_id", "publisher", "size_bytes", "type"],
+      "required": [
+        "id",
+        "input_id",
+        "overcollateralization_factor",
+        "publisher",
+        "shard",
+        "size_bytes",
+        "type"
+      ],
       "type": "object"
     },
     "GeneratedVote": {

sim-rs/sim-core/src/config.rs

@@ -75,6 +75,7 @@ pub struct RawParameters {
     // Transaction configuration
     pub tx_generation_distribution: DistributionConfig,
     pub tx_size_bytes_distribution: DistributionConfig,
+    pub tx_overcollateralization_factor_distribution: DistributionConfig,
     pub tx_validation_cpu_time_ms: f64,
     pub tx_max_size_bytes: u64,
     pub tx_conflict_fraction: Option<f64>,
@@ -408,6 +409,9 @@ impl TransactionConfig {
                 max_size: params.tx_max_size_bytes,
                 frequency_ms: params.tx_generation_distribution.into(),
                 size_bytes: params.tx_size_bytes_distribution.into(),
+                overcollateralization_factor: params
+                    .tx_overcollateralization_factor_distribution
+                    .into(),
                 conflict_fraction: params.tx_conflict_fraction.unwrap_or_default(),
                 start_time: params
                     .tx_start_time
@@ -431,6 +435,7 @@ pub(crate) struct RealTransactionConfig {
     pub max_size: u64,
     pub frequency_ms: FloatDistribution,
     pub size_bytes: FloatDistribution,
+    pub overcollateralization_factor: FloatDistribution,
     pub conflict_fraction: f64,
     pub start_time: Option<Timestamp>,
     pub stop_time: Option<Timestamp>,
@@ -453,6 +458,7 @@ impl MockTransactionConfig {
             shard: 0,
             bytes,
             input_id: id,
+            overcollateralization_factor: 0,
         }
     }
 }

sim-rs/sim-core/src/events.rs

@@ -104,6 +104,7 @@ pub enum Event {
         size_bytes: u64,
         shard: u64,
         input_id: u64,
+        overcollateralization_factor: u64,
     },
     TXSent {
         id: TransactionId,
@@ -430,6 +431,7 @@ impl EventTracker {
             size_bytes: transaction.bytes,
             shard: transaction.shard,
             input_id: transaction.input_id,
+            overcollateralization_factor: transaction.overcollateralization_factor,
         });
     }
 

sim-rs/sim-core/src/model.rs

@@ -92,6 +92,7 @@ pub struct Transaction {
     pub shard: u64,
     pub bytes: u64,
     pub input_id: u64,
+    pub overcollateralization_factor: u64,
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]

sim-rs/sim-core/src/sim/node.rs

@@ -1407,9 +1407,20 @@ impl Node {
             vec![Arc::new(tx)]
         } else {
             let ledger_state = self.resolve_ledger_state(rb_ref);
+            let ib_shards = self.sim_config.ib_shards;
+            let tx_may_use_shard = |tx: &Transaction, ib_shard: u64| {
+                for shard in tx.shard..=tx.shard + tx.overcollateralization_factor {
+                    let shard = shard % ib_shards;
+                    if shard == ib_shard {
+                        return true;
+                    }
+                }
+                false
+            };
             self.select_txs(
                 |seen| {
-                    seen.tx.shard == shard && !ledger_state.spent_inputs.contains(&seen.tx.input_id)
+                    tx_may_use_shard(&seen.tx, shard)
+                        && !ledger_state.spent_inputs.contains(&seen.tx.input_id)
                 },
                 self.sim_config.max_ib_size,
             )