Add support for conflicting transactions (#415)

* sim-rs: add input_id to traces

* sim-rs: include rb-ref on output

* sim-rs: do not create IBs with conflicting transactions

* sim-rs: do not add conflicting blocks to mempool

* sim-rs: support configuring tx-conflict-fraction

* sim-rs: support leios-aggressive-mempool-pruning setting

* sim-rs: fix intermittent test failure

Author: SupernaviX

data/simulation/config.d.ts

@@ -53,6 +53,10 @@ export interface Config {
    * The strategy to use when selecting TXs from the Leios mempool.
    */
   "leios-mempool-sampling-strategy": MempoolSamplingStrategy;
+  /**
+   * If true, transactions will be removed from the Leios mempool if they conflict with in-flight IBs.
+   */
+  "leios-mempool-aggressive-pruning": boolean;
   /**
    * Praos blockchain quality parameter.
    * This is η from the Leios paper.
@@ -75,6 +79,10 @@ export interface Config {
   "tx-validation-cpu-time-ms": number;
   /** Only supported by Rust simulation. */
   "tx-max-size-bytes": bigint;
+  /**
+   * What fraction of TXs (from 0 to 1) should introduce conflicts with transactions which were produced before?
+   * Only supported by Rust simulation.  */
+  "tx-conflict-fraction": number | null;
   /**
    * When the first transaction should appear.
    * Only supported by Rust simulation.  */

data/simulation/config.default.yaml

@@ -30,6 +30,7 @@ leios-vote-send-recv-stages: false
 leios-late-ib-inclusion: true
 leios-header-diffusion-time-ms: 1000.0
 leios-mempool-sampling-strategy: ordered-by-id
+leios-mempool-aggressive-pruning: false
 # TODO: revise default
 praos-chain-quality: 40
 praos-fallback-enabled: true
@@ -48,6 +49,7 @@ tx-size-bytes-distribution:
   sigma: 1.127
 tx-validation-cpu-time-ms: 1.5
 tx-max-size-bytes: 16384
+tx-conflict-fraction: 0
 
 ################################################################################
 # Ranking Block Configuration

data/simulation/config.schema.json

@@ -355,6 +355,10 @@
       "description": "When `true`, any delays and message sizes are calculated as if\neach block contained as much data as the expected average, rounded up.\nIn particular, for the sake of the above, we consider that:\n  - Each RB includes a certificate.\n  - Certificates contain votes from `vote-threshold` nodes.\n  - Vote bundles vote for `ceil eb-generation-probability` EBs.\n  - EBs reference `ceil (ib-generation-probability * leios-stage-length-slots)` IBs.\nOnly supported by Haskell simulation.",
       "type": "boolean"
     },
+    "tx-conflict-fraction": {
+      "description": "What fraction of TXs (from 0 to 1) should introduce conflicts with transactions which were produced before?\nOnly supported by Rust simulation.",
+      "type": "number"
+    },
     "tx-generation-distribution": {
       "$ref": "#/definitions/Distribution",
       "description": "Only supported by Rust simulation."

data/simulation/example.rust.jsonl

@@ -7,7 +7,7 @@
 {"time_s":0.0,"message":{"type":"CpuTaskScheduled","task":{"node":"node-0","index":1},"task_type":"GenIB","subtasks":1}}
 {"time_s":0.0,"message":{"type":"Cpu","task":{"node":"node-0","index":1},"node":"node-0","cpu_time_s":0.13,"task_label":"GenIB: node-0-1-0","task_type":"GenIB","id":"node-0-1-0"}}
 {"time_s":0.0,"message":{"type":"VTLotteryWon","id":"0-node-1","slot":0,"pipeline":0,"producer":"node-1"}}
-{"time_s":0.0,"message":{"type":"TXGenerated","id":"0","publisher":"node-11","size_bytes":156}}
+{"time_s":0.0,"message":{"type":"TXGenerated","id":"0","publisher":"node-11","size_bytes":156,"input_id":0}}
 {"time_s":0.0,"message":{"type":"VTLotteryWon","id":"0-node-3","slot":0,"pipeline":0,"producer":"node-3"}}
 {"time_s":0.0,"message":{"type":"VTLotteryWon","id":"0-node-4","slot":0,"pipeline":0,"producer":"node-4"}}
 {"time_s":0.0,"message":{"type":"EBLotteryWon","id":"0-node-4","slot":0,"pipeline":1,"producer":"node-4"}}

data/simulation/topology.d.ts

@@ -9,12 +9,12 @@
  */
 export interface Topology {
   nodes:
-    | {
-        [name: NodeName]: Node<Cluster>;
-      }
-    | {
-        [name: NodeName]: Node<Coord2D>;
-      };
+  | {
+    [name: NodeName]: Node<Cluster>;
+  }
+  | {
+    [name: NodeName]: Node<Coord2D>;
+  };
 }
 
 /** A node. */
@@ -23,6 +23,10 @@ export interface Node<Location> {
   "cpu-core-count"?: bigint | null;
   location: Location;
   producers: { [producer: NodeName]: LinkInfo };
+  /**
+   * What fraction of TXs (from 0 to 1) should introduce conflicts with transactions which were produced before?
+   * Only supported by Rust simulation.  */
+  "tx-conflict-fraction"?: number | null;
   /** If not null, the node will behave according to the given Behaviour.
   *
   * Only supported by Haskell simulation.

data/simulation/topology.schema.json

@@ -48,6 +48,10 @@
           "additionalProperties": false,
           "properties": {},
           "type": "number"
+        },
+        "tx-conflict-fraction": {
+          "description": "What fraction of TXs (from 0 to 1) should introduce conflicts with transactions which were produced before?\nOnly supported by Rust simulation.",
+          "type": "number"
         }
       },
       "type": "object"
@@ -87,6 +91,10 @@
           "additionalProperties": false,
           "properties": {},
           "type": "number"
+        },
+        "tx-conflict-fraction": {
+          "description": "What fraction of TXs (from 0 to 1) should introduce conflicts with transactions which were produced before?\nOnly supported by Rust simulation.",
+          "type": "number"
         }
       },
       "type": "object"

data/simulation/trace.rust.d.ts

@@ -56,6 +56,7 @@ interface GeneratedTransaction {
     id: string;
     publisher: string;
     size_bytes: number;
+    input_id: number;
 }
 
 interface LostTransaction {

data/simulation/trace.rust.schema.json

@@ -239,6 +239,9 @@
         "id": {
           "type": "string"
         },
+        "input_id": {
+          "type": "number"
+        },
         "publisher": {
           "type": "string"
         },
@@ -250,7 +253,7 @@
           "type": "string"
         }
       },
-      "required": ["id", "publisher", "size_bytes", "type"],
+      "required": ["id", "input_id", "publisher", "size_bytes", "type"],
       "type": "object"
     },
     "GeneratedVote": {

sim-rs/sim-cli/src/bin/gen-test-data/strategy/utils.rs

@@ -62,8 +62,8 @@ impl GraphBuilder {
             let loc1 = to_netsim_location(self.location_of(node));
             let loc2 = to_netsim_location(self.location_of(producer));
             latency_between_locations(loc1, loc2, 1.)
-                .unwrap()
-                .to_duration()
+                .map(|d| d.to_duration())
+                .unwrap_or_default()
                 .max(Duration::from_millis(1))
         });
         self.links.push(RawLinkConfig {
@@ -170,6 +170,7 @@ impl GraphBuilder {
                     stake: n.stake,
                     location: RawNodeLocation::Coords(n.location),
                     cpu_core_count: n.cores,
+                    tx_conflict_fraction: None,
                     producers: BTreeMap::new(),
                 };
                 (name, node)

sim-rs/sim-core/src/config.rs

@@ -68,6 +68,7 @@ pub struct RawParameters {
     pub leios_late_ib_inclusion: bool,
     pub leios_header_diffusion_time_ms: f64,
     pub leios_mempool_sampling_strategy: MempoolSamplingStrategy,
+    pub leios_mempool_aggressive_pruning: bool,
     pub praos_chain_quality: u64,
     pub praos_fallback_enabled: bool,
 
@@ -76,6 +77,7 @@ pub struct RawParameters {
     pub tx_size_bytes_distribution: DistributionConfig,
     pub tx_validation_cpu_time_ms: f64,
     pub tx_max_size_bytes: u64,
+    pub tx_conflict_fraction: Option<f64>,
     pub tx_start_time: Option<f64>,
     pub tx_stop_time: Option<f64>,
 
@@ -177,6 +179,8 @@ pub struct RawNode {
     pub location: RawNodeLocation,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub cpu_core_count: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tx_conflict_fraction: Option<f64>,
     pub producers: BTreeMap<String, RawLinkInfo>,
 }
 
@@ -254,6 +258,7 @@ impl From<RawTopology> for Topology {
                     stake: node.stake.unwrap_or_default(),
                     cpu_multiplier: 1.0,
                     cores: node.cpu_core_count,
+                    tx_conflict_fraction: node.tx_conflict_fraction,
                     consumers: vec![],
                 },
             );
@@ -403,6 +408,7 @@ impl TransactionConfig {
                 max_size: params.tx_max_size_bytes,
                 frequency_ms: params.tx_generation_distribution.into(),
                 size_bytes: params.tx_size_bytes_distribution.into(),
+                conflict_fraction: params.tx_conflict_fraction.unwrap_or_default(),
                 start_time: params
                     .tx_start_time
                     .map(|t| Timestamp::zero() + Duration::from_secs_f64(t)),
@@ -425,6 +431,7 @@ pub(crate) struct RealTransactionConfig {
     pub max_size: u64,
     pub frequency_ms: FloatDistribution,
     pub size_bytes: FloatDistribution,
+    pub conflict_fraction: f64,
     pub start_time: Option<Timestamp>,
     pub stop_time: Option<Timestamp>,
 }
@@ -437,11 +444,16 @@ pub(crate) struct MockTransactionConfig {
 }
 
 impl MockTransactionConfig {
-    pub fn next_id(&self) -> TransactionId {
+    pub fn mock_tx(&self, bytes: u64) -> Transaction {
         let id = self
             .next_id
             .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-        TransactionId::new(id)
+        Transaction {
+            id: TransactionId::new(id),
+            shard: 0,
+            bytes,
+            input_id: id,
+        }
     }
 }
 
@@ -464,6 +476,7 @@ pub struct SimConfiguration {
     pub(crate) header_diffusion_time: Duration,
     pub(crate) relay_strategy: RelayStrategy,
     pub(crate) mempool_strategy: MempoolSamplingStrategy,
+    pub(crate) mempool_aggressive_pruning: bool,
     pub(crate) praos_chain_quality: u64,
     pub(crate) block_generation_probability: f64,
     pub(crate) ib_generation_probability: f64,
@@ -509,6 +522,7 @@ impl SimConfiguration {
             header_diffusion_time: duration_ms(params.leios_header_diffusion_time_ms),
             relay_strategy: params.relay_strategy,
             mempool_strategy: params.leios_mempool_sampling_strategy,
+            mempool_aggressive_pruning: params.leios_mempool_aggressive_pruning,
             praos_chain_quality: params.praos_chain_quality,
             block_generation_probability: params.rb_generation_probability,
             ib_generation_probability: params.ib_generation_probability,
@@ -542,6 +556,7 @@ pub struct NodeConfiguration {
     pub stake: u64,
     pub cpu_multiplier: f64,
     pub cores: Option<u64>,
+    pub tx_conflict_fraction: Option<f64>,
     pub consumers: Vec<NodeId>,
 }