[antithesis] randomize checkpoint split limit for reachability assert

crates/sui-core/src/authority/authority_per_epoch_store.rs

@@ -1111,7 +1111,9 @@ impl AuthorityPerEpochStore {
             chain_from_id, chain.1
         );
 
-        let protocol_config = ProtocolConfig::get_for_version(protocol_version, chain.1);
+        let mut protocol_config = ProtocolConfig::get_for_version(protocol_version, chain.1);
+        protocol_config
+            .apply_seeded_test_overrides(epoch_start_configuration.epoch_digest().inner());
 
         let execution_component = ExecutionComponents::new(
             &protocol_config,

crates/sui-core/src/authority/snapshots/sui_core__authority__execution_time_estimator__tests__execution_time_observer_v113.snap

@@ -0,0 +1,157 @@
+---
+source: crates/sui-core/src/authority/execution_time_estimator.rs
+expression: snapshot_data
+---
+protocol_version: 113
+consensus_observations:
+  - - MakeMoveVec
+    - observations:
+        - - 2
+          - secs: 0
+            nanos: 27000000
+        - - 7
+          - secs: 0
+            nanos: 18000000
+        - - 10
+          - secs: 0
+            nanos: 20000000
+        - - 1
+          - secs: 0
+            nanos: 25000000
+      stake_weighted_median:
+        secs: 0
+        nanos: 25000000
+  - - MergeCoins
+    - observations:
+        - - 9
+          - secs: 0
+            nanos: 64000000
+        - - 2
+          - secs: 0
+            nanos: 62000000
+        - - 2
+          - secs: 0
+            nanos: 32000000
+        - - 7
+          - secs: 0
+            nanos: 0
+      stake_weighted_median:
+        secs: 0
+        nanos: 62000000
+  - - SplitCoins
+    - observations:
+        - - 6
+          - secs: 0
+            nanos: 42000000
+        - - 3
+          - secs: 0
+            nanos: 25000000
+        - - 8
+          - secs: 0
+            nanos: 67000000
+        - - 3
+          - secs: 0
+            nanos: 61000000
+      stake_weighted_median:
+        secs: 0
+        nanos: 61000000
+  - - TransferObjects
+    - observations:
+        - - 1
+          - secs: 0
+            nanos: 13000000
+        - - 10
+          - secs: 0
+            nanos: 80000000
+        - - 0
+          - ~
+        - - 9
+          - secs: 0
+            nanos: 39000000
+      stake_weighted_median:
+        secs: 0
+        nanos: 39000000
+  - - Upgrade
+    - observations:
+        - - 0
+          - ~
+        - - 8
+          - secs: 0
+            nanos: 389000000
+        - - 10
+          - secs: 0
+            nanos: 274000000
+        - - 4
+          - secs: 0
+            nanos: 587000000
+      stake_weighted_median:
+        secs: 0
+        nanos: 389000000
+  - - MoveEntryPoint:
+        package: "0x0000000000000000000000000000000000000000000000000000000000000001"
+        module: coin
+        function: transfer
+        type_arguments: []
+    - observations:
+        - - 7
+          - secs: 0
+            nanos: 404000000
+        - - 6
+          - secs: 0
+            nanos: 162000000
+        - - 9
+          - secs: 0
+            nanos: 268000000
+        - - 3
+          - secs: 0
+            nanos: 254000000
+      stake_weighted_median:
+        secs: 0
+        nanos: 268000000
+  - - MoveEntryPoint:
+        package: "0x0000000000000000000000000000000000000000000000000000000000000002"
+        module: nft
+        function: mint
+        type_arguments: []
+    - observations:
+        - - 0
+          - ~
+        - - 0
+          - ~
+        - - 2
+          - secs: 0
+            nanos: 142000000
+        - - 8
+          - secs: 0
+            nanos: 499000000
+      stake_weighted_median:
+        secs: 0
+        nanos: 499000000
+transaction_estimates:
+  - - coin_transfer_call
+    - secs: 0
+      nanos: 268000000
+  - - mixed_move_calls
+    - secs: 0
+      nanos: 767000000
+  - - native_commands_with_observations
+    - secs: 0
+      nanos: 374000000
+  - - transfer_objects_2_items
+    - secs: 0
+      nanos: 117000000
+  - - split_coins_3_amounts
+    - secs: 0
+      nanos: 244000000
+  - - merge_coins_3_sources
+    - secs: 0
+      nanos: 248000000
+  - - make_move_vec_4_elements
+    - secs: 0
+      nanos: 125000000
+  - - mixed_commands
+    - secs: 0
+      nanos: 240000000
+  - - upgrade_package
+    - secs: 0
+      nanos: 389000000

crates/sui-core/src/consensus_handler.rs

@@ -659,7 +659,9 @@ impl CheckpointQueue {
             let next_tx_count = queued.roots.tx_roots.len();
 
             if current_tx_count + next_tx_count > max_tx && !current_roots.is_empty() {
-                assert_reachable!("checkpoint split due to transaction limit");
+                assert_reachable!(
+                    "checkpoint flush split due to batched roots exceeding transaction limit"
+                );
                 let details = current_details.unwrap();
                 pending_checkpoints.push(PendingCheckpointV2 {
                     roots: std::mem::take(&mut current_roots),
@@ -1739,7 +1741,17 @@ impl<C: CheckpointServiceNotify + Send + Sync> ConsensusHandler<C> {
                     .collect()
             };
 
+        let num_schedulables = schedulables.len();
         let chunked_schedulables = build_chunks(schedulables, &mut checkpoint_queue);
+        if chunked_schedulables.len() > 1 {
+            info!(
+                "Splitting transactions into {} checkpoint chunks (num_schedulables={}, max_tx={})",
+                chunked_schedulables.len(),
+                num_schedulables,
+                max_transactions_per_checkpoint
+            );
+            assert_reachable!("checkpoint split due to transaction limit");
+        }
         let chunked_randomness_schedulables = if should_write_random_checkpoint {
             build_chunks(randomness_schedulables, &mut checkpoint_queue)
         } else {

crates/sui-open-rpc/tests/snapshots/generate_spec__openrpc.snap.json

@@ -1294,7 +1294,7 @@
             "name": "Result",
             "value": {
               "minSupportedProtocolVersion": "1",
-              "maxSupportedProtocolVersion": "112",
+              "maxSupportedProtocolVersion": "113",
               "protocolVersion": "6",
               "featureFlags": {
                 "abstract_size_in_object_runtime": false,
@@ -1406,6 +1406,7 @@
                 "prepend_prologue_tx_in_consensus_commit_in_checkpoints": false,
                 "private_generics_verifier_v2": false,
                 "random_beacon": false,
+                "randomize_checkpoint_tx_limit_in_tests": false,
                 "receive_objects": false,
                 "recompute_has_public_transfer_in_execution": false,
                 "record_additional_state_digest_in_prologue": false,

crates/sui-protocol-config/Cargo.toml

@@ -22,6 +22,7 @@ fastcrypto = { workspace = true }
 move-core-types.workspace = true
 move-binary-format.workspace = true
 mysten-common.workspace = true
+rand.workspace = true
 
 [dev-dependencies]
 insta.workspace = true

crates/sui-protocol-config/src/lib.rs

@@ -24,7 +24,7 @@ use tracing::{info, warn};
 
 /// The minimum and maximum protocol versions supported by this build.
 const MIN_PROTOCOL_VERSION: u64 = 1;
-const MAX_PROTOCOL_VERSION: u64 = 112;
+const MAX_PROTOCOL_VERSION: u64 = 113;
 
 // Record history of protocol version allocations here:
 //
@@ -297,6 +297,7 @@ const MAX_PROTOCOL_VERSION: u64 = 112;
 //              Enable additional validation on zkLogin public identifier.
 // Version 111: Validator metadata
 // Version 112: Enable Ristretto255 in devnet.
+// Version 113: Gate seeded test overrides for checkpoint tx limit behind feature flag.
 
 #[derive(Copy, Clone, Debug, Hash, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 pub struct ProtocolVersion(u64);
@@ -989,6 +990,9 @@ struct FeatureFlags {
     // If true perform consistent verification of metadata
     #[serde(skip_serializing_if = "is_false")]
     validator_metadata_verify_v2: bool,
+
+    #[serde(skip_serializing_if = "is_false")]
+    randomize_checkpoint_tx_limit_in_tests: bool,
 }
 
 fn is_false(b: &bool) -> bool {
@@ -4588,6 +4592,9 @@ impl ProtocolConfig {
                         cfg.feature_flags.enable_ristretto255_group_ops = true;
                     }
                 }
+                113 => {
+                    cfg.feature_flags.randomize_checkpoint_tx_limit_in_tests = true;
+                }
                 // Use this template when making changes:
                 //
                 //     // modify an existing constant.
@@ -4602,15 +4609,28 @@ impl ProtocolConfig {
             }
         }
 
-        // Simtest specific overrides.
-        if cfg!(msim) {
-            // Trigger checkpoint splitting more often.
-            // cfg.max_transactions_per_checkpoint = Some(10);
-            // FIXME: Re-introduce this once we resolve the checkpoint splitting issue
-            // in the quarantine output.
+        cfg
+    }
+
+    pub fn apply_seeded_test_overrides(&mut self, seed: &[u8; 32]) {
+        if !self.feature_flags.randomize_checkpoint_tx_limit_in_tests {
+            return;
         }
 
-        cfg
+        let should_apply = if cfg!(msim) {
+            true
+        } else {
+            mysten_common::in_antithesis()
+        };
+        if !should_apply {
+            return;
+        }
+
+        use rand::{Rng, SeedableRng, rngs::StdRng};
+        let mut rng = StdRng::from_seed(*seed);
+        let max_txns = rng.gen_range(10..=100u64);
+        info!("seeded test override: max_transactions_per_checkpoint = {max_txns}");
+        self.max_transactions_per_checkpoint = Some(max_txns);
     }
 
     // Extract the bytecode verifier config from this protocol config.
@@ -4924,6 +4944,7 @@ impl ProtocolConfig {
         self.feature_flags.enable_authenticated_event_streams = true;
         self.feature_flags
             .include_checkpoint_artifacts_digest_in_summary = true;
+        self.feature_flags.split_checkpoints_in_consensus_handler = true;
     }
 
     pub fn disable_authenticated_event_streams_for_testing(&mut self) {