Merge branch 'devnet-ready' into hybrid-node

Author: liamaharon

pallets/admin-utils/src/benchmarking.rs

@@ -335,5 +335,16 @@ mod benchmarks {
 		_(RawOrigin::Root, 1u16.into()/*netuid*/, true/*enabled*/)/*set_commit_reveal_weights_enabled*/;
     }
 
+    #[benchmark]
+    fn sudo_set_commit_reveal_version() {
+        pallet_subtensor::Pallet::<T>::init_new_network(
+            1u16.into(), /*netuid*/
+            1u16,        /*sudo_tempo*/
+        );
+
+        #[extrinsic_call]
+		_(RawOrigin::Root, 5u16/*version*/)/*sudo_set_commit_reveal_version()*/;
+    }
+
     //impl_benchmark_test_suite!(AdminUtils, crate::mock::new_test_ext(), crate::mock::Test);
 }

pallets/admin-utils/src/lib.rs

@@ -405,7 +405,7 @@ pub mod pallet {
         /// It is only callable by the root account or subnet owner.
         /// The extrinsic will call the Subtensor pallet to set the adjustment beta.
         #[pallet::call_index(12)]
-        #[pallet::weight(Weight::from_parts(19_240_000, 0)
+        #[pallet::weight(Weight::from_parts(14_850_000, 0)
         .saturating_add(<T as frame_system::Config>::DbWeight::get().reads(1_u64))
         .saturating_add(<T as frame_system::Config>::DbWeight::get().writes(1_u64)))]
         pub fn sudo_set_max_weight_limit(
@@ -480,7 +480,7 @@ pub mod pallet {
         /// It is only callable by the root account.
         /// The extrinsic will call the Subtensor pallet to set the maximum allowed UIDs for a subnet.
         #[pallet::call_index(15)]
-        #[pallet::weight(Weight::from_parts(23_820_000, 0)
+        #[pallet::weight(Weight::from_parts(18_770_000, 0)
         .saturating_add(<T as frame_system::Config>::DbWeight::get().reads(2_u64))
         .saturating_add(<T as frame_system::Config>::DbWeight::get().writes(1_u64)))]
         pub fn sudo_set_max_allowed_uids(
@@ -578,7 +578,7 @@ pub mod pallet {
         /// The extrinsic will call the Subtensor pallet to set the network registration allowed.
         #[pallet::call_index(19)]
         #[pallet::weight((
-			Weight::from_parts(8_696_000, 0)
+			Weight::from_parts(6_722_000, 0)
                 .saturating_add(<T as frame_system::Config>::DbWeight::get().reads(0))
 				.saturating_add(<T as frame_system::Config>::DbWeight::get().writes(1)),
 			DispatchClass::Operational,
@@ -705,7 +705,7 @@ pub mod pallet {
         /// It is only callable by the root account or subnet owner.
         /// The extrinsic will call the Subtensor pallet to set the difficulty.
         #[pallet::call_index(24)]
-        #[pallet::weight(Weight::from_parts(17_040_000, 0)
+        #[pallet::weight(Weight::from_parts(15_540_000, 0)
         .saturating_add(<T as frame_system::Config>::DbWeight::get().reads(1_u64))
         .saturating_add(<T as frame_system::Config>::DbWeight::get().writes(1_u64)))]
         pub fn sudo_set_difficulty(
@@ -727,7 +727,7 @@ pub mod pallet {
         /// It is only callable by the root account.
         /// The extrinsic will call the Subtensor pallet to set the maximum allowed validators.
         #[pallet::call_index(25)]
-        #[pallet::weight(Weight::from_parts(19_710_000, 0)
+        #[pallet::weight(Weight::from_parts(23_860_000, 0)
         .saturating_add(<T as frame_system::Config>::DbWeight::get().reads(2_u64))
         .saturating_add(<T as frame_system::Config>::DbWeight::get().writes(1_u64)))]
         pub fn sudo_set_max_allowed_validators(
@@ -1102,7 +1102,7 @@ pub mod pallet {
         /// It is only callable by the root account or subnet owner.
         /// The extrinsic will call the Subtensor pallet to set the value.
         #[pallet::call_index(49)]
-        #[pallet::weight(Weight::from_parts(19_480_000, 0)
+        #[pallet::weight(Weight::from_parts(14_780_000, 0)
         .saturating_add(<T as frame_system::Config>::DbWeight::get().reads(1_u64))
         .saturating_add(<T as frame_system::Config>::DbWeight::get().writes(1_u64)))]
         pub fn sudo_set_commit_reveal_weights_enabled(
@@ -1655,6 +1655,24 @@ pub mod pallet {
             );
             Ok(())
         }
+
+        /// Sets the commit-reveal weights version for all subnets
+        #[pallet::call_index(71)]
+        #[pallet::weight((
+            Weight::from_parts(6_171_000, 0)
+                .saturating_add(<T as frame_system::Config>::DbWeight::get().writes(1))
+                .saturating_add(<T as frame_system::Config>::DbWeight::get().reads(0_u64)),
+            DispatchClass::Operational,
+            Pays::No
+        ))]
+        pub fn sudo_set_commit_reveal_version(
+            origin: OriginFor<T>,
+            version: u16,
+        ) -> DispatchResult {
+            ensure_root(origin)?;
+            pallet_subtensor::Pallet::<T>::set_commit_reveal_weights_version(version);
+            Ok(())
+        }
     }
 }
 

pallets/admin-utils/src/tests/mod.rs

@@ -1930,3 +1930,24 @@ fn test_sudo_set_yuma3_enabled() {
         assert_eq!(SubtensorModule::get_yuma3_enabled(netuid), !to_be_set);
     });
 }
+
+#[test]
+fn test_sudo_set_commit_reveal_version() {
+    new_test_ext().execute_with(|| {
+        add_network(NetUid::from(1), 10);
+
+        let to_be_set: u16 = 5;
+        let init_value: u16 = SubtensorModule::get_commit_reveal_weights_version();
+
+        assert_ok!(AdminUtils::sudo_set_commit_reveal_version(
+            <<Test as Config>::RuntimeOrigin>::root(),
+            to_be_set
+        ));
+
+        assert!(init_value != to_be_set);
+        assert_eq!(
+            SubtensorModule::get_commit_reveal_weights_version(),
+            to_be_set
+        );
+    });
+}

pallets/drand/src/lib.rs

@@ -58,6 +58,7 @@ use sp_runtime::{
 };
 
 pub mod bls12_381;
+pub mod migrations;
 pub mod types;
 pub mod utils;
 pub mod verifier;
@@ -91,6 +92,8 @@ pub const QUICKNET_CHAIN_HASH: &str =
 const CHAIN_HASH: &str = QUICKNET_CHAIN_HASH;
 
 pub const MAX_PULSES_TO_FETCH: u64 = 50;
+pub const MAX_KEPT_PULSES: u64 = 216_000; // 1 week
+pub const MAX_REMOVED_PULSES: u64 = 100;
 
 /// Defines application identifier for crypto keys of this module.
 ///
@@ -212,12 +215,24 @@ pub mod pallet {
         }
     }
 
+    /// Define a maximum length for the migration key
+    type MigrationKeyMaxLen = ConstU32<128>;
+
+    /// Storage for migration run status
+    #[pallet::storage]
+    pub type HasMigrationRun<T: Config> =
+        StorageMap<_, Identity, BoundedVec<u8, MigrationKeyMaxLen>, bool, ValueQuery>;
+
     /// map round number to pulse
     #[pallet::storage]
     pub type Pulses<T: Config> = StorageMap<_, Blake2_128Concat, RoundNumber, Pulse, OptionQuery>;
 
     #[pallet::storage]
-    pub(super) type LastStoredRound<T: Config> = StorageValue<_, RoundNumber, ValueQuery>;
+    pub type LastStoredRound<T: Config> = StorageValue<_, RoundNumber, ValueQuery>;
+
+    /// oldest stored round
+    #[pallet::storage]
+    pub type OldestStoredRound<T: Config> = StorageValue<_, RoundNumber, ValueQuery>;
 
     /// Defines the block when next unsigned transaction will be accepted.
     ///
@@ -261,6 +276,13 @@ pub mod pallet {
                 log::debug!("Drand: Failed to fetch pulse from drand. {e:?}");
             }
         }
+        fn on_runtime_upgrade() -> frame_support::weights::Weight {
+            let mut weight = frame_support::weights::Weight::from_parts(0, 0);
+
+            weight = weight.saturating_add(migrations::migrate_set_oldest_round::<T>());
+
+            weight
+        }
     }
 
     #[pallet::validate_unsigned]
@@ -308,8 +330,8 @@ pub mod pallet {
         /// Verify and write a pulse from the beacon into the runtime
         #[pallet::call_index(0)]
         #[pallet::weight(Weight::from_parts(5_708_000_000, 0)
-        .saturating_add(T::DbWeight::get().reads(2_u64))
-        .saturating_add(T::DbWeight::get().writes(3_u64)))]
+        .saturating_add(T::DbWeight::get().reads(3_u64))
+        .saturating_add(T::DbWeight::get().writes(4_u64)))]
         pub fn write_pulse(
             origin: OriginFor<T>,
             pulses_payload: PulsesPayload<T::Public, BlockNumberFor<T>>,
@@ -321,6 +343,10 @@ pub mod pallet {
             let mut last_stored_round = LastStoredRound::<T>::get();
             let mut new_rounds = Vec::new();
 
+            let oldest_stored_round = OldestStoredRound::<T>::get();
+            let is_first_storage = last_stored_round == 0 && oldest_stored_round == 0;
+            let mut first_new_round: Option<RoundNumber> = None;
+
             for pulse in &pulses_payload.pulses {
                 let is_verified = T::Verifier::verify(config.clone(), pulse.clone())
                     .map_err(|_| Error::<T>::PulseVerificationError)?;
@@ -339,12 +365,25 @@ pub mod pallet {
 
                     // Collect the new round
                     new_rounds.push(pulse.round);
+
+                    // Set the first new round if this is the initial storage
+                    if is_first_storage && first_new_round.is_none() {
+                        first_new_round = Some(pulse.round);
+                    }
                 }
             }
 
             // Update LastStoredRound storage
             LastStoredRound::<T>::put(last_stored_round);
 
+            // Set OldestStoredRound if this was the first storage
+            if let Some(first_round) = first_new_round {
+                OldestStoredRound::<T>::put(first_round);
+            }
+
+            // Prune old pulses
+            Self::prune_old_pulses(last_stored_round);
+
             // Update the next unsigned block number
             let current_block = frame_system::Pallet::<T>::block_number();
             <NextUnsignedAt<T>>::put(current_block);
@@ -628,6 +667,24 @@ impl<T: Config> Pallet<T> {
             .propagate(true)
             .build()
     }
+
+    fn prune_old_pulses(last_stored_round: RoundNumber) {
+        let mut oldest = OldestStoredRound::<T>::get();
+        if oldest == 0 {
+            return;
+        }
+
+        let mut removed: u64 = 0;
+        while last_stored_round.saturating_sub(oldest).saturating_add(1) > MAX_KEPT_PULSES
+            && removed < MAX_REMOVED_PULSES
+        {
+            Pulses::<T>::remove(oldest);
+            oldest = oldest.saturating_add(1);
+            removed = removed.saturating_add(1);
+        }
+
+        OldestStoredRound::<T>::put(oldest);
+    }
 }
 
 /// construct a message (e.g. signed by drand)

pallets/drand/src/migrations/migrate_prune_old_pulses.rs

@@ -0,0 +1,63 @@
+use crate::*;
+use frame_support::{traits::Get, weights::Weight};
+use log;
+
+pub fn migrate_prune_old_pulses<T: Config>() -> Weight {
+    let migration_name = BoundedVec::truncate_from(b"migrate_prune_old_pulses".to_vec());
+
+    // Initialize the weight with one read operation.
+    let mut weight = T::DbWeight::get().reads(1);
+
+    // Check if the migration has already run
+    if HasMigrationRun::<T>::get(&migration_name) {
+        log::info!(
+            "Migration '{:?}' has already run. Skipping.",
+            String::from_utf8_lossy(&migration_name)
+        );
+        return weight;
+    }
+    log::info!(
+        "Running migration '{}'",
+        String::from_utf8_lossy(&migration_name)
+    );
+
+    // Collect all round numbers
+    let mut rounds: Vec<RoundNumber> = Pulses::<T>::iter_keys().collect();
+    weight = weight.saturating_add(T::DbWeight::get().reads(rounds.len() as u64));
+
+    if rounds.is_empty() {
+        OldestStoredRound::<T>::put(0u64);
+        LastStoredRound::<T>::put(0u64);
+        weight = weight.saturating_add(T::DbWeight::get().writes(2));
+    } else {
+        rounds.sort();
+        let num_pulses = rounds.len() as u64;
+
+        let mut new_oldest = rounds[0];
+        if num_pulses > MAX_KEPT_PULSES {
+            let num_to_delete = num_pulses.saturating_sub(MAX_KEPT_PULSES);
+            new_oldest = rounds[num_to_delete as usize];
+
+            for &round in &rounds[0..num_to_delete as usize] {
+                Pulses::<T>::remove(round);
+                weight = weight.saturating_add(T::DbWeight::get().writes(1));
+            }
+        }
+
+        OldestStoredRound::<T>::put(new_oldest);
+        LastStoredRound::<T>::put(*rounds.last().unwrap());
+        weight = weight.saturating_add(T::DbWeight::get().writes(2));
+    }
+
+    // Mark the migration as completed
+    HasMigrationRun::<T>::insert(&migration_name, true);
+    weight = weight.saturating_add(T::DbWeight::get().writes(1));
+
+    log::info!(
+        "Migration '{:?}' completed.",
+        String::from_utf8_lossy(&migration_name)
+    );
+
+    // Return the migration weight.
+    weight
+}

pallets/drand/src/migrations/migrate_set_oldest_round.rs

@@ -0,0 +1,57 @@
+use crate::*;
+use frame_support::weights::Weight;
+use log;
+
+/// Migration to set `OldestStoredRound` to the oldest round in storage.
+pub fn migrate_set_oldest_round<T: Config>() -> Weight {
+    use frame_support::traits::Get;
+
+    let migration_name = BoundedVec::truncate_from(b"migrate_set_oldest_round".to_vec());
+
+    // Start with one read for HasMigrationRun
+    let mut weight = T::DbWeight::get().reads(1);
+
+    // Skip if already run.
+    if HasMigrationRun::<T>::get(&migration_name) {
+        log::info!(
+            "Migration '{}' has already run. Skipping.",
+            String::from_utf8_lossy(&migration_name)
+        );
+        return weight;
+    }
+    log::info!(
+        "Running migration '{}'",
+        String::from_utf8_lossy(&migration_name)
+    );
+
+    // Single-pass over keys: track min and how many keys we read.
+    let mut reads: u64 = 0;
+    let mut min_round: Option<RoundNumber> = None;
+
+    for r in Pulses::<T>::iter_keys() {
+        reads = reads.saturating_add(1);
+        if min_round.is_none_or(|m| r < m) {
+            min_round = Some(r);
+        }
+    }
+
+    // Account for all key reads
+    weight = weight.saturating_add(T::DbWeight::get().reads(reads));
+
+    let oldest = min_round.unwrap_or(0u64);
+    OldestStoredRound::<T>::put(oldest);
+    weight = weight.saturating_add(T::DbWeight::get().writes(1));
+
+    // Mark as completed.
+    HasMigrationRun::<T>::insert(&migration_name, true);
+    weight = weight.saturating_add(T::DbWeight::get().writes(1));
+
+    log::info!(
+        "Migration '{}' completed. OldestStoredRound set to {} (scanned {} rounds).",
+        String::from_utf8_lossy(&migration_name),
+        oldest,
+        reads
+    );
+
+    weight
+}

pallets/drand/src/migrations/mod.rs

@@ -0,0 +1,4 @@
+pub mod migrate_prune_old_pulses;
+pub use migrate_prune_old_pulses::*;
+pub mod migrate_set_oldest_round;
+pub use migrate_set_oldest_round::*;