fix(catalyst-toolbox): Count missing voting power | NPG-0000 (#541)

# Description

Catalyst toolbox `snapshot` and `sve-snapshot` commands were both (with
different code paths) excluding individual registrations with <
threshold ada voting power.

This meant that multiple registrations to the same voting key would have
missing voting power if any of those individual registrations were < the
threshold. This change does that check only on the cumulative total of
all voting power.

## Type of change

Please delete options that are not relevant.

- [ x ] Bug fix (non-breaking change which fixes an issue)

## How Has This Been Tested?

The `utilities/snapshot-check/snapshot-check.py` was sued to validate
the voting power is properly being calculated.
The `utilities/snapshot-check/compar_snapshot.py` was used to compare a
full processed snapshot with an SVE one.

Old snapshots fail to validate, new ones after these changes validate
correctly.

## Checklist

- [ x ] My code follows the style guidelines of this project
- [ x ] I have performed a self-review of my code
- [ x ] I have commented my code, particularly in hard-to-understand
areas
- [ ] I have made corresponding changes to the documentation
- [ x ] My changes generate no new warnings
- [ x ] I have added tests that prove my fix is effective or that my
feature works
- [ x ] New and existing unit tests pass locally with my changes
- [ ] Any dependent changes have been merged and published in downstream
modules

Author: stevenj

src/catalyst-toolbox/snapshot-lib/src/lib.rs

@@ -103,6 +103,7 @@ pub struct Snapshot {
 }
 
 impl Snapshot {
+    #[allow(clippy::missing_errors_doc)]
     pub fn from_raw_snapshot(
         raw_snapshot: RawSnapshot,
         stake_threshold: Value,
@@ -114,8 +115,8 @@ impl Snapshot {
             .0
             .into_iter()
             // Discard registrations with 0 voting power since they don't influence
-            // snapshot anyway
-            .filter(|reg| reg.voting_power >= std::cmp::max(stake_threshold, 1.into()))
+            // snapshot anyway. But can not throw any others away, even if less than the stake threshold.
+            .filter(|reg| reg.voting_power >= 1.into())
             // TODO: add capability to select voting purpose for a snapshot.
             // At the moment Catalyst is the only one in use
             .filter(|reg| {
@@ -185,7 +186,12 @@ impl Snapshot {
                 },
                 contributions,
             })
+            // Because of multiple registrations to the same voting key,  we can only
+            // filter once all registrations for the same key are known.
+            // `stake_threshold` is the minimum stake for all registrations COMBINED.
+            .filter(|entry| entry.hir.voting_power >= stake_threshold)
             .collect();
+
         Ok(Self {
             inner: Self::apply_voting_power_cap(entries, cap)?
                 .into_iter()
@@ -204,17 +210,20 @@ impl Snapshot {
             .collect())
     }
 
+    #[must_use]
     pub fn stake_threshold(&self) -> Value {
         self.stake_threshold
     }
 
+    #[must_use]
     pub fn to_voter_hir(&self) -> Vec<VoterHIR> {
         self.inner
             .values()
             .map(|entry| entry.hir.clone())
             .collect::<Vec<_>>()
     }
 
+    #[must_use]
     pub fn to_full_snapshot_info(&self) -> Vec<SnapshotInfo> {
         self.inner.values().cloned().collect()
     }
@@ -223,6 +232,7 @@ impl Snapshot {
         self.inner.keys()
     }
 
+    #[must_use]
     pub fn contributions_for_voting_key<I: Borrow<Identifier>>(
         &self,
         voting_public_key: I,
@@ -252,6 +262,7 @@ pub mod tests {
     }
 
     impl Snapshot {
+        #[must_use]
         pub fn to_block0_initials(&self, discrimination: Discrimination) -> Vec<InitialUTxO> {
             self.inner
                 .iter()

src/catalyst-toolbox/snapshot-lib/src/sve.rs

@@ -16,10 +16,12 @@ pub struct Snapshot {
 }
 
 impl Snapshot {
+    #[must_use]
+    #[allow(clippy::missing_panics_doc)] // The one possible panic shouldn't happen in reality.
     pub fn new(raw_snapshot: RawSnapshot, min_stake_threshold: Value) -> (Self, usize) {
         let mut total_rejected_registrations: usize = 0;
 
-        let inner = raw_snapshot
+        let mut inner = raw_snapshot
             .0
             .into_iter()
             .filter(|r| {
@@ -37,7 +39,6 @@ impl Snapshot {
 
                 true
             })
-            .filter(|r| r.voting_power >= min_stake_threshold)
             .fold(HashMap::<Identifier, Vec<_>>::new(), |mut acc, r| {
                 let k = match &r.delegations {
                     Delegations::New(ds) => ds.first().unwrap().0.clone(),
@@ -48,9 +49,28 @@ impl Snapshot {
                 acc
             });
 
+        // Because of multiple registrations to the same voting key,  we can only
+        // filter once all registrations for the same key are known.
+        // `min_stake_threshold` is the minimum stake for all registrations COMBINED.
+        inner.retain(|_, regs| {
+            let value: Value = regs
+                .iter()
+                .map(|reg| u64::from(reg.voting_power))
+                .sum::<u64>()
+                .into();
+
+            // If the total stake across all registrations is < threshold, then they are all rejects.
+            if value < min_stake_threshold {
+                total_rejected_registrations += regs.len();
+            }
+
+            value >= min_stake_threshold
+        });
+
         (Self { inner }, total_rejected_registrations)
     }
 
+    #[must_use]
     pub fn to_block0_initials(
         &self,
         discrimination: Discrimination,

utilities/snapshot-check/snapshot-check.py

@@ -15,7 +15,8 @@
 import json
 
 from pathlib import Path
-from typing import Any
+from typing import Any, Iterable, List, Optional, Tuple, Union
+
 
 def is_dir(dirpath: str | Path):
     """Check if the directory is a directory."""
@@ -43,6 +44,21 @@ def compare_reg_error(reg:dict, error:dict) -> bool:
     except:
         return False
 
+def index_processed_snapshot(snapshot) -> dict:
+    #
+    indexed={}
+
+    if isinstance(snapshot, list):
+        for rec in snapshot:
+            indexed["0x" + rec["hir"]["voting_key"]] = rec
+    else:
+        # legacy snapshot
+        print("Legacy Snapshot not supported.  Use the 'compare_snapshot.py' tool to compare a legacy with a full processed snapshot.")
+        exit(1)
+
+    return indexed
+
+
 def analyze_snapshot(args: argparse.Namespace):
     """Convert a snapshot into a format supported by SVE1."""
 
@@ -61,25 +77,82 @@ def analyze_snapshot(args: argparse.Namespace):
     cip_36_single: list[dict[str, Any]] = []
     cip_36_multi: list[dict[str, Any]] = []
 
+    vkey_power: dict[str, list[int]] = {}
+
     total_rejects = 0
     total_registered_value = 0
 
+    rewards_payable = 0
+    rewards_pointer = 0
+    rewards_unpayable = 0
+    rewards_invalid = 0
+    rewards_types = {}
+    unique_rewards = {}
+
+
     for registration in snapshot:
         # Index the registrations
         stake_pub_key = registration["stake_public_key"]
         snapshot_index[stake_pub_key] = registration
 
-        total_registered_value += registration["voting_power"]
+        v_power = registration["voting_power"]
+
+        total_registered_value += v_power
+
+        rewards_addr = registration["rewards_address"]
+
+        long_addr_length = 116
+        short_addr_length = 60
+
+        if len(rewards_addr) > 4 and rewards_addr[0:2] == "0x" and rewards_addr[2] in "01234567ef" and rewards_addr[3] == "1":
+            rewards_type = rewards_addr[2]
+
+            if rewards_type in "0123":
+                if len(rewards_addr) == long_addr_length:
+                    rewards_payable += 1
+                    unique_rewards[rewards_addr] = unique_rewards.get(rewards_addr, 0) + 1
+                else:
+                    rewards_invalid += 1
+            elif rewards_type in "45":
+                if len(rewards_addr) == long_addr_length:
+                    rewards_pointer += 1
+                    unique_rewards[rewards_addr] = unique_rewards.get(rewards_addr, 0) + 1
+                else:
+                    rewards_invalid += 1
+            elif rewards_type in "67":
+                if len(rewards_addr) == short_addr_length:
+                    rewards_payable += 1
+                    unique_rewards[rewards_addr] = unique_rewards.get(rewards_addr, 0) + 1
+                else:
+                    rewards_invalid += 1
+            elif rewards_type in "ef":
+                if len(rewards_addr) == short_addr_length:
+                    rewards_unpayable += 1
+                else:
+                    rewards_invalid += 1
+
+            rewards_types[rewards_type] = rewards_types.get(rewards_type,0) + 1
+        else:
+            rewards_invalid += 1
 
         # Check if the delegation is a simple string.
         # If so, assume its a CIP-15 registration.
         delegation = registration["delegations"]
 
         if isinstance(delegation, str):
             cip_15_snapshot.append(registration)
+
+            if delegation not in vkey_power:
+                vkey_power[delegation] =[]
+            vkey_power[delegation].append(v_power)
+
         elif isinstance(delegation, list):
             if len(delegation) == 1:
                 cip_36_single.append(registration)
+
+                if delegation[0][0] not in vkey_power:
+                    vkey_power[delegation[0][0]] =[]
+                vkey_power[delegation[0][0]].append(v_power)
             else:
                 cip_36_multi.append(registration)
         else:
@@ -89,6 +162,30 @@ def analyze_snapshot(args: argparse.Namespace):
             )
             total_rejects += 1
 
+    # Read the processed snapshot.
+    total_processed_vpower = None
+    processed_snapshot = None
+    if args.processed is not None:
+        processed_snapshot = index_processed_snapshot(json.loads(args.processed.read_text()))
+
+        for rec in processed_snapshot.items():
+            rec_vpower = 0
+            for contribution in rec[1]["contributions"]:
+
+                if contribution["stake_public_key"] in snapshot_index:
+                    snap = snapshot_index[contribution["stake_public_key"]]
+                    if snap["voting_power"] != contribution["value"]:
+                        print(f"Mismatched Contribution Value for {contribution['stake_public_key']}")
+                    else:
+                        rec_vpower += contribution["value"]
+            if rec_vpower != rec[1]["hir"]["voting_power"]:
+                print(f"Mismatched Voting Power for {rec}")
+            else:
+                if total_processed_vpower is None:
+                    total_processed_vpower = rec_vpower
+                else:
+                    total_processed_vpower += rec_vpower
+
     # Index Errors
     registration_obsolete: dict[str, Any] = {}
     decode_errors: list[Any] = []
@@ -121,8 +218,11 @@ def analyze_snapshot(args: argparse.Namespace):
     mismatched: dict[str, Any] = {}
     equal_snapshots = 0
 
+
+
     if args.compare is not None:
         raw_compare = json.loads(args.compare.read_text())
+
         for comp in raw_compare:
             # Index all records being compared.
             stake_pub_key = comp["stake_public_key"]
@@ -150,14 +250,21 @@ def analyze_snapshot(args: argparse.Namespace):
                 missing_registrations.append(registration)
 
     print("Snapshot Analysis:")
-    print(f"  Total Registrations : {len(snapshot)}")
-    print(f"  Total CIP-15        : {len(cip_15_snapshot)}")
-    print(f"  Total CIP-36 Single : {len(cip_36_single)}")
-    print(f"  Total CIP-36 Multi  : {len(cip_36_multi)}")
-    print(f"  Total Rejects       : {total_rejects}")
+    print(f"  Total Registrations : {len(snapshot):10}")
+    print(f"  Total CIP-15        : {len(cip_15_snapshot):10}")
+    print(f"  Total CIP-36 Single : {len(cip_36_single):10}")
+    print(f"  Total CIP-36 Multi  : {len(cip_36_multi):10}")
+    print(f"  Total Rejects       : {total_rejects:10}")
 
     print()
-    print("Stake Address Types:")
+    print("Reward Address Types:")
+    print(f"  Total Payable        : {rewards_payable:10}")
+    print(f"  Total Pointer        : {rewards_pointer:10}")
+    print(f"  Total Unpayable      : {rewards_unpayable:10}")
+    print(f"  Total Invalid        : {rewards_invalid:10}")
+    print(f"  Total Types          : {len(rewards_types):10}")
+    print(f"    Types = {','.join(rewards_types.keys())}")
+    print(f"  Total Unique Rewards : {len(unique_rewards):10}")
 
     #if len(registration_errors) > 0:
     #    print()
@@ -198,22 +305,62 @@ def analyze_snapshot(args: argparse.Namespace):
         for reg in missing_registrations:
             print(f"        {reg}")
 
-        total_unregistered = len(snapshot_unregistered)
-        value_unregistered = 0
-        for value in snapshot_unregistered.values():
-            value_unregistered += value
+    total_unregistered = len(snapshot_unregistered)
+    value_unregistered = 0
+    for value in snapshot_unregistered.values():
+        value_unregistered += value
+
+    total_threshold_voting_power = 0
+    total_threshold_registrations = 0
+    multi_reg_voting_keys = 0
+
+    print()
+    print("Multiple Registrations to same voting key")
+    for key in vkey_power:
+        this_power = 0
+        for v in vkey_power[key]:
+            this_power += v
+        if this_power >= 450000000:
+            total_threshold_registrations += 1
+            total_threshold_voting_power += this_power
+
+            if processed_snapshot is not None:
+                if key not in processed_snapshot:
+                    print(f" Key {key} not in processed snapshot.")
+                elif this_power != processed_snapshot[key]["hir"]["voting_power"]:
+                    print(f" Key {key} voting power mismatch. Processed = {processed_snapshot[key]['hir']['voting_power']} Actual = {this_power}")
+
 
-        print(f"  Total Registrations = Total Voting Power  : {len(snapshot):>10}  = {total_registered_value/1000000:>25} ADA")
-        print(f"  Total Unregistered  = Total Voting Power  : {total_unregistered:>10}  = {value_unregistered/1000000:>25} ADA")
+        elif key in processed_snapshot:
+            print(f" Key {key} is in processed snapshot?")
 
-        staked_total = len(snapshot) + total_unregistered
-        staked_total_value = total_registered_value + value_unregistered
+        if len(vkey_power[key]) > 1:
+            multi_reg_voting_keys += 1
+            print(f"  {multi_reg_voting_keys:3} {key} = {this_power/1000000:>25.6f} ADA")
+            powers = ", ".join([f"{x/1000000:0.6f}".rstrip("0").rstrip(".") for x in sorted(vkey_power[key])])
+            print(f"      {len(vkey_power[key])} Stake Addresses : ADA = {powers} ")
 
-        reg_pct = 100.0 / staked_total * len(snapshot)
-        val_pct = 100.0 / staked_total_value * total_registered_value
 
-        print(f"  Registered%         = VotingPower %       : {reg_pct:>10.4}% = {val_pct:>23.4}%")
+    print("")
 
+    if total_processed_vpower is not None:
+        print(f"  Total Processed Registrations = Total Voting Power  : {len(processed_snapshot.keys()):>10}  = {total_processed_vpower/1000000:>25.6f} ADA - Validates : {total_processed_vpower == total_threshold_voting_power}")
+    print(f"  Total Threshold Registrations = Total Voting Power  : {total_threshold_registrations:>10}  = {total_threshold_voting_power/1000000:>25.6f} ADA")
+    print(f"  Total Registrations           = Total Voting Power  : {len(snapshot):>10}  = {total_registered_value/1000000:>25.6f} ADA")
+    print(f"  Total Unregistered            = Total Voting Power  : {total_unregistered:>10}  = {value_unregistered/1000000:>25.6f} ADA")
+
+    staked_total = len(snapshot) + total_unregistered
+    staked_total_value = total_registered_value + value_unregistered
+
+    reg_pct = 100.0 / staked_total * len(snapshot)
+    val_pct = 100.0 / staked_total_value * total_registered_value
+
+    print(f"  Registered%                   = VotingPower %       : {reg_pct:>10.4f}% =   {val_pct:>23.4f} %")
+
+    thresh_reg_pct = 100.0 / staked_total * total_threshold_registrations
+    thresh_val_pct = 100.0 / staked_total_value * total_threshold_voting_power
+
+    print(f"  Threshold Registered% (450 A) = VotingPower %       : {thresh_reg_pct:>10.4f}% =   {thresh_val_pct:>23.4f} %")
 
 
 def main() -> int:
@@ -235,6 +382,13 @@ def main() -> int:
         type=is_file,
     )
 
+    parser.add_argument(
+        "--processed",
+        help="Processed Snapshot file to compare with.",
+        required=False,
+        type=is_file,
+    )
+
     args = parser.parse_args()
     analyze_snapshot(args)
     return 0