diff --git a/Cargo.lock b/Cargo.lock index 8f1eec6842..baa9220a5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -582,6 +582,12 @@ dependencies = [ "tower-service", ] +[[package]] +name = "az" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" + [[package]] name = "backtrace" version = "0.3.69" @@ -1216,6 +1222,7 @@ dependencies = [ "hex", "imhamt", "lazy_static", + "num", "proptest", "quickcheck", "quickcheck_macros", @@ -1223,6 +1230,7 @@ dependencies = [ "rand_chacha 0.3.1", "rand_core 0.6.4", "rayon", + "rug", "serde", "serde_json", "sparse-array", @@ -1303,10 +1311,12 @@ dependencies = [ "const_format", "criterion", "cryptoxide 0.4.4", + "num", "rand 0.8.5", "rand_chacha 0.3.1", "rand_core 0.6.4", "rayon", + "rug", "smoke", "thiserror", ] @@ -2799,6 +2809,16 @@ dependencies = [ "walkdir", ] +[[package]] +name = "gmp-mpfr-sys" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0205cd82059bc63b63cf516d714352a30c44f2c74da9961dfda2617ae6b5918" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "goblin" version = "0.5.4" @@ -4739,25 +4759,24 @@ dependencies = [ [[package]] name = "num" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ "num-bigint", "num-complex", "num-integer", "num-iter", - "num-rational 0.4.1", + "num-rational 0.4.2", "num-traits", ] [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "autocfg", "num-integer", "num-traits", "serde", @@ -4765,9 +4784,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", "serde", @@ -4786,19 +4805,18 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -4818,11 +4836,10 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "autocfg", "num-bigint", "num-integer", "num-traits", @@ -4831,9 +4848,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", @@ -6287,6 +6304,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rug" +version = "1.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ae2c1089ec0575193eb9222881310cc1ed8bce3646ef8b81b44b518595b79d" +dependencies = [ + "az", + "gmp-mpfr-sys", + "libc", + "libm", +] + [[package]] name = "rust_decimal" version = "1.33.1" @@ -7865,9 +7894,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -7889,9 +7918,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", @@ -7900,9 +7929,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -9257,6 +9286,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -9287,6 +9325,22 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -9299,6 +9353,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -9311,6 +9371,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -9323,6 +9389,18 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -9335,6 +9413,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -9347,6 +9431,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -9359,6 +9449,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -9371,6 +9467,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "0.5.19" diff --git a/src/audit/README.md b/src/audit/README.md index f7f3629579..939c100216 100644 --- a/src/audit/README.md +++ b/src/audit/README.md @@ -1,42 +1,93 @@ -# Audit Tooling: +# Audit Tooling -## Offline audit +Independent verification tools for Catalyst voting results. These tools allow anyone to audit and verify the integrity of voting outcomes without needing to trust centralized authorities. -### Download Fund State -Download historical fund state from [*here*](https://github.com/input-output-hk/catalyst-core) in order to replay and audit the voting event. +## Quick Start -The official published results can be found in this file in the form of **activevoteplans.json**. +### 1. Download Fund State -**activevoteplans.json** = FINAL RESULTS. +Download historical fund state from [here](https://github.com/input-output-hk/catalyst-core) to replay and audit the voting event. -If you would like to re-generate **activevoteplans.json** yourself, via a live node and historical fragments - [*see here for instructions*](./balance/README.md) +The official published results are found in **activevoteplans.json**. -If not, you can begin the audit with the following steps. +**activevoteplans.json** = FINAL RESULTS. -*Example usage:* +### 2. Build the Audit Tool -``` +```bash cargo build --release -p audit -``` +``` -*Cross reference offline tallies with published catalyst tallies.* +### 3. Run Offline Audit -```bash +#### Option A: Cross-reference with Official Results +Compare your audit results with published Catalyst tallies: + +```bash OFFICIAL_RESULTS=/tmp/activevoteplans.json BLOCK0=/tmp/fund9-leader-1/artifacts/block0.bin FRAGMENTS_STORAGE=/tmp/fund9-leader-1/persist/leader-1 ./target/release/offline --fragments $FRAGMENTS_STORAGE --block0 $BLOCK0 --official-results $OFFICIAL_RESULTS +``` + +#### Option B: Generate Encrypted Tally with Gamma Scaling +```bash +BLOCK0=/tmp/fund9-leader-1/artifacts/block0.bin +FRAGMENTS_STORAGE=/tmp/fund9-leader-1/persist/leader-1 +GAMMA=0.5 +PRECISION=5 + +./target/release/offline --fragments $FRAGMENTS_STORAGE --block0 $BLOCK0 --gamma $GAMMA --precision $PRECISION ``` -This will create three files: -- *ledger_after_tally.json* **(decrypted ledger state after tally)** *should match official results!* -- *ledger_before_tally.json* **(encrypted ledger state before tally)** -- *decryption_shares.json* **(decryption shares for each proposal)** +### 4. Generated Files + +The offline audit creates three critical files: -[*See here for next steps of audit process*](src/tally/README.md) +- **ledger_after_tally.json** - Decrypted ledger state after tally *(should match official results!)* +- **ledger_before_tally.json** - Encrypted ledger state before tally +- **decryption_shares.json** - Decryption shares for each proposal -### Find my vote -[*See here for instructions on how to find your voting history*](src/find/README.md) \ No newline at end of file +--- + +## ➡️ **NEXT STEP: Verify Individual Proposals** + +**🔍 [Complete the audit process - Verify specific proposal results →](src/tally/README.md)** + +After generating the audit files above, the next crucial step is to **independently verify individual proposal results** using cryptographic proof. This step: + +- ✅ **Validates each proposal's decryption** was performed correctly +- ✅ **Provides mathematical proof** of result integrity +- ✅ **Requires no trust** in election officials or committee members +- ✅ **Can be run by anyone** using publicly available data + +**Why this step is essential:** +- The offline audit gives you the raw encrypted data +- The tally verification proves the decryption was legitimate +- Together, they provide complete end-to-end verification + +--- + +## Additional Tools + +### Find Your Vote +[See instructions on how to find your voting history →](src/find/README.md) + +### Regenerate Results from Live Node +If you want to regenerate **activevoteplans.json** yourself via a live node and historical fragments: +[See instructions here →](./balance/README.md) + +## Overview + +This audit tooling provides: + +1. **Offline Verification** - Replay voting events from blockchain data +2. **Cryptographic Proof** - Mathematically verify decryption integrity +3. **Individual Vote Tracking** - Find and verify your specific votes +4. **Complete Transparency** - No black boxes or trusted components + +The audit process is designed to be completely independent and reproducible, ensuring that anyone can verify Catalyst voting results without needing to trust any centralized authority. +``` diff --git a/src/audit/src/offline/bin/main.rs b/src/audit/src/offline/bin/main.rs index 39402585d7..9e98b0a9df 100644 --- a/src/audit/src/offline/bin/main.rs +++ b/src/audit/src/offline/bin/main.rs @@ -19,6 +19,7 @@ use lib::offline::{ use chain_core::packer::Codec; use color_eyre::{eyre::Context, Report}; +use std::env; use std::{error::Error, path::PathBuf}; /// @@ -36,6 +37,12 @@ pub struct Args { /// cross reference official results #[clap(short, long)] official_results: Option, + /// Gamma value for Quadratic scaling + #[clap(short, long)] + gamma: Option, + /// Rounding precision for arithmetic + #[clap(short, long)] + precision: Option, } fn main() -> Result<(), Box> { @@ -61,6 +68,16 @@ fn main() -> Result<(), Box> { info!("Audit Tool."); info!("Starting Offline Tally"); + if let Some(gamma) = args.gamma { + const GAMMA: &str = "QUADRATIC_VOTING_GAMMA"; + std::env::set_var(GAMMA, gamma); + } + + if let Some(precision) = args.precision { + const PRECISION: &str = "QUADRATIC_VOTING_PRECISION"; + std::env::set_var(PRECISION, precision); + } + // Load and replay fund fragments from storage let storage_path = PathBuf::from(args.fragments); diff --git a/src/audit/src/tally/README.md b/src/audit/src/tally/README.md index 6f13ede580..c3a8707036 100644 --- a/src/audit/src/tally/README.md +++ b/src/audit/src/tally/README.md @@ -42,6 +42,34 @@ SHARES_ALICE='WDDMb68A6JCVR5UdhDtl7QYrQHSMOFqg44lHcmtB/Q3IfSoqusq+obtC/JJOtDYWad #### Public use: Validate results #### `decrypt_tally_from_shares(pub_keys, encrypted_tally, decrypt_shares) -> tallyResultPlaintext` +**Public Verification: Independently Validate Voting Results** +This function allows anyone to independently verify that encrypted voting results have been correctly decrypted without requiring access to any private keys or secret information. It uses publicly available data to cryptographically prove the integrity of the decryption process. +**What it does:** +- Takes encrypted tallies, decryption shares, and public keys as input +- Validates that the decryption shares are legitimate and correspond to the correct public keys +- Combines the shares to reproduce the plaintext voting results +- Provides cryptographic proof that the decryption was performed correctly + +**Why it's important:** +- **Transparency**: Anyone can verify results using only publicly available information +- **Trust**: No need to trust election officials or committee members +- **Auditability**: Results can be independently verified by multiple parties +- **Security**: The verification process is cryptographically sound and tamper-proof + +**Who can use it:** +- Voters wanting to verify their voting event results +- Independent auditors and researchers +- Media organizations and watchdog groups +- Anyone interested in election integrity + +**What makes it "public":** +- No secret keys or private information required +- All necessary data is publicly available after the election +- The verification process is transparent and reproducible +- Results are deterministic - anyone running the same verification will get identical results + +This verification method ensures that the democratic process remains transparent and that election results can be independently validated by the community. + ```bash SHARES_ALICE='0DDHBs4TnabGQjvhIiQP2S53mTThxqilR+ogY8MpIRV6PdDb+5NWVZYQvAQQZUIe8e/rzeZjGX5QkCpd84b/CyvrivhD4u7zvhccz6zSgOfLx3EVjY9PXBXOhPYkrUoE0DDHBs4TnabGQjvhIiQP2S53mTThxqilR+ogY8MpIRXOp9W5weElw0uZSyz4oCkRMKiRv2L1kfrOuNLOXtobBnWorfj2FLdBb2jZ5Cb0tqYvMKj+WLTTs2hrohjlSC0D' diff --git a/src/catalyst-toolbox/catalyst-toolbox/scripts/python/proposers_rewards.py b/src/catalyst-toolbox/catalyst-toolbox/scripts/python/proposers_rewards.py index a3dacc0d2e..4c9fcb3c21 100755 --- a/src/catalyst-toolbox/catalyst-toolbox/scripts/python/proposers_rewards.py +++ b/src/catalyst-toolbox/catalyst-toolbox/scripts/python/proposers_rewards.py @@ -1,5 +1,16 @@ # coding: utf-8 -from typing import Dict, Optional, List, Tuple, Generator, TextIO, Union, Any, Set, Mapping +from typing import ( + Dict, + Optional, + List, + Tuple, + Generator, + TextIO, + Union, + Any, + Set, + Mapping, +) import sys import asyncio @@ -21,6 +32,7 @@ from rich import print from asyncio import run as aiorun from copy import deepcopy +from fractions import Fraction # VIT servicing station models @@ -34,6 +46,7 @@ NOT_FUNDED_APPROVAL_THRESHOLD = "Not Funded - Approval Threshold" LOVELACE_FACTOR = 1000000 + class Challenge(pydantic.BaseModel): id: int challenge_type: str @@ -43,6 +56,7 @@ class Challenge(pydantic.BaseModel): fund_id: int challenge_url: str + class Proposal(pydantic.BaseModel): internal_id: int proposal_id: str @@ -57,12 +71,13 @@ class Proposal(pydantic.BaseModel): challenge_id: int challenge_type: str challenge: Challenge - + @pydantic.computed_field @property def ideascale_url(self) -> str: return f"https://cardano.ideascale.com/c/idea/{self.proposal_id}" + class Author(pydantic.BaseModel): """Represents an author.""" @@ -71,8 +86,10 @@ class Author(pydantic.BaseModel): email: str user_name: str = pydantic.Field(alias="userName") + # Ideascale models + class IdeascaleProposal(pydantic.BaseModel): id: int title: str @@ -91,8 +108,10 @@ def assign_authors_if_any(cls, values): values["authors"] = authors return values + # Jormungandr models + class Options(pydantic.BaseModel): start: int end: int @@ -170,6 +189,7 @@ class Result(pydantic.BaseModel): votes_cast: int vote_result: Optional[int] = None + class Winner(pydantic.BaseModel): internal_id: int proposal_id: str @@ -187,16 +207,18 @@ def dict(self, **kwargs): # Override std dict to list all authors in different columns output = super().dict(**kwargs) _output = {} - for k,v in output.items(): - if k == 'authors': + for k, v in output.items(): + if k == "authors": for idx, author in enumerate(v): - _output[f"{k}_{idx}"] = author['email'] + _output[f"{k}_{idx}"] = author["email"] else: _output[k] = v return _output + # Ideascale interface + class JsonHttpClient: """HTTP Client for JSON APIs.""" @@ -222,6 +244,7 @@ async def get(self, path: str, headers: Mapping[str, str] = {}): else: raise GetFailed(r.status, r.reason, content) + class GetFailed(Exception): """Raised when a request fails.""" @@ -229,10 +252,13 @@ def __init__(self, status, reason, content): """Initialize a new instance of GetFailed.""" super().__init__(f"{status} {reason}\n{content})") + class IdeascaleImporter: """Interface with IdeaScale API.""" - def __init__(self, api_key: str, api_url: str = "https://temp-cardano-sandbox.ideascale.com"): + def __init__( + self, api_key: str, api_url: str = "https://temp-cardano-sandbox.ideascale.com" + ): """Initialize entities.""" self.api_key = api_key self.api_url = api_url @@ -240,7 +266,7 @@ def __init__(self, api_key: str, api_url: str = "https://temp-cardano-sandbox.id self.N_WORKERS = 3 self.proposals: List[IdeascaleProposal] = [] - + async def import_proposals(self, stage_ids: List[int], page_size: int = 50): """Get all ideas from the stage with the given id. @@ -264,7 +290,9 @@ async def worker(d: WorkerData, stage_id: int): p = d.page d.page += 1 - res = await self._get(f"/a/rest/v1/stages/{stage_id}/ideas/{p}/{page_size}") + res = await self._get( + f"/a/rest/v1/stages/{stage_id}/ideas/{p}/{page_size}" + ) res_proposals: List[IdeascaleProposal] = [] for i in res: @@ -275,20 +303,24 @@ async def worker(d: WorkerData, stage_id: int): if len(res_proposals) < page_size: d.done = True + d = {} - for stage_id in stage_ids: + for stage_id in stage_ids: print(f"Start proposal requests for stage: {stage_id}") d = WorkerData(stage_id) - worker_tasks = [asyncio.create_task(worker(d, stage_id)) for _ in range(self.N_WORKERS)] + worker_tasks = [ + asyncio.create_task(worker(d, stage_id)) for _ in range(self.N_WORKERS) + ] for task in worker_tasks: await task self.proposals.extend(d.proposals) - + async def _get(self, path: str): """Execute a GET request.""" headers = {"api_token": self.api_key} return await self.inner.get(path, headers) + # File loaders @@ -297,9 +329,11 @@ def load_json_from_file(file_path: str) -> Dict: return json.load(f) -def get_proposals_from_file(proposals_file_path: str, challenges: Dict[int, Challenge]) -> Dict[str, Proposal]: +def get_proposals_from_file( + proposals_file_path: str, challenges: Dict[int, Challenge] +) -> Dict[str, Proposal]: proposals: Generator[Proposal, None, None] = ( - Proposal(**proposal_data, challenge=challenges[proposal_data['challenge_id']]) + Proposal(**proposal_data, challenge=challenges[proposal_data["challenge_id"]]) for proposal_data in load_json_from_file(proposals_file_path) ) proposals_dict = {proposal.chain_proposal_id: proposal for proposal in proposals} @@ -411,8 +445,7 @@ def load_block0_data(block0_path: str) -> Dict[str, Any]: # Checkers -class SanityException(Exception): - ... +class SanityException(Exception): ... def sanity_check_data( @@ -436,10 +469,12 @@ def sanity_check_data( # Analyse and compute needed data + class WinnerSelectionRule(enum.Enum): YES_ONLY: str = "yes_only" YES_NO_DIFF: str = "yes_no_diff" + def extract_choices_votes(proposal: Proposal, voteplan_proposal: ProposalStatus): yes_index = int(proposal.chain_vote_options["yes"]) no_index = int(proposal.chain_vote_options["no"]) @@ -454,16 +489,22 @@ def calc_approval_threshold( voteplan_proposal: ProposalStatus, total_stake_threshold: float, winner_selection_rule: WinnerSelectionRule, - relative_threshold: float + relative_threshold: float, ) -> Tuple[int, bool]: - yes_result, second_choice_result = extract_choices_votes(proposal, voteplan_proposal) - pass_relative_threshold = ((yes_result - second_choice_result) / (yes_result + second_choice_result)) >= float(relative_threshold) + yes_result, second_choice_result = extract_choices_votes( + proposal, voteplan_proposal + ) + pass_relative_threshold = ( + (yes_result - second_choice_result) / (yes_result + second_choice_result) + ) >= float(relative_threshold) if winner_selection_rule == WinnerSelectionRule.YES_ONLY: vote_result = yes_result pass_total_threshold = yes_result >= float(total_stake_threshold) elif winner_selection_rule == WinnerSelectionRule.YES_NO_DIFF: vote_result = yes_result - second_choice_result - pass_total_threshold = (yes_result + second_choice_result) >= float(total_stake_threshold) + pass_total_threshold = (yes_result + second_choice_result) >= float( + total_stake_threshold + ) threshold_rules = pass_total_threshold and pass_relative_threshold return vote_result, threshold_rules @@ -473,7 +514,7 @@ def calc_vote_value_and_threshold_success( voteplan_proposals: Dict[str, ProposalStatus], total_stake_threshold: float, winner_selection_rule: WinnerSelectionRule, - relative_threshold: float + relative_threshold: float, ) -> Dict[str, Tuple[int, bool]]: full_ids = set(proposals.keys()) result = { @@ -482,7 +523,7 @@ def calc_vote_value_and_threshold_success( voteplan_proposals[proposal_id], total_stake_threshold, winner_selection_rule, - relative_threshold + relative_threshold, ) for proposal_id in full_ids } @@ -495,10 +536,14 @@ def calc_results( funds: float, total_stake_threshold: float, winner_selection_rule: WinnerSelectionRule, - relative_threshold: float + relative_threshold: float, ) -> List[Result]: success_results = calc_vote_value_and_threshold_success( - proposals, voteplan_proposals, total_stake_threshold, winner_selection_rule, relative_threshold + proposals, + voteplan_proposals, + total_stake_threshold, + winner_selection_rule, + relative_threshold, ) sorted_ids = sorted( success_results.keys(), key=lambda x: success_results[x][0], reverse=True @@ -509,7 +554,9 @@ def calc_results( proposal = proposals[proposal_id] voteplan_proposal = voteplan_proposals[proposal_id] vote_result, threshold_success = success_results[proposal_id] - yes_result, second_choice_result = extract_choices_votes(proposal, voteplan_proposal) + yes_result, second_choice_result = extract_choices_votes( + proposal, voteplan_proposal + ) funded = all( (threshold_success, depletion > 0, depletion >= proposal.proposal_funds) ) @@ -541,7 +588,7 @@ def calc_results( ideascale_url=proposal.ideascale_url, challenge_id=proposal.challenge.id, challenge_title=proposal.challenge.title, - votes_cast=voteplan_proposal.votes_cast + votes_cast=voteplan_proposal.votes_cast, ) if winner_selection_rule == WinnerSelectionRule.YES_ONLY: @@ -584,70 +631,83 @@ def filter_excluded_proposals( def calculate_total_stake_from_block0_configuration( - block0_config: Dict[str, Dict], committee_keys: List[str] + block0_config: Dict[str, Dict], committee_keys: List[str], gamma: Fraction ): funds = ( initial["fund"] for initial in block0_config["initial"] if "fund" in initial ) return sum( - fund["value"] + fund["value"] ** gamma for fund in itertools.chain.from_iterable(funds) if fund["address"] not in [key for key in committee_keys] ) + def extract_relevant_choice(x, winner_selection_rule): if winner_selection_rule == WinnerSelectionRule.YES_ONLY: return x.yes elif winner_selection_rule == WinnerSelectionRule.YES_NO_DIFF: return x.vote_result -def calc_leftovers(results, remaining_funds, excluded_categories, winner_selection_rule): - leftovers_candidates = sorted([ - result - for result in deepcopy(results) - if ( - result.status == NOT_FUNDED and - result.meets_threshold == YES and - result.challenge_id not in excluded_categories - ) - ], key=lambda x: extract_relevant_choice(x, winner_selection_rule), reverse=True) + +def calc_leftovers( + results, remaining_funds, excluded_categories, winner_selection_rule +): + leftovers_candidates = sorted( + [ + result + for result in deepcopy(results) + if ( + result.status == NOT_FUNDED + and result.meets_threshold == YES + and result.challenge_id not in excluded_categories + ) + ], + key=lambda x: extract_relevant_choice(x, winner_selection_rule), + reverse=True, + ) depletion = remaining_funds for candidate in leftovers_candidates: funded = depletion >= candidate.requested_funds - not_funded_reason = ( - "" - if funded - else NOT_FUNDED_OVER_BUDGET - ) + not_funded_reason = "" if funded else NOT_FUNDED_OVER_BUDGET if funded: depletion -= candidate.requested_funds candidate.status = FUNDED if funded else NOT_FUNDED candidate.fund_depletion = depletion candidate.not_funded_reason = not_funded_reason - + return leftovers_candidates, depletion + def pick_milestones_qty(winner, limits, qty): idx = next((i for i, l in enumerate(limits) if winner.requested_funds > l), None) return qty[idx] -def generate_winners(results, fund_prefix, milestones_limit, milestones_qty, _ideascale_proposals): + +def generate_winners( + results, fund_prefix, milestones_limit, milestones_qty, _ideascale_proposals +): ideascale_proposals = {p.id: p for p in _ideascale_proposals} winners = [] - _winners = sorted([r for r in results if r.status == FUNDED], key=lambda r: r.proposal.lower()) + _winners = sorted( + [r for r in results if r.status == FUNDED], key=lambda r: r.proposal.lower() + ) for idx, _winner in enumerate(_winners): winner = Winner( **_winner.dict(), proposal_title=_winner.proposal, project_id=fund_prefix + idx, - milestone_qty=pick_milestones_qty(_winner, milestones_limit, milestones_qty) + milestone_qty=pick_milestones_qty( + _winner, milestones_limit, milestones_qty + ), ) if winner.internal_id in ideascale_proposals.keys(): winner.authors = ideascale_proposals[winner.internal_id].authors winners.append(winner) return winners + # Output results @@ -666,24 +726,27 @@ def output_json(results: List[Result], f: TextIO): # CLI + class OutputFormat(enum.Enum): CSV: str = "csv" JSON: str = "json" -def build_path_for_challenge(file_path: str, challenge_name: str, output_format: OutputFormat) -> str: +def build_path_for_challenge( + file_path: str, challenge_name: str, output_format: OutputFormat +) -> str: path, suffix = os.path.splitext(file_path) - suffix = 'json' if (output_format == OutputFormat.JSON) else 'csv' + suffix = "json" if (output_format == OutputFormat.JSON) else "csv" return f"{path}_{challenge_name}.{suffix}" -def save_results(output_path: str, title: str, output_format: OutputFormat, results: List[Results]): +def save_results( + output_path: str, title: str, output_format: OutputFormat, results: List[Results] +): challenge_output_file_path = build_path_for_challenge( output_path, - re.sub( - r"(?u)[^-\w.]", "", title.replace(" ", "_").replace(":", "_") - ), - output_format + re.sub(r"(?u)[^-\w.]", "", title.replace(" ", "_").replace(":", "_")), + output_format, ) with open( @@ -698,16 +761,22 @@ def save_results(output_path: str, title: str, output_format: OutputFormat, resu def calculate_rewards( output_file: str = typer.Option(...), block0_path: str = typer.Option(...), + gamma: str = typer.Option( + "1", + help=""" + The gamma value applied for the calculation of the total stake threshold. It is applied to every single voting value before the sum is executed. + """ + ), total_stake_threshold: float = typer.Option( 0.01, help=""" This value indicates the minimum percentage of voting needed by projects to be eligible for funding. Voting choices considered for this depends by the winner rule. - """ + """, ), relative_threshold: float = typer.Option( - 0, - help="This value indicates the relative threshold between Yes/No votes needed by projects to be eligible for funding." + -1, + help="This value indicates the relative threshold between Yes/No votes needed by projects to be eligible for funding.", ), output_format: OutputFormat = typer.Option("csv", help="Output format"), winner_selection_rule: WinnerSelectionRule = typer.Option( @@ -717,7 +786,7 @@ def calculate_rewards( Possible choices are: - `yes_only` Fuzzy threshold voting: only YES votes are considered for ranking. Only YES votes are considered to calculate thresholds. - `yes_no_diff` Fuzzy threshold voting: YES/NO difference is considered for ranking. Sum of YES/NO is considered to calculate thresholds. - """ + """, ), proposals_path: Optional[str] = typer.Option(None), excluded_proposals_path: Optional[str] = typer.Option(None), @@ -725,21 +794,26 @@ def calculate_rewards( challenges_path: Optional[str] = typer.Option(None), vit_station_url: str = typer.Option("https://servicing-station.vit.iohk.io"), committee_keys_path: Optional[str] = typer.Option(None), - fund_prefix: int = typer.Option(1100001, help="This number will be used to assign progressively project ids to winners."), + fund_prefix: int = typer.Option( + 1100001, + help="This number will be used to assign progressively project ids to winners.", + ), leftovers_excluded_categories: List[int] = typer.Option( [], - help="List of categories IDs that are not considered in leftovers winners calculation." + help="List of categories IDs that are not considered in leftovers winners calculation.", ), milestones_limit: List[int] = typer.Option( [0, 75000, 150000, 300000], - help="Map of budgets to assign number of milestones. Lenght must coincide with `milestones_qty` parameter." + help="Map of budgets to assign number of milestones. Lenght must coincide with `milestones_qty` parameter.", ), milestones_qty: List[int] = typer.Option( [3, 4, 5, 6], - help="Map of milestones qty to assign number of milestones. Lenght must coincide with `milestones_limit` parameter." + help="Map of milestones qty to assign number of milestones. Lenght must coincide with `milestones_limit` parameter.", ), ideascale_api_key: str = typer.Option(None, help="IdeaScale API key"), - ideascale_api_url: str = typer.Option("https://temp-cardano-sandbox.ideascale.com", help="IdeaScale API url"), + ideascale_api_url: str = typer.Option( + "https://temp-cardano-sandbox.ideascale.com", help="IdeaScale API url" + ), stage_ids: List[int] = typer.Option([], help="Stage IDs"), ): """ @@ -784,11 +858,20 @@ def calculate_rewards( committee_keys = ( load_json_from_file(committee_keys_path) if committee_keys_path else [] ) + + # minimum amount of stake needed for a proposal to be accepted + _total_stake = calculate_total_stake_from_block0_configuration( + block0_config, committee_keys, Fraction(1) + ) + print(f"\nTotal stake before gamma applied {_total_stake}") + print(f"Gamma as fractional exponent: {gamma}") + total_stake = calculate_total_stake_from_block0_configuration( - block0_config, committee_keys + block0_config, committee_keys, Fraction(gamma) ) - # minimum amount of stake needed for a proposal to be accepted + total_stake_approval_threshold = float(total_stake_threshold) * float(total_stake) + print(f"Total stake after gamma applied {total_stake}\n") total_remaining_funds = 0 @@ -804,19 +887,24 @@ def calculate_rewards( challenge.rewards_total, total_stake_approval_threshold, winner_selection_rule, - relative_threshold + relative_threshold, ) total_remaining_funds += remaining_funds all_results += results save_results(output_file, challenge.title, output_format, results) - - leftover_results, final_remaining_funds = calc_leftovers(all_results, total_remaining_funds, leftovers_excluded_categories, winner_selection_rule) - save_results(output_file, 'leftovers', output_format, leftover_results) + + leftover_results, final_remaining_funds = calc_leftovers( + all_results, + total_remaining_funds, + leftovers_excluded_categories, + winner_selection_rule, + ) + save_results(output_file, "leftovers", output_format, leftover_results) ideascale_proposals = [] - if (ideascale_api_key): + if ideascale_api_key: ideascale = IdeascaleImporter(ideascale_api_key, ideascale_api_url) async def _get_proposals(): @@ -827,8 +915,14 @@ async def _get_proposals(): milestones_limit.reverse() milestones_qty.reverse() - winners = generate_winners(all_results + leftover_results, fund_prefix, milestones_limit, milestones_qty, ideascale_proposals) - save_results(output_file, 'winners', output_format, winners) + winners = generate_winners( + all_results + leftover_results, + fund_prefix, + milestones_limit, + milestones_qty, + ideascale_proposals, + ) + save_results(output_file, "winners", output_format, winners) print("[bold green]Winners generated.[/bold green]") print(f"Total Stake: {total_stake}") @@ -837,5 +931,6 @@ async def _get_proposals(): print(f"Unallocated budget: {final_remaining_funds}") print(f"Funded projects: {len(winners)}") + if __name__ == "__main__": typer.run(calculate_rewards) diff --git a/src/chain-libs/chain-impl-mockchain/Cargo.toml b/src/chain-libs/chain-impl-mockchain/Cargo.toml index 83f4b2b74d..bbe60b1aca 100644 --- a/src/chain-libs/chain-impl-mockchain/Cargo.toml +++ b/src/chain-libs/chain-impl-mockchain/Cargo.toml @@ -32,6 +32,9 @@ criterion = { version = "0.3.0", optional = true } rand = "0.8" cryptoxide = "0.4" tracing.workspace = true +rug = "1.26.1" +num = "0.4.3" + [features] property-test-api = [ diff --git a/src/chain-libs/chain-vote/Cargo.toml b/src/chain-libs/chain-vote/Cargo.toml index 6d31e20b84..aa5dc95200 100644 --- a/src/chain-libs/chain-vote/Cargo.toml +++ b/src/chain-libs/chain-vote/Cargo.toml @@ -14,6 +14,10 @@ thiserror = "1.0" cryptoxide = "^0.4.2" const_format = "0.2" base64 = "0.21.0" +rug = "1.26.1" +num = "0.4.3" + + [dev-dependencies] rand_chacha = "0.3" diff --git a/src/chain-libs/chain-vote/src/tally.rs b/src/chain-libs/chain-vote/src/tally.rs index d28543be17..020347f8ec 100644 --- a/src/chain-libs/chain-vote/src/tally.rs +++ b/src/chain-libs/chain-vote/src/tally.rs @@ -1,5 +1,11 @@ use std::num::NonZeroU64; +use core::cmp::Ordering; +use num::FromPrimitive; +use rug::{float::Round, ops::Pow, Float, Rational}; + +use std::str::FromStr; + use crate::GroupElement; use crate::{ committee::*, @@ -9,8 +15,14 @@ use crate::{ TallyOptimizationTable, }; use base64::{engine::general_purpose, Engine as _}; +use num::Rational32; +use rug::Integer; + use cryptoxide::blake2b::Blake2b; use cryptoxide::digest::Digest; + +use std::env; + use rand_core::{CryptoRng, RngCore}; /// Secret key for opening vote @@ -155,6 +167,38 @@ impl EncryptedTally { pub fn add(&mut self, ballot: &Ballot, weight: u64) { assert_eq!(ballot.vote().len(), self.r.len()); assert_eq!(ballot.fingerprint(), &self.fingerprint); + + const GAMMA: &str = "QUADRATIC_VOTING_GAMMA"; + const PRECISION: &str = "QUADRATIC_VOTING_PRECISION"; + + // Apply quadratic scaling if gamma value specified in env var. Else gamma is 1 and has no effect. + let mut gamma = f64::from_str(&env::var(GAMMA).unwrap_or(1.0.to_string())).unwrap(); + // Gamma must be between 0 and 1, anything else is treated as bad input; defaulting gamma to 1. + if gamma < 0.0 || gamma > 1.0 { + gamma = 1.0; + } + + let precision = u32::from_str(&env::var(PRECISION).unwrap_or(1.to_string())).unwrap_or(1); + + let gamma = Rational32::from_f64(gamma).unwrap_or(Rational32::from_integer(1)); + let denom = gamma.denom(); + let numer = gamma.numer(); + + let stake = Float::with_val(precision, weight); + + // rational = gamma in rational form i.e fraction + // 0.5 = 1/2 + let gamma = Float::with_val(precision, &Rational::from((*numer, *denom))); + + let stake_with_gamma_scaling = stake.clone().pow(&gamma); + + let weight = stake_with_gamma_scaling + .to_integer_round(Round::Down) + .unwrap_or((Integer::from(weight), Ordering::Less)) + .0 + .to_u64() + .unwrap_or(weight); + for (ri, ci) in self.r.iter_mut().zip(ballot.vote().iter()) { *ri = &*ri + &(ci * weight); }