diff --git a/CHANGELOG.md b/CHANGELOG.md index 2258ad9d..44d637c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ All notable changes to this project will be documented in this file. +## 1.7.0 + +### Added + +#### DeFi Wrapper + +- use case distributor now has `claim` command and `distribute` command now supports modes for `snapshot` generation + +#### Metrics + +- added `statistic-by-reports-full` command for full report-by-report metrics breakdown +- improved metrics calculation accuracy: refined `report-statistic`, `rebase-rewards`, and chart dataset utilities + +#### Testing & Build + +- added ~90 unit and integration tests covering utilities, consolidation checks, cache, report freshness, quarantine, and minting capacity +- migrated ESLint from v8 to v10 with flat config; added SonarJS, unicorn, vitest, and promise plugins + +### Fixed + +- IPFS utils now support CIDv1 +- fixed help message display for distributor commands + ## 1.6.0 ### Fixed diff --git a/docs/cli/commands/contracts/lazy-oracle.md b/docs/cli/commands/contracts/lazy-oracle.md index 1020b02d..fbd0dd88 100644 --- a/docs/cli/commands/contracts/lazy-oracle.md +++ b/docs/cli/commands/contracts/lazy-oracle.md @@ -24,29 +24,29 @@ yarn start contracts lazy-oracle -h ### Read -| Command | Description | -| ------------------------------------ | ----------------------------------------------------------------------------------- | -| info | get lazy oracle base info | -| DEFAULT_ADMIN_ROLE | Calls the read-only function "DEFAULT_ADMIN_ROLE" on the contract. | -| LIDO_LOCATOR | Calls the read-only function "LIDO_LOCATOR" on the contract. | -| MAX_LIDO_FEE_RATE_PER_SECOND | Calls the read-only function "MAX_LIDO_FEE_RATE_PER_SECOND" on the contract. | -| MAX_QUARANTINE_PERIOD | Calls the read-only function "MAX_QUARANTINE_PERIOD" on the contract. | -| MAX_REWARD_RATIO | Calls the read-only function "MAX_REWARD_RATIO" on the contract. | -| UPDATE_SANITY_PARAMS_ROLE | Calls the read-only function "UPDATE_SANITY_PARAMS_ROLE" on the contract. | -| batch-validator-statuses \ | get batch to mass check the validator statuses in PredepositGuarantee contract | -| batch-vaults-info \ \ | get batch vaults info | -| getRoleAdmin \ | Calls the read-only function "getRoleAdmin" on the contract. | -| getRoleMember \ \ | Calls the read-only function "getRoleMember" on the contract. | -| getRoleMemberCount \ | Calls the read-only function "getRoleMemberCount" on the contract. | -| getRoleMembers \ | Calls the read-only function "getRoleMembers" on the contract. | -| hasRole \ \ | Calls the read-only function "hasRole" on the contract. | -| latest-report-data (lrd) | get latest report data | -| latest-report-timestamp (lrt) | get latest report timestamp | -| max-lido-fee-rate-per-second (max-lfs) | get the max Lido fee rate per second, in ether | -| max-reward-ratio-bp (mrr) | get max reward ratio | -| quarantine-period (qp) | get quarantine period | -| quarantine-value (qv) \ | get the amount of total value that is pending in the quarantine for the given vault | -| supportsInterface \ | Calls the read-only function "supportsInterface" on the contract. | -| vault-info (vi) \ | get the vault data info | -| vault-quarantine (vq) \ | get vault quarantine | -| vaults-count (vc) | get vaults count | +| Command | Description | +| -------------------------------------- | ----------------------------------------------------------------------------------- | +| info | get lazy oracle base info | +| DEFAULT_ADMIN_ROLE | Calls the read-only function "DEFAULT_ADMIN_ROLE" on the contract. | +| LIDO_LOCATOR | Calls the read-only function "LIDO_LOCATOR" on the contract. | +| MAX_LIDO_FEE_RATE_PER_SECOND | Calls the read-only function "MAX_LIDO_FEE_RATE_PER_SECOND" on the contract. | +| MAX_QUARANTINE_PERIOD | Calls the read-only function "MAX_QUARANTINE_PERIOD" on the contract. | +| MAX_REWARD_RATIO | Calls the read-only function "MAX_REWARD_RATIO" on the contract. | +| UPDATE_SANITY_PARAMS_ROLE | Calls the read-only function "UPDATE_SANITY_PARAMS_ROLE" on the contract. | +| batch-validator-statuses \ | get batch to mass check the validator statuses in PredepositGuarantee contract | +| batch-vaults-info \ \ | get batch vaults info | +| getRoleAdmin \ | Calls the read-only function "getRoleAdmin" on the contract. | +| getRoleMember \ \ | Calls the read-only function "getRoleMember" on the contract. | +| getRoleMemberCount \ | Calls the read-only function "getRoleMemberCount" on the contract. | +| getRoleMembers \ | Calls the read-only function "getRoleMembers" on the contract. | +| hasRole \ \ | Calls the read-only function "hasRole" on the contract. | +| latest-report-data (lrd) | get latest report data | +| latest-report-timestamp (lrt) | get latest report timestamp | +| max-lido-fee-rate-per-second (max-lfs) | get the max Lido fee rate per second, in ether | +| max-reward-ratio-bp (mrr) | get max reward ratio | +| quarantine-period (qp) | get quarantine period | +| quarantine-value (qv) \ | get the amount of total value that is pending in the quarantine for the given vault | +| supportsInterface \ | Calls the read-only function "supportsInterface" on the contract. | +| vault-info (vi) \ | get the vault data info | +| vault-quarantine (vq) \ | get vault quarantine | +| vaults-count (vc) | get vaults count | diff --git a/docs/cli/commands/contracts/vault-viewer.md b/docs/cli/commands/contracts/vault-viewer.md index ad1fa055..3f6ebc0d 100644 --- a/docs/cli/commands/contracts/vault-viewer.md +++ b/docs/cli/commands/contracts/vault-viewer.md @@ -24,22 +24,22 @@ yarn start contracts v-v -h ### Read -| Command | Description | -| --------------------------------------------------------------- | --------------------------------------------------------------------------------- | -| DEFAULT_ADMIN_ROLE | Calls the read-only function "DEFAULT_ADMIN_ROLE" on the contract. | -| LAZY_ORACLE | Calls the read-only function "LAZY_ORACLE" on the contract. | -| LIDO_LOCATOR | Calls the read-only function "LIDO_LOCATOR" on the contract. | -| VAULT_HUB | Calls the read-only function "VAULT_HUB" on the contract. | -| has-role \ \ \ | check if an address has a role in a vault | -| isContract \ | Calls the read-only function "isContract" on the contract. | -| is-vault-owner \ \ | checks if a given address is the owner of a connection vault | -| role-members \ \ | get the VaultMembers for each specified role on a single vault | -| role-members-batch \ \ | get VaultMembers for each role on multiple vaults | -| vault-data \ | get aggregated data for a single vault | -| by-owner-batch \ \ \ | get vaults owned by `_owner` using batch pagination over the global vault list | +| Command | Description | +| ------------------------------------------------------------------ | --------------------------------------------------------------------------------- | +| DEFAULT_ADMIN_ROLE | Calls the read-only function "DEFAULT_ADMIN_ROLE" on the contract. | +| LAZY_ORACLE | Calls the read-only function "LAZY_ORACLE" on the contract. | +| LIDO_LOCATOR | Calls the read-only function "LIDO_LOCATOR" on the contract. | +| VAULT_HUB | Calls the read-only function "VAULT_HUB" on the contract. | +| has-role \ \ \ | check if an address has a role in a vault | +| isContract \ | Calls the read-only function "isContract" on the contract. | +| is-vault-owner \ \ | checks if a given address is the owner of a connection vault | +| role-members \ \ | get the VaultMembers for each specified role on a single vault | +| role-members-batch \ \ | get VaultMembers for each role on multiple vaults | +| vault-data \ | get aggregated data for a single vault | +| by-owner-batch \ \ \ | get vaults owned by `_owner` using batch pagination over the global vault list | | by-role-address-batch (by-ra) \ \ \ \ | get vaults where `_member` has `_role`, scanning a batch of the global vault list | -| vaults-count | get the number of vaults connected to the VaultHub | -| vaults-data-batch \ \ | get aggregated data for a batch of vaults | -| my | get all my vaults | -| my-by-role \ | get all vaults where I have a role | -| all | get all vaults connected to vault hub | +| vaults-count | get the number of vaults connected to the VaultHub | +| vaults-data-batch \ \ | get aggregated data for a batch of vaults | +| my | get all my vaults | +| my-by-role \ | get all vaults where I have a role | +| all | get all vaults connected to vault hub | diff --git a/docs/cli/commands/defi-wrapper/distributor.md b/docs/cli/commands/defi-wrapper/distributor.md index 05a3fdb3..a900a937 100644 --- a/docs/cli/commands/defi-wrapper/distributor.md +++ b/docs/cli/commands/defi-wrapper/distributor.md @@ -42,6 +42,7 @@ The Distributor commands are used to manage the distribution of side tokens via | add-token | add a new token to distribute | | set-merkle-root | updates merkle root and CID on the distributor contract | | distribute | generates and optionally uploads new distribution data | +| claim | permissionlessly claim rewards to recipients | ## Command Details @@ -125,18 +126,20 @@ Generates a new rewards distribution based on pool shares, and provides options **Options:** -| Option | Description | Default | -| ---------------------------- | ----------------------------------------------------------------------------------------------- | ----------------------------------- | -| `--blacklist [addresses...]` | A list of addresses to exclude from the distribution. | `[]` | -| `--from-block [block]` | The starting block for calculating pool share balances. | `undefined` | -| `--to-block [block]` | The ending block for calculating pool share balances. | `undefined` | -| `--max-batch-size [size]` | The maximum number of blocks to fetch events for in a single batch. | `50000` | -| `--output-path [path]` | The local file path to save the generated distribution data. | `./distribution-[merkle-root].json` | -| `--skip-write` | If set, the distribution data will not be written to a local file. | `false` | -| `--skip-transfer` | If set, the command will not attempt to transfer the reward tokens to the distributor contract. | `false` | -| `--ipfs-gateway [gateway]` | A custom IPFS gateway to fetch previous distribution data from. | `undefined` | -| `--upload [pinningUrl]` | A URL for a pinning service to upload the distribution data to. | `false` | -| `--skip-set-root` | If set, the new Merkle root will not be set on the distributor contract. | `false` | +| Option | Description | Default | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------- | +| `--blacklist [addresses...]` | A list of addresses to exclude from the distribution. | `[]` | +| `--mode [mode]` | distribution calculation mode, `integral` or `snapshot`. `integral` mode calculates based on historical holding share while `snapshot` calculates by balances on toBlock | `integral` | +| `--from-block [block]` | The starting block for calculating pool share balances. | `undefined` | +| `--to-block [block]` | The ending block for calculating pool share balances. | `undefined` | +| `--max-batch-size [size]` | The maximum number of blocks to fetch events for in a single batch. | `50000` | +| `--output-path [path]` | The local file path to save the generated distribution data. | `./distribution-[merkle-root].json` | +| `--skip-write` | If set, the distribution data will not be written to a local file. | `false` | +| `--skip-transfer` | If set, the command will not attempt to transfer the reward tokens to the distributor contract. | `false` | +| `--ipfs-gateway [gateway]` | A custom IPFS gateway to fetch previous distribution data from. | `undefined` | +| `--upload [pinningUrl]` | (unstable) uploading distribution data to provided pinning service URL | `undefined` | +| `--upload-authorization [token]` | Authorization token for uploading distribution data to pinning service, used as `Authorization: Bearer ` | `undefined` | +| `--skip-set-root` | If set, the new Merkle root will not be set on the distributor contract. | `false` | **Example:** @@ -146,3 +149,24 @@ yarn start dw uc d w distribute 0x... 0xtoken1... 1000 0xtoken2... 500.5 --uploa ``` **Use Case:** To perform a complete rewards distribution run. This command can calculate rewards, generate the necessary data, upload it to IPFS, and update the smart contract all in one go, making it a powerful tool for automating rewards distribution. + +### claim + +Allows to permissionlessly claim distributed tokens to set or all recipients. + +**Arguments:** + +- ``: The contract address of the pool. +- `--recipients [addresses...]`: (optional) The addresses(space separated) of the recipients to claim the distributed tokens for, if not provided, all recipients will be claimed. +- `--tokens [addresses]`: (optional) The addresses(space separated) of the tokens to claim, if not provided, all tokens in the distribution will be claimed. +- `--ipfs-gateway [gateway]`: (optional) A custom IPFS gateway to fetch distribution data from, if not provided, the default gateway will be used. +- `--print-only`: (optional) If set, the claim data will be printed to the console without sending any transactions. + +**Example:** + +```bash +# distribute rewards to specific recipients for specific tokens +yarn start dw uc d w claim 0x --recipients 0x 0x --tokens 0x 0x +``` + +**Use Case:** Manually claim personal rewards or force claim for all recipients diff --git a/docs/cli/commands/deposits.md b/docs/cli/commands/deposits.md index f0398b24..b09f6be3 100644 --- a/docs/cli/commands/deposits.md +++ b/docs/cli/commands/deposits.md @@ -29,30 +29,30 @@ Deposits commands handle validator deposits for Lido Staking Vaults. They work w ### Read -| Command | Description | -| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | -| info | get PredepositGuarantee base info | -| roles | get PredepositGuarantee roles | -| validator-status (v-status) \ | get validator status | -| no-balance (no-bal) | get total, locked & unlocked (the amount of ether that NO can lock for predeposit or withdraw) balances for the NO in PDG | -| no-info | get info about the NO in PDG | -| pending-activations (pd) | get the number of validators in PREDEPOSITED and PROVEN states but not ACTIVATED yet | +| Command | Description | +| ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| info | get PredepositGuarantee base info | +| roles | get PredepositGuarantee roles | +| validator-status (v-status) \ | get validator status | +| no-balance (no-bal) | get total, locked & unlocked (the amount of ether that NO can lock for predeposit or withdraw) balances for the NO in PDG | +| no-info | get info about the NO in PDG | +| pending-activations (pd) | get the number of validators in PREDEPOSITED and PROVEN states but not ACTIVATED yet | ### Write -| Command | Description | -| ------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------ | -| predeposit \ | deposits NO's validators with PREDEPOSIT_AMOUNT ether from StakingVault and locks up NO's balance | -| prove-and-activate (prove) | permissionless method to prove correct Withdrawal Credentials for the validator and to send the activation deposit | -| prove-and-top-up \ \ | prove validators to unlock NO balance, activate the validators from stash, and optionally top up NO balance | -| top-up-existing-validators (top-up-val) \ | deposits ether to proven validators from staking vault. | -| top-up-no \ | top up Node Operator balance | -| withdraw-no-balance \ | withdraw Node Operator balance | -| set-no-guarantor (set-no-g) | set Node Operator guarantor | -| claim-guarantor-refund (claim-g-refund) | claims refund for the previous guarantor of the NO | -| activate-validator (activate) \ | permissionless method to activate the proven validator depositing 31 ETH from the staged balance of StakingVault | -| set-no-depositor (set-no-d) | sets the depositor for the NO | -| unguaranteed-deposit-to-beacon-chain (unguaranteed-deposit) \ | withdraws ether from vault and deposits directly to provided validators bypassing the default PDG process | +| Command | Description | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| predeposit \ | deposits NO's validators with PREDEPOSIT_AMOUNT ether from StakingVault and locks up NO's balance | +| prove-and-activate (prove) | permissionless method to prove correct Withdrawal Credentials for the validator and to send the activation deposit | +| prove-and-top-up \ \ | prove validators to unlock NO balance, activate the validators from stash, and optionally top up NO balance | +| top-up-existing-validators (top-up-val) \ | deposits ether to proven validators from staking vault. | +| top-up-no \ | top up Node Operator balance | +| withdraw-no-balance \ | withdraw Node Operator balance | +| set-no-guarantor (set-no-g) | set Node Operator guarantor | +| claim-guarantor-refund (claim-g-refund) | claims refund for the previous guarantor of the NO | +| activate-validator (activate) \ | permissionless method to activate the proven validator depositing 31 ETH from the staged balance of StakingVault | +| set-no-depositor (set-no-d) | sets the depositor for the NO | +| unguaranteed-deposit-to-beacon-chain (unguaranteed-deposit) \ | withdraws ether from vault and deposits directly to provided validators bypassing the default PDG process | ## Command Details diff --git a/docs/cli/commands/pdg-helpers.md b/docs/cli/commands/pdg-helpers.md index a5c8c318..b36f4ffb 100644 --- a/docs/cli/commands/pdg-helpers.md +++ b/docs/cli/commands/pdg-helpers.md @@ -22,15 +22,15 @@ PredepositGuarantee Helper commands provide utilities for working with Beacon Ch ## API -| Command | Description | -| --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | -| proof-and-check (proof-check) | make predeposit proof by validator index and check by the PDG contract | -| proof | make predeposit proof by validator index | -| verify-predeposit-bls (verify-bls) \ | Verifies BLS signature of the deposit | -| fv-gindex \ | get first validator gindex | -| compute-deposit-data-root (compute-dd-root) \ \ \ \ | compute deposit data root | -| compute-deposit-domain (compute-d-domain) \ | compute deposit domain | -| validator-info \ | get validator info from Consensus Layer | +| Command | Description | +| ------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------- | +| proof-and-check (proof-check) | make predeposit proof by validator index and check by the PDG contract | +| proof | make predeposit proof by validator index | +| verify-predeposit-bls (verify-bls) \ | Verifies BLS signature of the deposit | +| fv-gindex \ | get first validator gindex | +| compute-deposit-data-root (compute-dd-root) \ \ \ \ | compute deposit data root | +| compute-deposit-domain (compute-d-domain) \ | compute deposit domain | +| validator-info \ | get validator info from Consensus Layer | ## Command Details diff --git a/docs/cli/commands/report.md b/docs/cli/commands/report.md index b0ad6b87..d309dfd2 100644 --- a/docs/cli/commands/report.md +++ b/docs/cli/commands/report.md @@ -29,21 +29,21 @@ Report commands handle vault accounting reports and oracle data updates for Lido ### Read -| Command | Description | -| ---------------------- | -------------------------- | -| latest-report-data (lrd) | get the latest report data | -| by-vault | get report by vault | -| proof-by-vault | get proof by vault | -| all | get all reports | -| check-cid | check ipfs CID | +| Command | Description | +| ------------------------ | -------------------------- | +| latest-report-data (lrd) | get the latest report data | +| by-vault | get report by vault | +| proof-by-vault | get proof by vault | +| all | get all reports | +| check-cid | check ipfs CID | ### Write -| Command | Description | -| ------------------------------ | ---------------------------- | -| by-vault-submit (submit) | submit report by vault | -| by-vaults-submit \ | submit report for vaults | -| submit-all | submit report for all vaults | +| Command | Description | +| ----------------------------- | ---------------------------- | +| by-vault-submit (submit) | submit report by vault | +| by-vaults-submit \ | submit report for vaults | +| submit-all | submit report for all vaults | ## Command Details diff --git a/features/defi-wrapper/distributor.ts b/features/defi-wrapper/distributor.ts index f984f2d8..71756377 100644 --- a/features/defi-wrapper/distributor.ts +++ b/features/defi-wrapper/distributor.ts @@ -27,6 +27,7 @@ type GenerateDistributionParams = { toBlock?: bigint; fromBlock?: bigint; maxBatchSize?: bigint; + mode: 'integral' | 'snapshot'; }; const treeFromData = (data: any) => { @@ -152,7 +153,8 @@ const getUserShares = async ( blackListSet: Set
, fromBlock: bigint, toBlock: bigint, - batchSize?: bigint, + batchSize: bigint | undefined, + mode: 'integral' | 'snapshot', ) => { const publicClient = await getPublicClient(); const userSharesMap: Map< @@ -252,8 +254,16 @@ const getUserShares = async ( const lastBalance = info.balance; const lastBlock = info.lastBlock; - const newShare = - lastBalance * (toBlock - lastBlock) + info.accumulatedShare; + let newShare: bigint; + if (mode === 'integral') { + // for integral mode, we calculate share based on historical holding share, so we accumulate share since last block until toBlock + newShare = lastBalance * (toBlock - lastBlock) + info.accumulatedShare; + } else if (mode === 'snapshot') { + // for snapshot mode, we calculate share based on snapshot of balances on toBlock, so we only take the last balance as share + newShare = lastBalance; + } else { + throw new Error(`Unsupported distribution mode: ${mode}`); + } userSharesMap.set(address, { accumulatedShare: newShare, balance: lastBalance, @@ -265,6 +275,18 @@ const getUserShares = async ( return { userSharesMap, denominator }; }; +export const fetchDistributionTree = async ( + cid: string, + ipfsGateway?: string, +) => { + const dataPrev = await fetchIPFS({ + cid, + bigNumberType: 'bigint', + gateway: ipfsGateway, + }); + return treeFromData(dataPrev); +}; + export const generateDistribution = async ({ tokens: tokensArg, poolAddress, @@ -273,6 +295,7 @@ export const generateDistribution = async ({ fromBlock, ipfsGateway, maxBatchSize, + mode, }: GenerateDistributionParams) => { // Contracts const publicClient = await getPublicClient(); @@ -320,12 +343,7 @@ export const generateDistribution = async ({ const merkleRootPrev = await distributor.read.root(); let treePrev: ReturnType | null = null; if (cidPrev === '' && merkleRootPrev !== zeroHash) { - const dataPrev = await fetchIPFS({ - cid: cidPrev, - bigNumberType: 'bigint', - gateway: ipfsGateway, - }); - treePrev = treeFromData(dataPrev); + treePrev = await fetchDistributionTree(cidPrev, ipfsGateway); if (merkleRootPrev !== treePrev.root) { throw new Error( @@ -358,6 +376,7 @@ export const generateDistribution = async ({ fromBlockNumber + 1n, BigInt(currentBlock.number), maxBatchSize, + mode, ); const newMerkleValues: [Address, Address, bigint][] = []; diff --git a/programs/defi-wrapper/use-cases/distributor/write.ts b/programs/defi-wrapper/use-cases/distributor/write.ts index c4fc82e8..efe85fcb 100644 --- a/programs/defi-wrapper/use-cases/distributor/write.ts +++ b/programs/defi-wrapper/use-cases/distributor/write.ts @@ -1,11 +1,14 @@ import { Address, + encodeFunctionData, erc20Abi, formatUnits, getContract, Hash, + Hex, isAddressEqual, parseUnits, + zeroHash, } from 'viem'; import { Option } from 'commander'; import { @@ -20,6 +23,9 @@ import { stringToHash, stringToBigInt, stringArrayToTokenPairs, + pinToIPFS, + logResult, + callWriteMethodWithReceiptBatchCalls, } from 'utils'; import { getDistributorContract, @@ -28,7 +34,10 @@ import { import { distributorUseCases } from './main.js'; import { getPublicClient } from 'providers/wallet.js'; -import { generateDistribution } from 'features/defi-wrapper/distributor.js'; +import { + fetchDistributionTree, + generateDistribution, +} from 'features/defi-wrapper/distributor.js'; import path from 'node:path'; import fs from 'node:fs/promises'; @@ -116,13 +125,15 @@ type DistributeOptions = { blacklist: Address[]; outputPath?: string; skipTransfer: boolean; - pinningUrl?: string; + upload?: string; + uploadAuthorization?: string; skipSetRoot: boolean; skipWrite: boolean; fromBlock?: bigint; toBlock?: bigint; ipfsGateway?: string; - maxBatchSize: bigint; + maxBatchSize?: bigint; + mode: 'integral' | 'snapshot'; }; distributorWrite @@ -134,6 +145,11 @@ distributorWrite 'ERC20 token addresses and amounts to distribute,e.g. 0x... 123.15 0x... 456.78', stringArrayToTokenPairs, ) + .option( + '--mode [mode]', + 'distribution calculation mode, `integral`(default) or `snapshot`. `integral` mode calculates based on historical holding share while `snapshot` calculates by balances on toBlock', + 'integral', + ) .option( '--blacklist [addresses...]', 'addresses to blacklist from distribution', @@ -144,9 +160,8 @@ distributorWrite .option('--to-block [block]', 'to block number', stringToBigInt) .option( '--max-batch-size [size]', - 'maximum batch size for fetching events', + '(default 50000) maximum batch size for fetching events ', stringToBigInt, - 50_000n, ) .option('--output-path [path]', 'path to save distribution data') .option('--skip-write', 'skip writing distribution data to file', false) @@ -157,8 +172,11 @@ distributorWrite ) .option( '--upload [pinningUrl]', - 'uploading distribution data to provided pinning service URL', - false, + '(unstable) uploading distribution data to provided pinning service URL', + ) + .option( + '--upload-authorization [token]', + 'authorization token for uploading distribution data to pinning service, used as `Authorization: Bearer `', ) .option('--skip-set-root', 'skip setting merkle root on distributor', false) .action( @@ -169,13 +187,16 @@ distributorWrite blacklist, outputPath, skipTransfer, - pinningUrl, + upload, + uploadAuthorization, skipSetRoot, skipWrite, fromBlock, toBlock, ipfsGateway, - maxBatchSize, + // commander does not support bigint default value + maxBatchSize = 50000n, + mode, }: DistributeOptions, ) => { const publicClient = await getPublicClient(); @@ -215,6 +236,7 @@ distributorWrite toBlock, fromBlock, ipfsGateway, + mode, maxBatchSize, tokens: tokens.map((t) => ({ address: t.contract.address, @@ -231,16 +253,25 @@ distributorWrite ...merkleTree.dump(), }; - const writeString = JSON.stringify( - dataToWrite, - (_, value) => (typeof value === 'bigint' ? value.toString() : value), - 2, + const writeString = JSON.stringify(dataToWrite, (_, value) => + typeof value === 'bigint' ? value.toString() : value, ); const encoder = new TextEncoder(); - const CID = await calculateIPFSAddCID(encoder.encode(writeString)); + const distributionCID = await calculateIPFSAddCID( + encoder.encode(writeString), + ); + const CidV0 = distributionCID.toV0().toString(); - logInfo(`Calculated new distribution data CID: ${CID.toString()}`); + logInfo(`Calculated new distribution data CID: ${CidV0}`); + logResult({ + data: [ + ['Merkle Root', merkleTree.root], + ['CID', distributionCID.toString()], + ['CID (v0)', CidV0], + ['CID (v1)', distributionCID.toV1().toString()], + ], + }); // writing distribution data to file if (!skipWrite) { @@ -290,41 +321,172 @@ distributorWrite logInfo(`Skipping transfer of tokens to distributor as requested.`); } - if (pinningUrl) { - const fetchResponse = await fetch(pinningUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: writeString, - }); - - if (!fetchResponse.ok) { - logError( - `Failed to upload distribution data to pinning service at ${pinningUrl}. Status: ${fetchResponse.status} ${fetchResponse.statusText}`, - ); - return; - } - const responseData = await fetchResponse.json(); + if (upload) { logInfo( - `Successfully uploaded distribution data to pinning service at ${pinningUrl}. Response: ${JSON.stringify( - responseData, - )}`, + '⚠️⚠️⚠️ Uploading distribution API is unstable, use manual upload to IPFS and choose CIDv0. ⚠️⚠️⚠️', ); + try { + const uploadResponse = await pinToIPFS({ + fileContent: writeString, + uploadUrl: upload, + uploadAuthorization, + fileName: `distribution-${merkleTree.root}.json`, + }); + logInfo('Distribution uploaded to IPFS provider', uploadResponse); + } catch (error) { + logError('Failed to upload distribution data to IPFS', error); + } } if (!skipSetRoot) { + const cidV0 = distributionCID.toV0().toString(); + const confirm = await confirmOperation( - `Set new Merkle root ${merkleTree.root} and CID ${CID.toString()} on distributor at ${distributorAddress}?`, + `Set new Merkle root ${merkleTree.root} and CID ${cidV0} on distributor at ${distributorAddress}?`, ); if (confirm) { await callWriteMethodWithReceipt({ contract: await getDistributorContract(distributorAddress), methodName: 'setMerkleRoot', - payload: [merkleTree.root as Hash, CID.toString()], + payload: [merkleTree.root as Hash, cidV0], }); } } }, ); + +type ClaimOptions = { + recipients?: Address[]; + tokens?: Address[]; + printOnly: boolean; + ipfsGateway?: string; +}; + +distributorWrite + .command('claim') + .description('claim tokens from distributor to recipients permissionlessly') + .argument('', 'pool contract address', stringToAddress) + .option( + '--recipients [addresses...]', + 'recipient addresses to claim for, empty to claim for all recipients(might call many transactions)', + stringArrayToAddressArray, + ) + .option( + '--tokens [addresses...]', + 'only claim for specified token addresses', + stringArrayToAddressArray, + ) + .option( + '--ipfs-gateway [gateway]', + 'IPFS gateway to fetch previous data from', + ) + .option( + '--print-only', + 'only print claim data without sending transactions', + false, + ) + .action(async (poolAddress: Address, options: ClaimOptions) => { + const poolContract = await getStvPoolContract(poolAddress); + const distributorAddress = await poolContract.read.DISTRIBUTOR(); + const distributor = await getDistributorContract(distributorAddress); + + const [cid, root] = await Promise.all([ + distributor.read.cid(), + distributor.read.root(), + ]); + + logResult({ + data: [ + ['Merkle Root', root], + ['CID', cid], + ], + }); + + if (root === zeroHash || cid === '') { + logError(`No distribution present. Merkle Root: ${root}, CID: ${cid}`); + return; + } + + const distribution = await fetchDistributionTree(cid, options.ipfsGateway); + + const userAddressSet = options.recipients + ? new Set(options.recipients.map((addr) => addr.toLowerCase())) + : null; + + const tokenFilterSet = options.tokens + ? new Set(options.tokens.map((addr) => addr.toLowerCase())) + : null; + + const claims: { + leafIndex: number; + recipient: Address; + token: Address; + cumulativeAmount: bigint; + claimableAmount: bigint; + proof: Hex[]; + }[] = []; + + for (const leaf of distribution.entries()) { + const [index, [recipientRaw, token, cumulativeAmount]] = leaf; + const recipient = recipientRaw.toLowerCase() as Address; + + if (userAddressSet && !userAddressSet.has(recipient)) continue; + if (tokenFilterSet && !tokenFilterSet.has(token.toLowerCase())) continue; + + const proof = distribution.getProof(index) as Hex[]; + + const claimableAmount = await distributor.read.previewClaim([ + recipient, + token, + cumulativeAmount, + proof, + ]); + + if (claimableAmount == 0n) continue; + + claims.push({ + leafIndex: index, + recipient, + token, + cumulativeAmount, + claimableAmount, + proof, + }); + } + + if (claims.length === 0) { + logInfo('No claims found for the given filters.'); + return; + } + + if (options.printOnly) { + logResult({ + params: { + head: ['Recipient', 'Token', 'Claimable Amount'], + }, + data: claims.map((claim) => [ + claim.recipient, + claim.token, + claim.claimableAmount, + ]), + }); + return; + } + + await callWriteMethodWithReceiptBatchCalls({ + calls: claims.map((claim) => ({ + to: distributor.address, + data: encodeFunctionData({ + abi: distributor.abi, + functionName: 'claim', + args: [ + claim.recipient, + claim.token, + claim.cumulativeAmount, + claim.proof, + ], + }), + })), + }); + }); diff --git a/programs/defi-wrapper/use-cases/timelock-governance/common/read.ts b/programs/defi-wrapper/use-cases/timelock-governance/common/read.ts index b1886221..c455b792 100644 --- a/programs/defi-wrapper/use-cases/timelock-governance/common/read.ts +++ b/programs/defi-wrapper/use-cases/timelock-governance/common/read.ts @@ -23,12 +23,14 @@ import { import { getPublicClient } from 'providers'; import { DashboardAbi } from 'abi'; -import { StvPoolAbi } from 'abi/defi-wrapper/StvPool.js'; -import { StvStETHPoolAbi } from 'abi/defi-wrapper/StvStEthPool.js'; -import { WithdrawalQueueAbi } from 'abi/defi-wrapper/WithdrawalQueue.js'; -import { OssifiableProxyAbi } from 'abi/defi-wrapper/OssifiableProxy.js'; -import { TimeLockAbi } from 'abi/defi-wrapper/TimeLock.js'; -import { DistributorAbi } from 'abi/defi-wrapper/Distributor.js'; +import { + StvPoolAbi, + StvStETHPoolAbi, + WithdrawalQueueAbi, + TimeLockAbi, + OssifiableProxyAbi, + DistributorAbi, +} from 'abi/defi-wrapper/index.js'; // all abis of expected timelock governed contracts const mixAbi = [ diff --git a/tests/fixtures/consolidation-fixtures.ts b/tests/fixtures/consolidation-fixtures.ts index 4b388142..6be3d884 100644 --- a/tests/fixtures/consolidation-fixtures.ts +++ b/tests/fixtures/consolidation-fixtures.ts @@ -19,8 +19,7 @@ export const VALID_DASHBOARD = '0x318FcB0CCE93aBA9C21a1B4B38dbACcCEfF091E0' as Hex; export const VALID_REFUND_RECIPIENT = '0x463f500FCb218d38FB35BECD20475ea75a79B7A9' as Hex; -export const ZERO_ADDRESS = - '0x0000000000000000000000000000000000000000' as Hex; +export const ZERO_ADDRESS = '0x0000000000000000000000000000000000000000' as Hex; export const createValidatorInfo = ( overrides: Partial = {}, @@ -81,8 +80,7 @@ export const createCLValidatorData = ( activation_eligibility_epoch: '0', activation_epoch: overrides.activation_epoch ?? '0', exit_epoch: overrides.exit_epoch ?? '18446744073709551615', - withdrawable_epoch: - overrides.withdrawable_epoch ?? '18446744073709551615', + withdrawable_epoch: overrides.withdrawable_epoch ?? '18446744073709551615', }, }); diff --git a/tests/integration/report-fresh.test.ts b/tests/integration/report-fresh.test.ts index 680e13dd..8fd4884d 100644 --- a/tests/integration/report-fresh.test.ts +++ b/tests/integration/report-fresh.test.ts @@ -1,9 +1,6 @@ import { describe, test, expect, beforeAll, vi } from 'vitest'; import { type Address } from 'viem'; -import { - checkIsReportFresh, - checkIsReportFreshThrowError, -} from 'features'; +import { checkIsReportFresh, checkIsReportFreshThrowError } from 'features'; import { loadTestConfig } from './helpers/test-config.js'; import { setupIntegrationTestsBeforeAll } from './helpers/test-setup.js'; diff --git a/tests/utils/confirmations.test.ts b/tests/utils/confirmations.test.ts index cc610fb7..d2b1be86 100644 --- a/tests/utils/confirmations.test.ts +++ b/tests/utils/confirmations.test.ts @@ -32,10 +32,7 @@ describe('formatConfirmationArgs', () => { it('should format transferVaultOwnership with new owner', () => { const newOwner = '0xabcdefabcdefabcdefabcdefabcdefabcdefabcd' as Address; - const result = formatConfirmationArgs( - [newOwner], - 'transferVaultOwnership', - ); + const result = formatConfirmationArgs([newOwner], 'transferVaultOwnership'); expect(result).toBe(`new owner: ${newOwner}`); }); diff --git a/tests/utils/consolidation-validators-checks.test.ts b/tests/utils/consolidation-validators-checks.test.ts index a3913775..32a5461d 100644 --- a/tests/utils/consolidation-validators-checks.test.ts +++ b/tests/utils/consolidation-validators-checks.test.ts @@ -86,7 +86,9 @@ describe('checkSourceValidators', () => { }), ]; expect(() => checkSourceValidators(validators, FINALIZED_EPOCH)).toThrow( - new RegExp(`${VALID_PUBKEY_1}.*${VALID_PUBKEY_2}|${VALID_PUBKEY_2}.*${VALID_PUBKEY_1}`), + new RegExp( + `${VALID_PUBKEY_1}.*${VALID_PUBKEY_2}|${VALID_PUBKEY_2}.*${VALID_PUBKEY_1}`, + ), ); }); }); diff --git a/tests/utils/ipfs.test.ts b/tests/utils/ipfs.test.ts index 47f473cc..cfe6537a 100644 --- a/tests/utils/ipfs.test.ts +++ b/tests/utils/ipfs.test.ts @@ -137,6 +137,7 @@ describe('ipfs helpers', () => { const other = CID.parse( 'bafkreib3m4q5x2fr2di5m3lgvq4hzj4qkgjq2d0k8vh7y6xfxkrmwrkduy', ); + const dataCid = `bafkreigh2akiscaildcjk2d6gtrevhb7f7esg6k4t4u5p4sqkgfa6vlriu`; const jsonData = '{"test":456}'; const encoder = new TextEncoder(); const fileContent = encoder.encode(jsonData); @@ -154,6 +155,8 @@ describe('ipfs helpers', () => { await expect( ipfs.fetchIPFSDirectAndVerify(fakeCid.toString()), - ).rejects.toThrow('CID mismatch'); + ).rejects.toThrow( + `❌ File hash mismatch! Expected ${dataCid}, but got ${other}`, + ); }); }); diff --git a/utils/ipfs.ts b/utils/ipfs.ts index 49657501..c59938bd 100644 --- a/utils/ipfs.ts +++ b/utils/ipfs.ts @@ -1,4 +1,4 @@ -import { CID } from 'multiformats/cid'; +import { CID, Version } from 'multiformats/cid'; import { MemoryBlockstore } from 'blockstore-core'; import { importer } from 'ipfs-unixfs-importer'; import jsonBigInt from 'json-bigint'; @@ -71,12 +71,14 @@ export const fetchIPFSBuffer = async ( // Recalculate CID using full UnixFS logic (like `ipfs add`) export const calculateIPFSAddCID = async ( fileContent: Uint8Array, + version: Version = 0, ): Promise => { const blockstore = new MemoryBlockstore(); const entries = importer([{ content: fileContent }], blockstore, { - cidVersion: 0, - rawLeaves: false, // important! otherwise CID will be v1 + cidVersion: version, + // important! otherwise CID will be v1 + rawLeaves: version === 0 ? false : undefined, }); let lastCid: CID | null = null; @@ -99,19 +101,22 @@ export const fetchIPFSDirectAndVerify = async ( const originalCID = CID.parse(cid); const fileContent = await fetchIPFSBuffer({ cid, gateway }); - const calculatedCID = await calculateIPFSAddCID(fileContent); + const calculatedCID = await calculateIPFSAddCID( + fileContent, + originalCID.version, + ); if (!calculatedCID.equals(originalCID)) { throw new Error( - `❌ CID mismatch! Expected ${originalCID}, but got ${calculatedCID}`, + `❌ File hash mismatch! Expected ${originalCID}, but got ${calculatedCID}`, ); } logTable({ data: [ ['✅ CID verified, file matches IPFS hash'], - ['Original CID', originalCID.toString()], - ['Calculated CID', calculatedCID.toString()], + [`Original CIDv${originalCID.version}`, originalCID.toString()], + [`Calculated CIDv${calculatedCID.version}`, calculatedCID.toString()], ], params: { head: ['Type', 'CID'], @@ -142,3 +147,44 @@ export const fetchIPFSWithCacheAndVerify = async ( return json; } }; + +type PinToIPFSArgs = { + uploadUrl: string; + uploadAuthorization?: string; + fileContent: string; + fileName?: string; + uploadType?: 'pinata'; +}; + +export const pinToIPFS = async ({ + fileContent, + fileName = 'file.json', + uploadAuthorization, + uploadUrl, +}: PinToIPFSArgs): Promise => { + const blob = new Blob([fileContent]); + const file = new File([blob], fileName, { + type: 'application/json', + }); + const formData = new FormData(); + formData.append('file', file); + + const fetchResponse = await fetch(uploadUrl, { + method: 'POST', + headers: { + ...(uploadAuthorization + ? { Authorization: `Bearer ${uploadAuthorization}` } + : {}), + }, + + body: formData, + }); + + if (!fetchResponse.ok) { + throw new Error( + `Failed to upload distribution data to pinning service at ${uploadUrl}. Status: ${fetchResponse.status} ${fetchResponse.statusText}`, + ); + } + const responseData = await fetchResponse.json(); + return responseData; +};