From 7e82e7064635707d29b42839d23c9a53eb5eff56 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Fri, 9 Jan 2026 00:40:52 +0300 Subject: [PATCH 01/65] Add update redeemable positions command Signed-off-by: cyc60 --- src/commands/internal/__init__.py | 0 .../internal/update_redeemable_positions.py | 187 ++++++++++++++++++ src/common/abi/Erc20Token.json | 56 ++++++ src/common/clients.py | 42 ++++ src/common/contracts.py | 9 + src/config/networks.py | 10 + src/config/settings.py | 41 +++- src/main.py | 4 + src/redeem/__init__.py | 0 src/redeem/api_client.py | 52 +++++ src/redeem/graph.py | 64 ++++++ src/redeem/typings.py | 31 +++ src/reward_splitter/graph.py | 4 + 13 files changed, 497 insertions(+), 3 deletions(-) create mode 100644 src/commands/internal/__init__.py create mode 100644 src/commands/internal/update_redeemable_positions.py create mode 100644 src/common/abi/Erc20Token.json create mode 100644 src/redeem/__init__.py create mode 100644 src/redeem/api_client.py create mode 100644 src/redeem/graph.py create mode 100644 src/redeem/typings.py diff --git a/src/commands/internal/__init__.py b/src/commands/internal/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py new file mode 100644 index 00000000..f817ca33 --- /dev/null +++ b/src/commands/internal/update_redeemable_positions.py @@ -0,0 +1,187 @@ +import asyncio +import logging +import sys +from collections import defaultdict +from pathlib import Path + +import click +from eth_typing import BlockNumber, ChecksumAddress +from web3 import Web3 +from web3.types import Wei + +from src.common.clients import ( + build_ipfs_upload_clients, + execution_client, + setup_clients, +) +from src.common.contracts import Erc20Contract +from src.common.logging import LOG_LEVELS, setup_logging +from src.common.utils import log_verbose +from src.config.networks import AVAILABLE_NETWORKS, ZERO_CHECKSUM_ADDRESS +from src.config.settings import settings +from src.redeem.api_client import APIClient +from src.redeem.graph import graph_get_allocators, graph_get_leverage_positions_proxies +from src.redeem.typings import Allocator, RedeemablePosition + +logger = logging.getLogger(__name__) + + +@click.option( + '--wallet-password-file', + type=click.Path(exists=True, file_okay=True, dir_okay=False), + envvar='WALLET_PASSWORD_FILE', + help='Absolute path to the wallet password file. ' + 'Default is the file generated with "create-wallet" command.', +) +@click.option( + '--wallet-file', + type=click.Path(exists=True, file_okay=True, dir_okay=False), + envvar='WALLET_FILE', + help='Absolute path to the wallet. ' + 'Default is the file generated with "create-wallet" command.', +) +@click.option( + '--execution-endpoints', + type=str, + envvar='EXECUTION_ENDPOINTS', + prompt='Enter the comma separated list of API endpoints for execution nodes', + help='Comma separated list of API endpoints for execution nodes.', +) +@click.option( + '--execution-jwt-secret', + type=str, + envvar='EXECUTION_JWT_SECRET', + help='JWT secret key used for signing and verifying JSON Web Tokens' + ' when connecting to execution nodes.', +) +@click.option( + '--graph-endpoint', + type=str, + envvar='GRAPH_ENDPOINT', + help='API endpoint for graph node.', +) +@click.option( + '--log-level', + type=click.Choice( + LOG_LEVELS, + case_sensitive=False, + ), + default='INFO', + envvar='LOG_LEVEL', + help='The log level.', +) +@click.option( + '-v', + '--verbose', + help='Enable debug mode. Default is false.', + envvar='VERBOSE', + is_flag=True, +) +@click.option( + '--network', + help='The network of the meta vaults.', + prompt='Enter the network name', + envvar='NETWORK', + type=click.Choice( + AVAILABLE_NETWORKS, + case_sensitive=False, + ), +) +@click.command( + help='Performs a vault validators consolidation from 0x01 validators to 0x02 validator. ' + 'Switches a validator from 0x01 to 0x02 if the source and target keys are identical.', +) +# pylint: disable-next=too-many-arguments +def update_redeemable_positions( + execution_endpoints: str, + execution_jwt_secret: str | None, + graph_endpoint: str, + network: str, + verbose: bool, + log_level: str, + wallet_file: str | None, + wallet_password_file: str | None, +) -> None: + settings.set( + vault=ZERO_CHECKSUM_ADDRESS, + vault_dir=Path.home() / '.stakewise', + execution_endpoints=execution_endpoints, + execution_jwt_secret=execution_jwt_secret, + graph_endpoint=graph_endpoint, + verbose=verbose, + network=network, + wallet_file=wallet_file, + wallet_password_file=wallet_password_file, + log_level=log_level, + ) + try: + asyncio.run(main()) + except Exception as e: + log_verbose(e) + sys.exit(1) + + +async def main() -> None: + """ + Fetch redeemable positions, calculate kept os token amounts and upload to IPFS. + """ + setup_logging() + await setup_clients() + block_number = await execution_client.eth.block_number + allocators = await graph_get_allocators(block_number) + + # filter + boost_proxies = await graph_get_leverage_positions_proxies(block_number) + logger.info('Found %s boost positions to exclude', len(boost_proxies)) + allocators = [a for a in allocators if a.minted_shares > 0 and a.address not in boost_proxies] + logger.info('Filtered allocators count: %s', len(allocators)) + + user_addresses = set(allocator.address for allocator in allocators) + logger.info('Fetching kept tokens for %s addresses...', len(user_addresses)) + + kept_tokens = await get_kept_tokens(list(user_addresses), block_number) + redeemable_positions: list[RedeemablePosition] = [] + for allocator in allocators: + kept_token = kept_tokens.get(allocator.address, Wei(0)) # 0? + amount = min(allocator.minted_shares, kept_token) + if amount > 0: + redeemable_positions.append( + RedeemablePosition( + owner=allocator.address, + vault=allocator.vault, + amount=Wei(allocator.minted_shares - amount), + ) + ) + kept_tokens[allocator.address] = Wei(kept_tokens[allocator.address] - amount) + logger.info('Fetched kept tokens for %s addresses...', {len(user_addresses)}) + + click.confirm( + 'Proceed consolidation?', + default=True, + abort=True, + ) + ipfs_upload_client = build_ipfs_upload_clients() + ipfs_hash = await ipfs_upload_client.upload_json([p.as_dict() for p in redeemable_positions]) + click.echo(f'Redeemable position uploaded to IPFS: hash={ipfs_hash}') + + +async def get_kept_tokens( + user_addresses: list[ChecksumAddress], block_number: BlockNumber +) -> dict[ChecksumAddress, Wei]: + wallet_balances = {} + + contract = Erc20Contract(settings.network_config.OS_TOKEN_CONTRACT_ADDRESS) + for address in user_addresses: + wallet_balances[address] = await contract.balance(address, block_number) + api_client = APIClient() + locked_oseth_per_user: dict[ChecksumAddress, Wei] = {} + for address in user_addresses: + locked_os_token = await api_client.get_protocols_locked_locked_os_token(address=address) + locked_oseth_per_user[address] = locked_os_token + + kept_token = defaultdict(lambda: Wei(0)) + for address, amount in locked_oseth_per_user.items(): + kept_token[address] = amount + for address, amount in wallet_balances.items(): + kept_token[address] = Wei(kept_token[address] + amount) + return kept_token diff --git a/src/common/abi/Erc20Token.json b/src/common/abi/Erc20Token.json new file mode 100644 index 00000000..6271a1a7 --- /dev/null +++ b/src/common/abi/Erc20Token.json @@ -0,0 +1,56 @@ +[ + { + "inputs": [], + "name": "symbol", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "account", + "type": "address" + } + ], + "name": "balanceOf", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "name": "from", + "type": "address" + }, + { + "indexed": true, + "name": "to", + "type": "address" + }, + { + "indexed": false, + "name": "value", + "type": "uint256" + } + ], + "name": "Transfer", + "type": "event" + } +] diff --git a/src/common/clients.py b/src/common/clients.py index 6e856d6e..b120fa4a 100644 --- a/src/common/clients.py +++ b/src/common/clients.py @@ -5,8 +5,12 @@ from typing import cast from sw_utils import ( + BaseUploadClient, ExtendedAsyncBeacon, IpfsFetchClient, + IpfsMultiUploadClient, + IpfsUploadClient, + PinataUploadClient, get_consensus_client, get_execution_client, ) @@ -118,6 +122,44 @@ async def fetch_json(self, ipfs_hash: str) -> dict | list: return await self.client.fetch_json(ipfs_hash) +def build_ipfs_upload_clients() -> IpfsMultiUploadClient: + clients: list[BaseUploadClient] = [] + + if settings.ipfs_local_client_endpoint: + clients.append( + IpfsUploadClient( + settings.ipfs_local_client_endpoint, + username=settings.ipfs_local_username, + password=settings.ipfs_local_password, + timeout=settings.ipfs_upload_client_timeout, + ) + ) + + if settings.ipfs_infura_client_username and settings.ipfs_infura_client_password: + ipfs_client = IpfsUploadClient( + settings.ipfs_infura_client_endpoint, + settings.ipfs_infura_client_username, + settings.ipfs_infura_client_password, + timeout=settings.ipfs_upload_client_timeout, + ) + clients.append(ipfs_client) + + if settings.ipfs_pinata_api_key and settings.ipfs_pinata_secret_key: + pinata_client = PinataUploadClient( + settings.ipfs_pinata_api_key, + settings.ipfs_pinata_secret_key, + timeout=settings.ipfs_upload_client_timeout, + ) + clients.append(pinata_client) + + if not clients: + raise ValueError( + 'No IPFS clients settings. ' + 'Please provide IPFS_LOCAL_CLIENT_ENDPOINT or third party IPFS services settings.' + ) + return IpfsMultiUploadClient(clients) + + db_client = Database() execution_client = cast(AsyncWeb3, ExecutionClient()) execution_non_retry_client = cast(AsyncWeb3, ExecutionClient(use_retries=False)) diff --git a/src/common/contracts.py b/src/common/contracts.py index b551656f..15803a40 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -257,6 +257,15 @@ async def _get_public_keys_chunk( return [Web3.to_hex(event['args']['publicKey']) for event in events] +class Erc20Contract(ContractWrapper): + abi_path = 'abi/Erc20Token.json' + + async def balance( + self, address: ChecksumAddress, block_number: BlockNumber | None = None + ) -> Wei: + return await self.contract.functions.balanceOf(address).call(block_identifier=block_number) + + class ValidatorsRegistryContract(ContractWrapper): abi_path = 'abi/IValidatorsRegistry.json' settings_key = 'VALIDATORS_REGISTRY_CONTRACT_ADDRESS' diff --git a/src/config/networks.py b/src/config/networks.py index 2b2af976..9f80c7a2 100644 --- a/src/config/networks.py +++ b/src/config/networks.py @@ -26,6 +26,7 @@ class NetworkConfig(BaseNetworkConfig): VALIDATORS_CHECKER_CONTRACT_ADDRESS: ChecksumAddress CONSOLIDATION_CONTRACT_ADDRESS: ChecksumAddress WITHDRAWAL_CONTRACT_ADDRESS: ChecksumAddress + OS_TOKEN_CONTRACT_ADDRESS: ChecksumAddress WALLET_MIN_BALANCE: Wei STAKEWISE_API_URL: str STAKEWISE_GRAPH_ENDPOINT: str @@ -92,6 +93,9 @@ def INITIAL_SYNC_ETA(self) -> int: WITHDRAWAL_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x00000961Ef480Eb55e80D19ad83579A64c007002' ), + OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0xf1C9acDc66974dFB6dEcB12aA385b9cD01190E38' + ), WALLET_MIN_BALANCE=Web3.to_wei('0.03', 'ether'), STAKEWISE_API_URL='https://mainnet-api.stakewise.io/graphql', STAKEWISE_GRAPH_ENDPOINT=( @@ -142,6 +146,9 @@ def INITIAL_SYNC_ETA(self) -> int: WITHDRAWAL_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x00000961Ef480Eb55e80D19ad83579A64c007002' ), + OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0x7345fC8268459413beE9e9dd327f31283C65Ee7e' + ), WALLET_MIN_BALANCE=Web3.to_wei('0.03', 'ether'), STAKEWISE_API_URL='https://hoodi-api.stakewise.io/graphql', STAKEWISE_GRAPH_ENDPOINT='https://graphs.stakewise.io/hoodi/subgraphs/name/stakewise/prod', @@ -190,6 +197,9 @@ def INITIAL_SYNC_ETA(self) -> int: WITHDRAWAL_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x00000961Ef480Eb55e80D19ad83579A64c007002' ), + OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0xF490c80aAE5f2616d3e3BDa2483E30C4CB21d1A0' + ), WALLET_MIN_BALANCE=Web3.to_wei('0.01', 'ether'), STAKEWISE_API_URL='https://gnosis-api.stakewise.io/graphql', STAKEWISE_GRAPH_ENDPOINT=( diff --git a/src/config/settings.py b/src/config/settings.py index c3f0dada..57ca2ffa 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -89,6 +89,17 @@ class Settings(metaclass=Singleton): ipfs_fetch_endpoints: list[str] ipfs_timeout: int ipfs_retry_timeout: int + + ipfs_upload_client_timeout: int + ipfs_local_client_endpoint: str + ipfs_local_username: str | None + ipfs_local_password: str | None + ipfs_infura_client_endpoint: str + ipfs_infura_client_username: str | None + ipfs_infura_client_password: str | None + ipfs_pinata_api_key: str | None + ipfs_pinata_secret_key: str | None + genesis_validators_ipfs_timeout: int genesis_validators_ipfs_retry_timeout: int validators_fetch_chunk_size: int @@ -290,6 +301,30 @@ def set( self.ipfs_timeout = decouple_config('IPFS_TIMEOUT', default=60, cast=int) self.ipfs_retry_timeout = decouple_config('IPFS_RETRY_TIMEOUT', default=120, cast=int) + self.ipfs_upload_client_timeout = decouple_config( + 'IPFS_UPLOAD_CLIENT_TIMEOUT', default=30, cast=int + ) + + # local IPFS + self.ipfs_local_client_endpoint: str = decouple_config( + 'IPFS_LOCAL_CLIENT_ENDPOINT', default='' + ) + self.ipfs_local_username: str | None = decouple_config('IPFS_LOCAL_USERNAME', default=None) + self.ipfs_local_password: str | None = decouple_config('IPFS_LOCAL_PASSWORD', default=None) + + # infura + self.ipfs_infura_client_endpoint: str = '/dns/ipfs.infura.io/tcp/5001/https' + self.ipfs_infura_client_username: str = decouple_config( + 'IPFS_INFURA_CLIENT_USERNAME', default='' + ) + self.ipfs_infura_client_password: str = decouple_config( + 'IPFS_INFURA_CLIENT_PASSWORD', default='' + ) + + # pinata + self.ipfs_pinata_api_key: str = decouple_config('IPFS_PINATA_API_KEY', default='') + self.ipfs_pinata_secret_key: str = decouple_config('IPFS_PINATA_SECRET_KEY', default='') + # Genesis validators ipfs fetch may have larger timeouts self.genesis_validators_ipfs_timeout = decouple_config( 'GENESIS_VALIDATORS_IPFS_TIMEOUT', default=300, cast=int @@ -318,9 +353,9 @@ def set( self.consensus_retry_timeout = decouple_config( 'CONSENSUS_RETRY_TIMEOUT', default=120, cast=int ) - self.graph_request_timeout = decouple_config('GRAPH_REQUEST_TIMEOUT', default=10, cast=int) - self.graph_retry_timeout = decouple_config('GRAPH_RETRY_TIMEOUT', default=60, cast=int) - self.graph_page_size = decouple_config('GRAPH_PAGE_SIZE', default=100, cast=int) + self.graph_request_timeout = decouple_config('GRAPH_REQUEST_TIMEOUT', default=30, cast=int) + self.graph_retry_timeout = decouple_config('GRAPH_RETRY_TIMEOUT', default=120, cast=int) + self.graph_page_size = decouple_config('GRAPH_PAGE_SIZE', default=500, cast=int) self.relayer_endpoint = relayer_endpoint or '' self.relayer_timeout = decouple_config('RELAYER_TIMEOUT', default=10, cast=int) diff --git a/src/main.py b/src/main.py index 4cc7c9e8..5a121870 100644 --- a/src/main.py +++ b/src/main.py @@ -13,6 +13,9 @@ from src.commands.create_wallet import create_wallet from src.commands.exit_validators import exit_validators from src.commands.init import init +from src.commands.internal.update_redeemable_positions import ( + update_redeemable_positions, +) from src.commands.nodes.node_install import node_install from src.commands.nodes.node_start import node_start from src.commands.nodes.node_status import node_status_command as node_status @@ -57,6 +60,7 @@ def cli() -> None: cli.add_command(node_install) cli.add_command(node_start) cli.add_command(node_status) +cli.add_command(update_redeemable_positions) if __name__ == '__main__': diff --git a/src/redeem/__init__.py b/src/redeem/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/redeem/api_client.py b/src/redeem/api_client.py new file mode 100644 index 00000000..45aeac48 --- /dev/null +++ b/src/redeem/api_client.py @@ -0,0 +1,52 @@ +import aiohttp +from eth_typing import ChecksumAddress +from sw_utils.common import urljoin +from web3 import Web3 +from web3.types import Wei + +from src.config.settings import settings + +API_ENDPOINT = 'https://api.rabby.io/' +DEFAULT_USER_AGENT = ( + 'Mozilla/5.0 (X11; Linux x86_64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36' +) + + +class APIClient: + + base_url = API_ENDPOINT + + async def get_protocols_locked_locked_os_token(self, address: ChecksumAddress) -> Wei: + url = urljoin(self.base_url, 'v1/user/complex_protocol_list') + params = { + 'id': address, + } + + protocol_data = [] + async with aiohttp.ClientSession() as session: + async with session.get( + url=url, + params=params, + headers={'user-agent:': DEFAULT_USER_AGENT}, + ) as response: + response.raise_for_status() + protocol_data = await response.json() + + total_locked_oseth = Wei(0) + for protocol in protocol_data: + if protocol['id'] == 'stakewise': + continue + portfolio_item_list = protocol.get('portfolio_item_list', []) + for item in portfolio_item_list: + for assets in item.get('asset_token_list', []): + if not Web3.is_address(assets['id']): + continue + if ( + Web3.to_checksum_address(assets['id']) + == settings.network_config.OS_TOKEN_CONTRACT_ADDRESS + ): + total_locked_oseth = Wei( + total_locked_oseth + Web3.to_wei(float(assets['amount']), 'ether') + ) + return total_locked_oseth diff --git a/src/redeem/graph.py b/src/redeem/graph.py new file mode 100644 index 00000000..1f404da0 --- /dev/null +++ b/src/redeem/graph.py @@ -0,0 +1,64 @@ +import logging + +from eth_typing import BlockNumber, ChecksumAddress +from gql import gql +from web3 import Web3 + +from src.common.clients import graph_client +from src.redeem.typings import Allocator + +logger = logging.getLogger(__name__) + + +async def graph_get_allocators(block_number: BlockNumber) -> list[Allocator]: + """ + Returns mapping from sub-vault address to list of ExitRequest objects + Skips claimed exit requests and those with exitedAssets == 0 + """ + query = gql( + """ + query getAllocators($block: Int, $first: Int, $lastID: String){ + allocators( + block: {number: $block}, + where: { + id_gt: $lastID + }, + orderBy: id + first: $first + ){ + vault { + id + } + id + address + mintedOsTokenShares + } + } + """ + ) + params = { + 'block': block_number, + } + response = await graph_client.fetch_pages(query, params=params, cursor_pagination=True) + return [Allocator.from_graph(item) for item in response] + + +async def graph_get_leverage_positions_proxies(block_number: BlockNumber) -> list[ChecksumAddress]: + query = gql( + """ + query PositionsQuery($block: Int, $first: Int, $skip: Int) { + leverageStrategyPositions( + block: { number: $block }, + orderBy: borrowLtv, + orderDirection: desc, + first: $first + skip: $skip + ) { + proxy + } + } + """ + ) + params = {'block': block_number} + response = await graph_client.fetch_pages(query, params=params) + return [Web3.to_checksum_address(item['proxy']) for item in response] diff --git a/src/redeem/typings.py b/src/redeem/typings.py new file mode 100644 index 00000000..60375ef3 --- /dev/null +++ b/src/redeem/typings.py @@ -0,0 +1,31 @@ +import dataclasses +from dataclasses import dataclass + +from eth_typing import ChecksumAddress +from web3 import Web3 +from web3.types import Wei + + +@dataclass +class Allocator: + address: ChecksumAddress + vault: ChecksumAddress + minted_shares: Wei + + @classmethod + def from_graph(cls, data: dict) -> 'Allocator': + return Allocator( + vault=Web3.to_checksum_address(data['vault']['id']), + address=Web3.to_checksum_address(data['address']), + minted_shares=Wei(int(data['mintedOsTokenShares'])), + ) + + +@dataclass +class RedeemablePosition: + owner: ChecksumAddress + vault: ChecksumAddress + amount: Wei + + def as_dict(self) -> dict: + return dataclasses.asdict(self) diff --git a/src/reward_splitter/graph.py b/src/reward_splitter/graph.py index 37eb7aac..5e4eef62 100644 --- a/src/reward_splitter/graph.py +++ b/src/reward_splitter/graph.py @@ -23,6 +23,8 @@ async def graph_get_reward_splitters( vault: $vault, version_gte: 3, } + first: $first, + skip: $skip ) { id vault { @@ -77,6 +79,8 @@ async def graph_get_claimable_exit_requests( isClaimable: true, isClaimed: false } + first: $first, + skip: $skip ) { id positionTicket From 488c6bceaa6e49c5ff51c18a3962e5b33d61b96f Mon Sep 17 00:00:00 2001 From: cyc60 Date: Fri, 9 Jan 2026 01:36:10 +0300 Subject: [PATCH 02/65] Cleanup Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 14 +++++--------- src/redeem/api_client.py | 4 ++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index f817ca33..9f3797e4 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -6,7 +6,6 @@ import click from eth_typing import BlockNumber, ChecksumAddress -from web3 import Web3 from web3.types import Wei from src.common.clients import ( @@ -21,7 +20,7 @@ from src.config.settings import settings from src.redeem.api_client import APIClient from src.redeem.graph import graph_get_allocators, graph_get_leverage_positions_proxies -from src.redeem.typings import Allocator, RedeemablePosition +from src.redeem.typings import RedeemablePosition logger = logging.getLogger(__name__) @@ -87,10 +86,7 @@ case_sensitive=False, ), ) -@click.command( - help='Performs a vault validators consolidation from 0x01 validators to 0x02 validator. ' - 'Switches a validator from 0x01 to 0x02 if the source and target keys are identical.', -) +@click.command(help='Updates redeemable positions for leverage positions') # pylint: disable-next=too-many-arguments def update_redeemable_positions( execution_endpoints: str, @@ -153,10 +149,10 @@ async def main() -> None: ) ) kept_tokens[allocator.address] = Wei(kept_tokens[allocator.address] - amount) - logger.info('Fetched kept tokens for %s addresses...', {len(user_addresses)}) + logger.info('Fetched kept tokens for %s addresses...', len(user_addresses)) click.confirm( - 'Proceed consolidation?', + 'Proceed with uploading redeemable positions to IPFS?', default=True, abort=True, ) @@ -176,7 +172,7 @@ async def get_kept_tokens( api_client = APIClient() locked_oseth_per_user: dict[ChecksumAddress, Wei] = {} for address in user_addresses: - locked_os_token = await api_client.get_protocols_locked_locked_os_token(address=address) + locked_os_token = await api_client.get_protocols_locked_os_token(address=address) locked_oseth_per_user[address] = locked_os_token kept_token = defaultdict(lambda: Wei(0)) diff --git a/src/redeem/api_client.py b/src/redeem/api_client.py index 45aeac48..868087f9 100644 --- a/src/redeem/api_client.py +++ b/src/redeem/api_client.py @@ -17,7 +17,7 @@ class APIClient: base_url = API_ENDPOINT - async def get_protocols_locked_locked_os_token(self, address: ChecksumAddress) -> Wei: + async def get_protocols_locked_os_token(self, address: ChecksumAddress) -> Wei: url = urljoin(self.base_url, 'v1/user/complex_protocol_list') params = { 'id': address, @@ -28,7 +28,7 @@ async def get_protocols_locked_locked_os_token(self, address: ChecksumAddress) - async with session.get( url=url, params=params, - headers={'user-agent:': DEFAULT_USER_AGENT}, + headers={'user-agent': DEFAULT_USER_AGENT}, ) as response: response.raise_for_status() protocol_data = await response.json() From fac01348f86bebc1add223746a5d37851c782cba Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 12 Jan 2026 13:59:56 +0300 Subject: [PATCH 03/65] Update api client Signed-off-by: cyc60 --- src/config/networks.py | 6 + src/redeem/api_client.py | 51 +- src/redeem/tests/__init__.py | 0 src/redeem/tests/api_samples/protocols.json | 869 ++++++++++++++++++++ src/redeem/tests/test_api_client.py | 80 ++ 5 files changed, 986 insertions(+), 20 deletions(-) create mode 100644 src/redeem/tests/__init__.py create mode 100644 src/redeem/tests/api_samples/protocols.json create mode 100644 src/redeem/tests/test_api_client.py diff --git a/src/config/networks.py b/src/config/networks.py index 9f80c7a2..ead76982 100644 --- a/src/config/networks.py +++ b/src/config/networks.py @@ -27,6 +27,7 @@ class NetworkConfig(BaseNetworkConfig): CONSOLIDATION_CONTRACT_ADDRESS: ChecksumAddress WITHDRAWAL_CONTRACT_ADDRESS: ChecksumAddress OS_TOKEN_CONTRACT_ADDRESS: ChecksumAddress + OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS: ChecksumAddress WALLET_MIN_BALANCE: Wei STAKEWISE_API_URL: str STAKEWISE_GRAPH_ENDPOINT: str @@ -96,6 +97,9 @@ def INITIAL_SYNC_ETA(self) -> int: OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xf1C9acDc66974dFB6dEcB12aA385b9cD01190E38' ), + OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0xf7d4e7273E5015C96728A6b02f31C505eE184603' + ), WALLET_MIN_BALANCE=Web3.to_wei('0.03', 'ether'), STAKEWISE_API_URL='https://mainnet-api.stakewise.io/graphql', STAKEWISE_GRAPH_ENDPOINT=( @@ -149,6 +153,7 @@ def INITIAL_SYNC_ETA(self) -> int: OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x7345fC8268459413beE9e9dd327f31283C65Ee7e' ), + OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS=ZERO_CHECKSUM_ADDRESS, WALLET_MIN_BALANCE=Web3.to_wei('0.03', 'ether'), STAKEWISE_API_URL='https://hoodi-api.stakewise.io/graphql', STAKEWISE_GRAPH_ENDPOINT='https://graphs.stakewise.io/hoodi/subgraphs/name/stakewise/prod', @@ -200,6 +205,7 @@ def INITIAL_SYNC_ETA(self) -> int: OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xF490c80aAE5f2616d3e3BDa2483E30C4CB21d1A0' ), + OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS=ZERO_CHECKSUM_ADDRESS, WALLET_MIN_BALANCE=Web3.to_wei('0.01', 'ether'), STAKEWISE_API_URL='https://gnosis-api.stakewise.io/graphql', STAKEWISE_GRAPH_ENDPOINT=( diff --git a/src/redeem/api_client.py b/src/redeem/api_client.py index 868087f9..1ee42510 100644 --- a/src/redeem/api_client.py +++ b/src/redeem/api_client.py @@ -4,6 +4,7 @@ from web3 import Web3 from web3.types import Wei +from src.config.networks import ZERO_CHECKSUM_ADDRESS from src.config.settings import settings API_ENDPOINT = 'https://api.rabby.io/' @@ -11,6 +12,7 @@ 'Mozilla/5.0 (X11; Linux x86_64) ' 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36' ) +SUPPORTED_CHAINS = {'eth', 'arb'} class APIClient: @@ -23,30 +25,39 @@ async def get_protocols_locked_os_token(self, address: ChecksumAddress) -> Wei: 'id': address, } - protocol_data = [] - async with aiohttp.ClientSession() as session: - async with session.get( - url=url, - params=params, - headers={'user-agent': DEFAULT_USER_AGENT}, - ) as response: - response.raise_for_status() - protocol_data = await response.json() - + protocol_data = await self._fetch_json(url, params=params) total_locked_oseth = Wei(0) for protocol in protocol_data: - if protocol['id'] == 'stakewise': + if protocol['id'] in ['stakewise', 'xdai_stakewise']: continue - portfolio_item_list = protocol.get('portfolio_item_list', []) - for item in portfolio_item_list: - for assets in item.get('asset_token_list', []): - if not Web3.is_address(assets['id']): + for portfolio_item in protocol.get('portfolio_item_list', []): + supply_token_list = portfolio_item.get('detail', {}).get('supply_token_list', []) + for supply_token in supply_token_list: + if supply_token['chain'] not in SUPPORTED_CHAINS: + continue + if not Web3.is_address(supply_token['id']): continue - if ( - Web3.to_checksum_address(assets['id']) - == settings.network_config.OS_TOKEN_CONTRACT_ADDRESS - ): + if self._is_os_token(Web3.to_checksum_address(supply_token['id'])): total_locked_oseth = Wei( - total_locked_oseth + Web3.to_wei(float(assets['amount']), 'ether') + total_locked_oseth + Web3.to_wei(float(supply_token['amount']), 'ether') ) + return total_locked_oseth + + async def _fetch_json(self, url: str, params: dict | None = None) -> dict | list: + async with aiohttp.ClientSession() as session: + async with session.get( + url=url, + params=params, + headers={'user-agent': DEFAULT_USER_AGENT}, + ) as response: + response.raise_for_status() + return await response.json() + + def _is_os_token(self, token_address: ChecksumAddress) -> bool: + if token_address == ZERO_CHECKSUM_ADDRESS: + return False + return token_address in [ + settings.network_config.OS_TOKEN_CONTRACT_ADDRESS, + settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS, + ] diff --git a/src/redeem/tests/__init__.py b/src/redeem/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/redeem/tests/api_samples/protocols.json b/src/redeem/tests/api_samples/protocols.json new file mode 100644 index 00000000..bc85aa05 --- /dev/null +++ b/src/redeem/tests/api_samples/protocols.json @@ -0,0 +1,869 @@ +[ + { + "id": "aave3", + "chain": "eth", + "name": "Aave V3", + "site_url": "https://app.aave.com", + "logo_url": "https://static.debank.com/image/project/logo_url/aave3/54df7839ab09493ba7540ab832590255.png", + "has_supported_portfolio": true, + "tvl": 27881218389.2258, + "portfolio_item_list": [ + { + "stats": { + "asset_usd_value": 6.536441090850581, + "debt_usd_value": 0, + "net_usd_value": 6.536441090850581 + }, + "asset_dict": { + "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38": 0.002000000000000038 + }, + "asset_token_list": [ + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3268.220545425228, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 0.002000000000000038 + } + ], + "withdraw_actions": [ + { + "type": "withdraw", + "contract_id": "0x87870bca3f3fd6335c3f4ce8392d69350b4fa4e2", + "func": "withdraw(address,uint256,address)(uint256)", + "params": [ + "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + 1.157920892373162e+77, + "0x50d3e5bcc40d6ef539eb9ec74b18e936b4cd7a42" + ], + "str_params": [ + "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + "0x50d3e5bcc40d6ef539eb9ec74b18e936b4cd7a42" + ] + } + ], + "update_at": 1768086955, + "name": "Lending", + "detail_types": [ + "lending" + ], + "detail": { + "supply_token_list": [ + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3268.220545425228, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 0.002000000000000038 + } + ], + "health_rate": 1.157920892373162e+59 + }, + "proxy_detail": {}, + "pool": { + "id": "0x87870bca3f3fd6335c3f4ce8392d69350b4fa4e2", + "chain": "eth", + "project_id": "aave3", + "adapter_id": "aave3_proxy_lending", + "controller": "0x87870bca3f3fd6335c3f4ce8392d69350b4fa4e2", + "index": null, + "time_at": 1672325495 + } + } + ] + }, + { + "id": "arb_balancer3", + "chain": "arb", + "name": "Balancer V3", + "site_url": "https://balancer.fi", + "logo_url": "https://static.debank.com/image/project/logo_url/xdai_balancer3/82d080651ec38c39a896b768a5a9b8cf.png", + "has_supported_portfolio": true, + "tvl": 11944021.93948108, + "portfolio_item_list": [ + { + "stats": { + "asset_usd_value": 3.8536199798725375, + "debt_usd_value": 0, + "net_usd_value": 3.8536199798725375 + }, + "asset_dict": { + "0x82af49447d8a07e3bd95bd0d56f35241523fbab1": 0.00029401523588944563, + "0xf7d4e7273e5015c96728a6b02f31c505ee184603": 0.0009017311937502758 + }, + "asset_token_list": [ + { + "id": "0x82af49447d8a07e3bd95bd0d56f35241523fbab1", + "chain": "arb", + "name": "Wrapped Ether", + "symbol": "WETH", + "display_symbol": null, + "optimized_symbol": "WETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/arb_token/logo_url/0x82af49447d8a07e3bd95bd0d56f35241523fbab1/61844453e63cf81301f845d7864236f6.png", + "protocol_id": "", + "price": 3084.76, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1622346702, + "amount": 0.00029401523588944563 + }, + { + "id": "0xf7d4e7273e5015c96728a6b02f31c505ee184603", + "chain": "arb", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "", + "price": 3267.7737680950777, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1716285151, + "amount": 0.0009017311937502758 + } + ], + "withdraw_actions": [ + { + "type": "withdraw", + "contract_id": "0xc1a64500e035d9159c8826e982dfb802003227f0", + "func": "removeLiquidityProportionalFromERC4626Pool(address,bool[],uint256,uint256[],bool,bytes)(address[],uint256[])", + "params": [ + "0x1a36195044abbad8807ceea78ec645c0c5d09593", + [ + true, + false + ], + 1234011286651171, + [ + 286652887280261, + 879187913906519 + ], + false, + "0x" + ], + "need_approve": { + "token_id": "0x1a36195044abbad8807ceea78ec645c0c5d09593", + "to": "0xc1a64500e035d9159c8826e982dfb802003227f0", + "raw_amount": 1234011286651171, + "str_raw_amount": "1234011286651171" + }, + "str_params": [ + "0x1a36195044abbad8807ceea78ec645c0c5d09593", + [ + true, + false + ], + "1234011286651171", + [ + "286652887280261", + "879187913906519" + ], + false, + "0x" + ] + } + ], + "update_at": 1768011485, + "name": "Liquidity Pool", + "detail_types": [ + "common" + ], + "detail": { + "supply_token_list": [ + { + "id": "0x82af49447d8a07e3bd95bd0d56f35241523fbab1", + "chain": "arb", + "name": "Wrapped Ether", + "symbol": "WETH", + "display_symbol": null, + "optimized_symbol": "WETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/arb_token/logo_url/0x82af49447d8a07e3bd95bd0d56f35241523fbab1/61844453e63cf81301f845d7864236f6.png", + "protocol_id": "", + "price": 3084.76, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1622346702, + "amount": 0.00029401523588944563 + }, + { + "id": "0xf7d4e7273e5015c96728a6b02f31c505ee184603", + "chain": "arb", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "", + "price": 3267.7737680950777, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1716285151, + "amount": 0.0009017311937502758 + } + ], + "description": "osETH-WETH" + }, + "proxy_detail": {}, + "pool": { + "id": "0x1a36195044abbad8807ceea78ec645c0c5d09593", + "chain": "arb", + "project_id": "arb_balancer3", + "adapter_id": "balancer3_liquidity", + "controller": "0x1a36195044abbad8807ceea78ec645c0c5d09593", + "index": null, + "time_at": 1745503757 + } + } + ] + }, + { + "id": "balancer3", + "chain": "eth", + "name": "Balancer V3", + "site_url": "https://balancer.fi", + "logo_url": "https://static.debank.com/image/project/logo_url/xdai_balancer3/82d080651ec38c39a896b768a5a9b8cf.png", + "has_supported_portfolio": true, + "tvl": 53094052.744415, + "portfolio_item_list": [ + { + "stats": { + "asset_usd_value": 3.9048665756590535, + "debt_usd_value": 0, + "net_usd_value": 3.9048665756590535 + }, + "asset_dict": { + "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2": 0.0003030616995899286, + "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38": 0.000908766246664519 + }, + "asset_token_list": [ + { + "id": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", + "chain": "eth", + "name": "Wrapped Ether", + "symbol": "WETH", + "display_symbol": null, + "optimized_symbol": "WETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2/61844453e63cf81301f845d7864236f6.png", + "protocol_id": "weth", + "price": 3084.58, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1513077455, + "amount": 0.0003030616995899286 + }, + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3268.220545425228, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 0.000908766246664519 + } + ], + "withdraw_actions": [ + { + "type": "withdraw", + "contract_id": "0xb21a277466e7db6934556a1ce12eb3f032815c8a", + "func": "removeLiquidityProportionalFromERC4626Pool(address,bool[],uint256,uint256[],bool,bytes)(address[],uint256[])", + "params": [ + "0x57c23c58b1d8c3292c15becf07c62c5c52457a42", + [ + true, + false + ], + 1239643144912738, + [ + 295485157100180, + 886047090497906 + ], + false, + "0x" + ], + "need_approve": { + "token_id": "0x57c23c58b1d8c3292c15becf07c62c5c52457a42", + "to": "0xb21a277466e7db6934556a1ce12eb3f032815c8a", + "raw_amount": 1239643144912738, + "str_raw_amount": "1239643144912738" + }, + "str_params": [ + "0x57c23c58b1d8c3292c15becf07c62c5c52457a42", + [ + true, + false + ], + "1239643144912738", + [ + "295485157100180", + "886047090497906" + ], + false, + "0x" + ] + } + ], + "update_at": 1768086952, + "name": "Liquidity Pool", + "detail_types": [ + "common" + ], + "detail": { + "supply_token_list": [ + { + "id": "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2", + "chain": "eth", + "name": "Wrapped Ether", + "symbol": "WETH", + "display_symbol": null, + "optimized_symbol": "WETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2/61844453e63cf81301f845d7864236f6.png", + "protocol_id": "weth", + "price": 3084.58, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1513077455, + "amount": 0.0003030616995899286 + }, + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3268.220545425228, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 0.000908766246664519 + } + ], + "description": "osETH-waWETH" + }, + "proxy_detail": {}, + "pool": { + "id": "0x57c23c58b1d8c3292c15becf07c62c5c52457a42", + "chain": "eth", + "project_id": "balancer3", + "adapter_id": "balancer3_liquidity", + "controller": "0x57c23c58b1d8c3292c15becf07c62c5c52457a42", + "index": null, + "time_at": 1733866823 + } + } + ] + }, + { + "id": "compound3", + "chain": "eth", + "name": "Compound V3", + "site_url": "https://v3-app.compound.finance", + "logo_url": "https://static.debank.com/image/project/logo_url/compound3/28e2b958e38eb0c49d600633b9ce0969.png", + "has_supported_portfolio": true, + "tvl": 1501571985.3018792, + "portfolio_item_list": [ + { + "stats": { + "asset_usd_value": 3.268220545425228, + "debt_usd_value": 0, + "net_usd_value": 3.268220545425228 + }, + "asset_dict": { + "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38": 0.001 + }, + "asset_token_list": [ + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3268.220545425228, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 0.001 + } + ], + "withdraw_actions": [ + { + "type": "withdraw", + "contract_id": "0xa17581a9e3356d9a858b789d68b4d866e593ae94", + "func": "withdraw(address,uint256)()", + "params": [ + "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + 1000000000000000 + ], + "str_params": [ + "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "1000000000000000" + ] + } + ], + "update_at": 1768086954, + "name": "Lending", + "detail_types": [ + "lending" + ], + "detail": { + "supply_token_list": [ + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3268.220545425228, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 0.001 + } + ] + }, + "proxy_detail": {}, + "pool": { + "id": "0xa17581a9e3356d9a858b789d68b4d866e593ae94:lending", + "chain": "eth", + "project_id": "compound3", + "adapter_id": "compound_lending2", + "controller": "0xa17581a9e3356d9a858b789d68b4d866e593ae94", + "index": "lending", + "time_at": 1673647103 + } + } + ] + }, + { + "id": "lido", + "chain": "eth", + "name": "LIDO", + "site_url": "https://stake.lido.fi", + "logo_url": "https://static.debank.com/image/project/logo_url/lido/081388ebc44fa042561749bd5338d49e.png", + "has_supported_portfolio": true, + "tvl": 27178509597.199047, + "portfolio_item_list": [ + { + "stats": { + "asset_usd_value": 3.08458e-15, + "debt_usd_value": 0, + "net_usd_value": 3.08458e-15 + }, + "asset_dict": { + "eth": 1e-18 + }, + "asset_token_list": [ + { + "id": "eth", + "chain": "eth", + "name": "ETH", + "symbol": "ETH", + "display_symbol": null, + "optimized_symbol": "ETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/coin/logo_url/eth/6443cdccced33e204d90cb723c632917.png", + "protocol_id": "", + "price": 3084.58, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1483200000, + "amount": 1e-18 + } + ], + "withdraw_actions": [], + "update_at": 1767127127, + "name": "Staked", + "detail_types": [ + "common" + ], + "detail": { + "supply_token_list": [ + { + "id": "eth", + "chain": "eth", + "name": "ETH", + "symbol": "ETH", + "display_symbol": null, + "optimized_symbol": "ETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/coin/logo_url/eth/6443cdccced33e204d90cb723c632917.png", + "protocol_id": "", + "price": 3084.58, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1483200000, + "amount": 1e-18 + } + ], + "description": "stETH" + }, + "proxy_detail": {}, + "pool": { + "id": "0xae7ab96520de3a18e5e111b5eaab095312d7fe84", + "chain": "eth", + "project_id": "lido", + "adapter_id": "lido_staked", + "controller": "0xae7ab96520de3a18e5e111b5eaab095312d7fe84", + "index": null, + "time_at": 1608242396 + } + } + ] + }, + { + "id": "stakewise", + "chain": "eth", + "name": "StakeWise", + "site_url": "https://app.stakewise.io", + "logo_url": "https://static.debank.com/image/project/logo_url/stakewise/c23c49fcf9096ab73cf0001d78f570a3.png", + "has_supported_portfolio": true, + "tvl": 325287267.5753829, + "portfolio_item_list": [ + { + "stats": { + "asset_usd_value": 201.72813874111105, + "debt_usd_value": 169.86438052271313, + "net_usd_value": 31.86375821839792 + }, + "asset_dict": { + "eth": 0.06539889992838929, + "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38": -0.05197457704024441 + }, + "asset_token_list": [ + { + "id": "eth", + "chain": "eth", + "name": "ETH", + "symbol": "ETH", + "display_symbol": null, + "optimized_symbol": "ETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/coin/logo_url/eth/6443cdccced33e204d90cb723c632917.png", + "protocol_id": "", + "price": 3084.58, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1483200000, + "amount": 0.06539889992838929 + }, + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3268.220545425228, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": -0.05197457704024441 + } + ], + "withdraw_actions": [], + "update_at": 1768037733, + "name": "Lending", + "detail_types": [ + "lending" + ], + "detail": { + "supply_token_list": [ + { + "id": "eth", + "chain": "eth", + "name": "ETH", + "symbol": "ETH", + "display_symbol": null, + "optimized_symbol": "ETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/coin/logo_url/eth/6443cdccced33e204d90cb723c632917.png", + "protocol_id": "", + "price": 3084.58, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1483200000, + "amount": 0.06539889992838929 + } + ], + "borrow_token_list": [ + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3268.220545425228, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 0.05197457704024441 + } + ], + "health_rate": 1.0688251669261737 + }, + "proxy_detail": {}, + "pool": { + "id": "0xac0f906e433d58fa868f936e8a43230473652885", + "chain": "eth", + "project_id": "stakewise", + "adapter_id": "stakewise_lending", + "controller": "0xac0f906e433d58fa868f936e8a43230473652885", + "index": null, + "time_at": 1698755747 + } + } + ] + }, + { + "id": "symbiotic", + "chain": "eth", + "name": "Symbiotic", + "site_url": "https://app.symbiotic.fi", + "logo_url": "https://static.debank.com/image/project/logo_url/symbiotic/77a8a373f6082ff7da5df5c67ae450a2.png", + "has_supported_portfolio": true, + "tvl": 545497671.9370697, + "portfolio_item_list": [ + { + "stats": { + "asset_usd_value": 3.268220545425228, + "debt_usd_value": 0, + "net_usd_value": 3.268220545425228 + }, + "asset_dict": { + "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38": 0.001 + }, + "asset_token_list": [ + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3268.220545425228, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 0.001 + } + ], + "withdraw_actions": [ + { + "type": "withdraw", + "contract_id": "0x9ec7175541948494db7831c95868dd97d2e0f742", + "func": "withdraw(address,uint256)()", + "params": [ + "0x50d3e5bcc40d6ef539eb9ec74b18e936b4cd7a42", + 1000000000000000 + ], + "str_params": [ + "0x50d3e5bcc40d6ef539eb9ec74b18e936b4cd7a42", + "1000000000000000" + ] + } + ], + "update_at": 1768086954, + "name": "Staked", + "detail_types": [ + "common" + ], + "detail": { + "supply_token_list": [ + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3268.220545425228, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 0.001 + } + ], + "description": "Migrate" + }, + "proxy_detail": {}, + "pool": { + "id": "0x9ec7175541948494db7831c95868dd97d2e0f742", + "chain": "eth", + "project_id": "symbiotic", + "adapter_id": "symbiotic_staked3", + "controller": "0x9ec7175541948494db7831c95868dd97d2e0f742", + "index": null, + "time_at": 1740345023 + } + } + ] + }, + { + "id": "xdai_stakewise", + "chain": "xdai", + "name": "StakeWise", + "site_url": "https://app.stakewise.io", + "logo_url": "https://static.debank.com/image/project/logo_url/xdai_stakewise/c23c49fcf9096ab73cf0001d78f570a3.png", + "has_supported_portfolio": true, + "tvl": 24284920.18498818, + "portfolio_item_list": [ + { + "stats": { + "asset_usd_value": 141.01223963341488, + "debt_usd_value": 0, + "net_usd_value": 141.01223963341488 + }, + "asset_dict": { + "0x9c58bacc331c9aa871afd802db6379a98e80cedb": 1.017217404800869 + }, + "asset_token_list": [ + { + "id": "0x9c58bacc331c9aa871afd802db6379a98e80cedb", + "chain": "xdai", + "name": "Gnosis Token on xDai", + "symbol": "GNO", + "display_symbol": null, + "optimized_symbol": "GNO", + "decimals": 18, + "logo_url": "https://static.debank.com/image/xdai_token/logo_url/0x9c58bacc331c9aa871afd802db6379a98e80cedb/69e5fedeca09913fe078a8dca5b7e48c.png", + "protocol_id": "xdai_gnosis", + "price": 138.62546882101324, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1598213055, + "amount": 1.017217404800869 + } + ], + "withdraw_actions": [ + { + "type": "queue", + "contract_id": "0x4b4406ed8659d03423490d8b62a1639206da0a7a", + "func": "enterExitQueue(uint256,address)(uint256)", + "params": [ + 912630135170511000, + "0x50d3e5bcc40d6ef539eb9ec74b18e936b4cd7a42" + ], + "str_params": [ + "912630135170511001", + "0x50d3e5bcc40d6ef539eb9ec74b18e936b4cd7a42" + ] + } + ], + "update_at": 1768072817, + "name": "Lending", + "detail_types": [ + "lending" + ], + "detail": { + "supply_token_list": [ + { + "id": "0x9c58bacc331c9aa871afd802db6379a98e80cedb", + "chain": "xdai", + "name": "Gnosis Token on xDai", + "symbol": "GNO", + "display_symbol": null, + "optimized_symbol": "GNO", + "decimals": 18, + "logo_url": "https://static.debank.com/image/xdai_token/logo_url/0x9c58bacc331c9aa871afd802db6379a98e80cedb/69e5fedeca09913fe078a8dca5b7e48c.png", + "protocol_id": "xdai_gnosis", + "price": 138.62546882101324, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1598213055, + "amount": 1.017217404800869 + } + ] + }, + "proxy_detail": {}, + "pool": { + "id": "0x4b4406ed8659d03423490d8b62a1639206da0a7a", + "chain": "xdai", + "project_id": "xdai_stakewise", + "adapter_id": "stakewise_lending", + "controller": "0x4b4406ed8659d03423490d8b62a1639206da0a7a", + "index": null, + "time_at": 1720015340 + } + } + ] + } +] diff --git a/src/redeem/tests/test_api_client.py b/src/redeem/tests/test_api_client.py new file mode 100644 index 00000000..9d74f29b --- /dev/null +++ b/src/redeem/tests/test_api_client.py @@ -0,0 +1,80 @@ +import json +from unittest.mock import patch + +import pytest +from web3 import Web3 +from web3.types import Wei + +from src.config.settings import settings +from src.redeem.api_client import APIClient + + +class TestGetOraclesRequest: + @pytest.mark.usefixtures('fake_settings') + async def test_zero_when_no_protocol_data(self): + client = APIClient() + with patch('src.redeem.api_client.APIClient._fetch_json', return_value=[]): + result = await client.get_protocols_locked_os_token( + Web3.to_checksum_address('0x1234567890abcdef1234567890abcdef12345678') + ) + assert result == Wei(0) + + @pytest.mark.usefixtures('fake_settings') + async def test_excludes_stakewise_protocol_from_total(self): + mock_protocol_data = [ + { + 'id': 'stakewise', + 'portfolio_item_list': [ + { + 'detail': { + 'supply_token_list': [ + { + 'id': '0x1234567890abcdef1234567890abcdef12345678', + 'chain': 'eth', + 'amount': '57', + } + ] + } + } + ], + }, + { + 'id': 'other', + 'portfolio_item_list': [ + { + 'detail': { + 'supply_token_list': [ + { + 'id': settings.network_config.OS_TOKEN_CONTRACT_ADDRESS, + 'chain': 'eth', + 'amount': '5', + } + ] + } + } + ], + }, + ] + with patch('src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data): + client = APIClient() + result = await client.get_protocols_locked_os_token( + Web3.to_checksum_address('0x1234567890abcdef1234567890abcdef12345678') + ) + assert result == Wei(Web3.to_wei(5, 'ether')) + + @pytest.mark.usefixtures('fake_settings') + async def test_real_data(self): + with open('src/redeem/tests/api_samples/protocols.json', 'r') as f: + mock_protocol_data = json.load(f) + settings.network_config.OS_TOKEN_CONTRACT_ADDRESS = ( + '0xf1C9acDc66974dFB6dEcB12aA385b9cD01190E38' + ) + settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS = ( + '0xf7d4e7273E5015C96728A6b02f31C505eE184603' + ) + with patch('src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data): + client = APIClient() + result = await client.get_protocols_locked_os_token( + Web3.to_checksum_address('0x1234567890abcdef1234567890abcdef12345678') + ) + assert result == Wei(5810497440414831) From cc6010f3e4472ca41d2d25bf4b0b8e69be9a6fe8 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 12 Jan 2026 22:30:38 +0300 Subject: [PATCH 04/65] Check arbitrum balance, add min shares arg Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 94 +++++++++++++++---- src/redeem/graph.py | 9 +- 2 files changed, 77 insertions(+), 26 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 9f3797e4..ef56d2f0 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -3,20 +3,23 @@ import sys from collections import defaultdict from pathlib import Path +from typing import cast import click from eth_typing import BlockNumber, ChecksumAddress -from web3.types import Wei +from web3 import Web3 +from web3.types import Gwei, Wei from src.common.clients import ( build_ipfs_upload_clients, execution_client, + get_execution_client, setup_clients, ) from src.common.contracts import Erc20Contract from src.common.logging import LOG_LEVELS, setup_logging from src.common.utils import log_verbose -from src.config.networks import AVAILABLE_NETWORKS, ZERO_CHECKSUM_ADDRESS +from src.config.networks import AVAILABLE_NETWORKS, MAINNET, ZERO_CHECKSUM_ADDRESS from src.config.settings import settings from src.redeem.api_client import APIClient from src.redeem.graph import graph_get_allocators, graph_get_leverage_positions_proxies @@ -39,6 +42,14 @@ help='Absolute path to the wallet. ' 'Default is the file generated with "create-wallet" command.', ) +@click.option( + '--min-os-token-position-amount-gwei', + type=int, + default=0, + envvar='MIN_OS_TOKEN_POSITION_AMOUNT_GWEI', + help='Process positions only if the amount of minted osETH' + ' is greater than the specified value.', +) @click.option( '--execution-endpoints', type=str, @@ -59,6 +70,12 @@ envvar='GRAPH_ENDPOINT', help='API endpoint for graph node.', ) +@click.option( + '--arbitrum-endpoint', + type=str, + envvar='ARBITRUM_ENDPOINT', + help='API endpoint for the execution node on Arbitrum.', +) @click.option( '--log-level', type=click.Choice( @@ -92,12 +109,16 @@ def update_redeemable_positions( execution_endpoints: str, execution_jwt_secret: str | None, graph_endpoint: str, + arbitrum_endpoint: str | None, network: str, verbose: bool, log_level: str, wallet_file: str | None, wallet_password_file: str | None, + min_os_token_position_amount_gwei: int, ) -> None: + if network == MAINNET and not arbitrum_endpoint: + raise click.BadParameter('arbitrum-endpoint is required for mainnet network') settings.set( vault=ZERO_CHECKSUM_ADDRESS, vault_dir=Path.home() / '.stakewise', @@ -111,13 +132,19 @@ def update_redeemable_positions( log_level=log_level, ) try: - asyncio.run(main()) + asyncio.run( + main( + arbitrum_endpoint=arbitrum_endpoint, + min_os_token_position_amount_gwei=Gwei(min_os_token_position_amount_gwei), + ) + ) except Exception as e: log_verbose(e) sys.exit(1) -async def main() -> None: +# pylint: disable-next=too-many-locals +async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: Gwei) -> None: """ Fetch redeemable positions, calculate kept os token amounts and upload to IPFS. """ @@ -129,13 +156,24 @@ async def main() -> None: # filter boost_proxies = await graph_get_leverage_positions_proxies(block_number) logger.info('Found %s boost positions to exclude', len(boost_proxies)) - allocators = [a for a in allocators if a.minted_shares > 0 and a.address not in boost_proxies] + min_minted_shares = Web3.to_wei(min_os_token_position_amount_gwei, 'gwei') + allocators = [ + a + for a in allocators + if a.minted_shares > min_minted_shares and a.address not in boost_proxies + ] + allocators = sorted(allocators, key=lambda x: x.minted_shares, reverse=True) logger.info('Filtered allocators count: %s', len(allocators)) + address_to_minted_shares = {a.address: a.minted_shares for a in allocators} user_addresses = set(allocator.address for allocator in allocators) - logger.info('Fetching kept tokens for %s addresses...', len(user_addresses)) - - kept_tokens = await get_kept_tokens(list(user_addresses), block_number) + logger.info('Fetching kept tokens for %s addresses...', len(address_to_minted_shares)) + kept_tokens = await get_kept_tokens(address_to_minted_shares, block_number, arbitrum_endpoint) + filled = 0 + for allocator in allocators: + if allocator.minted_shares == kept_tokens.get(allocator.address, Wei(0)): + filled += 1 + logger.info('Found %s fully filled positions', filled) redeemable_positions: list[RedeemablePosition] = [] for allocator in allocators: kept_token = kept_tokens.get(allocator.address, Wei(0)) # 0? @@ -162,22 +200,38 @@ async def main() -> None: async def get_kept_tokens( - user_addresses: list[ChecksumAddress], block_number: BlockNumber + address_to_minted_shares: dict[ChecksumAddress, Wei], + block_number: BlockNumber, + arbitrum_endpoint: str | None, ) -> dict[ChecksumAddress, Wei]: - wallet_balances = {} - + kept_token = defaultdict(lambda: Wei(0)) contract = Erc20Contract(settings.network_config.OS_TOKEN_CONTRACT_ADDRESS) - for address in user_addresses: - wallet_balances[address] = await contract.balance(address, block_number) + for address in address_to_minted_shares.keys(): + kept_token[address] = await contract.balance(address, block_number) + + # arb wallet balance + if settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS != ZERO_CHECKSUM_ADDRESS: + arbitrum_endpoint = cast(str, arbitrum_endpoint) + arb_execution_client = get_execution_client([arbitrum_endpoint]) + + arb_contract = Erc20Contract( + settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS, + execution_client=arb_execution_client, + ) + for address in address_to_minted_shares.keys(): + arb_balance = await arb_contract.balance(address) + kept_token[address] = Wei(kept_token[address] + arb_balance) + + # do not fetch data from api if all os token are on the wallet + api_addresses = [] + for address in address_to_minted_shares.keys(): + if address_to_minted_shares[address] > kept_token[address]: + api_addresses.append(address) + api_client = APIClient() locked_oseth_per_user: dict[ChecksumAddress, Wei] = {} - for address in user_addresses: + for address in api_addresses: locked_os_token = await api_client.get_protocols_locked_os_token(address=address) locked_oseth_per_user[address] = locked_os_token - - kept_token = defaultdict(lambda: Wei(0)) - for address, amount in locked_oseth_per_user.items(): - kept_token[address] = amount - for address, amount in wallet_balances.items(): - kept_token[address] = Wei(kept_token[address] + amount) + kept_token[address] = Wei(kept_token[address] + locked_os_token) return kept_token diff --git a/src/redeem/graph.py b/src/redeem/graph.py index 1f404da0..0d3c7d85 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -17,14 +17,11 @@ async def graph_get_allocators(block_number: BlockNumber) -> list[Allocator]: """ query = gql( """ - query getAllocators($block: Int, $first: Int, $lastID: String){ + query getAllocators($block: Int, $first: Int, $skip: Int){ allocators( block: {number: $block}, - where: { - id_gt: $lastID - }, - orderBy: id first: $first + skip: $skip ){ vault { id @@ -39,7 +36,7 @@ async def graph_get_allocators(block_number: BlockNumber) -> list[Allocator]: params = { 'block': block_number, } - response = await graph_client.fetch_pages(query, params=params, cursor_pagination=True) + response = await graph_client.fetch_pages(query, params=params) return [Allocator.from_graph(item) for item in response] From 743f179f3dcd96839700505e16087752d5dab013 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 12 Jan 2026 23:45:44 +0300 Subject: [PATCH 05/65] Check boosted shares via subgraph Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 49 +++++++++++++++---- src/common/contracts.py | 5 ++ src/redeem/api_client.py | 1 + src/redeem/graph.py | 15 ++++-- src/redeem/typings.py | 19 +++++++ 5 files changed, 74 insertions(+), 15 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index ef56d2f0..e72c3b7d 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -16,14 +16,14 @@ get_execution_client, setup_clients, ) -from src.common.contracts import Erc20Contract +from src.common.contracts import Erc20Contract, VaultContract from src.common.logging import LOG_LEVELS, setup_logging from src.common.utils import log_verbose from src.config.networks import AVAILABLE_NETWORKS, MAINNET, ZERO_CHECKSUM_ADDRESS from src.config.settings import settings from src.redeem.api_client import APIClient -from src.redeem.graph import graph_get_allocators, graph_get_leverage_positions_proxies -from src.redeem.typings import RedeemablePosition +from src.redeem.graph import graph_get_allocators, graph_get_leverage_positions +from src.redeem.typings import LeverageStrategyPosition, RedeemablePosition logger = logging.getLogger(__name__) @@ -153,21 +153,30 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: block_number = await execution_client.eth.block_number allocators = await graph_get_allocators(block_number) - # filter - boost_proxies = await graph_get_leverage_positions_proxies(block_number) + # # filter + leverage_positions = await graph_get_leverage_positions(block_number) + boost_proxies = {pos.proxy for pos in leverage_positions} logger.info('Found %s boost positions to exclude', len(boost_proxies)) min_minted_shares = Web3.to_wei(min_os_token_position_amount_gwei, 'gwei') - allocators = [ - a - for a in allocators - if a.minted_shares > min_minted_shares and a.address not in boost_proxies - ] + allocators = [a for a in allocators if a.address not in boost_proxies] + allocators = sorted(allocators, key=lambda x: x.minted_shares, reverse=True) logger.info('Filtered allocators count: %s', len(allocators)) address_to_minted_shares = {a.address: a.minted_shares for a in allocators} user_addresses = set(allocator.address for allocator in allocators) logger.info('Fetching kept tokens for %s addresses...', len(address_to_minted_shares)) + + # filter boosted positions + boosted_amounts = await get_boosted_amounts( + address_to_minted_shares, leverage_positions=leverage_positions, block_number=block_number + ) + for allocator in allocators: + allocator.minted_shares = Wei( + allocator.minted_shares - boosted_amounts.get(allocator.address, Wei(0)) + ) + allocators = [a for a in allocators if a.minted_shares >= min_minted_shares] + kept_tokens = await get_kept_tokens(address_to_minted_shares, block_number, arbitrum_endpoint) filled = 0 for allocator in allocators: @@ -235,3 +244,23 @@ async def get_kept_tokens( locked_oseth_per_user[address] = locked_os_token kept_token[address] = Wei(kept_token[address] + locked_os_token) return kept_token + + +async def get_boosted_amounts( + address_to_minted_shares: dict[ChecksumAddress, Wei], + leverage_positions: list[LeverageStrategyPosition], + block_number: BlockNumber, +) -> dict[ChecksumAddress, Wei]: + boosted_os_token_shares: defaultdict[ChecksumAddress, Wei] = defaultdict(lambda: Wei(0)) + for position in leverage_positions: + if position.user not in address_to_minted_shares: + continue + vault_contract = VaultContract(position.vault) + position_os_token_shares = ( + position.os_token_shares + + await vault_contract.convert_to_shares(position.assets, block_number) + ) + boosted_os_token_shares[position.user] = Wei( + boosted_os_token_shares[position.user] + position_os_token_shares + ) + return boosted_os_token_shares diff --git a/src/common/contracts.py b/src/common/contracts.py index 15803a40..32886d5f 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -220,6 +220,11 @@ async def validators_manager(self) -> ChecksumAddress: async def get_exit_queue_index(self, position_ticket: int) -> int: return await self.contract.functions.getExitQueueIndex(position_ticket).call() + async def convert_to_shares(self, assets: Wei, block_number: BlockNumber | None = None) -> Wei: + return await self.contract.functions.convertToShares(assets).call( + block_identifier=block_number + ) + async def get_validator_withdrawal_submitted_events( self, from_block: BlockNumber, diff --git a/src/redeem/api_client.py b/src/redeem/api_client.py index 1ee42510..1e04f653 100644 --- a/src/redeem/api_client.py +++ b/src/redeem/api_client.py @@ -28,6 +28,7 @@ async def get_protocols_locked_os_token(self, address: ChecksumAddress) -> Wei: protocol_data = await self._fetch_json(url, params=params) total_locked_oseth = Wei(0) for protocol in protocol_data: + # boosted OsEth handled via graph separately if protocol['id'] in ['stakewise', 'xdai_stakewise']: continue for portfolio_item in protocol.get('portfolio_item_list', []): diff --git a/src/redeem/graph.py b/src/redeem/graph.py index 0d3c7d85..88c42eac 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -1,11 +1,10 @@ import logging -from eth_typing import BlockNumber, ChecksumAddress +from eth_typing import BlockNumber from gql import gql -from web3 import Web3 from src.common.clients import graph_client -from src.redeem.typings import Allocator +from src.redeem.typings import Allocator, LeverageStrategyPosition logger = logging.getLogger(__name__) @@ -40,7 +39,7 @@ async def graph_get_allocators(block_number: BlockNumber) -> list[Allocator]: return [Allocator.from_graph(item) for item in response] -async def graph_get_leverage_positions_proxies(block_number: BlockNumber) -> list[ChecksumAddress]: +async def graph_get_leverage_positions(block_number: BlockNumber) -> list[LeverageStrategyPosition]: query = gql( """ query PositionsQuery($block: Int, $first: Int, $skip: Int) { @@ -51,11 +50,17 @@ async def graph_get_leverage_positions_proxies(block_number: BlockNumber) -> lis first: $first skip: $skip ) { + user proxy + vault { + id + } + osTokenShares + assets } } """ ) params = {'block': block_number} response = await graph_client.fetch_pages(query, params=params) - return [Web3.to_checksum_address(item['proxy']) for item in response] + return [LeverageStrategyPosition.from_graph(item) for item in response] diff --git a/src/redeem/typings.py b/src/redeem/typings.py index 60375ef3..bbc94f75 100644 --- a/src/redeem/typings.py +++ b/src/redeem/typings.py @@ -21,6 +21,25 @@ def from_graph(cls, data: dict) -> 'Allocator': ) +@dataclass +class LeverageStrategyPosition: + user: ChecksumAddress + vault: ChecksumAddress + proxy: ChecksumAddress + os_token_shares: Wei + assets: Wei + + @classmethod + def from_graph(cls, data: dict) -> 'LeverageStrategyPosition': + return LeverageStrategyPosition( + user=Web3.to_checksum_address(data['user']), + vault=Web3.to_checksum_address(data['vault']['id']), + proxy=Web3.to_checksum_address(data['proxy']), + os_token_shares=Wei(int(data['osTokenShares'])), + assets=Wei(int(data['assets'])), + ) + + @dataclass class RedeemablePosition: owner: ChecksumAddress From f314a013ec552ee9027ceafaf7d69703c2abb40a Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 13 Jan 2026 14:08:33 +0300 Subject: [PATCH 06/65] Add vault proportions Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 106 ++++++++++-------- src/redeem/graph.py | 21 +++- src/redeem/typings.py | 28 +++-- 3 files changed, 99 insertions(+), 56 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index e72c3b7d..750e0dfb 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -23,7 +23,7 @@ from src.config.settings import settings from src.redeem.api_client import APIClient from src.redeem.graph import graph_get_allocators, graph_get_leverage_positions -from src.redeem.typings import LeverageStrategyPosition, RedeemablePosition +from src.redeem.typings import Allocator, LeverageStrategyPosition, RedeemablePosition logger = logging.getLogger(__name__) @@ -153,50 +153,37 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: block_number = await execution_client.eth.block_number allocators = await graph_get_allocators(block_number) - # # filter + # filter boost proxy positions leverage_positions = await graph_get_leverage_positions(block_number) boost_proxies = {pos.proxy for pos in leverage_positions} logger.info('Found %s boost positions to exclude', len(boost_proxies)) min_minted_shares = Web3.to_wei(min_os_token_position_amount_gwei, 'gwei') allocators = [a for a in allocators if a.address not in boost_proxies] - - allocators = sorted(allocators, key=lambda x: x.minted_shares, reverse=True) - logger.info('Filtered allocators count: %s', len(allocators)) - - address_to_minted_shares = {a.address: a.minted_shares for a in allocators} - user_addresses = set(allocator.address for allocator in allocators) - logger.info('Fetching kept tokens for %s addresses...', len(address_to_minted_shares)) + address_to_minted_shares = {a.address: a.total_shares for a in allocators} # filter boosted positions - boosted_amounts = await get_boosted_amounts( - address_to_minted_shares, leverage_positions=leverage_positions, block_number=block_number + boosted_positions = await get_boosted_positions( + users=list(address_to_minted_shares.keys()), + leverage_positions=leverage_positions, + block_number=block_number, ) for allocator in allocators: - allocator.minted_shares = Wei( - allocator.minted_shares - boosted_amounts.get(allocator.address, Wei(0)) - ) - allocators = [a for a in allocators if a.minted_shares >= min_minted_shares] + if allocator.address not in boosted_positions: + continue + for vault_address, boosted_amount in boosted_positions[allocator.address].items(): + for vault_share in allocator.vault_shares: + if vault_share.address == vault_address: + vault_share.minted_shares = Wei(vault_share.minted_shares - boosted_amount) + + # filter zero positions + allocators = [a for a in allocators if a.total_shares >= min_minted_shares] + logger.info('Fetching kept tokens for %s addresses', len(allocators)) kept_tokens = await get_kept_tokens(address_to_minted_shares, block_number, arbitrum_endpoint) - filled = 0 - for allocator in allocators: - if allocator.minted_shares == kept_tokens.get(allocator.address, Wei(0)): - filled += 1 - logger.info('Found %s fully filled positions', filled) - redeemable_positions: list[RedeemablePosition] = [] - for allocator in allocators: - kept_token = kept_tokens.get(allocator.address, Wei(0)) # 0? - amount = min(allocator.minted_shares, kept_token) - if amount > 0: - redeemable_positions.append( - RedeemablePosition( - owner=allocator.address, - vault=allocator.vault, - amount=Wei(allocator.minted_shares - amount), - ) - ) - kept_tokens[allocator.address] = Wei(kept_tokens[allocator.address] - amount) - logger.info('Fetched kept tokens for %s addresses...', len(user_addresses)) + logger.info('Fetched kept tokens for %s addresses...', len(address_to_minted_shares)) + + redeemable_positions = create_redeemable_positions(allocators, kept_tokens) + logger.info('Created %s redeemable positions', len(redeemable_positions)) click.confirm( 'Proceed with uploading redeemable positions to IPFS?', @@ -246,21 +233,52 @@ async def get_kept_tokens( return kept_token -async def get_boosted_amounts( - address_to_minted_shares: dict[ChecksumAddress, Wei], +async def get_boosted_positions( + users: list[ChecksumAddress], leverage_positions: list[LeverageStrategyPosition], block_number: BlockNumber, -) -> dict[ChecksumAddress, Wei]: - boosted_os_token_shares: defaultdict[ChecksumAddress, Wei] = defaultdict(lambda: Wei(0)) +) -> dict[ChecksumAddress, dict[ChecksumAddress, Wei]]: + boosted_positions: defaultdict[ChecksumAddress, dict[ChecksumAddress, Wei]] = defaultdict(dict) for position in leverage_positions: - if position.user not in address_to_minted_shares: + if position.user not in users: continue vault_contract = VaultContract(position.vault) - position_os_token_shares = ( + position_os_token_shares = Wei( position.os_token_shares + await vault_contract.convert_to_shares(position.assets, block_number) ) - boosted_os_token_shares[position.user] = Wei( - boosted_os_token_shares[position.user] + position_os_token_shares - ) - return boosted_os_token_shares + boosted_positions[position.user][position.vault] = position_os_token_shares + + return boosted_positions + + +def create_redeemable_positions( + allocators: list[Allocator], kept_tokens: dict[ChecksumAddress, Wei] +) -> list[RedeemablePosition]: + # calculate vault proportions # create redeemable positions + filled = 0 + redeemable_positions: list[RedeemablePosition] = [] + for allocator in allocators: + kept_token = kept_tokens.get(allocator.address, Wei(0)) + amount = min(allocator.total_shares, kept_token) + if amount <= 0: + continue + + filled = 0 + for index, (vault_address, proportion) in enumerate(allocator.vaults_proportions.items()): + # dust handling + if index == len(allocator.vaults_proportions) - 1: + vault_amount = int(amount - filled) + else: + vault_amount = int(amount * proportion) + + redeemable_positions.append( + RedeemablePosition( + owner=allocator.address, + vault=vault_address, + amount=Wei(vault_amount), + ) + ) + filled = filled + vault_amount + + return redeemable_positions diff --git a/src/redeem/graph.py b/src/redeem/graph.py index 88c42eac..c27cd4af 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -1,10 +1,13 @@ import logging +from collections import defaultdict from eth_typing import BlockNumber from gql import gql +from web3 import Web3 +from web3.types import Wei from src.common.clients import graph_client -from src.redeem.typings import Allocator, LeverageStrategyPosition +from src.redeem.typings import Allocator, LeverageStrategyPosition, VaultShares logger = logging.getLogger(__name__) @@ -36,7 +39,21 @@ async def graph_get_allocators(block_number: BlockNumber) -> list[Allocator]: 'block': block_number, } response = await graph_client.fetch_pages(query, params=params) - return [Allocator.from_graph(item) for item in response] + tmp_allocators: defaultdict[str, dict[str, Wei]] = defaultdict(dict) + allocators = [] + for item in response: + tmp_allocators[item['address']][item['vault']['id']] = Wei(int(item['mintedOsTokenShares'])) + for allocator_address, vaults in tmp_allocators.items(): + vault_shares = [ + VaultShares(address=Web3.to_checksum_address(vault_address), minted_shares=shares) + for vault_address, shares in vaults.items() + ] + allocators.append( + Allocator( + address=Web3.to_checksum_address(allocator_address), vault_shares=vault_shares + ) + ) + return allocators async def graph_get_leverage_positions(block_number: BlockNumber) -> list[LeverageStrategyPosition]: diff --git a/src/redeem/typings.py b/src/redeem/typings.py index bbc94f75..0cfaad3b 100644 --- a/src/redeem/typings.py +++ b/src/redeem/typings.py @@ -7,18 +7,26 @@ @dataclass -class Allocator: +class VaultShares: address: ChecksumAddress - vault: ChecksumAddress minted_shares: Wei - @classmethod - def from_graph(cls, data: dict) -> 'Allocator': - return Allocator( - vault=Web3.to_checksum_address(data['vault']['id']), - address=Web3.to_checksum_address(data['address']), - minted_shares=Wei(int(data['mintedOsTokenShares'])), - ) + +@dataclass +class Allocator: + address: ChecksumAddress + vault_shares: list[VaultShares] + + @property + def total_shares(self) -> Wei: + return Wei(sum(s.minted_shares for s in self.vault_shares)) + + @property + def vaults_proportions(self) -> dict[ChecksumAddress, float]: + total = self.total_shares + if total == 0: + return {} + return {s.address: s.minted_shares / total for s in self.vault_shares} @dataclass @@ -42,7 +50,7 @@ def from_graph(cls, data: dict) -> 'LeverageStrategyPosition': @dataclass class RedeemablePosition: - owner: ChecksumAddress + owner: ChecksumAddress # noqa vault: ChecksumAddress amount: Wei From bf5d407d8909029117a579aad19ae2fff6185924 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 13 Jan 2026 17:24:55 +0300 Subject: [PATCH 07/65] Add logs Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 31 +++++++++++++++---- src/redeem/graph.py | 16 +++++++--- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 750e0dfb..92bae093 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -151,19 +151,20 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: setup_logging() await setup_clients() block_number = await execution_client.eth.block_number + logger.info('Fetching allocators from the subgraph...') allocators = await graph_get_allocators(block_number) + logger.info('Fetched %s allocators from the subgraph', len(allocators)) # filter boost proxy positions leverage_positions = await graph_get_leverage_positions(block_number) boost_proxies = {pos.proxy for pos in leverage_positions} - logger.info('Found %s boost positions to exclude', len(boost_proxies)) - min_minted_shares = Web3.to_wei(min_os_token_position_amount_gwei, 'gwei') + logger.info('Found %s proxy positions to exclude', len(boost_proxies)) allocators = [a for a in allocators if a.address not in boost_proxies] - address_to_minted_shares = {a.address: a.total_shares for a in allocators} # filter boosted positions + logger.info('Fetching boosted positions from the subgraph...') boosted_positions = await get_boosted_positions( - users=list(address_to_minted_shares.keys()), + users=[a.address for a in allocators], leverage_positions=leverage_positions, block_number=block_number, ) @@ -176,9 +177,11 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: vault_share.minted_shares = Wei(vault_share.minted_shares - boosted_amount) # filter zero positions + min_minted_shares = Web3.to_wei(min_os_token_position_amount_gwei, 'gwei') allocators = [a for a in allocators if a.total_shares >= min_minted_shares] logger.info('Fetching kept tokens for %s addresses', len(allocators)) + address_to_minted_shares = {a.address: a.total_shares for a in allocators} kept_tokens = await get_kept_tokens(address_to_minted_shares, block_number, arbitrum_endpoint) logger.info('Fetched kept tokens for %s addresses...', len(address_to_minted_shares)) @@ -201,12 +204,20 @@ async def get_kept_tokens( arbitrum_endpoint: str | None, ) -> dict[ChecksumAddress, Wei]: kept_token = defaultdict(lambda: Wei(0)) + logger.info('Fetching OsETH from wallet balances...') + + index = 0 contract = Erc20Contract(settings.network_config.OS_TOKEN_CONTRACT_ADDRESS) for address in address_to_minted_shares.keys(): + if index % 50 == 0: + logger.info( + 'Fetched wallet balances for %d/%d addresses', index, len(address_to_minted_shares) + ) kept_token[address] = await contract.balance(address, block_number) - + index += 1 # arb wallet balance if settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS != ZERO_CHECKSUM_ADDRESS: + logger.info('Fetching OsETH from Arbitrum wallet balances...') arbitrum_endpoint = cast(str, arbitrum_endpoint) arb_execution_client = get_execution_client([arbitrum_endpoint]) @@ -214,16 +225,24 @@ async def get_kept_tokens( settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS, execution_client=arb_execution_client, ) + index = 0 for address in address_to_minted_shares.keys(): + if index % 50 == 0: + logger.info( + 'Fetched wallet balances for %d/%d addresses', + index, + len(address_to_minted_shares), + ) arb_balance = await arb_contract.balance(address) kept_token[address] = Wei(kept_token[address] + arb_balance) - + index += 1 # do not fetch data from api if all os token are on the wallet api_addresses = [] for address in address_to_minted_shares.keys(): if address_to_minted_shares[address] > kept_token[address]: api_addresses.append(address) + logger.info('Fetching locked OsETH from DeBank API...') api_client = APIClient() locked_oseth_per_user: dict[ChecksumAddress, Wei] = {} for address in api_addresses: diff --git a/src/redeem/graph.py b/src/redeem/graph.py index c27cd4af..423ee50b 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -19,17 +19,22 @@ async def graph_get_allocators(block_number: BlockNumber) -> list[Allocator]: """ query = gql( """ - query getAllocators($block: Int, $first: Int, $skip: Int){ + query getAllocators($block: Int, $first: Int, $lastID: String){ allocators( block: {number: $block}, + where: { + id_gt: $lastID + }, + orderBy: id first: $first - skip: $skip ){ vault { id } id address + shares + assets mintedOsTokenShares } } @@ -38,11 +43,14 @@ async def graph_get_allocators(block_number: BlockNumber) -> list[Allocator]: params = { 'block': block_number, } - response = await graph_client.fetch_pages(query, params=params) + response = await graph_client.fetch_pages(query, params=params, cursor_pagination=True) tmp_allocators: defaultdict[str, dict[str, Wei]] = defaultdict(dict) allocators = [] for item in response: - tmp_allocators[item['address']][item['vault']['id']] = Wei(int(item['mintedOsTokenShares'])) + if int(item['mintedOsTokenShares']) > 0: + tmp_allocators[item['address']][item['vault']['id']] = Wei( + int(item['mintedOsTokenShares']) + ) for allocator_address, vaults in tmp_allocators.items(): vault_shares = [ VaultShares(address=Web3.to_checksum_address(vault_address), minted_shares=shares) From baf4d2f580bd4c690f3a54f87c1ada0c8af4c73d Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 13 Jan 2026 18:09:12 +0300 Subject: [PATCH 08/65] Small refactoring Signed-off-by: cyc60 --- poetry.lock | 48 +++++++++---------- pyproject.toml | 3 +- .../internal/update_redeemable_positions.py | 34 ++++++++----- 3 files changed, 47 insertions(+), 38 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4c491f90..338a3c1b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1798,42 +1798,42 @@ files = [ [[package]] name = "gql" -version = "3.5.0" +version = "3.5.3" description = "GraphQL client for Python" optional = false python-versions = "*" files = [ - {file = "gql-3.5.0-py2.py3-none-any.whl", hash = "sha256:70dda5694a5b194a8441f077aa5fb70cc94e4ec08016117523f013680901ecb7"}, - {file = "gql-3.5.0.tar.gz", hash = "sha256:ccb9c5db543682b28f577069950488218ed65d4ac70bb03b6929aaadaf636de9"}, + {file = "gql-3.5.3-py2.py3-none-any.whl", hash = "sha256:e1fcbde2893fcafdd28114ece87ff47f1cc339a31db271fc4e1d528f5a1d4fbc"}, + {file = "gql-3.5.3.tar.gz", hash = "sha256:393b8c049d58e0d2f5461b9d738a2b5f904186a40395500b4a84dd092d56e42b"}, ] [package.dependencies] aiohttp = {version = ">=3.9.0b0,<4", optional = true, markers = "python_version > \"3.11\" and extra == \"aiohttp\""} anyio = ">=3.0,<5" backoff = ">=1.11.1,<3.0" -graphql-core = ">=3.2,<3.3" +graphql-core = ">=3.2,<3.2.7" yarl = ">=1.6,<2.0" [package.extras] aiohttp = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)"] all = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "websockets (>=10,<12)"] botocore = ["botocore (>=1.21,<2)"] -dev = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "httpx (>=0.23.1,<1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "sphinx (>=5.3.0,<6)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +dev = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "httpx (>=0.23.1,<1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "sphinx (>=5.3.0,<6)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "vcrpy (==4.4.0)", "vcrpy (==7.0.0)", "websockets (>=10,<12)"] httpx = ["httpx (>=0.23.1,<1)"] requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)"] -test = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] -test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.4.0)"] +test = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "vcrpy (==4.4.0)", "vcrpy (==7.0.0)", "websockets (>=10,<12)"] +test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.4.0)", "vcrpy (==7.0.0)"] websockets = ["websockets (>=10,<12)"] [[package]] name = "graphql-core" -version = "3.2.7" +version = "3.2.6" description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." optional = false -python-versions = "<4,>=3.7" +python-versions = "<4,>=3.6" files = [ - {file = "graphql_core-3.2.7-py3-none-any.whl", hash = "sha256:17fc8f3ca4a42913d8e24d9ac9f08deddf0a0b2483076575757f6c412ead2ec0"}, - {file = "graphql_core-3.2.7.tar.gz", hash = "sha256:27b6904bdd3b43f2a0556dad5d579bdfdeab1f38e8e8788e555bdcb586a6f62c"}, + {file = "graphql_core-3.2.6-py3-none-any.whl", hash = "sha256:78b016718c161a6fb20a7d97bbf107f331cd1afe53e45566c59f776ed7f0b45f"}, + {file = "graphql_core-3.2.6.tar.gz", hash = "sha256:c08eec22f9e40f0bd61d805907e3b3b1b9a320bc606e23dc145eebca07c8fbab"}, ] [[package]] @@ -1854,13 +1854,13 @@ test = ["eth_utils (>=2.0.0)", "hypothesis (>=3.44.24)", "pytest (>=7.0.0)", "py [[package]] name = "identify" -version = "2.6.15" +version = "2.6.16" description = "File identification library for Python" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" files = [ - {file = "identify-2.6.15-py2.py3-none-any.whl", hash = "sha256:1181ef7608e00704db228516541eb83a88a9f94433a8c80bb9b5bd54b1d81757"}, - {file = "identify-2.6.15.tar.gz", hash = "sha256:e4f4864b96c6557ef2a1e1c951771838f4edc9df3a72ec7118b338801b11c7bf"}, + {file = "identify-2.6.16-py2.py3-none-any.whl", hash = "sha256:391ee4d77741d994189522896270b787aed8670389bfd60f326d677d64a6dfb0"}, + {file = "identify-2.6.16.tar.gz", hash = "sha256:846857203b5511bbe94d5a352a48ef2359532bc8f6727b5544077a0dcfb24980"}, ] [package.extras] @@ -3878,7 +3878,7 @@ test = ["py-ecc (==6.0.0)", "pytest (>=6.2.5)", "pytest-benchmark (>=3.2.3)"] [[package]] name = "sw-utils" -version = "v0.10.8" +version = "v0.12.1" description = "StakeWise Python utils" optional = false python-versions = "^3.10" @@ -3886,7 +3886,7 @@ files = [] develop = false [package.dependencies] -gql = {version = "==3.5.0", extras = ["aiohttp"]} +gql = {version = "==3.5.3", extras = ["aiohttp"]} ipfshttpclient = "^0.8.0a2" py-ecc = "^8.0.0" pyjwt = "==2.8.0" @@ -3898,8 +3898,8 @@ web3 = "==7.13.0" [package.source] type = "git" url = "https://github.com/stakewise/sw-utils.git" -reference = "v0.10.8" -resolved_reference = "e9342e989f0d062cc9520937bb20e5792d71d59b" +reference = "v0.12.1" +resolved_reference = "2db244e487825e2339c3d7d1d3cb1b78a2bf78ce" [[package]] name = "tenacity" @@ -3973,13 +3973,13 @@ files = [ [[package]] name = "tomlkit" -version = "0.13.3" +version = "0.14.0" description = "Style preserving TOML library" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0"}, - {file = "tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1"}, + {file = "tomlkit-0.14.0-py3-none-any.whl", hash = "sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680"}, + {file = "tomlkit-0.14.0.tar.gz", hash = "sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064"}, ] [[package]] @@ -4419,4 +4419,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.12,<3.13" -content-hash = "06a9d292477d9eae99eaa75831d169aadda52dbccc49f2c19671eb5d5f0d55b5" +content-hash = "5f5345f335692f90b3a646322bd1f339b31758708dc3db0fb6cfb713a1f2a4b7" diff --git a/pyproject.toml b/pyproject.toml index 4829af6e..56131a11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,8 +10,7 @@ python = ">=3.12,<3.13" python-decouple = "==3.8" sentry-sdk = "==1.45.1" py-ecc = "==8.0.0" -gql = {extras = ["aiohttp"], version = "==3.5.0"} -sw-utils = {git = "https://github.com/stakewise/sw-utils.git", rev = "v0.10.8"} +sw-utils = {git = "https://github.com/stakewise/sw-utils.git", rev = "v0.12.1"} staking-deposit = { git = "https://github.com/ethereum/staking-deposit-cli.git", rev = "v2.8.0" } pycryptodomex = "3.19.1" click = "==8.2.1" diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 92bae093..d2fe8a65 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -180,13 +180,24 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: min_minted_shares = Web3.to_wei(min_os_token_position_amount_gwei, 'gwei') allocators = [a for a in allocators if a.total_shares >= min_minted_shares] + if not allocators: + logger.info('No allocators with minted shares above the threshold found, exiting...') + return + logger.info('Fetching kept tokens for %s addresses', len(allocators)) address_to_minted_shares = {a.address: a.total_shares for a in allocators} kept_tokens = await get_kept_tokens(address_to_minted_shares, block_number, arbitrum_endpoint) logger.info('Fetched kept tokens for %s addresses...', len(address_to_minted_shares)) redeemable_positions = create_redeemable_positions(allocators, kept_tokens) - logger.info('Created %s redeemable positions', len(redeemable_positions)) + if not redeemable_positions: + logger.info('No redeemable positions to upload, exiting...') + return + logger.info( + 'Created %s redeemable positions. Total redeemed OsEth amount: %s', + len(redeemable_positions), + sum(p.amount for p in redeemable_positions), + ) click.confirm( 'Proceed with uploading redeemable positions to IPFS?', @@ -206,15 +217,13 @@ async def get_kept_tokens( kept_token = defaultdict(lambda: Wei(0)) logger.info('Fetching OsETH from wallet balances...') - index = 0 contract = Erc20Contract(settings.network_config.OS_TOKEN_CONTRACT_ADDRESS) - for address in address_to_minted_shares.keys(): - if index % 50 == 0: + for index, address in enumerate(address_to_minted_shares.keys()): + if index and index % 50 == 0: logger.info( 'Fetched wallet balances for %d/%d addresses', index, len(address_to_minted_shares) ) kept_token[address] = await contract.balance(address, block_number) - index += 1 # arb wallet balance if settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS != ZERO_CHECKSUM_ADDRESS: logger.info('Fetching OsETH from Arbitrum wallet balances...') @@ -225,9 +234,8 @@ async def get_kept_tokens( settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS, execution_client=arb_execution_client, ) - index = 0 - for address in address_to_minted_shares.keys(): - if index % 50 == 0: + for index, address in enumerate(address_to_minted_shares.keys()): + if index and index % 50 == 0: logger.info( 'Fetched wallet balances for %d/%d addresses', index, @@ -235,14 +243,17 @@ async def get_kept_tokens( ) arb_balance = await arb_contract.balance(address) kept_token[address] = Wei(kept_token[address] + arb_balance) - index += 1 + # do not fetch data from api if all os token are on the wallet api_addresses = [] for address in address_to_minted_shares.keys(): if address_to_minted_shares[address] > kept_token[address]: api_addresses.append(address) - logger.info('Fetching locked OsETH from DeBank API...') + if not api_addresses: + return kept_token + + logger.info('Fetching locked OsETH from DeBank API for %s addresses...', len(api_addresses)) api_client = APIClient() locked_oseth_per_user: dict[ChecksumAddress, Wei] = {} for address in api_addresses: @@ -274,8 +285,7 @@ async def get_boosted_positions( def create_redeemable_positions( allocators: list[Allocator], kept_tokens: dict[ChecksumAddress, Wei] ) -> list[RedeemablePosition]: - # calculate vault proportions # create redeemable positions - filled = 0 + """Calculate vault proportions and create redeemable positions""" redeemable_positions: list[RedeemablePosition] = [] for allocator in allocators: kept_token = kept_tokens.get(allocator.address, Wei(0)) From 6c85b199977f8ece963be2bd9a69c4fb0f9a351a Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 13 Jan 2026 18:12:26 +0300 Subject: [PATCH 09/65] Add API rate limits Signed-off-by: cyc60 --- src/commands/internal/update_redeemable_positions.py | 3 ++- src/redeem/api_client.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index d2fe8a65..6ccda884 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -21,7 +21,7 @@ from src.common.utils import log_verbose from src.config.networks import AVAILABLE_NETWORKS, MAINNET, ZERO_CHECKSUM_ADDRESS from src.config.settings import settings -from src.redeem.api_client import APIClient +from src.redeem.api_client import API_SLEEP_TIMEOUT, APIClient from src.redeem.graph import graph_get_allocators, graph_get_leverage_positions from src.redeem.typings import Allocator, LeverageStrategyPosition, RedeemablePosition @@ -260,6 +260,7 @@ async def get_kept_tokens( locked_os_token = await api_client.get_protocols_locked_os_token(address=address) locked_oseth_per_user[address] = locked_os_token kept_token[address] = Wei(kept_token[address] + locked_os_token) + await asyncio.sleep(API_SLEEP_TIMEOUT) # to avoid rate limiting return kept_token diff --git a/src/redeem/api_client.py b/src/redeem/api_client.py index 1e04f653..37c4a070 100644 --- a/src/redeem/api_client.py +++ b/src/redeem/api_client.py @@ -13,6 +13,7 @@ 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36' ) SUPPORTED_CHAINS = {'eth', 'arb'} +API_SLEEP_TIMEOUT = 1 class APIClient: From 503c959be340c5497fdaf7932adbe4b32f111944 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 13 Jan 2026 19:52:09 +0300 Subject: [PATCH 10/65] Update comment Signed-off-by: cyc60 --- src/config/settings.py | 4 ++-- src/redeem/graph.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/config/settings.py b/src/config/settings.py index 57ca2ffa..c940fd0f 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -353,8 +353,8 @@ def set( self.consensus_retry_timeout = decouple_config( 'CONSENSUS_RETRY_TIMEOUT', default=120, cast=int ) - self.graph_request_timeout = decouple_config('GRAPH_REQUEST_TIMEOUT', default=30, cast=int) - self.graph_retry_timeout = decouple_config('GRAPH_RETRY_TIMEOUT', default=120, cast=int) + self.graph_request_timeout = decouple_config('GRAPH_REQUEST_TIMEOUT', default=10, cast=int) + self.graph_retry_timeout = decouple_config('GRAPH_RETRY_TIMEOUT', default=60, cast=int) self.graph_page_size = decouple_config('GRAPH_PAGE_SIZE', default=500, cast=int) self.relayer_endpoint = relayer_endpoint or '' self.relayer_timeout = decouple_config('RELAYER_TIMEOUT', default=10, cast=int) diff --git a/src/redeem/graph.py b/src/redeem/graph.py index 423ee50b..9f25fa46 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -14,8 +14,8 @@ async def graph_get_allocators(block_number: BlockNumber) -> list[Allocator]: """ - Returns mapping from sub-vault address to list of ExitRequest objects - Skips claimed exit requests and those with exitedAssets == 0 + Fetch allocators at the given block and return them as a list of Allocator objects. + Filter record to include only those with mintedOsTokenShares > 0. """ query = gql( """ From 6600e0e554f02c673f5d79e8f697f2ce80e67fbd Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 13 Jan 2026 19:58:23 +0300 Subject: [PATCH 11/65] Rename var Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 6ccda884..10ba302c 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -214,7 +214,7 @@ async def get_kept_tokens( block_number: BlockNumber, arbitrum_endpoint: str | None, ) -> dict[ChecksumAddress, Wei]: - kept_token = defaultdict(lambda: Wei(0)) + kept_tokens = defaultdict(lambda: Wei(0)) logger.info('Fetching OsETH from wallet balances...') contract = Erc20Contract(settings.network_config.OS_TOKEN_CONTRACT_ADDRESS) @@ -223,7 +223,7 @@ async def get_kept_tokens( logger.info( 'Fetched wallet balances for %d/%d addresses', index, len(address_to_minted_shares) ) - kept_token[address] = await contract.balance(address, block_number) + kept_tokens[address] = await contract.balance(address, block_number) # arb wallet balance if settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS != ZERO_CHECKSUM_ADDRESS: logger.info('Fetching OsETH from Arbitrum wallet balances...') @@ -242,16 +242,16 @@ async def get_kept_tokens( len(address_to_minted_shares), ) arb_balance = await arb_contract.balance(address) - kept_token[address] = Wei(kept_token[address] + arb_balance) + kept_tokens[address] = Wei(kept_tokens[address] + arb_balance) # do not fetch data from api if all os token are on the wallet api_addresses = [] for address in address_to_minted_shares.keys(): - if address_to_minted_shares[address] > kept_token[address]: + if address_to_minted_shares[address] > kept_tokens[address]: api_addresses.append(address) if not api_addresses: - return kept_token + return kept_tokens logger.info('Fetching locked OsETH from DeBank API for %s addresses...', len(api_addresses)) api_client = APIClient() @@ -259,9 +259,9 @@ async def get_kept_tokens( for address in api_addresses: locked_os_token = await api_client.get_protocols_locked_os_token(address=address) locked_oseth_per_user[address] = locked_os_token - kept_token[address] = Wei(kept_token[address] + locked_os_token) + kept_tokens[address] = Wei(kept_tokens[address] + locked_os_token) await asyncio.sleep(API_SLEEP_TIMEOUT) # to avoid rate limiting - return kept_token + return kept_tokens async def get_boosted_positions( From 14465fc3434ede70cad28c2943aaf866a91f2844 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 13 Jan 2026 21:23:35 +0300 Subject: [PATCH 12/65] Fix typo Signed-off-by: cyc60 --- src/redeem/graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redeem/graph.py b/src/redeem/graph.py index 9f25fa46..1053d701 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -15,7 +15,7 @@ async def graph_get_allocators(block_number: BlockNumber) -> list[Allocator]: """ Fetch allocators at the given block and return them as a list of Allocator objects. - Filter record to include only those with mintedOsTokenShares > 0. + Filter records to include only those with mintedOsTokenShares > 0. """ query = gql( """ From 248a9be6881a0dc020d77189fb7e9ba8072e370a Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 19 Jan 2026 12:00:32 +0300 Subject: [PATCH 13/65] Add tests Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 3 +- src/commands/tests/test_internal/__init__.py | 0 .../test_update_redeemable_positions.py | 88 ++++++ src/redeem/tests/api_samples/with_boost.json | 278 ++++++++++++++++++ src/redeem/tests/test_api_client.py | 19 +- src/redeem/tests/test_graph.py | 53 ++++ 6 files changed, 439 insertions(+), 2 deletions(-) create mode 100644 src/commands/tests/test_internal/__init__.py create mode 100644 src/commands/tests/test_internal/test_update_redeemable_positions.py create mode 100644 src/redeem/tests/api_samples/with_boost.json create mode 100644 src/redeem/tests/test_graph.py diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 10ba302c..833a3c50 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -290,7 +290,8 @@ def create_redeemable_positions( redeemable_positions: list[RedeemablePosition] = [] for allocator in allocators: kept_token = kept_tokens.get(allocator.address, Wei(0)) - amount = min(allocator.total_shares, kept_token) + kept_amount = min(allocator.total_shares, kept_token) + amount = int(allocator.total_shares - kept_amount) if amount <= 0: continue diff --git a/src/commands/tests/test_internal/__init__.py b/src/commands/tests/test_internal/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/commands/tests/test_internal/test_update_redeemable_positions.py b/src/commands/tests/test_internal/test_update_redeemable_positions.py new file mode 100644 index 00000000..74d7579d --- /dev/null +++ b/src/commands/tests/test_internal/test_update_redeemable_positions.py @@ -0,0 +1,88 @@ +from sw_utils.tests import faker +from web3 import Web3 +from web3.types import Wei + +from src.commands.internal.update_redeemable_positions import ( + create_redeemable_positions, +) +from src.redeem.typings import Allocator, RedeemablePosition, VaultShares + + +def test_create_redeemable_positions(): + address_1 = faker.eth_address() + address_2 = faker.eth_address() + vault_1 = faker.eth_address() + vault_2 = faker.eth_address() + + # test zero allocators + result = create_redeemable_positions([], {}) + assert result == [] + + # test single vault + allocators = [ + Allocator( + address=Web3.to_checksum_address(address_1), + vault_shares=[ + VaultShares(address=Web3.to_checksum_address(vault_1), minted_shares=Wei(150)), + ], + ) + ] + kept_tokens = { + address_1: Wei(0), + } + result = create_redeemable_positions(allocators, kept_tokens) + assert result == [RedeemablePosition(owner=address_1, vault=vault_1, amount=Wei(150))] + + allocators = [ + Allocator( + address=Web3.to_checksum_address(address_1), + vault_shares=[ + VaultShares(address=Web3.to_checksum_address(vault_1), minted_shares=Wei(150)), + ], + ) + ] + kept_tokens = { + address_1: Wei(100), + } + result = create_redeemable_positions(allocators, kept_tokens) + assert result == [RedeemablePosition(owner=address_1, vault=vault_1, amount=Wei(50))] + + allocators = [ + Allocator( + address=Web3.to_checksum_address(address_1), + vault_shares=[ + VaultShares(address=Web3.to_checksum_address(vault_1), minted_shares=Wei(150)), + ], + ), + Allocator( + address=Web3.to_checksum_address(address_2), + vault_shares=[ + VaultShares(address=Web3.to_checksum_address(vault_1), minted_shares=Wei(75)), + ], + ), + ] + kept_tokens = { + address_1: Wei(0), + address_2: Wei(75), + } + result = create_redeemable_positions(allocators, kept_tokens) + assert result == [RedeemablePosition(owner=address_1, vault=vault_1, amount=Wei(150))] + + # test multiple vaults + allocators = [ + Allocator( + address=Web3.to_checksum_address(address_1), + vault_shares=[ + VaultShares(address=Web3.to_checksum_address(vault_1), minted_shares=Wei(150)), + VaultShares(address=Web3.to_checksum_address(vault_2), minted_shares=Wei(150)), + ], + ) + ] + kept_tokens = { + address_1: Wei(0), + } + result = create_redeemable_positions(allocators, kept_tokens) + assert result == [ + RedeemablePosition(owner=address_1, vault=vault_1, amount=Wei(150)), + RedeemablePosition(owner=address_1, vault=vault_2, amount=Wei(150)), + ] diff --git a/src/redeem/tests/api_samples/with_boost.json b/src/redeem/tests/api_samples/with_boost.json new file mode 100644 index 00000000..68c1faca --- /dev/null +++ b/src/redeem/tests/api_samples/with_boost.json @@ -0,0 +1,278 @@ +[ { + "id": "stakewise", + "chain": "eth", + "name": "StakeWise", + "site_url": "https://app.stakewise.io", + "logo_url": "https://static.debank.com/image/project/logo_url/stakewise/c23c49fcf9096ab73cf0001d78f570a3.png", + "has_supported_portfolio": true, + "tvl": 331114416.4403308, + "portfolio_item_list": [ + { + "stats": { + "asset_usd_value": 4596325.0420510955, + "debt_usd_value": 0, + "net_usd_value": 4596325.0420510955 + }, + "asset_dict": { + "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38": 1373.5740496458861 + }, + "asset_token_list": [ + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3346.252095572168, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 1373.5740496458861 + } + ], + "withdraw_actions": [ + { + "type": "queue", + "contract_id": "0x48cd14fdb8e72a03c8d952af081dbb127d6281fc", + "func": "enterExitQueue(address,uint256)(uint256)", + "params": [ + "0xe6d8d8ac54461b1c5ed15740eee322043f696c08", + 1000000000000000000 + ], + "str_params": [ + "0xe6d8d8ac54461b1c5ed15740eee322043f696c08", + "1000000000000000000" + ] + } + ], + "update_at": 1768182769, + "name": "Staked", + "detail_types": [ + "common" + ], + "detail": { + "supply_token_list": [ + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3346.252095572168, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 1373.5740496458861 + } + ], + "description": "Boost" + }, + "proxy_detail": {}, + "position_index": "boost", + "pool": { + "id": "0x48cd14fdb8e72a03c8d952af081dbb127d6281fc:0xe6d8d8ac54461b1c5ed15740eee322043f696c08", + "chain": "eth", + "project_id": "stakewise", + "adapter_id": "stakewise_staked2", + "controller": "0x48cd14fdb8e72a03c8d952af081dbb127d6281fc", + "index": "0xe6d8d8ac54461b1c5ed15740eee322043f696c08", + "time_at": 1732828487 + } + }, + { + "stats": { + "asset_usd_value": 98487.71209985697, + "debt_usd_value": 0, + "net_usd_value": 98487.71209985697 + }, + "asset_dict": { + "eth": 31.179528512996352 + }, + "asset_token_list": [ + { + "id": "eth", + "chain": "eth", + "name": "ETH", + "symbol": "ETH", + "display_symbol": null, + "optimized_symbol": "ETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/coin/logo_url/eth/6443cdccced33e204d90cb723c632917.png", + "protocol_id": "", + "price": 3158.73, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1483200000, + "amount": 31.179528512996352 + } + ], + "withdraw_actions": [ + { + "type": "queue", + "contract_id": "0xdbdee04c72a02a740b9f26ada9203582c8a99daf", + "func": "enterExitQueue(uint256,address)(uint256)", + "params": [ + 29995550572473540000, + "0x8e57bc446f76b2054089cc5c8fa6f0f5b72fc59a" + ], + "str_params": [ + "29995550572473538720", + "0x8e57bc446f76b2054089cc5c8fa6f0f5b72fc59a" + ] + } + ], + "update_at": 1768182769, + "name": "Lending", + "detail_types": [ + "lending" + ], + "detail": { + "supply_token_list": [ + { + "id": "eth", + "chain": "eth", + "name": "ETH", + "symbol": "ETH", + "display_symbol": null, + "optimized_symbol": "ETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/coin/logo_url/eth/6443cdccced33e204d90cb723c632917.png", + "protocol_id": "", + "price": 3158.73, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1483200000, + "amount": 31.179528512996352 + } + ] + }, + "proxy_detail": {}, + "pool": { + "id": "0xdbdee04c72a02a740b9f26ada9203582c8a99daf", + "chain": "eth", + "project_id": "stakewise", + "adapter_id": "stakewise_lending", + "controller": "0xdbdee04c72a02a740b9f26ada9203582c8a99daf", + "index": null, + "time_at": 1719414779 + } + }, + { + "stats": { + "asset_usd_value": 5031224.823366908, + "debt_usd_value": 4999997.98723114, + "net_usd_value": 31226.83613576833 + }, + "asset_dict": { + "eth": 1592.7998984930362, + "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38": -1494.2083992557655 + }, + "asset_token_list": [ + { + "id": "eth", + "chain": "eth", + "name": "ETH", + "symbol": "ETH", + "display_symbol": null, + "optimized_symbol": "ETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/coin/logo_url/eth/6443cdccced33e204d90cb723c632917.png", + "protocol_id": "", + "price": 3158.73, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1483200000, + "amount": 1592.7998984930362 + }, + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3346.252095572168, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": -1494.2083992557655 + } + ], + "withdraw_actions": [], + "update_at": 1768182769, + "name": "Lending", + "detail_types": [ + "lending" + ], + "detail": { + "supply_token_list": [ + { + "id": "eth", + "chain": "eth", + "name": "ETH", + "symbol": "ETH", + "display_symbol": null, + "optimized_symbol": "ETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/coin/logo_url/eth/6443cdccced33e204d90cb723c632917.png", + "protocol_id": "", + "price": 3158.73, + "is_verified": true, + "is_core": true, + "is_wallet": true, + "time_at": 1483200000, + "amount": 1592.7998984930362 + } + ], + "borrow_token_list": [ + { + "id": "0xf1c9acdc66974dfb6decb12aa385b9cd01190e38", + "chain": "eth", + "name": "Staked ETH", + "symbol": "osETH", + "display_symbol": null, + "optimized_symbol": "osETH", + "decimals": 18, + "logo_url": "https://static.debank.com/image/eth_token/logo_url/0xf1c9acdc66974dfb6decb12aa385b9cd01190e38/4b9f533a91f012c8b142dd3b0755adae.png", + "protocol_id": "stakewise", + "price": 3346.252095572168, + "is_verified": false, + "is_core": true, + "is_wallet": true, + "time_at": 1698755027, + "amount": 1494.2083992557655 + } + ], + "health_rate": 0.9056208327671258 + }, + "proxy_detail": {}, + "pool": { + "id": "0xe6d8d8ac54461b1c5ed15740eee322043f696c08", + "chain": "eth", + "project_id": "stakewise", + "adapter_id": "stakewise_lending", + "controller": "0xe6d8d8ac54461b1c5ed15740eee322043f696c08", + "index": null, + "time_at": 1700813123 + } + } + ] + } +] diff --git a/src/redeem/tests/test_api_client.py b/src/redeem/tests/test_api_client.py index 9d74f29b..790fd4ea 100644 --- a/src/redeem/tests/test_api_client.py +++ b/src/redeem/tests/test_api_client.py @@ -9,7 +9,7 @@ from src.redeem.api_client import APIClient -class TestGetOraclesRequest: +class TestAPIClient: @pytest.mark.usefixtures('fake_settings') async def test_zero_when_no_protocol_data(self): client = APIClient() @@ -78,3 +78,20 @@ async def test_real_data(self): Web3.to_checksum_address('0x1234567890abcdef1234567890abcdef12345678') ) assert result == Wei(5810497440414831) + + @pytest.mark.usefixtures('fake_settings') + async def test_real_data_with_boost(self): + with open('src/redeem/tests/api_samples/with_boost.json', 'r') as f: + mock_protocol_data = json.load(f) + settings.network_config.OS_TOKEN_CONTRACT_ADDRESS = ( + '0xf1C9acDc66974dFB6dEcB12aA385b9cD01190E38' + ) + settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS = ( + '0xf7d4e7273E5015C96728A6b02f31C505eE184603' + ) + with patch('src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data): + client = APIClient() + result = await client.get_protocols_locked_os_token( + Web3.to_checksum_address('0x1234567890abcdef1234567890abcdef12345678') + ) + assert result == Wei(0) diff --git a/src/redeem/tests/test_graph.py b/src/redeem/tests/test_graph.py new file mode 100644 index 00000000..20c93555 --- /dev/null +++ b/src/redeem/tests/test_graph.py @@ -0,0 +1,53 @@ +import random +from unittest.mock import patch + +import pytest +from sw_utils.tests import faker +from web3 import Web3 +from web3.types import Wei + +from src.redeem.graph import graph_get_allocators +from src.redeem.typings import Allocator, VaultShares + + +@pytest.mark.usefixtures('fake_settings') +async def test_graph_get_allocators(): + address_1 = faker.eth_address().lower() + address_2 = faker.eth_address().lower() + vault_1 = faker.eth_address().lower() + vault_2 = faker.eth_address().lower() + + with patch('src.redeem.graph.graph_client.fetch_pages', return_value=[]): + result = await graph_get_allocators(random.randint(1, 1000000)) + assert result == [] + + mock_response = [ + {'address': address_1, 'vault': {'id': vault_1}, 'mintedOsTokenShares': '0'}, + {'address': address_2, 'vault': {'id': vault_2}, 'mintedOsTokenShares': '1000'}, + ] + with patch('src.redeem.graph.graph_client.fetch_pages', return_value=mock_response): + result = await graph_get_allocators(random.randint(1, 1000000)) + assert result == [ + Allocator( + address=Web3.to_checksum_address(address_2), + vault_shares=[ + VaultShares(address=Web3.to_checksum_address(vault_2), minted_shares=Wei(1000)) + ], + ) + ] + + mock_response = [ + {'address': address_1, 'vault': {'id': vault_1}, 'mintedOsTokenShares': '150'}, + {'address': address_1, 'vault': {'id': vault_2}, 'mintedOsTokenShares': '1000'}, + ] + with patch('src.redeem.graph.graph_client.fetch_pages', return_value=mock_response): + result = await graph_get_allocators(random.randint(1, 1000000)) + assert result == [ + Allocator( + address=Web3.to_checksum_address(address_1), + vault_shares=[ + VaultShares(address=Web3.to_checksum_address(vault_1), minted_shares=Wei(150)), + VaultShares(address=Web3.to_checksum_address(vault_2), minted_shares=Wei(1000)), + ], + ) + ] From 5d4eea453b9d1cc3a8c511c207630cb0f852ebd6 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 19 Jan 2026 12:56:42 +0300 Subject: [PATCH 14/65] Review fixes Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 21 +++++++++---------- src/common/contracts.py | 2 +- src/redeem/api_client.py | 8 +++---- src/redeem/graph.py | 2 +- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 833a3c50..996fa66f 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -103,7 +103,7 @@ case_sensitive=False, ), ) -@click.command(help='Updates redeemable positions for leverage positions') +@click.command(help='Updates redeemable positions') # pylint: disable-next=too-many-arguments def update_redeemable_positions( execution_endpoints: str, @@ -146,7 +146,7 @@ def update_redeemable_positions( # pylint: disable-next=too-many-locals async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: Gwei) -> None: """ - Fetch redeemable positions, calculate kept os token amounts and upload to IPFS. + Fetch redeemable positions, calculate kept osToken amounts and upload to IPFS. """ setup_logging() await setup_clients() @@ -223,7 +223,7 @@ async def get_kept_tokens( logger.info( 'Fetched wallet balances for %d/%d addresses', index, len(address_to_minted_shares) ) - kept_tokens[address] = await contract.balance(address, block_number) + kept_tokens[address] = await contract.get_balance(address, block_number) # arb wallet balance if settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS != ZERO_CHECKSUM_ADDRESS: logger.info('Fetching OsETH from Arbitrum wallet balances...') @@ -241,7 +241,7 @@ async def get_kept_tokens( index, len(address_to_minted_shares), ) - arb_balance = await arb_contract.balance(address) + arb_balance = await arb_contract.get_balance(address) kept_tokens[address] = Wei(kept_tokens[address] + arb_balance) # do not fetch data from api if all os token are on the wallet @@ -255,10 +255,10 @@ async def get_kept_tokens( logger.info('Fetching locked OsETH from DeBank API for %s addresses...', len(api_addresses)) api_client = APIClient() - locked_oseth_per_user: dict[ChecksumAddress, Wei] = {} + locked_os_token_per_user: dict[ChecksumAddress, Wei] = {} for address in api_addresses: locked_os_token = await api_client.get_protocols_locked_os_token(address=address) - locked_oseth_per_user[address] = locked_os_token + locked_os_token_per_user[address] = locked_os_token kept_tokens[address] = Wei(kept_tokens[address] + locked_os_token) await asyncio.sleep(API_SLEEP_TIMEOUT) # to avoid rate limiting return kept_tokens @@ -290,18 +290,17 @@ def create_redeemable_positions( redeemable_positions: list[RedeemablePosition] = [] for allocator in allocators: kept_token = kept_tokens.get(allocator.address, Wei(0)) - kept_amount = min(allocator.total_shares, kept_token) - amount = int(allocator.total_shares - kept_amount) - if amount <= 0: + redeemable_amount = max(0, allocator.total_shares - kept_token) + if redeemable_amount <= 0: continue filled = 0 for index, (vault_address, proportion) in enumerate(allocator.vaults_proportions.items()): # dust handling if index == len(allocator.vaults_proportions) - 1: - vault_amount = int(amount - filled) + vault_amount = int(redeemable_amount - filled) else: - vault_amount = int(amount * proportion) + vault_amount = int(redeemable_amount * proportion) redeemable_positions.append( RedeemablePosition( diff --git a/src/common/contracts.py b/src/common/contracts.py index cca1afd4..049518c4 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -264,7 +264,7 @@ async def _get_public_keys_chunk( class Erc20Contract(ContractWrapper): abi_path = 'abi/Erc20Token.json' - async def balance( + async def get_balance( self, address: ChecksumAddress, block_number: BlockNumber | None = None ) -> Wei: return await self.contract.functions.balanceOf(address).call(block_identifier=block_number) diff --git a/src/redeem/api_client.py b/src/redeem/api_client.py index 37c4a070..9dcdad81 100644 --- a/src/redeem/api_client.py +++ b/src/redeem/api_client.py @@ -27,7 +27,7 @@ async def get_protocols_locked_os_token(self, address: ChecksumAddress) -> Wei: } protocol_data = await self._fetch_json(url, params=params) - total_locked_oseth = Wei(0) + total_locked_os_token = Wei(0) for protocol in protocol_data: # boosted OsEth handled via graph separately if protocol['id'] in ['stakewise', 'xdai_stakewise']: @@ -40,11 +40,11 @@ async def get_protocols_locked_os_token(self, address: ChecksumAddress) -> Wei: if not Web3.is_address(supply_token['id']): continue if self._is_os_token(Web3.to_checksum_address(supply_token['id'])): - total_locked_oseth = Wei( - total_locked_oseth + Web3.to_wei(float(supply_token['amount']), 'ether') + total_locked_os_token = Wei( + total_locked_os_token + Web3.to_wei(supply_token['amount'], 'ether') ) - return total_locked_oseth + return total_locked_os_token async def _fetch_json(self, url: str, params: dict | None = None) -> dict | list: async with aiohttp.ClientSession() as session: diff --git a/src/redeem/graph.py b/src/redeem/graph.py index 1053d701..7d92f7b9 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -45,7 +45,7 @@ async def graph_get_allocators(block_number: BlockNumber) -> list[Allocator]: } response = await graph_client.fetch_pages(query, params=params, cursor_pagination=True) tmp_allocators: defaultdict[str, dict[str, Wei]] = defaultdict(dict) - allocators = [] + allocators: list[Allocator] = [] for item in response: if int(item['mintedOsTokenShares']) > 0: tmp_allocators[item['address']][item['vault']['id']] = Wei( From dc38e91ebeeadb371899392964cfdc238b7acbad Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 19 Jan 2026 15:39:11 +0300 Subject: [PATCH 15/65] Review fixes #2 Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 42 ++++++++++++------- src/config/networks.py | 4 ++ src/redeem/tests/test_api_client.py | 36 +++++++++------- 3 files changed, 53 insertions(+), 29 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 996fa66f..5982617d 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -47,7 +47,7 @@ type=int, default=0, envvar='MIN_OS_TOKEN_POSITION_AMOUNT_GWEI', - help='Process positions only if the amount of minted osETH' + help='Process positions only if the amount of minted os token in Gwei' ' is greater than the specified value.', ) @click.option( @@ -164,7 +164,7 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: # filter boosted positions logger.info('Fetching boosted positions from the subgraph...') boosted_positions = await get_boosted_positions( - users=[a.address for a in allocators], + users={a.address for a in allocators}, leverage_positions=leverage_positions, block_number=block_number, ) @@ -174,7 +174,9 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: for vault_address, boosted_amount in boosted_positions[allocator.address].items(): for vault_share in allocator.vault_shares: if vault_share.address == vault_address: - vault_share.minted_shares = Wei(vault_share.minted_shares - boosted_amount) + vault_share.minted_shares = Wei( + max(0, vault_share.minted_shares - boosted_amount) + ) # filter zero positions min_minted_shares = Web3.to_wei(min_os_token_position_amount_gwei, 'gwei') @@ -194,8 +196,9 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: logger.info('No redeemable positions to upload, exiting...') return logger.info( - 'Created %s redeemable positions. Total redeemed OsEth amount: %s', + 'Created %s redeemable positions. Total redeemed %s amount: %s', len(redeemable_positions), + settings.network_config.OS_TOKEN_BALANCE_SYMBOL, sum(p.amount for p in redeemable_positions), ) @@ -215,7 +218,9 @@ async def get_kept_tokens( arbitrum_endpoint: str | None, ) -> dict[ChecksumAddress, Wei]: kept_tokens = defaultdict(lambda: Wei(0)) - logger.info('Fetching OsETH from wallet balances...') + logger.info( + 'Fetching %s from wallet balances...', settings.network_config.OS_TOKEN_BALANCE_SYMBOL + ) contract = Erc20Contract(settings.network_config.OS_TOKEN_CONTRACT_ADDRESS) for index, address in enumerate(address_to_minted_shares.keys()): @@ -226,7 +231,10 @@ async def get_kept_tokens( kept_tokens[address] = await contract.get_balance(address, block_number) # arb wallet balance if settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS != ZERO_CHECKSUM_ADDRESS: - logger.info('Fetching OsETH from Arbitrum wallet balances...') + logger.info( + 'Fetching %s from Arbitrum wallet balances...', + settings.network_config.OS_TOKEN_BALANCE_SYMBOL, + ) arbitrum_endpoint = cast(str, arbitrum_endpoint) arb_execution_client = get_execution_client([arbitrum_endpoint]) @@ -244,28 +252,32 @@ async def get_kept_tokens( arb_balance = await arb_contract.get_balance(address) kept_tokens[address] = Wei(kept_tokens[address] + arb_balance) - # do not fetch data from api if all os token are on the wallet + # do not fetch data from api if all os token are in the wallet api_addresses = [] for address in address_to_minted_shares.keys(): - if address_to_minted_shares[address] > kept_tokens[address]: + if address_to_minted_shares[address] >= kept_tokens[address]: api_addresses.append(address) if not api_addresses: return kept_tokens - logger.info('Fetching locked OsETH from DeBank API for %s addresses...', len(api_addresses)) + logger.info( + 'Fetching locked %s from DeBank API for %s addresses...', + settings.network_config.OS_TOKEN_BALANCE_SYMBOL, + len(api_addresses), + ) api_client = APIClient() - locked_os_token_per_user: dict[ChecksumAddress, Wei] = {} + locked_os_token_per_address: dict[ChecksumAddress, Wei] = {} for address in api_addresses: locked_os_token = await api_client.get_protocols_locked_os_token(address=address) - locked_os_token_per_user[address] = locked_os_token + locked_os_token_per_address[address] = locked_os_token kept_tokens[address] = Wei(kept_tokens[address] + locked_os_token) await asyncio.sleep(API_SLEEP_TIMEOUT) # to avoid rate limiting return kept_tokens async def get_boosted_positions( - users: list[ChecksumAddress], + users: set[ChecksumAddress], leverage_positions: list[LeverageStrategyPosition], block_number: BlockNumber, ) -> dict[ChecksumAddress, dict[ChecksumAddress, Wei]]: @@ -294,11 +306,11 @@ def create_redeemable_positions( if redeemable_amount <= 0: continue - filled = 0 + allocated_amount = 0 for index, (vault_address, proportion) in enumerate(allocator.vaults_proportions.items()): # dust handling if index == len(allocator.vaults_proportions) - 1: - vault_amount = int(redeemable_amount - filled) + vault_amount = int(redeemable_amount - allocated_amount) else: vault_amount = int(redeemable_amount * proportion) @@ -309,6 +321,6 @@ def create_redeemable_positions( amount=Wei(vault_amount), ) ) - filled = filled + vault_amount + allocated_amount += vault_amount return redeemable_positions diff --git a/src/config/networks.py b/src/config/networks.py index ed582e17..c012b0d0 100644 --- a/src/config/networks.py +++ b/src/config/networks.py @@ -22,6 +22,7 @@ class NetworkConfig(BaseNetworkConfig): WALLET_BALANCE_SYMBOL: str VAULT_BALANCE_SYMBOL: str + OS_TOKEN_BALANCE_SYMBOL: str DEPOSIT_DATA_REGISTRY_CONTRACT_ADDRESS: ChecksumAddress VALIDATORS_CHECKER_CONTRACT_ADDRESS: ChecksumAddress CONSOLIDATION_CONTRACT_ADDRESS: ChecksumAddress @@ -82,6 +83,7 @@ def INITIAL_SYNC_ETA(self) -> int: **asdict(BASE_NETWORKS[MAINNET]), WALLET_BALANCE_SYMBOL='ETH', VAULT_BALANCE_SYMBOL='ETH', + OS_TOKEN_BALANCE_SYMBOL='osETH', DEPOSIT_DATA_REGISTRY_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x75AB6DdCe07556639333d3Df1eaa684F5735223e' ), @@ -138,6 +140,7 @@ def INITIAL_SYNC_ETA(self) -> int: **asdict(BASE_NETWORKS[HOODI]), WALLET_BALANCE_SYMBOL='HoodiETH', VAULT_BALANCE_SYMBOL='HoodiETH', + OS_TOKEN_BALANCE_SYMBOL='osETH', DEPOSIT_DATA_REGISTRY_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x93a3f880E07B27dacA6Ef2d3C23E77DBd6294487' ), @@ -190,6 +193,7 @@ def INITIAL_SYNC_ETA(self) -> int: **asdict(BASE_NETWORKS[GNOSIS]), WALLET_BALANCE_SYMBOL='xDAI', VAULT_BALANCE_SYMBOL='GNO', + OS_TOKEN_BALANCE_SYMBOL='osGNO', DEPOSIT_DATA_REGISTRY_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x58e16621B5c0786D6667D2d54E28A20940269E16' ), diff --git a/src/redeem/tests/test_api_client.py b/src/redeem/tests/test_api_client.py index 790fd4ea..e167f4e9 100644 --- a/src/redeem/tests/test_api_client.py +++ b/src/redeem/tests/test_api_client.py @@ -66,13 +66,17 @@ async def test_excludes_stakewise_protocol_from_total(self): async def test_real_data(self): with open('src/redeem/tests/api_samples/protocols.json', 'r') as f: mock_protocol_data = json.load(f) - settings.network_config.OS_TOKEN_CONTRACT_ADDRESS = ( - '0xf1C9acDc66974dFB6dEcB12aA385b9cD01190E38' - ) - settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS = ( - '0xf7d4e7273E5015C96728A6b02f31C505eE184603' - ) - with patch('src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data): + with patch( + 'src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data + ), patch.object( + settings.network_config, + 'OS_TOKEN_CONTRACT_ADDRESS', + '0xf1C9acDc66974dFB6dEcB12aA385b9cD01190E38', + ), patch.object( + settings.network_config, + 'OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS', + '0xf7d4e7273E5015C96728A6b02f31C505eE184603', + ): client = APIClient() result = await client.get_protocols_locked_os_token( Web3.to_checksum_address('0x1234567890abcdef1234567890abcdef12345678') @@ -83,13 +87,17 @@ async def test_real_data(self): async def test_real_data_with_boost(self): with open('src/redeem/tests/api_samples/with_boost.json', 'r') as f: mock_protocol_data = json.load(f) - settings.network_config.OS_TOKEN_CONTRACT_ADDRESS = ( - '0xf1C9acDc66974dFB6dEcB12aA385b9cD01190E38' - ) - settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS = ( - '0xf7d4e7273E5015C96728A6b02f31C505eE184603' - ) - with patch('src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data): + with patch( + 'src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data + ), patch.object( + settings.network_config, + 'OS_TOKEN_CONTRACT_ADDRESS', + '0xf1C9acDc66974dFB6dEcB12aA385b9cD01190E38', + ), patch.object( + settings.network_config, + 'OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS', + '0xf7d4e7273E5015C96728A6b02f31C505eE184603', + ): client = APIClient() result = await client.get_protocols_locked_os_token( Web3.to_checksum_address('0x1234567890abcdef1234567890abcdef12345678') From 89bd7fa0a4f5c9a6a680e59bc76ab42ca4dc6fa0 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 19 Jan 2026 18:14:49 +0300 Subject: [PATCH 16/65] Review fixes #3 Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 54 ++- src/common/abi/IOsTokenVaultController.json | 416 ++++++++++++++++++ src/common/contracts.py | 17 +- src/redeem/graph.py | 2 + src/redeem/typings.py | 4 + 5 files changed, 468 insertions(+), 25 deletions(-) create mode 100644 src/common/abi/IOsTokenVaultController.json diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 5982617d..f307657b 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -16,7 +16,7 @@ get_execution_client, setup_clients, ) -from src.common.contracts import Erc20Contract, VaultContract +from src.common.contracts import Erc20Contract, os_token_vault_controller_contract from src.common.logging import LOG_LEVELS, setup_logging from src.common.utils import log_verbose from src.config.networks import AVAILABLE_NETWORKS, MAINNET, ZERO_CHECKSUM_ADDRESS @@ -163,15 +163,15 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: # filter boosted positions logger.info('Fetching boosted positions from the subgraph...') - boosted_positions = await get_boosted_positions( + boost_ostoken_shares = await get_boost_ostoken_shares( users={a.address for a in allocators}, leverage_positions=leverage_positions, block_number=block_number, ) for allocator in allocators: - if allocator.address not in boosted_positions: + if allocator.address not in boost_ostoken_shares: continue - for vault_address, boosted_amount in boosted_positions[allocator.address].items(): + for vault_address, boosted_amount in boost_ostoken_shares[allocator.address].items(): for vault_share in allocator.vault_shares: if vault_share.address == vault_address: vault_share.minted_shares = Wei( @@ -188,10 +188,10 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: logger.info('Fetching kept tokens for %s addresses', len(allocators)) address_to_minted_shares = {a.address: a.total_shares for a in allocators} - kept_tokens = await get_kept_tokens(address_to_minted_shares, block_number, arbitrum_endpoint) + kept_shares = await get_kept_shares(address_to_minted_shares, block_number, arbitrum_endpoint) logger.info('Fetched kept tokens for %s addresses...', len(address_to_minted_shares)) - redeemable_positions = create_redeemable_positions(allocators, kept_tokens) + redeemable_positions = create_redeemable_positions(allocators, kept_shares) if not redeemable_positions: logger.info('No redeemable positions to upload, exiting...') return @@ -212,12 +212,12 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: click.echo(f'Redeemable position uploaded to IPFS: hash={ipfs_hash}') -async def get_kept_tokens( +async def get_kept_shares( address_to_minted_shares: dict[ChecksumAddress, Wei], block_number: BlockNumber, arbitrum_endpoint: str | None, ) -> dict[ChecksumAddress, Wei]: - kept_tokens = defaultdict(lambda: Wei(0)) + kept_shares = defaultdict(lambda: Wei(0)) logger.info( 'Fetching %s from wallet balances...', settings.network_config.OS_TOKEN_BALANCE_SYMBOL ) @@ -228,7 +228,7 @@ async def get_kept_tokens( logger.info( 'Fetched wallet balances for %d/%d addresses', index, len(address_to_minted_shares) ) - kept_tokens[address] = await contract.get_balance(address, block_number) + kept_shares[address] = await contract.get_balance(address, block_number) # arb wallet balance if settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS != ZERO_CHECKSUM_ADDRESS: logger.info( @@ -250,16 +250,16 @@ async def get_kept_tokens( len(address_to_minted_shares), ) arb_balance = await arb_contract.get_balance(address) - kept_tokens[address] = Wei(kept_tokens[address] + arb_balance) + kept_shares[address] = Wei(kept_shares[address] + arb_balance) # do not fetch data from api if all os token are in the wallet api_addresses = [] for address in address_to_minted_shares.keys(): - if address_to_minted_shares[address] >= kept_tokens[address]: + if address_to_minted_shares[address] >= kept_shares[address]: api_addresses.append(address) if not api_addresses: - return kept_tokens + return kept_shares logger.info( 'Fetching locked %s from DeBank API for %s addresses...', @@ -271,38 +271,52 @@ async def get_kept_tokens( for address in api_addresses: locked_os_token = await api_client.get_protocols_locked_os_token(address=address) locked_os_token_per_address[address] = locked_os_token - kept_tokens[address] = Wei(kept_tokens[address] + locked_os_token) + kept_shares[address] = Wei(kept_shares[address] + locked_os_token) await asyncio.sleep(API_SLEEP_TIMEOUT) # to avoid rate limiting - return kept_tokens + return kept_shares -async def get_boosted_positions( +async def get_boost_ostoken_shares( users: set[ChecksumAddress], leverage_positions: list[LeverageStrategyPosition], block_number: BlockNumber, ) -> dict[ChecksumAddress, dict[ChecksumAddress, Wei]]: boosted_positions: defaultdict[ChecksumAddress, dict[ChecksumAddress, Wei]] = defaultdict(dict) + if not leverage_positions: + return boosted_positions + + total_shares = await os_token_vault_controller_contract.total_shares(block_number) + total_assets = await os_token_vault_controller_contract.total_assets(block_number) for position in leverage_positions: if position.user not in users: continue - vault_contract = VaultContract(position.vault) position_os_token_shares = Wei( position.os_token_shares - + await vault_contract.convert_to_shares(position.assets, block_number) + + position.exiting_os_token_shares + + _assets_to_shares( + assets=position.assets, total_shares=total_shares, total_assets=total_assets + ) + + _assets_to_shares( + assets=position.exiting_assets, total_shares=total_shares, total_assets=total_assets + ) ) boosted_positions[position.user][position.vault] = position_os_token_shares return boosted_positions +def _assets_to_shares(assets: Wei, total_shares: Wei, total_assets: Wei) -> Wei: + return Wei(assets * total_shares // total_assets) + + def create_redeemable_positions( - allocators: list[Allocator], kept_tokens: dict[ChecksumAddress, Wei] + allocators: list[Allocator], kept_shares: dict[ChecksumAddress, Wei] ) -> list[RedeemablePosition]: """Calculate vault proportions and create redeemable positions""" redeemable_positions: list[RedeemablePosition] = [] for allocator in allocators: - kept_token = kept_tokens.get(allocator.address, Wei(0)) - redeemable_amount = max(0, allocator.total_shares - kept_token) + allocator_kept_shares = kept_shares.get(allocator.address, Wei(0)) + redeemable_amount = max(0, allocator.total_shares - allocator_kept_shares) if redeemable_amount <= 0: continue diff --git a/src/common/abi/IOsTokenVaultController.json b/src/common/abi/IOsTokenVaultController.json new file mode 100644 index 00000000..dddfcb2c --- /dev/null +++ b/src/common/abi/IOsTokenVaultController.json @@ -0,0 +1,416 @@ +[ + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint256", + "name": "avgRewardPerSecond", + "type": "uint256" + } + ], + "name": "AvgRewardPerSecondUpdated", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "vault", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "owner", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "assets", + "type": "uint256" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "name": "Burn", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint256", + "name": "capacity", + "type": "uint256" + } + ], + "name": "CapacityUpdated", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint16", + "name": "feePercent", + "type": "uint16" + } + ], + "name": "FeePercentUpdated", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "address", + "name": "keeper", + "type": "address" + } + ], + "name": "KeeperUpdated", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "vault", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "receiver", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "assets", + "type": "uint256" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "name": "Mint", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint256", + "name": "profitAccrued", + "type": "uint256" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "treasuryShares", + "type": "uint256" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "treasuryAssets", + "type": "uint256" + } + ], + "name": "StateUpdated", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "treasury", + "type": "address" + } + ], + "name": "TreasuryUpdated", + "type": "event" + }, + { + "inputs": [], + "name": "avgRewardPerSecond", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "owner", + "type": "address" + }, + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "name": "burnShares", + "outputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "capacity", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "name": "convertToAssets", + "outputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "name": "convertToShares", + "outputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "cumulativeFeePerShare", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "feePercent", + "outputs": [ + { + "internalType": "uint64", + "name": "", + "type": "uint64" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "keeper", + "outputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "receiver", + "type": "address" + }, + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "name": "mintShares", + "outputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_avgRewardPerSecond", + "type": "uint256" + } + ], + "name": "setAvgRewardPerSecond", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_capacity", + "type": "uint256" + } + ], + "name": "setCapacity", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint16", + "name": "_feePercent", + "type": "uint16" + } + ], + "name": "setFeePercent", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "_keeper", + "type": "address" + } + ], + "name": "setKeeper", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "_treasury", + "type": "address" + } + ], + "name": "setTreasury", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "totalAssets", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "totalShares", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "treasury", + "outputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "updateState", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + } +] diff --git a/src/common/contracts.py b/src/common/contracts.py index 049518c4..926bd639 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -219,11 +219,6 @@ async def validators_manager(self) -> ChecksumAddress: async def get_exit_queue_index(self, position_ticket: int) -> int: return await self.contract.functions.getExitQueueIndex(position_ticket).call() - async def convert_to_shares(self, assets: Wei, block_number: BlockNumber | None = None) -> Wei: - return await self.contract.functions.convertToShares(assets).call( - block_identifier=block_number - ) - async def get_validator_withdrawal_submitted_events( self, from_block: BlockNumber, @@ -348,6 +343,17 @@ async def can_harvest( ) +class OsTokenVaultControllerContract(ContractWrapper): + abi_path = 'abi/IOsTokenVaultController.json' + settings_key = 'OS_TOKEN_VAULT_CONTROLLER_CONTRACT_ADDRESS' + + async def total_assets(self, block_number: BlockNumber | None = None) -> Wei: + return await self.contract.functions.totalAssets().call(block_identifier=block_number) + + async def total_shares(self, block_number: BlockNumber | None = None) -> Wei: + return await self.contract.functions.totalShares().call(block_identifier=block_number) + + class RewardSplitterContract(ContractWrapper): abi_path = 'abi/IRewardSplitter.json' @@ -563,3 +569,4 @@ def _get_exit_queue_missing_assets_call(self, params: ExitQueueMissingAssetsPara keeper_contract = KeeperContract() multicall_contract = MulticallContract() validators_checker_contract = ValidatorsCheckerContract() +os_token_vault_controller_contract = OsTokenVaultControllerContract() diff --git a/src/redeem/graph.py b/src/redeem/graph.py index 7d92f7b9..4b4bedb1 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -81,7 +81,9 @@ async def graph_get_leverage_positions(block_number: BlockNumber) -> list[Levera id } osTokenShares + exitingOsTokenShares assets + exitingAssets } } """ diff --git a/src/redeem/typings.py b/src/redeem/typings.py index 0cfaad3b..b9e6dd4f 100644 --- a/src/redeem/typings.py +++ b/src/redeem/typings.py @@ -35,7 +35,9 @@ class LeverageStrategyPosition: vault: ChecksumAddress proxy: ChecksumAddress os_token_shares: Wei + exiting_os_token_shares: Wei assets: Wei + exiting_assets: Wei @classmethod def from_graph(cls, data: dict) -> 'LeverageStrategyPosition': @@ -44,7 +46,9 @@ def from_graph(cls, data: dict) -> 'LeverageStrategyPosition': vault=Web3.to_checksum_address(data['vault']['id']), proxy=Web3.to_checksum_address(data['proxy']), os_token_shares=Wei(int(data['osTokenShares'])), + exiting_os_token_shares=Wei(int(data['exitingOsTokenShares'])), assets=Wei(int(data['assets'])), + exiting_assets=Wei(int(data['exitingAssets'])), ) From cb7308d94142718fe4398d5ae4e40c4e3f16045f Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 19 Jan 2026 22:51:29 +0300 Subject: [PATCH 17/65] Add merkle root Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 16 +++++++++++++--- src/redeem/typings.py | 6 +++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index f307657b..376ee000 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -7,6 +7,7 @@ import click from eth_typing import BlockNumber, ChecksumAddress +from multiproof import StandardMerkleTree from web3 import Web3 from web3.types import Gwei, Wei @@ -146,7 +147,7 @@ def update_redeemable_positions( # pylint: disable-next=too-many-locals async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: Gwei) -> None: """ - Fetch redeemable positions, calculate kept osToken amounts and upload to IPFS. + Fetch redeemable positions, calculate kept os token amounts and upload to IPFS. """ setup_logging() await setup_clients() @@ -195,11 +196,15 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: if not redeemable_positions: logger.info('No redeemable positions to upload, exiting...') return + + total_redeemable = sum(p.amount for p in redeemable_positions) logger.info( - 'Created %s redeemable positions. Total redeemed %s amount: %s', + 'Created %s redeemable positions. Total redeemed %s amount: %s (%s %s)', len(redeemable_positions), settings.network_config.OS_TOKEN_BALANCE_SYMBOL, - sum(p.amount for p in redeemable_positions), + total_redeemable, + round(Web3.from_wei(total_redeemable, 'ether'), 5), + settings.network_config.WALLET_BALANCE_SYMBOL, ) click.confirm( @@ -211,6 +216,11 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: ipfs_hash = await ipfs_upload_client.upload_json([p.as_dict() for p in redeemable_positions]) click.echo(f'Redeemable position uploaded to IPFS: hash={ipfs_hash}') + # calculate merkle root + leaves = [r.merkle_leaf for r in redeemable_positions] + tree = StandardMerkleTree.of(leaves, ['address', 'address', 'uint160']) + logger.info('Generated Merkle Tree root: %s', tree.root) + async def get_kept_shares( address_to_minted_shares: dict[ChecksumAddress, Wei], diff --git a/src/redeem/typings.py b/src/redeem/typings.py index b9e6dd4f..0eb3a339 100644 --- a/src/redeem/typings.py +++ b/src/redeem/typings.py @@ -54,9 +54,13 @@ def from_graph(cls, data: dict) -> 'LeverageStrategyPosition': @dataclass class RedeemablePosition: - owner: ChecksumAddress # noqa + owner: ChecksumAddress vault: ChecksumAddress amount: Wei def as_dict(self) -> dict: return dataclasses.asdict(self) + + @property + def merkle_leaf(self) -> tuple[ChecksumAddress, ChecksumAddress, Wei]: + return self.owner, self.vault, self.amount From 625c0bafaea496219a3c5fe990277796f1c7c4aa Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 19 Jan 2026 22:58:58 +0300 Subject: [PATCH 18/65] Add progress bar Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 376ee000..95a58faf 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -278,11 +278,18 @@ async def get_kept_shares( ) api_client = APIClient() locked_os_token_per_address: dict[ChecksumAddress, Wei] = {} - for address in api_addresses: - locked_os_token = await api_client.get_protocols_locked_os_token(address=address) - locked_os_token_per_address[address] = locked_os_token - kept_shares[address] = Wei(kept_shares[address] + locked_os_token) - await asyncio.sleep(API_SLEEP_TIMEOUT) # to avoid rate limiting + # fetch locked os token from the api + with click.progressbar( + api_addresses, + label='Fetching os token amount locked in protocols from the api:\t\t', + show_percent=False, + show_pos=True, + ) as progress_bar: + for address in progress_bar: + locked_os_token = await api_client.get_protocols_locked_os_token(address=address) + locked_os_token_per_address[address] = locked_os_token + kept_shares[address] = Wei(kept_shares[address] + locked_os_token) + await asyncio.sleep(API_SLEEP_TIMEOUT) # to avoid rate limiting return kept_shares From c4558806ef26c6abafc8323395a7808dd112db8b Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 20 Jan 2026 11:51:45 +0300 Subject: [PATCH 19/65] Simplify boosted filters Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 47 ++++++++++++------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 95a58faf..73be29b7 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -162,26 +162,23 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: logger.info('Found %s proxy positions to exclude', len(boost_proxies)) allocators = [a for a in allocators if a.address not in boost_proxies] - # filter boosted positions + # reduce boosted positions logger.info('Fetching boosted positions from the subgraph...') boost_ostoken_shares = await get_boost_ostoken_shares( users={a.address for a in allocators}, leverage_positions=leverage_positions, block_number=block_number, ) - for allocator in allocators: - if allocator.address not in boost_ostoken_shares: - continue - for vault_address, boosted_amount in boost_ostoken_shares[allocator.address].items(): - for vault_share in allocator.vault_shares: - if vault_share.address == vault_address: - vault_share.minted_shares = Wei( - max(0, vault_share.minted_shares - boosted_amount) - ) + allocators = _reduce_boosted_amount(allocators, boost_ostoken_shares) # filter zero positions min_minted_shares = Web3.to_wei(min_os_token_position_amount_gwei, 'gwei') - allocators = [a for a in allocators if a.total_shares >= min_minted_shares] + for allocator in allocators: + allocator.vault_shares = [ + vault_share + for vault_share in allocator.vault_shares + if vault_share.minted_shares >= min_minted_shares + ] if not allocators: logger.info('No allocators with minted shares above the threshold found, exiting...') @@ -322,10 +319,6 @@ async def get_boost_ostoken_shares( return boosted_positions -def _assets_to_shares(assets: Wei, total_shares: Wei, total_assets: Wei) -> Wei: - return Wei(assets * total_shares // total_assets) - - def create_redeemable_positions( allocators: list[Allocator], kept_shares: dict[ChecksumAddress, Wei] ) -> list[RedeemablePosition]: @@ -338,9 +331,10 @@ def create_redeemable_positions( continue allocated_amount = 0 - for index, (vault_address, proportion) in enumerate(allocator.vaults_proportions.items()): + vaults_proportions = allocator.vaults_proportions.items() + for index, (vault_address, proportion) in enumerate(vaults_proportions): # dust handling - if index == len(allocator.vaults_proportions) - 1: + if index == len(vaults_proportions) - 1: vault_amount = int(redeemable_amount - allocated_amount) else: vault_amount = int(redeemable_amount * proportion) @@ -355,3 +349,22 @@ def create_redeemable_positions( allocated_amount += vault_amount return redeemable_positions + + +def _reduce_boosted_amount( + allocators: list[Allocator], + boost_ostoken_shares: dict[ChecksumAddress, dict[ChecksumAddress, Wei]], +) -> list[Allocator]: + for allocator in allocators: + if allocator.address not in boost_ostoken_shares: + continue + for vault_share in allocator.vault_shares: + boosted_amount = boost_ostoken_shares[allocator.address].get( + vault_share.address, Wei(0) + ) + vault_share.minted_shares = Wei(max(0, vault_share.minted_shares - boosted_amount)) + return allocators + + +def _assets_to_shares(assets: Wei, total_shares: Wei, total_assets: Wei) -> Wei: + return Wei(assets * total_shares // total_assets) From 1e861b2f5c9a58221a001faf2351d6d7857039bb Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 20 Jan 2026 12:40:27 +0300 Subject: [PATCH 20/65] Fetch oseth from graph Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 18 ++++++------ src/redeem/graph.py | 28 ++++++++++++++++++- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 73be29b7..027e8293 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -23,7 +23,11 @@ from src.config.networks import AVAILABLE_NETWORKS, MAINNET, ZERO_CHECKSUM_ADDRESS from src.config.settings import settings from src.redeem.api_client import API_SLEEP_TIMEOUT, APIClient -from src.redeem.graph import graph_get_allocators, graph_get_leverage_positions +from src.redeem.graph import ( + graph_get_allocators, + graph_get_leverage_positions, + graph_get_os_token_holders, +) from src.redeem.typings import Allocator, LeverageStrategyPosition, RedeemablePosition logger = logging.getLogger(__name__) @@ -226,16 +230,12 @@ async def get_kept_shares( ) -> dict[ChecksumAddress, Wei]: kept_shares = defaultdict(lambda: Wei(0)) logger.info( - 'Fetching %s from wallet balances...', settings.network_config.OS_TOKEN_BALANCE_SYMBOL + 'Fetching %s balances from the subgraph...', settings.network_config.OS_TOKEN_BALANCE_SYMBOL ) + os_token_holders = await graph_get_os_token_holders(block_number) + for address in address_to_minted_shares.keys(): + kept_shares[address] = os_token_holders.get(address, Wei(0)) - contract = Erc20Contract(settings.network_config.OS_TOKEN_CONTRACT_ADDRESS) - for index, address in enumerate(address_to_minted_shares.keys()): - if index and index % 50 == 0: - logger.info( - 'Fetched wallet balances for %d/%d addresses', index, len(address_to_minted_shares) - ) - kept_shares[address] = await contract.get_balance(address, block_number) # arb wallet balance if settings.network_config.OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS != ZERO_CHECKSUM_ADDRESS: logger.info( diff --git a/src/redeem/graph.py b/src/redeem/graph.py index 4b4bedb1..40f3437e 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -4,7 +4,7 @@ from eth_typing import BlockNumber from gql import gql from web3 import Web3 -from web3.types import Wei +from web3.types import ChecksumAddress, Wei from src.common.clients import graph_client from src.redeem.typings import Allocator, LeverageStrategyPosition, VaultShares @@ -91,3 +91,29 @@ async def graph_get_leverage_positions(block_number: BlockNumber) -> list[Levera params = {'block': block_number} response = await graph_client.fetch_pages(query, params=params) return [LeverageStrategyPosition.from_graph(item) for item in response] + + +async def graph_get_os_token_holders(block_number: BlockNumber) -> dict[ChecksumAddress, Wei]: + query = gql( + """ + query osTokenHoldersQuery($block: Int, $first: Int, $skip: Int) { + osTokenHolders( + block: { number: $block }, + where:{ + balance_gt: 0 + } + orderBy: balance, + first: $first + skip: $skip + ) { + id + balance + } + } + """ + ) + params = {'block': block_number} + response = await graph_client.fetch_pages(query, params=params) + return { + Web3.to_checksum_address(item['id']): Wei(int(item['balance'])) for item in response + } From 06e8f81a06826f49bc823acc9d127a6b209b72cb Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 20 Jan 2026 12:58:11 +0300 Subject: [PATCH 21/65] Add OsTokenConverter Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 18 ++++--------- src/redeem/graph.py | 4 +-- src/redeem/os_token_converter.py | 25 +++++++++++++++++++ 3 files changed, 31 insertions(+), 16 deletions(-) create mode 100644 src/redeem/os_token_converter.py diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 027e8293..9bb0aa7c 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -17,7 +17,7 @@ get_execution_client, setup_clients, ) -from src.common.contracts import Erc20Contract, os_token_vault_controller_contract +from src.common.contracts import Erc20Contract from src.common.logging import LOG_LEVELS, setup_logging from src.common.utils import log_verbose from src.config.networks import AVAILABLE_NETWORKS, MAINNET, ZERO_CHECKSUM_ADDRESS @@ -28,6 +28,7 @@ graph_get_leverage_positions, graph_get_os_token_holders, ) +from src.redeem.os_token_converter import create_os_token_converter from src.redeem.typings import Allocator, LeverageStrategyPosition, RedeemablePosition logger = logging.getLogger(__name__) @@ -299,20 +300,15 @@ async def get_boost_ostoken_shares( if not leverage_positions: return boosted_positions - total_shares = await os_token_vault_controller_contract.total_shares(block_number) - total_assets = await os_token_vault_controller_contract.total_assets(block_number) + os_token_converter = await create_os_token_converter(block_number) for position in leverage_positions: if position.user not in users: continue position_os_token_shares = Wei( position.os_token_shares + position.exiting_os_token_shares - + _assets_to_shares( - assets=position.assets, total_shares=total_shares, total_assets=total_assets - ) - + _assets_to_shares( - assets=position.exiting_assets, total_shares=total_shares, total_assets=total_assets - ) + + os_token_converter.to_shares(position.assets) + + os_token_converter.to_shares(position.exiting_assets) ) boosted_positions[position.user][position.vault] = position_os_token_shares @@ -364,7 +360,3 @@ def _reduce_boosted_amount( ) vault_share.minted_shares = Wei(max(0, vault_share.minted_shares - boosted_amount)) return allocators - - -def _assets_to_shares(assets: Wei, total_shares: Wei, total_assets: Wei) -> Wei: - return Wei(assets * total_shares // total_assets) diff --git a/src/redeem/graph.py b/src/redeem/graph.py index 40f3437e..9e3929b5 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -114,6 +114,4 @@ async def graph_get_os_token_holders(block_number: BlockNumber) -> dict[Checksum ) params = {'block': block_number} response = await graph_client.fetch_pages(query, params=params) - return { - Web3.to_checksum_address(item['id']): Wei(int(item['balance'])) for item in response - } + return {Web3.to_checksum_address(item['id']): Wei(int(item['balance'])) for item in response} diff --git a/src/redeem/os_token_converter.py b/src/redeem/os_token_converter.py new file mode 100644 index 00000000..a669ea7a --- /dev/null +++ b/src/redeem/os_token_converter.py @@ -0,0 +1,25 @@ +from web3.types import BlockNumber, Wei + +from src.common.contracts import os_token_vault_controller_contract + + +class OsTokenConverter: + """ + Convert between shares and assets based on total assets and total shares. + Helps to avoid repeating calls to the contract. + """ + + def __init__(self, total_assets: Wei, total_shares: Wei): + self.total_assets = total_assets + self.total_shares = total_shares + + def to_shares(self, assets: Wei) -> Wei: + if self.total_assets == 0: + return Wei(0) + return Wei((assets * self.total_shares) // self.total_assets) + + +async def create_os_token_converter(block_number: BlockNumber) -> OsTokenConverter: + total_assets = await os_token_vault_controller_contract.total_assets(block_number) + total_shares = await os_token_vault_controller_contract.total_shares(block_number) + return OsTokenConverter(total_assets=total_assets, total_shares=total_shares) From 32546bb9a7ea2e04c1c757e2427d806bef1742db Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 20 Jan 2026 15:33:57 +0300 Subject: [PATCH 22/65] Add tests Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 18 ++- .../test_update_redeemable_positions.py | 137 +++++++++++++++++- 2 files changed, 147 insertions(+), 8 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 9bb0aa7c..d58e687f 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -28,7 +28,7 @@ graph_get_leverage_positions, graph_get_os_token_holders, ) -from src.redeem.os_token_converter import create_os_token_converter +from src.redeem.os_token_converter import OsTokenConverter, create_os_token_converter from src.redeem.typings import Allocator, LeverageStrategyPosition, RedeemablePosition logger = logging.getLogger(__name__) @@ -169,10 +169,11 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: # reduce boosted positions logger.info('Fetching boosted positions from the subgraph...') - boost_ostoken_shares = await get_boost_ostoken_shares( + os_token_converter = await create_os_token_converter(block_number) + boost_ostoken_shares = await calculate_boost_ostoken_shares( users={a.address for a in allocators}, leverage_positions=leverage_positions, - block_number=block_number, + os_token_converter=os_token_converter, ) allocators = _reduce_boosted_amount(allocators, boost_ostoken_shares) @@ -291,16 +292,15 @@ async def get_kept_shares( return kept_shares -async def get_boost_ostoken_shares( +async def calculate_boost_ostoken_shares( users: set[ChecksumAddress], leverage_positions: list[LeverageStrategyPosition], - block_number: BlockNumber, + os_token_converter: OsTokenConverter, ) -> dict[ChecksumAddress, dict[ChecksumAddress, Wei]]: boosted_positions: defaultdict[ChecksumAddress, dict[ChecksumAddress, Wei]] = defaultdict(dict) if not leverage_positions: return boosted_positions - os_token_converter = await create_os_token_converter(block_number) for position in leverage_positions: if position.user not in users: continue @@ -310,7 +310,11 @@ async def get_boost_ostoken_shares( + os_token_converter.to_shares(position.assets) + os_token_converter.to_shares(position.exiting_assets) ) - boosted_positions[position.user][position.vault] = position_os_token_shares + if position.vault not in boosted_positions[position.user]: + boosted_positions[position.user][position.vault] = Wei(0) + boosted_positions[position.user][position.vault] = Wei( + boosted_positions[position.user][position.vault] + position_os_token_shares + ) return boosted_positions diff --git a/src/commands/tests/test_internal/test_update_redeemable_positions.py b/src/commands/tests/test_internal/test_update_redeemable_positions.py index 74d7579d..6a6235f8 100644 --- a/src/commands/tests/test_internal/test_update_redeemable_positions.py +++ b/src/commands/tests/test_internal/test_update_redeemable_positions.py @@ -3,9 +3,17 @@ from web3.types import Wei from src.commands.internal.update_redeemable_positions import ( + _reduce_boosted_amount, + calculate_boost_ostoken_shares, create_redeemable_positions, ) -from src.redeem.typings import Allocator, RedeemablePosition, VaultShares +from src.redeem.os_token_converter import OsTokenConverter +from src.redeem.typings import ( + Allocator, + LeverageStrategyPosition, + RedeemablePosition, + VaultShares, +) def test_create_redeemable_positions(): @@ -86,3 +94,130 @@ def test_create_redeemable_positions(): RedeemablePosition(owner=address_1, vault=vault_1, amount=Wei(150)), RedeemablePosition(owner=address_1, vault=vault_2, amount=Wei(150)), ] + + +async def test_calculate_boost_ostoken_shares(): + address_1 = faker.eth_address() + address_2 = faker.eth_address() + vault_1 = faker.eth_address() + vault_2 = faker.eth_address() + proxy = faker.eth_address() + + os_token_converter = OsTokenConverter(105, 100) + + # empty case + result = await calculate_boost_ostoken_shares(set(), [], os_token_converter) + assert result == {} + + # filter by users + leverage_positions = [ + LeverageStrategyPosition( + user=address_1, + vault=vault_1, + proxy=proxy, + os_token_shares=Wei(1000), + exiting_os_token_shares=Wei(500), + assets=Wei(200), + exiting_assets=Wei(100), + ), + LeverageStrategyPosition( + user=address_1, + vault=vault_1, + proxy=proxy, + os_token_shares=Wei(1000), + exiting_os_token_shares=Wei(500), + assets=Wei(200), + exiting_assets=Wei(100), + ), + LeverageStrategyPosition( + user=address_2, + vault=vault_1, + proxy=proxy, + os_token_shares=Wei(100), + exiting_os_token_shares=Wei(0), + assets=Wei(0), + exiting_assets=Wei(0), + ), + LeverageStrategyPosition( + user=address_2, + vault=vault_2, + proxy=proxy, + os_token_shares=Wei(3000), + exiting_os_token_shares=Wei(0), + assets=Wei(100), + exiting_assets=Wei(0), + ), + ] + result = await calculate_boost_ostoken_shares( + {address_1, address_2}, leverage_positions, os_token_converter + ) + assert result == {address_1: {vault_1: 3570}, address_2: {vault_1: 100, vault_2: 3095}} + + +def test_reduces_boosted_amount(): + address_1 = faker.eth_address() + address_2 = faker.eth_address() + vault_1 = faker.eth_address() + vault_2 = faker.eth_address() + # empty case + allocators = [ + Allocator( + address=address_1, + vault_shares=[ + VaultShares(address=vault_1, minted_shares=Wei(1000)), + VaultShares(address=vault_2, minted_shares=Wei(2000)), + ], + ) + ] + boost_ostoken_shares = {} + result = _reduce_boosted_amount(allocators, boost_ostoken_shares) + assert result == [ + Allocator( + address=address_1, + vault_shares=[ + VaultShares(address=vault_1, minted_shares=Wei(1000)), + VaultShares(address=vault_2, minted_shares=Wei(2000)), + ], + ) + ] + # basic reduction + allocators = [ + Allocator( + address=address_1, + vault_shares=[ + VaultShares(address=vault_1, minted_shares=Wei(500)), + ], + ), + Allocator( + address=address_2, + vault_shares=[ + VaultShares(address=vault_1, minted_shares=Wei(1000)), + VaultShares(address=vault_2, minted_shares=Wei(2000)), + ], + ), + ] + boost_ostoken_shares = { + address_1: { + vault_1: Wei(300), + }, + address_2: { + vault_1: Wei(500), + vault_2: Wei(1500), + }, + } + result = _reduce_boosted_amount(allocators, boost_ostoken_shares) + assert result == [ + Allocator( + address=address_1, + vault_shares=[ + VaultShares(address=vault_1, minted_shares=Wei(200)), + ], + ), + Allocator( + address=address_2, + vault_shares=[ + VaultShares(address=vault_1, minted_shares=Wei(500)), + VaultShares(address=vault_2, minted_shares=Wei(500)), + ], + ), + ] From 921dcd27384bc84c0fbcf3c098248c0d63678aca Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 21 Jan 2026 10:37:57 +0300 Subject: [PATCH 23/65] Install multiproof Signed-off-by: cyc60 --- poetry.lock | 284 ++++++++++++++++++++++++++++--------------------- pyproject.toml | 1 + 2 files changed, 161 insertions(+), 124 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8d1cda9a..e4eabe64 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1322,13 +1322,13 @@ test = ["pytest"] [[package]] name = "dill" -version = "0.4.0" +version = "0.4.1" description = "serialize all of Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049"}, - {file = "dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0"}, + {file = "dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d"}, + {file = "dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa"}, ] [package.extras] @@ -2444,6 +2444,26 @@ files = [ {file = "multidict-6.7.0.tar.gz", hash = "sha256:c6e99d9a65ca282e578dfea819cfa9c0a62b2499d8677392e09feaf305e9e6f5"}, ] +[[package]] +name = "multiproof" +version = "v0.1.10" +description = "A Python library to generate merkle trees and merkle proofs." +optional = false +python-versions = "^3.10" +files = [] +develop = false + +[package.dependencies] +eth-abi = "^5.0.1" +eth-hash = {version = "^0.7.0", extras = ["pycryptodome"]} +eth-utils = ">=5.1.0" + +[package.source] +type = "git" +url = "https://github.com/stakewise/multiproof.git" +reference = "v0.1.10" +resolved_reference = "c5378e44877c5442a486a8f4006f16961e42662b" + [[package]] name = "mypy" version = "1.14.1" @@ -3270,13 +3290,13 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2025.11" +version = "2026.0" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.8" files = [ - {file = "pyinstaller_hooks_contrib-2025.11-py3-none-any.whl", hash = "sha256:777e163e2942474aa41a8e6d31ac1635292d63422c3646c176d584d04d971c34"}, - {file = "pyinstaller_hooks_contrib-2025.11.tar.gz", hash = "sha256:dfe18632e06655fa88d218e0d768fd753e1886465c12a6d4bce04f1aaeec917d"}, + {file = "pyinstaller_hooks_contrib-2026.0-py3-none-any.whl", hash = "sha256:0590db8edeba3e6c30c8474937021f5cd39c0602b4d10f74a064c73911efaca5"}, + {file = "pyinstaller_hooks_contrib-2026.0.tar.gz", hash = "sha256:0120893de491a000845470ca9c0b39284731ac6bace26f6849dea9627aaed48e"}, ] [package.dependencies] @@ -3534,126 +3554,142 @@ files = [ [[package]] name = "regex" -version = "2025.11.3" +version = "2026.1.15" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.9" files = [ - {file = "regex-2025.11.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2b441a4ae2c8049106e8b39973bfbddfb25a179dda2bdb99b0eeb60c40a6a3af"}, - {file = "regex-2025.11.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2fa2eed3f76677777345d2f81ee89f5de2f5745910e805f7af7386a920fa7313"}, - {file = "regex-2025.11.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d8b4a27eebd684319bdf473d39f1d79eed36bf2cd34bd4465cdb4618d82b3d56"}, - {file = "regex-2025.11.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cf77eac15bd264986c4a2c63353212c095b40f3affb2bc6b4ef80c4776c1a28"}, - {file = "regex-2025.11.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b7f9ee819f94c6abfa56ec7b1dbab586f41ebbdc0a57e6524bd5e7f487a878c7"}, - {file = "regex-2025.11.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:838441333bc90b829406d4a03cb4b8bf7656231b84358628b0406d803931ef32"}, - {file = "regex-2025.11.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cfe6d3f0c9e3b7e8c0c694b24d25e677776f5ca26dce46fd6b0489f9c8339391"}, - {file = "regex-2025.11.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2ab815eb8a96379a27c3b6157fcb127c8f59c36f043c1678110cea492868f1d5"}, - {file = "regex-2025.11.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:728a9d2d173a65b62bdc380b7932dd8e74ed4295279a8fe1021204ce210803e7"}, - {file = "regex-2025.11.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:509dc827f89c15c66a0c216331260d777dd6c81e9a4e4f830e662b0bb296c313"}, - {file = "regex-2025.11.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:849202cd789e5f3cf5dcc7822c34b502181b4824a65ff20ce82da5524e45e8e9"}, - {file = "regex-2025.11.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b6f78f98741dcc89607c16b1e9426ee46ce4bf31ac5e6b0d40e81c89f3481ea5"}, - {file = "regex-2025.11.3-cp310-cp310-win32.whl", hash = "sha256:149eb0bba95231fb4f6d37c8f760ec9fa6fabf65bab555e128dde5f2475193ec"}, - {file = "regex-2025.11.3-cp310-cp310-win_amd64.whl", hash = "sha256:ee3a83ce492074c35a74cc76cf8235d49e77b757193a5365ff86e3f2f93db9fd"}, - {file = "regex-2025.11.3-cp310-cp310-win_arm64.whl", hash = "sha256:38af559ad934a7b35147716655d4a2f79fcef2d695ddfe06a06ba40ae631fa7e"}, - {file = "regex-2025.11.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eadade04221641516fa25139273505a1c19f9bf97589a05bc4cfcd8b4a618031"}, - {file = "regex-2025.11.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:feff9e54ec0dd3833d659257f5c3f5322a12eee58ffa360984b716f8b92983f4"}, - {file = "regex-2025.11.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3b30bc921d50365775c09a7ed446359e5c0179e9e2512beec4a60cbcef6ddd50"}, - {file = "regex-2025.11.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f99be08cfead2020c7ca6e396c13543baea32343b7a9a5780c462e323bd8872f"}, - {file = "regex-2025.11.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6dd329a1b61c0ee95ba95385fb0c07ea0d3fe1a21e1349fa2bec272636217118"}, - {file = "regex-2025.11.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4c5238d32f3c5269d9e87be0cf096437b7622b6920f5eac4fd202468aaeb34d2"}, - {file = "regex-2025.11.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10483eefbfb0adb18ee9474498c9a32fcf4e594fbca0543bb94c48bac6183e2e"}, - {file = "regex-2025.11.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:78c2d02bb6e1da0720eedc0bad578049cad3f71050ef8cd065ecc87691bed2b0"}, - {file = "regex-2025.11.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e6b49cd2aad93a1790ce9cffb18964f6d3a4b0b3dbdbd5de094b65296fce6e58"}, - {file = "regex-2025.11.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:885b26aa3ee56433b630502dc3d36ba78d186a00cc535d3806e6bfd9ed3c70ab"}, - {file = "regex-2025.11.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ddd76a9f58e6a00f8772e72cff8ebcff78e022be95edf018766707c730593e1e"}, - {file = "regex-2025.11.3-cp311-cp311-win32.whl", hash = "sha256:3e816cc9aac1cd3cc9a4ec4d860f06d40f994b5c7b4d03b93345f44e08cc68bf"}, - {file = "regex-2025.11.3-cp311-cp311-win_amd64.whl", hash = "sha256:087511f5c8b7dfbe3a03f5d5ad0c2a33861b1fc387f21f6f60825a44865a385a"}, - {file = "regex-2025.11.3-cp311-cp311-win_arm64.whl", hash = "sha256:1ff0d190c7f68ae7769cd0313fe45820ba07ffebfddfaa89cc1eb70827ba0ddc"}, - {file = "regex-2025.11.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bc8ab71e2e31b16e40868a40a69007bc305e1109bd4658eb6cad007e0bf67c41"}, - {file = "regex-2025.11.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:22b29dda7e1f7062a52359fca6e58e548e28c6686f205e780b02ad8ef710de36"}, - {file = "regex-2025.11.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3a91e4a29938bc1a082cc28fdea44be420bf2bebe2665343029723892eb073e1"}, - {file = "regex-2025.11.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b884f4226602ad40c5d55f52bf91a9df30f513864e0054bad40c0e9cf1afb7"}, - {file = "regex-2025.11.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3e0b11b2b2433d1c39c7c7a30e3f3d0aeeea44c2a8d0bae28f6b95f639927a69"}, - {file = "regex-2025.11.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87eb52a81ef58c7ba4d45c3ca74e12aa4b4e77816f72ca25258a85b3ea96cb48"}, - {file = "regex-2025.11.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a12ab1f5c29b4e93db518f5e3872116b7e9b1646c9f9f426f777b50d44a09e8c"}, - {file = "regex-2025.11.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7521684c8c7c4f6e88e35ec89680ee1aa8358d3f09d27dfbdf62c446f5d4c695"}, - {file = "regex-2025.11.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7fe6e5440584e94cc4b3f5f4d98a25e29ca12dccf8873679a635638349831b98"}, - {file = "regex-2025.11.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:8e026094aa12b43f4fd74576714e987803a315c76edb6b098b9809db5de58f74"}, - {file = "regex-2025.11.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:435bbad13e57eb5606a68443af62bed3556de2f46deb9f7d4237bc2f1c9fb3a0"}, - {file = "regex-2025.11.3-cp312-cp312-win32.whl", hash = "sha256:3839967cf4dc4b985e1570fd8d91078f0c519f30491c60f9ac42a8db039be204"}, - {file = "regex-2025.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:e721d1b46e25c481dc5ded6f4b3f66c897c58d2e8cfdf77bbced84339108b0b9"}, - {file = "regex-2025.11.3-cp312-cp312-win_arm64.whl", hash = "sha256:64350685ff08b1d3a6fff33f45a9ca183dc1d58bbfe4981604e70ec9801bbc26"}, - {file = "regex-2025.11.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c1e448051717a334891f2b9a620fe36776ebf3dd8ec46a0b877c8ae69575feb4"}, - {file = "regex-2025.11.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9b5aca4d5dfd7fbfbfbdaf44850fcc7709a01146a797536a8f84952e940cca76"}, - {file = "regex-2025.11.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:04d2765516395cf7dda331a244a3282c0f5ae96075f728629287dfa6f76ba70a"}, - {file = "regex-2025.11.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d9903ca42bfeec4cebedba8022a7c97ad2aab22e09573ce9976ba01b65e4361"}, - {file = "regex-2025.11.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:639431bdc89d6429f6721625e8129413980ccd62e9d3f496be618a41d205f160"}, - {file = "regex-2025.11.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f117efad42068f9715677c8523ed2be1518116d1c49b1dd17987716695181efe"}, - {file = "regex-2025.11.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4aecb6f461316adf9f1f0f6a4a1a3d79e045f9b71ec76055a791affa3b285850"}, - {file = "regex-2025.11.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3b3a5f320136873cc5561098dfab677eea139521cb9a9e8db98b7e64aef44cbc"}, - {file = "regex-2025.11.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:75fa6f0056e7efb1f42a1c34e58be24072cb9e61a601340cc1196ae92326a4f9"}, - {file = "regex-2025.11.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:dbe6095001465294f13f1adcd3311e50dd84e5a71525f20a10bd16689c61ce0b"}, - {file = "regex-2025.11.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:454d9b4ae7881afbc25015b8627c16d88a597479b9dea82b8c6e7e2e07240dc7"}, - {file = "regex-2025.11.3-cp313-cp313-win32.whl", hash = "sha256:28ba4d69171fc6e9896337d4fc63a43660002b7da53fc15ac992abcf3410917c"}, - {file = "regex-2025.11.3-cp313-cp313-win_amd64.whl", hash = "sha256:bac4200befe50c670c405dc33af26dad5a3b6b255dd6c000d92fe4629f9ed6a5"}, - {file = "regex-2025.11.3-cp313-cp313-win_arm64.whl", hash = "sha256:2292cd5a90dab247f9abe892ac584cb24f0f54680c73fcb4a7493c66c2bf2467"}, - {file = "regex-2025.11.3-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:1eb1ebf6822b756c723e09f5186473d93236c06c579d2cc0671a722d2ab14281"}, - {file = "regex-2025.11.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:1e00ec2970aab10dc5db34af535f21fcf32b4a31d99e34963419636e2f85ae39"}, - {file = "regex-2025.11.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a4cb042b615245d5ff9b3794f56be4138b5adc35a4166014d31d1814744148c7"}, - {file = "regex-2025.11.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:44f264d4bf02f3176467d90b294d59bf1db9fe53c141ff772f27a8b456b2a9ed"}, - {file = "regex-2025.11.3-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7be0277469bf3bd7a34a9c57c1b6a724532a0d235cd0dc4e7f4316f982c28b19"}, - {file = "regex-2025.11.3-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0d31e08426ff4b5b650f68839f5af51a92a5b51abd8554a60c2fbc7c71f25d0b"}, - {file = "regex-2025.11.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e43586ce5bd28f9f285a6e729466841368c4a0353f6fd08d4ce4630843d3648a"}, - {file = "regex-2025.11.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:0f9397d561a4c16829d4e6ff75202c1c08b68a3bdbfe29dbfcdb31c9830907c6"}, - {file = "regex-2025.11.3-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:dd16e78eb18ffdb25ee33a0682d17912e8cc8a770e885aeee95020046128f1ce"}, - {file = "regex-2025.11.3-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:ffcca5b9efe948ba0661e9df0fa50d2bc4b097c70b9810212d6b62f05d83b2dd"}, - {file = "regex-2025.11.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c56b4d162ca2b43318ac671c65bd4d563e841a694ac70e1a976ac38fcf4ca1d2"}, - {file = "regex-2025.11.3-cp313-cp313t-win32.whl", hash = "sha256:9ddc42e68114e161e51e272f667d640f97e84a2b9ef14b7477c53aac20c2d59a"}, - {file = "regex-2025.11.3-cp313-cp313t-win_amd64.whl", hash = "sha256:7a7c7fdf755032ffdd72c77e3d8096bdcb0eb92e89e17571a196f03d88b11b3c"}, - {file = "regex-2025.11.3-cp313-cp313t-win_arm64.whl", hash = "sha256:df9eb838c44f570283712e7cff14c16329a9f0fb19ca492d21d4b7528ee6821e"}, - {file = "regex-2025.11.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:9697a52e57576c83139d7c6f213d64485d3df5bf84807c35fa409e6c970801c6"}, - {file = "regex-2025.11.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e18bc3f73bd41243c9b38a6d9f2366cd0e0137a9aebe2d8ff76c5b67d4c0a3f4"}, - {file = "regex-2025.11.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:61a08bcb0ec14ff4e0ed2044aad948d0659604f824cbd50b55e30b0ec6f09c73"}, - {file = "regex-2025.11.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9c30003b9347c24bcc210958c5d167b9e4f9be786cb380a7d32f14f9b84674f"}, - {file = "regex-2025.11.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4e1e592789704459900728d88d41a46fe3969b82ab62945560a31732ffc19a6d"}, - {file = "regex-2025.11.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6538241f45eb5a25aa575dbba1069ad786f68a4f2773a29a2bd3dd1f9de787be"}, - {file = "regex-2025.11.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bce22519c989bb72a7e6b36a199384c53db7722fe669ba891da75907fe3587db"}, - {file = "regex-2025.11.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:66d559b21d3640203ab9075797a55165d79017520685fb407b9234d72ab63c62"}, - {file = "regex-2025.11.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:669dcfb2e38f9e8c69507bace46f4889e3abbfd9b0c29719202883c0a603598f"}, - {file = "regex-2025.11.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:32f74f35ff0f25a5021373ac61442edcb150731fbaa28286bbc8bb1582c89d02"}, - {file = "regex-2025.11.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e6c7a21dffba883234baefe91bc3388e629779582038f75d2a5be918e250f0ed"}, - {file = "regex-2025.11.3-cp314-cp314-win32.whl", hash = "sha256:795ea137b1d809eb6836b43748b12634291c0ed55ad50a7d72d21edf1cd565c4"}, - {file = "regex-2025.11.3-cp314-cp314-win_amd64.whl", hash = "sha256:9f95fbaa0ee1610ec0fc6b26668e9917a582ba80c52cc6d9ada15e30aa9ab9ad"}, - {file = "regex-2025.11.3-cp314-cp314-win_arm64.whl", hash = "sha256:dfec44d532be4c07088c3de2876130ff0fbeeacaa89a137decbbb5f665855a0f"}, - {file = "regex-2025.11.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ba0d8a5d7f04f73ee7d01d974d47c5834f8a1b0224390e4fe7c12a3a92a78ecc"}, - {file = "regex-2025.11.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:442d86cf1cfe4faabf97db7d901ef58347efd004934da045c745e7b5bd57ac49"}, - {file = "regex-2025.11.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fd0a5e563c756de210bb964789b5abe4f114dacae9104a47e1a649b910361536"}, - {file = "regex-2025.11.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf3490bcbb985a1ae97b2ce9ad1c0f06a852d5b19dde9b07bdf25bf224248c95"}, - {file = "regex-2025.11.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3809988f0a8b8c9dcc0f92478d6501fac7200b9ec56aecf0ec21f4a2ec4b6009"}, - {file = "regex-2025.11.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f4ff94e58e84aedb9c9fce66d4ef9f27a190285b451420f297c9a09f2b9abee9"}, - {file = "regex-2025.11.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eb542fd347ce61e1321b0a6b945d5701528dca0cd9759c2e3bb8bd57e47964d"}, - {file = "regex-2025.11.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d6c2d5919075a1f2e413c00b056ea0c2f065b3f5fe83c3d07d325ab92dce51d6"}, - {file = "regex-2025.11.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:3f8bf11a4827cc7ce5a53d4ef6cddd5ad25595d3c1435ef08f76825851343154"}, - {file = "regex-2025.11.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:22c12d837298651e5550ac1d964e4ff57c3f56965fc1812c90c9fb2028eaf267"}, - {file = "regex-2025.11.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:62ba394a3dda9ad41c7c780f60f6e4a70988741415ae96f6d1bf6c239cf01379"}, - {file = "regex-2025.11.3-cp314-cp314t-win32.whl", hash = "sha256:4bf146dca15cdd53224a1bf46d628bd7590e4a07fbb69e720d561aea43a32b38"}, - {file = "regex-2025.11.3-cp314-cp314t-win_amd64.whl", hash = "sha256:adad1a1bcf1c9e76346e091d22d23ac54ef28e1365117d99521631078dfec9de"}, - {file = "regex-2025.11.3-cp314-cp314t-win_arm64.whl", hash = "sha256:c54f768482cef41e219720013cd05933b6f971d9562544d691c68699bf2b6801"}, - {file = "regex-2025.11.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:81519e25707fc076978c6143b81ea3dc853f176895af05bf7ec51effe818aeec"}, - {file = "regex-2025.11.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3bf28b1873a8af8bbb58c26cc56ea6e534d80053b41fb511a35795b6de507e6a"}, - {file = "regex-2025.11.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:856a25c73b697f2ce2a24e7968285579e62577a048526161a2c0f53090bea9f9"}, - {file = "regex-2025.11.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a3d571bd95fade53c86c0517f859477ff3a93c3fde10c9e669086f038e0f207"}, - {file = "regex-2025.11.3-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:732aea6de26051af97b94bc98ed86448821f839d058e5d259c72bf6d73ad0fc0"}, - {file = "regex-2025.11.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:51c1c1847128238f54930edb8805b660305dca164645a9fd29243f5610beea34"}, - {file = "regex-2025.11.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22dd622a402aad4558277305350699b2be14bc59f64d64ae1d928ce7d072dced"}, - {file = "regex-2025.11.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f3b5a391c7597ffa96b41bd5cbd2ed0305f515fcbb367dfa72735679d5502364"}, - {file = "regex-2025.11.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:cc4076a5b4f36d849fd709284b4a3b112326652f3b0466f04002a6c15a0c96c1"}, - {file = "regex-2025.11.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:a295ca2bba5c1c885826ce3125fa0b9f702a1be547d821c01d65f199e10c01e2"}, - {file = "regex-2025.11.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:b4774ff32f18e0504bfc4e59a3e71e18d83bc1e171a3c8ed75013958a03b2f14"}, - {file = "regex-2025.11.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:22e7d1cdfa88ef33a2ae6aa0d707f9255eb286ffbd90045f1088246833223aee"}, - {file = "regex-2025.11.3-cp39-cp39-win32.whl", hash = "sha256:74d04244852ff73b32eeede4f76f51c5bcf44bc3c207bc3e6cf1c5c45b890708"}, - {file = "regex-2025.11.3-cp39-cp39-win_amd64.whl", hash = "sha256:7a50cd39f73faa34ec18d6720ee25ef10c4c1839514186fcda658a06c06057a2"}, - {file = "regex-2025.11.3-cp39-cp39-win_arm64.whl", hash = "sha256:43b4fb020e779ca81c1b5255015fe2b82816c76ec982354534ad9ec09ad7c9e3"}, - {file = "regex-2025.11.3.tar.gz", hash = "sha256:1fedc720f9bb2494ce31a58a1631f9c82df6a09b49c19517ea5cc280b4541e01"}, + {file = "regex-2026.1.15-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4e3dd93c8f9abe8aa4b6c652016da9a3afa190df5ad822907efe6b206c09896e"}, + {file = "regex-2026.1.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:97499ff7862e868b1977107873dd1a06e151467129159a6ffd07b66706ba3a9f"}, + {file = "regex-2026.1.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0bda75ebcac38d884240914c6c43d8ab5fb82e74cde6da94b43b17c411aa4c2b"}, + {file = "regex-2026.1.15-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7dcc02368585334f5bc81fc73a2a6a0bbade60e7d83da21cead622faf408f32c"}, + {file = "regex-2026.1.15-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:693b465171707bbe882a7a05de5e866f33c76aa449750bee94a8d90463533cc9"}, + {file = "regex-2026.1.15-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b0d190e6f013ea938623a58706d1469a62103fb2a241ce2873a9906e0386582c"}, + {file = "regex-2026.1.15-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ff818702440a5878a81886f127b80127f5d50563753a28211482867f8318106"}, + {file = "regex-2026.1.15-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f052d1be37ef35a54e394de66136e30fa1191fab64f71fc06ac7bc98c9a84618"}, + {file = "regex-2026.1.15-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6bfc31a37fd1592f0c4fc4bfc674b5c42e52efe45b4b7a6a14f334cca4bcebe4"}, + {file = "regex-2026.1.15-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3d6ce5ae80066b319ae3bc62fd55a557c9491baa5efd0d355f0de08c4ba54e79"}, + {file = "regex-2026.1.15-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1704d204bd42b6bb80167df0e4554f35c255b579ba99616def38f69e14a5ccb9"}, + {file = "regex-2026.1.15-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:e3174a5ed4171570dc8318afada56373aa9289eb6dc0d96cceb48e7358b0e220"}, + {file = "regex-2026.1.15-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:87adf5bd6d72e3e17c9cb59ac4096b1faaf84b7eb3037a5ffa61c4b4370f0f13"}, + {file = "regex-2026.1.15-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e85dc94595f4d766bd7d872a9de5ede1ca8d3063f3bdf1e2c725f5eb411159e3"}, + {file = "regex-2026.1.15-cp310-cp310-win32.whl", hash = "sha256:21ca32c28c30d5d65fc9886ff576fc9b59bbca08933e844fa2363e530f4c8218"}, + {file = "regex-2026.1.15-cp310-cp310-win_amd64.whl", hash = "sha256:3038a62fc7d6e5547b8915a3d927a0fbeef84cdbe0b1deb8c99bbd4a8961b52a"}, + {file = "regex-2026.1.15-cp310-cp310-win_arm64.whl", hash = "sha256:505831646c945e3e63552cc1b1b9b514f0e93232972a2d5bedbcc32f15bc82e3"}, + {file = "regex-2026.1.15-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1ae6020fb311f68d753b7efa9d4b9a5d47a5d6466ea0d5e3b5a471a960ea6e4a"}, + {file = "regex-2026.1.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:eddf73f41225942c1f994914742afa53dc0d01a6e20fe14b878a1b1edc74151f"}, + {file = "regex-2026.1.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e8cd52557603f5c66a548f69421310886b28b7066853089e1a71ee710e1cdc1"}, + {file = "regex-2026.1.15-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5170907244b14303edc5978f522f16c974f32d3aa92109fabc2af52411c9433b"}, + {file = "regex-2026.1.15-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2748c1ec0663580b4510bd89941a31560b4b439a0b428b49472a3d9944d11cd8"}, + {file = "regex-2026.1.15-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2f2775843ca49360508d080eaa87f94fa248e2c946bbcd963bb3aae14f333413"}, + {file = "regex-2026.1.15-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9ea2604370efc9a174c1b5dcc81784fb040044232150f7f33756049edfc9026"}, + {file = "regex-2026.1.15-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0dcd31594264029b57bf16f37fd7248a70b3b764ed9e0839a8f271b2d22c0785"}, + {file = "regex-2026.1.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c08c1f3e34338256732bd6938747daa3c0d5b251e04b6e43b5813e94d503076e"}, + {file = "regex-2026.1.15-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e43a55f378df1e7a4fa3547c88d9a5a9b7113f653a66821bcea4718fe6c58763"}, + {file = "regex-2026.1.15-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:f82110ab962a541737bd0ce87978d4c658f06e7591ba899192e2712a517badbb"}, + {file = "regex-2026.1.15-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:27618391db7bdaf87ac6c92b31e8f0dfb83a9de0075855152b720140bda177a2"}, + {file = "regex-2026.1.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bfb0d6be01fbae8d6655c8ca21b3b72458606c4aec9bbc932db758d47aba6db1"}, + {file = "regex-2026.1.15-cp311-cp311-win32.whl", hash = "sha256:b10e42a6de0e32559a92f2f8dc908478cc0fa02838d7dbe764c44dca3fa13569"}, + {file = "regex-2026.1.15-cp311-cp311-win_amd64.whl", hash = "sha256:e9bf3f0bbdb56633c07d7116ae60a576f846efdd86a8848f8d62b749e1209ca7"}, + {file = "regex-2026.1.15-cp311-cp311-win_arm64.whl", hash = "sha256:41aef6f953283291c4e4e6850607bd71502be67779586a61472beacb315c97ec"}, + {file = "regex-2026.1.15-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:4c8fcc5793dde01641a35905d6731ee1548f02b956815f8f1cab89e515a5bdf1"}, + {file = "regex-2026.1.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bfd876041a956e6a90ad7cdb3f6a630c07d491280bfeed4544053cd434901681"}, + {file = "regex-2026.1.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9250d087bc92b7d4899ccd5539a1b2334e44eee85d848c4c1aef8e221d3f8c8f"}, + {file = "regex-2026.1.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8a154cf6537ebbc110e24dabe53095e714245c272da9c1be05734bdad4a61aa"}, + {file = "regex-2026.1.15-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8050ba2e3ea1d8731a549e83c18d2f0999fbc99a5f6bd06b4c91449f55291804"}, + {file = "regex-2026.1.15-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf065240704cb8951cc04972cf107063917022511273e0969bdb34fc173456c"}, + {file = "regex-2026.1.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c32bef3e7aeee75746748643667668ef941d28b003bfc89994ecf09a10f7a1b5"}, + {file = "regex-2026.1.15-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d5eaa4a4c5b1906bd0d2508d68927f15b81821f85092e06f1a34a4254b0e1af3"}, + {file = "regex-2026.1.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:86c1077a3cc60d453d4084d5b9649065f3bf1184e22992bd322e1f081d3117fb"}, + {file = "regex-2026.1.15-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:2b091aefc05c78d286657cd4db95f2e6313375ff65dcf085e42e4c04d9c8d410"}, + {file = "regex-2026.1.15-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:57e7d17f59f9ebfa9667e6e5a1c0127b96b87cb9cede8335482451ed00788ba4"}, + {file = "regex-2026.1.15-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:c6c4dcdfff2c08509faa15d36ba7e5ef5fcfab25f1e8f85a0c8f45bc3a30725d"}, + {file = "regex-2026.1.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cf8ff04c642716a7f2048713ddc6278c5fd41faa3b9cab12607c7abecd012c22"}, + {file = "regex-2026.1.15-cp312-cp312-win32.whl", hash = "sha256:82345326b1d8d56afbe41d881fdf62f1926d7264b2fc1537f99ae5da9aad7913"}, + {file = "regex-2026.1.15-cp312-cp312-win_amd64.whl", hash = "sha256:4def140aa6156bc64ee9912383d4038f3fdd18fee03a6f222abd4de6357ce42a"}, + {file = "regex-2026.1.15-cp312-cp312-win_arm64.whl", hash = "sha256:c6c565d9a6e1a8d783c1948937ffc377dd5771e83bd56de8317c450a954d2056"}, + {file = "regex-2026.1.15-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e69d0deeb977ffe7ed3d2e4439360089f9c3f217ada608f0f88ebd67afb6385e"}, + {file = "regex-2026.1.15-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3601ffb5375de85a16f407854d11cca8fe3f5febbe3ac78fb2866bb220c74d10"}, + {file = "regex-2026.1.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4c5ef43b5c2d4114eb8ea424bb8c9cec01d5d17f242af88b2448f5ee81caadbc"}, + {file = "regex-2026.1.15-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:968c14d4f03e10b2fd960f1d5168c1f0ac969381d3c1fcc973bc45fb06346599"}, + {file = "regex-2026.1.15-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56a5595d0f892f214609c9f76b41b7428bed439d98dc961efafdd1354d42baae"}, + {file = "regex-2026.1.15-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf650f26087363434c4e560011f8e4e738f6f3e029b85d4904c50135b86cfa5"}, + {file = "regex-2026.1.15-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18388a62989c72ac24de75f1449d0fb0b04dfccd0a1a7c1c43af5eb503d890f6"}, + {file = "regex-2026.1.15-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6d220a2517f5893f55daac983bfa9fe998a7dbcaee4f5d27a88500f8b7873788"}, + {file = "regex-2026.1.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c9c08c2fbc6120e70abff5d7f28ffb4d969e14294fb2143b4b5c7d20e46d1714"}, + {file = "regex-2026.1.15-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7ef7d5d4bd49ec7364315167a4134a015f61e8266c6d446fc116a9ac4456e10d"}, + {file = "regex-2026.1.15-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:6e42844ad64194fa08d5ccb75fe6a459b9b08e6d7296bd704460168d58a388f3"}, + {file = "regex-2026.1.15-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:cfecdaa4b19f9ca534746eb3b55a5195d5c95b88cac32a205e981ec0a22b7d31"}, + {file = "regex-2026.1.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08df9722d9b87834a3d701f3fca570b2be115654dbfd30179f30ab2f39d606d3"}, + {file = "regex-2026.1.15-cp313-cp313-win32.whl", hash = "sha256:d426616dae0967ca225ab12c22274eb816558f2f99ccb4a1d52ca92e8baf180f"}, + {file = "regex-2026.1.15-cp313-cp313-win_amd64.whl", hash = "sha256:febd38857b09867d3ed3f4f1af7d241c5c50362e25ef43034995b77a50df494e"}, + {file = "regex-2026.1.15-cp313-cp313-win_arm64.whl", hash = "sha256:8e32f7896f83774f91499d239e24cebfadbc07639c1494bb7213983842348337"}, + {file = "regex-2026.1.15-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:ec94c04149b6a7b8120f9f44565722c7ae31b7a6d2275569d2eefa76b83da3be"}, + {file = "regex-2026.1.15-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:40c86d8046915bb9aeb15d3f3f15b6fd500b8ea4485b30e1bbc799dab3fe29f8"}, + {file = "regex-2026.1.15-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:726ea4e727aba21643205edad8f2187ec682d3305d790f73b7a51c7587b64bdd"}, + {file = "regex-2026.1.15-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1cb740d044aff31898804e7bf1181cc72c03d11dfd19932b9911ffc19a79070a"}, + {file = "regex-2026.1.15-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05d75a668e9ea16f832390d22131fe1e8acc8389a694c8febc3e340b0f810b93"}, + {file = "regex-2026.1.15-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d991483606f3dbec93287b9f35596f41aa2e92b7c2ebbb935b63f409e243c9af"}, + {file = "regex-2026.1.15-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:194312a14819d3e44628a44ed6fea6898fdbecb0550089d84c403475138d0a09"}, + {file = "regex-2026.1.15-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe2fda4110a3d0bc163c2e0664be44657431440722c5c5315c65155cab92f9e5"}, + {file = "regex-2026.1.15-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:124dc36c85d34ef2d9164da41a53c1c8c122cfb1f6e1ec377a1f27ee81deb794"}, + {file = "regex-2026.1.15-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:a1774cd1981cd212506a23a14dba7fdeaee259f5deba2df6229966d9911e767a"}, + {file = "regex-2026.1.15-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b5f7d8d2867152cdb625e72a530d2ccb48a3d199159144cbdd63870882fb6f80"}, + {file = "regex-2026.1.15-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:492534a0ab925d1db998defc3c302dae3616a2fc3fe2e08db1472348f096ddf2"}, + {file = "regex-2026.1.15-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c661fc820cfb33e166bf2450d3dadbda47c8d8981898adb9b6fe24e5e582ba60"}, + {file = "regex-2026.1.15-cp313-cp313t-win32.whl", hash = "sha256:99ad739c3686085e614bf77a508e26954ff1b8f14da0e3765ff7abbf7799f952"}, + {file = "regex-2026.1.15-cp313-cp313t-win_amd64.whl", hash = "sha256:32655d17905e7ff8ba5c764c43cb124e34a9245e45b83c22e81041e1071aee10"}, + {file = "regex-2026.1.15-cp313-cp313t-win_arm64.whl", hash = "sha256:b2a13dd6a95e95a489ca242319d18fc02e07ceb28fa9ad146385194d95b3c829"}, + {file = "regex-2026.1.15-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:d920392a6b1f353f4aa54328c867fec3320fa50657e25f64abf17af054fc97ac"}, + {file = "regex-2026.1.15-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:b5a28980a926fa810dbbed059547b02783952e2efd9c636412345232ddb87ff6"}, + {file = "regex-2026.1.15-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:621f73a07595d83f28952d7bd1e91e9d1ed7625fb7af0064d3516674ec93a2a2"}, + {file = "regex-2026.1.15-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3d7d92495f47567a9b1669c51fc8d6d809821849063d168121ef801bbc213846"}, + {file = "regex-2026.1.15-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8dd16fba2758db7a3780a051f245539c4451ca20910f5a5e6ea1c08d06d4a76b"}, + {file = "regex-2026.1.15-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1e1808471fbe44c1a63e5f577a1d5f02fe5d66031dcbdf12f093ffc1305a858e"}, + {file = "regex-2026.1.15-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0751a26ad39d4f2ade8fe16c59b2bf5cb19eb3d2cd543e709e583d559bd9efde"}, + {file = "regex-2026.1.15-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0f0c7684c7f9ca241344ff95a1de964f257a5251968484270e91c25a755532c5"}, + {file = "regex-2026.1.15-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:74f45d170a21df41508cb67165456538425185baaf686281fa210d7e729abc34"}, + {file = "regex-2026.1.15-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f1862739a1ffb50615c0fde6bae6569b5efbe08d98e59ce009f68a336f64da75"}, + {file = "regex-2026.1.15-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:453078802f1b9e2b7303fb79222c054cb18e76f7bdc220f7530fdc85d319f99e"}, + {file = "regex-2026.1.15-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:a30a68e89e5a218b8b23a52292924c1f4b245cb0c68d1cce9aec9bbda6e2c160"}, + {file = "regex-2026.1.15-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9479cae874c81bf610d72b85bb681a94c95722c127b55445285fb0e2c82db8e1"}, + {file = "regex-2026.1.15-cp314-cp314-win32.whl", hash = "sha256:d639a750223132afbfb8f429c60d9d318aeba03281a5f1ab49f877456448dcf1"}, + {file = "regex-2026.1.15-cp314-cp314-win_amd64.whl", hash = "sha256:4161d87f85fa831e31469bfd82c186923070fc970b9de75339b68f0c75b51903"}, + {file = "regex-2026.1.15-cp314-cp314-win_arm64.whl", hash = "sha256:91c5036ebb62663a6b3999bdd2e559fd8456d17e2b485bf509784cd31a8b1705"}, + {file = "regex-2026.1.15-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ee6854c9000a10938c79238de2379bea30c82e4925a371711af45387df35cab8"}, + {file = "regex-2026.1.15-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2c2b80399a422348ce5de4fe40c418d6299a0fa2803dd61dc0b1a2f28e280fcf"}, + {file = "regex-2026.1.15-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:dca3582bca82596609959ac39e12b7dad98385b4fefccb1151b937383cec547d"}, + {file = "regex-2026.1.15-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef71d476caa6692eea743ae5ea23cde3260677f70122c4d258ca952e5c2d4e84"}, + {file = "regex-2026.1.15-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c243da3436354f4af6c3058a3f81a97d47ea52c9bd874b52fd30274853a1d5df"}, + {file = "regex-2026.1.15-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8355ad842a7c7e9e5e55653eade3b7d1885ba86f124dd8ab1f722f9be6627434"}, + {file = "regex-2026.1.15-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f192a831d9575271a22d804ff1a5355355723f94f31d9eef25f0d45a152fdc1a"}, + {file = "regex-2026.1.15-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:166551807ec20d47ceaeec380081f843e88c8949780cd42c40f18d16168bed10"}, + {file = "regex-2026.1.15-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f9ca1cbdc0fbfe5e6e6f8221ef2309988db5bcede52443aeaee9a4ad555e0dac"}, + {file = "regex-2026.1.15-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b30bcbd1e1221783c721483953d9e4f3ab9c5d165aa709693d3f3946747b1aea"}, + {file = "regex-2026.1.15-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2a8d7b50c34578d0d3bf7ad58cde9652b7d683691876f83aedc002862a35dc5e"}, + {file = "regex-2026.1.15-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9d787e3310c6a6425eb346be4ff2ccf6eece63017916fd77fe8328c57be83521"}, + {file = "regex-2026.1.15-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:619843841e220adca114118533a574a9cd183ed8a28b85627d2844c500a2b0db"}, + {file = "regex-2026.1.15-cp314-cp314t-win32.whl", hash = "sha256:e90b8db97f6f2c97eb045b51a6b2c5ed69cedd8392459e0642d4199b94fabd7e"}, + {file = "regex-2026.1.15-cp314-cp314t-win_amd64.whl", hash = "sha256:5ef19071f4ac9f0834793af85bd04a920b4407715624e40cb7a0631a11137cdf"}, + {file = "regex-2026.1.15-cp314-cp314t-win_arm64.whl", hash = "sha256:ca89c5e596fc05b015f27561b3793dc2fa0917ea0d7507eebb448efd35274a70"}, + {file = "regex-2026.1.15-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:55b4ea996a8e4458dd7b584a2f89863b1655dd3d17b88b46cbb9becc495a0ec5"}, + {file = "regex-2026.1.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7e1e28be779884189cdd57735e997f282b64fd7ccf6e2eef3e16e57d7a34a815"}, + {file = "regex-2026.1.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0057de9eaef45783ff69fa94ae9f0fd906d629d0bd4c3217048f46d1daa32e9b"}, + {file = "regex-2026.1.15-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc7cd0b2be0f0269283a45c0d8b2c35e149d1319dcb4a43c9c3689fa935c1ee6"}, + {file = "regex-2026.1.15-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8db052bbd981e1666f09e957f3790ed74080c2229007c1dd67afdbf0b469c48b"}, + {file = "regex-2026.1.15-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:343db82cb3712c31ddf720f097ef17c11dab2f67f7a3e7be976c4f82eba4e6df"}, + {file = "regex-2026.1.15-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:55e9d0118d97794367309635df398bdfd7c33b93e2fdfa0b239661cd74b4c14e"}, + {file = "regex-2026.1.15-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:008b185f235acd1e53787333e5690082e4f156c44c87d894f880056089e9bc7c"}, + {file = "regex-2026.1.15-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fd65af65e2aaf9474e468f9e571bd7b189e1df3a61caa59dcbabd0000e4ea839"}, + {file = "regex-2026.1.15-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f42e68301ff4afee63e365a5fc302b81bb8ba31af625a671d7acb19d10168a8c"}, + {file = "regex-2026.1.15-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:f7792f27d3ee6e0244ea4697d92b825f9a329ab5230a78c1a68bd274e64b5077"}, + {file = "regex-2026.1.15-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:dbaf3c3c37ef190439981648ccbf0c02ed99ae066087dd117fcb616d80b010a4"}, + {file = "regex-2026.1.15-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:adc97a9077c2696501443d8ad3fa1b4fc6d131fc8fd7dfefd1a723f89071cf0a"}, + {file = "regex-2026.1.15-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:069f56a7bf71d286a6ff932a9e6fb878f151c998ebb2519a9f6d1cee4bffdba3"}, + {file = "regex-2026.1.15-cp39-cp39-win32.whl", hash = "sha256:ea4e6b3566127fda5e007e90a8fd5a4169f0cf0619506ed426db647f19c8454a"}, + {file = "regex-2026.1.15-cp39-cp39-win_amd64.whl", hash = "sha256:cda1ed70d2b264952e88adaa52eea653a33a1b98ac907ae2f86508eb44f65cdc"}, + {file = "regex-2026.1.15-cp39-cp39-win_arm64.whl", hash = "sha256:b325d4714c3c48277bfea1accd94e193ad6ed42b4bad79ad64f3b8f8a31260a5"}, + {file = "regex-2026.1.15.tar.gz", hash = "sha256:164759aa25575cbc0651bef59a0b18353e54300d79ace8084c818ad8ac72b7d5"}, ] [[package]] @@ -4419,4 +4455,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.12,<3.13" -content-hash = "27d79e7aa23ce1964e291ebd15b804f006a2e6149ad1113f24bb06e556160c1f" +content-hash = "505c74ff6ec6327eccebf589d0a7a0e376e585db84dbb96263a1dd367f268c62" diff --git a/pyproject.toml b/pyproject.toml index ee339ac0..2742d001 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ sentry-sdk = "==1.45.1" py-ecc = "==8.0.0" sw-utils = {git = "https://github.com/stakewise/sw-utils.git", rev = "v0.12.1"} staking-deposit = { git = "https://github.com/ethereum/staking-deposit-cli.git", rev = "v2.8.0" } +multiproof = { git = "https://github.com/stakewise/multiproof.git", rev = "v0.1.10" } pycryptodomex = "3.19.1" click = "==8.2.1" tomli = "~2" From ff10073999767c010545118c1c1cbc8054105ec5 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Thu, 22 Jan 2026 16:39:52 +0300 Subject: [PATCH 24/65] Add process-redeemer command Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 305 ++++++++++++++++++++++ src/common/contracts.py | 49 ++++ src/common/harvest.py | 10 +- src/common/startup_check.py | 6 +- src/config/networks.py | 6 + src/harvest/tasks.py | 2 +- src/main.py | 2 + src/redeem/os_token_converter.py | 5 + src/redeem/typings.py | 17 +- src/reward_splitter/tasks.py | 2 +- src/validators/execution.py | 6 +- src/validators/tasks.py | 4 +- src/withdrawals/assets.py | 2 +- 13 files changed, 402 insertions(+), 14 deletions(-) create mode 100644 src/commands/internal/process_redeemer.py diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py new file mode 100644 index 00000000..1212285c --- /dev/null +++ b/src/commands/internal/process_redeemer.py @@ -0,0 +1,305 @@ +import asyncio +import logging +import sys +from collections import defaultdict +from pathlib import Path +from typing import cast + +import click +from eth_typing import ChecksumAddress +from multiproof.standard import standard_leaf_hash +from web3 import Web3 +from web3.types import Wei + +from src.common.clients import execution_client, ipfs_fetch_client, setup_clients +from src.common.contracts import ( + VaultContract, + multicall_contract, + os_token_redeemer_contract, +) +from src.common.execution import transaction_gas_wrapper +from src.common.harvest import get_harvest_params +from src.common.logging import LOG_LEVELS, setup_logging +from src.common.typings import HarvestParams +from src.common.utils import log_verbose +from src.common.wallet import wallet +from src.config.networks import AVAILABLE_NETWORKS, ZERO_CHECKSUM_ADDRESS +from src.config.settings import settings +from src.redeem.os_token_converter import create_os_token_converter +from src.redeem.typings import OsTokenPosition, RedeemablePosition +from src.validators.execution import get_withdrawable_assets + +logger = logging.getLogger(__name__) + + +@click.option( + '--wallet-password-file', + type=click.Path(exists=True, file_okay=True, dir_okay=False), + envvar='WALLET_PASSWORD_FILE', + help='Absolute path to the wallet password file. ' + 'Default is the file generated with "create-wallet" command.', +) +@click.option( + '--wallet-file', + type=click.Path(exists=True, file_okay=True, dir_okay=False), + envvar='WALLET_FILE', + help='Absolute path to the wallet. ' + 'Default is the file generated with "create-wallet" command.', +) +@click.option( + '--execution-endpoints', + type=str, + envvar='EXECUTION_ENDPOINTS', + prompt='Enter the comma separated list of API endpoints for execution nodes', + help='Comma separated list of API endpoints for execution nodes.', +) +@click.option( + '--execution-jwt-secret', + type=str, + envvar='EXECUTION_JWT_SECRET', + help='JWT secret key used for signing and verifying JSON Web Tokens' + ' when connecting to execution nodes.', +) +@click.option( + '--graph-endpoint', + type=str, + envvar='GRAPH_ENDPOINT', + help='API endpoint for graph node.', +) +@click.option( + '--log-level', + type=click.Choice( + LOG_LEVELS, + case_sensitive=False, + ), + default='INFO', + envvar='LOG_LEVEL', + help='The log level.', +) +@click.option( + '-v', + '--verbose', + help='Enable debug mode. Default is false.', + envvar='VERBOSE', + is_flag=True, +) +@click.option( + '--network', + help='The network of the meta vaults.', + prompt='Enter the network name', + envvar='NETWORK', + type=click.Choice( + AVAILABLE_NETWORKS, + case_sensitive=False, + ), +) +@click.command(help='Updates redeemable positions') +# pylint: disable-next=too-many-arguments +def process_redeemer( + execution_endpoints: str, + execution_jwt_secret: str | None, + graph_endpoint: str, + network: str, + verbose: bool, + log_level: str, + wallet_file: str | None, + wallet_password_file: str | None, +) -> None: + settings.set( + vault=ZERO_CHECKSUM_ADDRESS, + vault_dir=Path.home() / '.stakewise', + execution_endpoints=execution_endpoints, + execution_jwt_secret=execution_jwt_secret, + graph_endpoint=graph_endpoint, + verbose=verbose, + network=network, + wallet_file=wallet_file, + wallet_password_file=wallet_password_file, + log_level=log_level, + ) + try: + asyncio.run(main()) + except Exception as e: + log_verbose(e) + sys.exit(1) + + +# pylint: disable-next=too-many-locals +async def main() -> None: + """ + Fetch redeemable positions, calculate kept os token amounts and upload to IPFS. + """ + setup_logging() + await setup_clients() + await _startup_check() + + while True: + block_number = await execution_client.eth.block_number + # Check Exit Queue Processing + can_process_exit_queue = await os_token_redeemer_contract.can_process_exit_queue( + block_number + ) + if can_process_exit_queue: + logger.info('Exit queue can be processed. Calling processExitQueue...') + tx_hash = await os_token_redeemer_contract.process_exit_queue() + logger.info('ProcessExitQueue transaction sent. Tx Hash: %s', tx_hash.hex()) + + # Check Queued Shares for Redemption + queued_shares = await os_token_redeemer_contract.queued_shares(block_number) + if queued_shares == 0: + logger.info('No queued shares for redemption. Skipping to next interval.') + await asyncio.sleep(300) # Sleep for 5 minutes before next check + continue + + os_token_converter = await create_os_token_converter(block_number) + queued_assets = os_token_converter.to_assets(queued_shares) + + # Fetch Positions from IPFS + redeemable_positions_meta = await os_token_redeemer_contract.redeemable_positions( + block_number + ) + redeemable_positions = await fetch_redeemable_positions(redeemable_positions_meta.ipfs_hash) + + # Calculate Redeemable Shares Per Position + for redeemable_position in redeemable_positions: + # Compute leaf hash + leaf_hash = get_redeemable_position_leaf_hash( + redeemable_position=redeemable_position, nonce=redeemable_positions_meta.nonce - 1 + ) + + # Get already processed shares + processed_shares = await os_token_redeemer_contract.leaf_to_processed_shares( + leaf_hash, block_number + ) + + # Calculate redeemable shares + redeemable_position.redeemable_shares = redeemable_position.amount - processed_shares + + # Filter Positions by Vault Withdrawable Assets + + # Group positions by vault + vault_to_positions: defaultdict[ChecksumAddress, list[RedeemablePosition]] = defaultdict( + list + ) + for position in redeemable_positions: + vault_to_positions[position.vault].append(position) + + processing_positions = [] + vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None] = {} + for vault_address, positions in vault_to_positions.items(): + # Check if state update is required + harvest_params = await get_harvest_params(vault_address, block_number) + vault_to_harvest_params[vault_address] = harvest_params + withdrawable_assets = await get_withdrawable_assets(vault_address, harvest_params) + + # Process each position in the vault + for position in positions: + # Convert redeemable shares to assets + redeemable_assets = os_token_converter.to_assets(position.redeemable_shares) + + if redeemable_assets <= withdrawable_assets: + shares_to_redeem = min(position.amount, queued_shares) + logger.info( + f"Position Owner: {position.owner}, " + f"Vault: {position.vault}, " + f"Shares to Redeem: {shares_to_redeem}" + ) + processing_positions.append( + OsTokenPosition( + vault=position.vault, + owner=position.owner, + leaf_shares=position.amount, + shares_to_redeem=shares_to_redeem, + ) + ) + withdrawable_assets -= redeemable_assets + queued_shares -= shares_to_redeem + + # Handle Meta-Vaults with Insufficient Withdrawable Assets + # Execute Redemption with Multicall + await execute_redemption( + redeemed_positions=processing_positions, + vault_to_harvest_params=vault_to_harvest_params, + ) + + +async def execute_redemption( + redeemed_positions: list[OsTokenPosition], + vault_to_harvest_params: HarvestParams | None, +) -> None: + calls = [] + + for vault in set(pos.vault for pos in redeemed_positions): + harvest_params = vault_to_harvest_params.get(vault) + if harvest_params: + vault_contract = VaultContract(vault) + calls.append( + [ + ( + vault_contract.contract_address, + vault_contract.get_update_state_call(harvest_params), + ) + ] + ) + + redeem_os_token_positions_call = os_token_redeemer_contract.encode_abi( + fn_name='redeemOsTokenPositions', + args=[redeemed_positions, proof, proofFlags], + ) + calls.append((os_token_redeemer_contract.address, redeem_os_token_positions_call)) + try: + tx_function = multicall_contract.functions.aggregate(calls) + tx = await transaction_gas_wrapper(tx_function=tx_function) + except Exception as e: + logger.error('Failed to redeem os token positions: %s', e) + logger.exception(e) + return None + + tx_hash = Web3.to_hex(tx) + logger.info('Waiting for transaction %s confirmation', tx_hash) + tx_receipt = await execution_client.eth.wait_for_transaction_receipt( + tx, timeout=settings.execution_transaction_timeout + ) + if not tx_receipt['status']: + logger.error(logger.error('Failed to redeem os token positions...')) + return None + + return tx_hash + + +def get_redeemable_position_leaf_hash(redeemable_position: RedeemablePosition, nonce: int) -> bytes: + """Get the leaf hash for a redeemable position.""" + vault = redeemable_position.vault + owner = redeemable_position.owner + amount = redeemable_position.amount + + leaf = standard_leaf_hash( + values=(nonce, vault, amount, owner), + types=['uint256', 'address', 'uint256', 'address'], + ) + return leaf + + +async def fetch_redeemable_positions(ipfs_hash: str) -> list[RedeemablePosition]: + # Fetch redeemable positions data from IPFS + data = cast(list[dict], await ipfs_fetch_client.fetch_json(ipfs_hash)) + + # data structure example: + # [{"owner:" 0x01, "amount": 100000, "vault": 0x02}, ...] + + return [ + RedeemablePosition( + owner=Web3.to_checksum_address(item['owner']), + vault=Web3.to_checksum_address(item['vault']), + amount=Wei(int(item['amount'])), + ) + for item in data + ] + + +async def _startup_check() -> None: + positions_manager = await os_token_redeemer_contract.positions_manager() + if positions_manager != wallet.account.address: + raise RuntimeError( + f'The Position Manager role must be assigned to the address {wallet.account.address}.' + ) diff --git a/src/common/contracts.py b/src/common/contracts.py index 926bd639..4481b9b2 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -27,6 +27,7 @@ settings, ) from src.meta_vault.typings import SubVaultExitRequest +from src.redeem.typings import RedeemablePositionsMeta from src.validators.typings import V2ValidatorEventData from src.withdrawals.typings import WithdrawalEvent @@ -484,6 +485,53 @@ async def tx_aggregate( return Web3.to_hex(tx_hash) +class OsTokenRedeemerContract(ContractWrapper): + abi_path = 'abi/IOsTokenRedeemer.json' + settings_key = 'OS_TOKEN_REDEEMER_CONTRACT_ADDRESS' + + async def redeemable_positions(self) -> RedeemablePositionsMeta: + merkle_root, ipfs_hash = await self.contract.functions.redeemablePositions().call() + return RedeemablePositionsMeta( + merkle_root=Web3.to_hex(merkle_root), + ipfs_hash=ipfs_hash, + ) + + async def get_exit_queue_cumulative_tickets(self, block_number: BlockNumber) -> int: + return await self.contract.functions.getExitQueueCumulativeTickets().call( + block_identifier=block_number + ) + + async def get_exit_queue_missing_assets(self, target_ticket: int) -> Wei: + return await self.contract.functions.getExitQueueMissingAssets(target_ticket).call() + + async def nonce(self) -> int: + return await self.contract.functions.nonce().call() + + ### + async def positions_manager(self) -> ChecksumAddress: + return await self.contract.functions.positionsManager().call() + + async def queued_shares(self) -> Wei: + return await self.contract.functions.queuedShares().call() + + async def can_process_exit_queue(self, block_number: BlockNumber | None = None) -> bool: + return await self.contract.functions.canProcessExitQueue().call( + block_identifier=block_number + ) + + async def leaf_to_processed_shares( + self, leaf: bytes, block_number: BlockNumber | None = None + ) -> Wei: + return await self.contract.functions.leafToProcessedShares(leaf).call( + block_identifier=block_number + ) + + async def process_exit_queue(self) -> HexStr: + tx_function = self.contract.functions.processExitQueue() + tx_hash = await transaction_gas_wrapper(tx_function) + return Web3.to_hex(tx_hash) + + class ValidatorsCheckerContract(ContractWrapper): abi_path = 'abi/IValidatorsChecker.json' settings_key = 'VALIDATORS_CHECKER_CONTRACT_ADDRESS' @@ -570,3 +618,4 @@ def _get_exit_queue_missing_assets_call(self, params: ExitQueueMissingAssetsPara multicall_contract = MulticallContract() validators_checker_contract = ValidatorsCheckerContract() os_token_vault_controller_contract = OsTokenVaultControllerContract() +os_token_redeemer_contract = OsTokenRedeemerContract() diff --git a/src/common/harvest.py b/src/common/harvest.py index e9f89967..554f94c8 100644 --- a/src/common/harvest.py +++ b/src/common/harvest.py @@ -1,7 +1,7 @@ from hexbytes import HexBytes from sw_utils.networks import GNO_NETWORKS from web3 import Web3 -from web3.types import BlockNumber, Wei +from web3.types import BlockNumber, ChecksumAddress, Wei from src.common.clients import ipfs_fetch_client from src.common.contracts import VaultContract, keeper_contract @@ -9,15 +9,17 @@ from src.config.settings import settings -async def get_harvest_params(block_number: BlockNumber | None = None) -> HarvestParams | None: - if not await keeper_contract.can_harvest(settings.vault, block_number): +async def get_harvest_params( + vault: ChecksumAddress, block_number: BlockNumber | None = None +) -> HarvestParams | None: + if not await keeper_contract.can_harvest(vault, block_number): return None last_rewards = await keeper_contract.get_last_rewards_update(block_number) if last_rewards is None: return None - vault_contract = VaultContract(settings.vault) + vault_contract = VaultContract(vault) harvest_params = await _fetch_harvest_params_from_ipfs( vault_contract=vault_contract, ipfs_hash=last_rewards.ipfs_hash, diff --git a/src/common/startup_check.py b/src/common/startup_check.py index 3b555188..72b076f5 100644 --- a/src/common/startup_check.py +++ b/src/common/startup_check.py @@ -484,8 +484,10 @@ async def _check_events_logs() -> None: async def _check_vault_withdrawable_assets() -> None: - harvest_params = await get_harvest_params() - withdrawable_assets = await get_withdrawable_assets(harvest_params=harvest_params) + harvest_params = await get_harvest_params(settings.vault) + withdrawable_assets = await get_withdrawable_assets( + settings.vault, harvest_params=harvest_params + ) # Note. We round down assets in the log message because of the case when assets # is slightly less than required amount to register validator. diff --git a/src/config/networks.py b/src/config/networks.py index c012b0d0..d187bbd6 100644 --- a/src/config/networks.py +++ b/src/config/networks.py @@ -25,6 +25,7 @@ class NetworkConfig(BaseNetworkConfig): OS_TOKEN_BALANCE_SYMBOL: str DEPOSIT_DATA_REGISTRY_CONTRACT_ADDRESS: ChecksumAddress VALIDATORS_CHECKER_CONTRACT_ADDRESS: ChecksumAddress + OS_TOKEN_REDEEMER_CONTRACT_ADDRESS: ChecksumAddress CONSOLIDATION_CONTRACT_ADDRESS: ChecksumAddress WITHDRAWAL_CONTRACT_ADDRESS: ChecksumAddress OS_TOKEN_CONTRACT_ADDRESS: ChecksumAddress @@ -90,6 +91,7 @@ def INITIAL_SYNC_ETA(self) -> int: VALIDATORS_CHECKER_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xA89629B41477560d49dd56ef1a59BD214362aCDC' ), + OS_TOKEN_REDEEMER_CONTRACT_ADDRESS=ZERO_CHECKSUM_ADDRESS, CONSOLIDATION_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x0000BBdDc7CE488642fb579F8B00f3a590007251' ), @@ -147,6 +149,9 @@ def INITIAL_SYNC_ETA(self) -> int: VALIDATORS_CHECKER_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xA89629B41477560d49dd56ef1a59BD214362aCDC' ), + OS_TOKEN_REDEEMER_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0x103A59fD85FB7aC085Cb4D59c805F11A1b5C1bFc' + ), CONSOLIDATION_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x0000BBdDc7CE488642fb579F8B00f3a590007251' ), @@ -200,6 +205,7 @@ def INITIAL_SYNC_ETA(self) -> int: VALIDATORS_CHECKER_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xA89629B41477560d49dd56ef1a59BD214362aCDC' ), + OS_TOKEN_REDEEMER_CONTRACT_ADDRESS=ZERO_CHECKSUM_ADDRESS, CONSOLIDATION_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x0000BBdDc7CE488642fb579F8B00f3a590007251' ), diff --git a/src/harvest/tasks.py b/src/harvest/tasks.py index 6d228edf..1467438a 100644 --- a/src/harvest/tasks.py +++ b/src/harvest/tasks.py @@ -19,7 +19,7 @@ async def process_block(self, interrupt_handler: InterruptHandler) -> None: """ # check current gas prices - harvest_params = await get_harvest_params() + harvest_params = await get_harvest_params(settings.vault) if not harvest_params: return diff --git a/src/main.py b/src/main.py index 6caeeece..41050432 100644 --- a/src/main.py +++ b/src/main.py @@ -13,6 +13,7 @@ from src.commands.create_wallet import create_wallet from src.commands.exit_validators import exit_validators from src.commands.init import init +from src.commands.internal.process_redeemer import process_redeemer from src.commands.internal.update_redeemable_positions import ( update_redeemable_positions, ) @@ -61,6 +62,7 @@ def cli() -> None: cli.add_command(node_start) cli.add_command(node_status) cli.add_command(update_redeemable_positions) +cli.add_command(process_redeemer) if __name__ == '__main__': diff --git a/src/redeem/os_token_converter.py b/src/redeem/os_token_converter.py index a669ea7a..01674490 100644 --- a/src/redeem/os_token_converter.py +++ b/src/redeem/os_token_converter.py @@ -18,6 +18,11 @@ def to_shares(self, assets: Wei) -> Wei: return Wei(0) return Wei((assets * self.total_shares) // self.total_assets) + def to_assets(self, shares: Wei) -> Wei: + if self.total_shares == 0: + return Wei(0) + return Wei((shares * self.total_assets) // self.total_shares) + async def create_os_token_converter(block_number: BlockNumber) -> OsTokenConverter: total_assets = await os_token_vault_controller_contract.total_assets(block_number) diff --git a/src/redeem/typings.py b/src/redeem/typings.py index 0eb3a339..a2f92229 100644 --- a/src/redeem/typings.py +++ b/src/redeem/typings.py @@ -1,7 +1,7 @@ import dataclasses from dataclasses import dataclass -from eth_typing import ChecksumAddress +from eth_typing import ChecksumAddress, HexStr from web3 import Web3 from web3.types import Wei @@ -57,6 +57,7 @@ class RedeemablePosition: owner: ChecksumAddress vault: ChecksumAddress amount: Wei + redeemable_shares: Wei = Wei(0) def as_dict(self) -> dict: return dataclasses.asdict(self) @@ -64,3 +65,17 @@ def as_dict(self) -> dict: @property def merkle_leaf(self) -> tuple[ChecksumAddress, ChecksumAddress, Wei]: return self.owner, self.vault, self.amount + + +@dataclass +class RedeemablePositionsMeta: + merkle_root: HexStr + ipfs_hash: str + + +@dataclass +class OsTokenPosition: + owner: ChecksumAddress + vault: ChecksumAddress + leaf_shares: Wei + shares_to_redeem: Wei diff --git a/src/reward_splitter/tasks.py b/src/reward_splitter/tasks.py index 30ab8bef..75fe1e22 100644 --- a/src/reward_splitter/tasks.py +++ b/src/reward_splitter/tasks.py @@ -62,7 +62,7 @@ async def process_block(self, interrupt_handler: InterruptHandler) -> None: 'Processing fee splitter %s ', reward_splitter.address, ) - harvest_params = await get_harvest_params() + harvest_params = await get_harvest_params(settings.vault) exit_requests = splitter_to_exit_requests.get(reward_splitter.address, []) # nosec reward_splitter_calls = await _get_reward_splitter_calls( diff --git a/src/validators/execution.py b/src/validators/execution.py index b241637a..bddcd563 100644 --- a/src/validators/execution.py +++ b/src/validators/execution.py @@ -206,9 +206,11 @@ def process_network_validator_event( return None -async def get_withdrawable_assets(harvest_params: HarvestParams | None) -> Wei: +async def get_withdrawable_assets( + vault: ChecksumAddress, harvest_params: HarvestParams | None +) -> Wei: """Fetches vault's available assets for staking.""" - vault_contract = VaultContract(settings.vault) + vault_contract = VaultContract(vault) if harvest_params is None: return await vault_contract.functions.withdrawableAssets().call() diff --git a/src/validators/tasks.py b/src/validators/tasks.py index 8274f84d..61a63b1a 100644 --- a/src/validators/tasks.py +++ b/src/validators/tasks.py @@ -49,7 +49,7 @@ async def process(self) -> None: await update_unused_validator_keys_metric( keystore=self.keystore, ) - harvest_params = await get_harvest_params() + harvest_params = await get_harvest_params(settings.vault) vault_assets = await get_vault_assets(harvest_params=harvest_params) vault_assets = Gwei(max(0, vault_assets - settings.vault_min_balance_gwei)) @@ -258,7 +258,7 @@ async def register_new_validators( async def get_vault_assets(harvest_params: HarvestParams | None) -> Gwei: - vault_assets = await get_withdrawable_assets(harvest_params=harvest_params) + vault_assets = await get_withdrawable_assets(settings.vault, harvest_params=harvest_params) if settings.network in GNO_NETWORKS: # apply GNO -> mGNO exchange rate vault_assets = convert_to_mgno(vault_assets) diff --git a/src/withdrawals/assets.py b/src/withdrawals/assets.py index 4a66544d..ceaa76fe 100644 --- a/src/withdrawals/assets.py +++ b/src/withdrawals/assets.py @@ -44,7 +44,7 @@ async def get_queued_assets( pending_partial_withdrawals: list[PendingPartialWithdrawal], chain_head: ChainHead, ) -> Gwei: - harvest_params = await get_harvest_params(chain_head.block_number) + harvest_params = await get_harvest_params(settings.vault, chain_head.block_number) # Get exit queue cumulative tickets exit_queue_cumulative_ticket = ( From 1e71e0f1be90c2d77090ef61972e12b9da55af3a Mon Sep 17 00:00:00 2001 From: cyc60 Date: Thu, 22 Jan 2026 17:32:13 +0300 Subject: [PATCH 25/65] Review fixes Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 32 +++++++++++++++---- src/common/contracts.py | 9 ++++++ src/config/networks.py | 16 ++++++++++ src/redeem/graph.py | 2 +- src/redeem/typings.py | 12 ++++--- 5 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index d58e687f..263e8be2 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -13,11 +13,12 @@ from src.common.clients import ( build_ipfs_upload_clients, + close_clients, execution_client, get_execution_client, setup_clients, ) -from src.common.contracts import Erc20Contract +from src.common.contracts import Erc20Contract, os_token_redeemer_contract from src.common.logging import LOG_LEVELS, setup_logging from src.common.utils import log_verbose from src.config.networks import AVAILABLE_NETWORKS, MAINNET, ZERO_CHECKSUM_ADDRESS @@ -149,13 +150,23 @@ def update_redeemable_positions( sys.exit(1) -# pylint: disable-next=too-many-locals async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: Gwei) -> None: + setup_logging() + await setup_clients() + try: + await process( + arbitrum_endpoint=arbitrum_endpoint, + min_os_token_position_amount_gwei=min_os_token_position_amount_gwei, + ) + finally: + await close_clients() + + +# pylint: disable-next=too-many-locals +async def process(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: Gwei) -> None: """ Fetch redeemable positions, calculate kept os token amounts and upload to IPFS. """ - setup_logging() - await setup_clients() block_number = await execution_client.eth.block_number logger.info('Fetching allocators from the subgraph...') allocators = await graph_get_allocators(block_number) @@ -220,8 +231,17 @@ async def main(arbitrum_endpoint: str | None, min_os_token_position_amount_gwei: click.echo(f'Redeemable position uploaded to IPFS: hash={ipfs_hash}') # calculate merkle root - leaves = [r.merkle_leaf for r in redeemable_positions] - tree = StandardMerkleTree.of(leaves, ['address', 'address', 'uint160']) + nonce = await os_token_redeemer_contract.nonce() + leaves = [r.merkle_leaf(nonce) for r in redeemable_positions] + tree = StandardMerkleTree.of( + leaves, + [ + 'uint256', + 'address', + 'uint256', + 'address', + ], + ) logger.info('Generated Merkle Tree root: %s', tree.root) diff --git a/src/common/contracts.py b/src/common/contracts.py index 926bd639..39df8978 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -484,6 +484,14 @@ async def tx_aggregate( return Web3.to_hex(tx_hash) +class OsTokenRedeemerContract(ContractWrapper): + abi_path = 'abi/IOsTokenRedeemer.json' + settings_key = 'OS_TOKEN_REDEEMER_CONTRACT_ADDRESS' + + async def nonce(self) -> int: + return await self.contract.functions.nonce().call() + + class ValidatorsCheckerContract(ContractWrapper): abi_path = 'abi/IValidatorsChecker.json' settings_key = 'VALIDATORS_CHECKER_CONTRACT_ADDRESS' @@ -570,3 +578,4 @@ def _get_exit_queue_missing_assets_call(self, params: ExitQueueMissingAssetsPara multicall_contract = MulticallContract() validators_checker_contract = ValidatorsCheckerContract() os_token_vault_controller_contract = OsTokenVaultControllerContract() +os_token_redeemer_contract = OsTokenRedeemerContract() diff --git a/src/config/networks.py b/src/config/networks.py index c012b0d0..474abbe9 100644 --- a/src/config/networks.py +++ b/src/config/networks.py @@ -25,8 +25,10 @@ class NetworkConfig(BaseNetworkConfig): OS_TOKEN_BALANCE_SYMBOL: str DEPOSIT_DATA_REGISTRY_CONTRACT_ADDRESS: ChecksumAddress VALIDATORS_CHECKER_CONTRACT_ADDRESS: ChecksumAddress + OS_TOKEN_REDEEMER_CONTRACT_ADDRESS: ChecksumAddress CONSOLIDATION_CONTRACT_ADDRESS: ChecksumAddress WITHDRAWAL_CONTRACT_ADDRESS: ChecksumAddress + OS_TOKEN_VAULT_CONTROLLER_CONTRACT_ADDRESS: ChecksumAddress OS_TOKEN_CONTRACT_ADDRESS: ChecksumAddress OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS: ChecksumAddress WALLET_MIN_BALANCE: Wei @@ -90,12 +92,16 @@ def INITIAL_SYNC_ETA(self) -> int: VALIDATORS_CHECKER_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xA89629B41477560d49dd56ef1a59BD214362aCDC' ), + OS_TOKEN_REDEEMER_CONTRACT_ADDRESS=ZERO_CHECKSUM_ADDRESS, CONSOLIDATION_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x0000BBdDc7CE488642fb579F8B00f3a590007251' ), WITHDRAWAL_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x00000961Ef480Eb55e80D19ad83579A64c007002' ), + OS_TOKEN_VAULT_CONTROLLER_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0x2A261e60FB14586B474C208b1B7AC6D0f5000306' + ), OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xf1C9acDc66974dFB6dEcB12aA385b9cD01190E38' ), @@ -147,12 +153,18 @@ def INITIAL_SYNC_ETA(self) -> int: VALIDATORS_CHECKER_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xA89629B41477560d49dd56ef1a59BD214362aCDC' ), + OS_TOKEN_REDEEMER_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0x103A59fD85FB7aC085Cb4D59c805F11A1b5C1bFc' + ), CONSOLIDATION_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x0000BBdDc7CE488642fb579F8B00f3a590007251' ), WITHDRAWAL_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x00000961Ef480Eb55e80D19ad83579A64c007002' ), + OS_TOKEN_VAULT_CONTROLLER_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0x140Fc69Eabd77fFF91d9852B612B2323256f7Ac1' + ), OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x7345fC8268459413beE9e9dd327f31283C65Ee7e' ), @@ -200,12 +212,16 @@ def INITIAL_SYNC_ETA(self) -> int: VALIDATORS_CHECKER_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xA89629B41477560d49dd56ef1a59BD214362aCDC' ), + OS_TOKEN_REDEEMER_CONTRACT_ADDRESS=ZERO_CHECKSUM_ADDRESS, CONSOLIDATION_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x0000BBdDc7CE488642fb579F8B00f3a590007251' ), WITHDRAWAL_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x00000961Ef480Eb55e80D19ad83579A64c007002' ), + OS_TOKEN_VAULT_CONTROLLER_CONTRACT_ADDRESS=Web3.to_checksum_address( + '0x60B2053d7f2a0bBa70fe6CDd88FB47b579B9179a' + ), OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xF490c80aAE5f2616d3e3BDa2483E30C4CB21d1A0' ), diff --git a/src/redeem/graph.py b/src/redeem/graph.py index 9e3929b5..c925b307 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -102,7 +102,7 @@ async def graph_get_os_token_holders(block_number: BlockNumber) -> dict[Checksum where:{ balance_gt: 0 } - orderBy: balance, + orderBy: id, first: $first skip: $skip ) { diff --git a/src/redeem/typings.py b/src/redeem/typings.py index 0eb3a339..1d5ada5e 100644 --- a/src/redeem/typings.py +++ b/src/redeem/typings.py @@ -1,4 +1,3 @@ -import dataclasses from dataclasses import dataclass from eth_typing import ChecksumAddress @@ -59,8 +58,11 @@ class RedeemablePosition: amount: Wei def as_dict(self) -> dict: - return dataclasses.asdict(self) + return { + 'owner': self.owner, + 'vault': self.vault, + 'amount': str(self.amount), + } - @property - def merkle_leaf(self) -> tuple[ChecksumAddress, ChecksumAddress, Wei]: - return self.owner, self.vault, self.amount + def merkle_leaf(self, nonce: int) -> tuple[int, ChecksumAddress, Wei, ChecksumAddress]: + return nonce, self.vault, self.amount, self.owner From 93b2f650a59267d9f00a30426ceb060ac5d6a993 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Thu, 22 Jan 2026 17:45:17 +0300 Subject: [PATCH 26/65] Rm OS_TOKEN_VAULT_CONTROLLER_CONTRACT_ADDRESS Signed-off-by: cyc60 --- src/config/networks.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/config/networks.py b/src/config/networks.py index 474abbe9..d187bbd6 100644 --- a/src/config/networks.py +++ b/src/config/networks.py @@ -28,7 +28,6 @@ class NetworkConfig(BaseNetworkConfig): OS_TOKEN_REDEEMER_CONTRACT_ADDRESS: ChecksumAddress CONSOLIDATION_CONTRACT_ADDRESS: ChecksumAddress WITHDRAWAL_CONTRACT_ADDRESS: ChecksumAddress - OS_TOKEN_VAULT_CONTROLLER_CONTRACT_ADDRESS: ChecksumAddress OS_TOKEN_CONTRACT_ADDRESS: ChecksumAddress OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS: ChecksumAddress WALLET_MIN_BALANCE: Wei @@ -99,9 +98,6 @@ def INITIAL_SYNC_ETA(self) -> int: WITHDRAWAL_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x00000961Ef480Eb55e80D19ad83579A64c007002' ), - OS_TOKEN_VAULT_CONTROLLER_CONTRACT_ADDRESS=Web3.to_checksum_address( - '0x2A261e60FB14586B474C208b1B7AC6D0f5000306' - ), OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xf1C9acDc66974dFB6dEcB12aA385b9cD01190E38' ), @@ -162,9 +158,6 @@ def INITIAL_SYNC_ETA(self) -> int: WITHDRAWAL_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x00000961Ef480Eb55e80D19ad83579A64c007002' ), - OS_TOKEN_VAULT_CONTROLLER_CONTRACT_ADDRESS=Web3.to_checksum_address( - '0x140Fc69Eabd77fFF91d9852B612B2323256f7Ac1' - ), OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x7345fC8268459413beE9e9dd327f31283C65Ee7e' ), @@ -219,9 +212,6 @@ def INITIAL_SYNC_ETA(self) -> int: WITHDRAWAL_CONTRACT_ADDRESS=Web3.to_checksum_address( '0x00000961Ef480Eb55e80D19ad83579A64c007002' ), - OS_TOKEN_VAULT_CONTROLLER_CONTRACT_ADDRESS=Web3.to_checksum_address( - '0x60B2053d7f2a0bBa70fe6CDd88FB47b579B9179a' - ), OS_TOKEN_CONTRACT_ADDRESS=Web3.to_checksum_address( '0xF490c80aAE5f2616d3e3BDa2483E30C4CB21d1A0' ), From 6d6b0fc91b89405b84a5f50c707e3f51394440bd Mon Sep 17 00:00:00 2001 From: cyc60 Date: Thu, 22 Jan 2026 23:26:49 +0300 Subject: [PATCH 27/65] Add integration test for update_redeemable_positions Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 38 +++- .../test_update_redeemable_positions.py | 181 ++++++++++++++++++ src/redeem/graph.py | 2 - 3 files changed, 210 insertions(+), 11 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 263e8be2..b160cfae 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -2,6 +2,7 @@ import logging import sys from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import cast @@ -139,12 +140,33 @@ def update_redeemable_positions( log_level=log_level, ) try: - asyncio.run( - main( - arbitrum_endpoint=arbitrum_endpoint, - min_os_token_position_amount_gwei=Gwei(min_os_token_position_amount_gwei), - ) - ) + # Try-catch to enable async calls in test - an event loop + # will already be running in that case + try: + asyncio.get_running_loop() + # we need to create a separate thread so we can block before returning + with ThreadPoolExecutor(1) as pool: + pool.submit( + lambda: asyncio.run( + main( + arbitrum_endpoint=arbitrum_endpoint, + min_os_token_position_amount_gwei=Gwei( + min_os_token_position_amount_gwei + ), + ) + ) + ).result() + except RuntimeError as e: + if 'no running event loop' == e.args[0]: + # no event loop running + asyncio.run( + main( + arbitrum_endpoint=arbitrum_endpoint, + min_os_token_position_amount_gwei=Gwei(min_os_token_position_amount_gwei), + ) + ) + else: + raise e except Exception as e: log_verbose(e) sys.exit(1) @@ -167,7 +189,7 @@ async def process(arbitrum_endpoint: str | None, min_os_token_position_amount_gw """ Fetch redeemable positions, calculate kept os token amounts and upload to IPFS. """ - block_number = await execution_client.eth.block_number + block_number = await execution_client.eth.get_block_number() logger.info('Fetching allocators from the subgraph...') allocators = await graph_get_allocators(block_number) logger.info('Fetched %s allocators from the subgraph', len(allocators)) @@ -177,7 +199,6 @@ async def process(arbitrum_endpoint: str | None, min_os_token_position_amount_gw boost_proxies = {pos.proxy for pos in leverage_positions} logger.info('Found %s proxy positions to exclude', len(boost_proxies)) allocators = [a for a in allocators if a.address not in boost_proxies] - # reduce boosted positions logger.info('Fetching boosted positions from the subgraph...') os_token_converter = await create_os_token_converter(block_number) @@ -210,7 +231,6 @@ async def process(arbitrum_endpoint: str | None, min_os_token_position_amount_gw if not redeemable_positions: logger.info('No redeemable positions to upload, exiting...') return - total_redeemable = sum(p.amount for p in redeemable_positions) logger.info( 'Created %s redeemable positions. Total redeemed %s amount: %s (%s %s)', diff --git a/src/commands/tests/test_internal/test_update_redeemable_positions.py b/src/commands/tests/test_internal/test_update_redeemable_positions.py index 6a6235f8..3e6c51a8 100644 --- a/src/commands/tests/test_internal/test_update_redeemable_positions.py +++ b/src/commands/tests/test_internal/test_update_redeemable_positions.py @@ -1,3 +1,9 @@ +import contextlib +from unittest import mock +from unittest.mock import AsyncMock + +import pytest +from click.testing import CliRunner from sw_utils.tests import faker from web3 import Web3 from web3.types import Wei @@ -6,7 +12,10 @@ _reduce_boosted_amount, calculate_boost_ostoken_shares, create_redeemable_positions, + update_redeemable_positions, ) +from src.config.networks import MAINNET, NETWORKS +from src.config.settings import settings from src.redeem.os_token_converter import OsTokenConverter from src.redeem.typings import ( Allocator, @@ -221,3 +230,175 @@ def test_reduces_boosted_amount(): ], ), ] + + +@pytest.mark.usefixtures('_init_config') +class TestUpdateRedeemablePositions: + @pytest.mark.usefixtures('fake_settings', 'setup_test_clients') + async def test_update_redeemable_positions( + self, + vault_address: str, + execution_endpoints: str, + runner: CliRunner, + ): + address_1 = faker.eth_address() + address_2 = faker.eth_address() + vault_1 = faker.eth_address() + vault_2 = faker.eth_address() + # mocked data + allocators = [ + { + 'vault': { + 'id': vault_1.lower(), + }, + 'id': address_1.lower(), + 'address': address_1, + 'mintedOsTokenShares': 1, + } + ] + leverage_positions = [ + { + 'user': address_1, + 'vault': { + 'id': vault_1.lower(), + }, + 'proxy': faker.eth_address(), + 'osTokenShares': '1000', + 'exitingOsTokenShares': '500', + 'assets': '200', + 'exitingAssets': '100', + }, + { + 'user': address_2, + 'vault': { + 'id': vault_2.lower(), + }, + 'proxy': faker.eth_address(), + 'osTokenShares': '3000', + 'exitingOsTokenShares': '0', + 'assets': '100', + 'exitingAssets': '0', + }, + ] + os_token_holders = [ + { + 'id': address_1.lower(), + 'balance': '1000', + }, + { + 'id': address_2.lower(), + 'balance': '5000', + }, + ] + mock_protocol_data = [ + { + 'id': 'stakewise', + 'portfolio_item_list': [ + { + 'detail': { + 'supply_token_list': [ + { + 'id': '0x1234567890abcdef1234567890abcdef12345678', + 'chain': 'eth', + 'amount': '57', + } + ] + } + } + ], + }, + { + 'id': 'other', + 'portfolio_item_list': [ + { + 'detail': { + 'supply_token_list': [ + { + 'id': settings.network_config.OS_TOKEN_CONTRACT_ADDRESS, + 'chain': 'eth', + 'amount': '555', + } + ] + } + } + ], + }, + ] + os_token_converter = OsTokenConverter(105, 100) + args = [ + '--network', + MAINNET, + '--execution-endpoints', + execution_endpoints, + '--arbitrum-endpoint', + execution_endpoints, + '--verbose', + ] + with ( + patch_ipfs_upload(), + patch_latest_block(11), + patch_get_erc_balance(11), + patch_os_token_redeemer_contract_nonce(6), + patch_os_token_arbitrum_contract_address(), + mock.patch( + 'src.redeem.graph.graph_client.fetch_pages', + side_effect=[allocators, leverage_positions, os_token_holders], + ), + mock.patch( + 'src.commands.internal.update_redeemable_positions.create_os_token_converter', + return_value=os_token_converter, + ), + mock.patch( + 'src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data + ), + ): + result = runner.invoke(update_redeemable_positions, args) + assert result.exit_code == 0 + assert '' == result.output.strip() + + +@contextlib.contextmanager +def patch_ipfs_upload(): + with mock.patch( + 'src.commands.internal.update_redeemable_positions.build_ipfs_upload_clients', + new=AsyncMock(), + ) as ipfs_mock: + ipfs_mock.upload_json = 'bafk...' + yield + + +@contextlib.contextmanager +def patch_latest_block(block_number): + with mock.patch( + 'src.commands.internal.update_redeemable_positions.execution_client', new=AsyncMock() + ) as execution_client_mock: + execution_client_mock.eth.get_block_number.return_value = block_number + yield + + +@contextlib.contextmanager +def patch_get_erc_balance(balance): + with mock.patch( + 'src.commands.internal.update_redeemable_positions.Erc20Contract.get_balance', + return_value=balance, + ): + yield + + +@contextlib.contextmanager +def patch_os_token_arbitrum_contract_address(): + with mock.patch.object( + settings.network_config, + 'OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS', + NETWORKS[MAINNET].OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS, + ): + yield + + +@contextlib.contextmanager +def patch_os_token_redeemer_contract_nonce(nonce): + with mock.patch( + 'src.commands.internal.update_redeemable_positions.os_token_redeemer_contract.nonce', + return_value=nonce, + ): + yield diff --git a/src/redeem/graph.py b/src/redeem/graph.py index c925b307..250f872b 100644 --- a/src/redeem/graph.py +++ b/src/redeem/graph.py @@ -33,8 +33,6 @@ async def graph_get_allocators(block_number: BlockNumber) -> list[Allocator]: } id address - shares - assets mintedOsTokenShares } } From 0ec716fc27baf944c811c6032e50d2cd39ebbbce Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 26 Jan 2026 13:04:44 +0300 Subject: [PATCH 28/65] Add integration test for update_redeemable_positions Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 2 +- .../test_update_redeemable_positions.py | 112 +++++++++++------- 2 files changed, 69 insertions(+), 45 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index b160cfae..9c7c4a52 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -262,7 +262,7 @@ async def process(arbitrum_endpoint: str | None, min_os_token_position_amount_gw 'address', ], ) - logger.info('Generated Merkle Tree root: %s', tree.root) + click.echo(f'Generated Merkle Tree root: {tree.root}') async def get_kept_shares( diff --git a/src/commands/tests/test_internal/test_update_redeemable_positions.py b/src/commands/tests/test_internal/test_update_redeemable_positions.py index 3e6c51a8..0fed82dd 100644 --- a/src/commands/tests/test_internal/test_update_redeemable_positions.py +++ b/src/commands/tests/test_internal/test_update_redeemable_positions.py @@ -6,7 +6,7 @@ from click.testing import CliRunner from sw_utils.tests import faker from web3 import Web3 -from web3.types import Wei +from web3.types import ChecksumAddress, Wei from src.commands.internal.update_redeemable_positions import ( _reduce_boosted_amount, @@ -241,11 +241,12 @@ async def test_update_redeemable_positions( execution_endpoints: str, runner: CliRunner, ): - address_1 = faker.eth_address() - address_2 = faker.eth_address() - vault_1 = faker.eth_address() - vault_2 = faker.eth_address() - # mocked data + # hardcoded to check merkle root + address_1 = '0x2242b8ab71521f6abEE4B4D83195E70AcB08727a' + address_2 = '0x24c8DBBC3d1C35C4159787b1f7a62bea1A814242' + vault_1 = '0xEd735de172272C03CA6F60c1d90D83D9CFB46D22' + vault_2 = '0xe8Ea1025b49D2B51C536cFBc0833F021ba4c6903' + os_token_contract_address = NETWORKS[MAINNET].OS_TOKEN_CONTRACT_ADDRESS allocators = [ { 'vault': { @@ -253,41 +254,38 @@ async def test_update_redeemable_positions( }, 'id': address_1.lower(), 'address': address_1, - 'mintedOsTokenShares': 1, - } - ] - leverage_positions = [ + 'mintedOsTokenShares': Web3.to_wei(10, 'ether'), + }, { - 'user': address_1, 'vault': { - 'id': vault_1.lower(), + 'id': vault_2.lower(), }, - 'proxy': faker.eth_address(), - 'osTokenShares': '1000', - 'exitingOsTokenShares': '500', - 'assets': '200', - 'exitingAssets': '100', + 'id': address_2.lower(), + 'address': address_2, + 'mintedOsTokenShares': Web3.to_wei(12, 'ether'), }, + ] + leverage_positions = [ { - 'user': address_2, + 'user': address_1, 'vault': { - 'id': vault_2.lower(), + 'id': vault_1.lower(), }, 'proxy': faker.eth_address(), - 'osTokenShares': '3000', - 'exitingOsTokenShares': '0', - 'assets': '100', - 'exitingAssets': '0', + 'osTokenShares': Web3.to_wei(1, 'ether'), + 'exitingOsTokenShares': Web3.to_wei(0.1, 'ether'), + 'assets': Web3.to_wei(0.1, 'ether'), + 'exitingAssets': Web3.to_wei(0.05, 'ether'), }, ] os_token_holders = [ { 'id': address_1.lower(), - 'balance': '1000', + 'balance': Web3.to_wei(3, 'ether'), }, { 'id': address_2.lower(), - 'balance': '5000', + 'balance': Web3.to_wei(12, 'ether'), }, ] mock_protocol_data = [ @@ -300,7 +298,7 @@ async def test_update_redeemable_positions( { 'id': '0x1234567890abcdef1234567890abcdef12345678', 'chain': 'eth', - 'amount': '57', + 'amount': '0.5', } ] } @@ -308,15 +306,31 @@ async def test_update_redeemable_positions( ], }, { - 'id': 'other', + 'id': 'aave3', 'portfolio_item_list': [ { 'detail': { 'supply_token_list': [ { - 'id': settings.network_config.OS_TOKEN_CONTRACT_ADDRESS, + 'id': os_token_contract_address, 'chain': 'eth', - 'amount': '555', + 'amount': '2', + } + ] + } + } + ], + }, + { + 'id': 'balancer', + 'portfolio_item_list': [ + { + 'detail': { + 'supply_token_list': [ + { + 'id': os_token_contract_address, + 'chain': 'eth', + 'amount': '0.2', } ] } @@ -324,7 +338,7 @@ async def test_update_redeemable_positions( ], }, ] - os_token_converter = OsTokenConverter(105, 100) + os_token_converter = OsTokenConverter(110, 100) args = [ '--network', MAINNET, @@ -335,11 +349,11 @@ async def test_update_redeemable_positions( '--verbose', ] with ( - patch_ipfs_upload(), patch_latest_block(11), - patch_get_erc_balance(11), + patch_get_erc_balance(Web3.to_wei(1, 'ether')), patch_os_token_redeemer_contract_nonce(6), patch_os_token_arbitrum_contract_address(), + patch_os_token_contract_address(os_token_contract_address), mock.patch( 'src.redeem.graph.graph_client.fetch_pages', side_effect=[allocators, leverage_positions, os_token_holders], @@ -351,20 +365,20 @@ async def test_update_redeemable_positions( mock.patch( 'src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data ), + mock.patch( + 'src.common.clients.IpfsMultiUploadClient.upload_json', + return_value=faker.ipfs_hash(), + ) as ipfs_mock, ): - result = runner.invoke(update_redeemable_positions, args) + result = runner.invoke(update_redeemable_positions, args, input='\n') assert result.exit_code == 0 - assert '' == result.output.strip() - - -@contextlib.contextmanager -def patch_ipfs_upload(): - with mock.patch( - 'src.commands.internal.update_redeemable_positions.build_ipfs_upload_clients', - new=AsyncMock(), - ) as ipfs_mock: - ipfs_mock.upload_json = 'bafk...' - yield + ipfs_mock.assert_called_once_with( + [{'owner': address_1, 'vault': vault_1, 'amount': '2563636363636363637'}] + ) + assert ( + '0x9bb2ee30813b89e23e6bbfa1b78706c008f71489750571c81d3b33289647bec1' + in result.output.strip() + ) @contextlib.contextmanager @@ -395,6 +409,16 @@ def patch_os_token_arbitrum_contract_address(): yield +@contextlib.contextmanager +def patch_os_token_contract_address(address: ChecksumAddress): + with mock.patch.object( + settings.network_config, + 'OS_TOKEN_CONTRACT_ADDRESS', + address, + ): + yield + + @contextlib.contextmanager def patch_os_token_redeemer_contract_nonce(nonce): with mock.patch( From 5b266cde340b013605b3093c909a53bd4876ed96 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 26 Jan 2026 13:26:22 +0300 Subject: [PATCH 29/65] Rename package Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 15 +++++++++++---- .../test_update_redeemable_positions.py | 8 ++++---- src/{redeem => redemptions}/__init__.py | 0 src/{redeem => redemptions}/api_client.py | 0 src/{redeem => redemptions}/graph.py | 2 +- .../os_token_converter.py | 0 src/{redeem => redemptions}/tests/__init__.py | 0 .../tests/api_samples/protocols.json | 0 .../tests/api_samples/with_boost.json | 0 .../tests/test_api_client.py | 16 +++++++++------- src/{redeem => redemptions}/tests/test_graph.py | 10 +++++----- src/{redeem => redemptions}/typings.py | 0 12 files changed, 30 insertions(+), 21 deletions(-) rename src/{redeem => redemptions}/__init__.py (100%) rename src/{redeem => redemptions}/api_client.py (100%) rename src/{redeem => redemptions}/graph.py (97%) rename src/{redeem => redemptions}/os_token_converter.py (100%) rename src/{redeem => redemptions}/tests/__init__.py (100%) rename src/{redeem => redemptions}/tests/api_samples/protocols.json (100%) rename src/{redeem => redemptions}/tests/api_samples/with_boost.json (100%) rename src/{redeem => redemptions}/tests/test_api_client.py (85%) rename src/{redeem => redemptions}/tests/test_graph.py (81%) rename src/{redeem => redemptions}/typings.py (100%) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 9c7c4a52..27bda8f1 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -24,14 +24,21 @@ from src.common.utils import log_verbose from src.config.networks import AVAILABLE_NETWORKS, MAINNET, ZERO_CHECKSUM_ADDRESS from src.config.settings import settings -from src.redeem.api_client import API_SLEEP_TIMEOUT, APIClient -from src.redeem.graph import ( +from src.redemptions.api_client import API_SLEEP_TIMEOUT, APIClient +from src.redemptions.graph import ( graph_get_allocators, graph_get_leverage_positions, graph_get_os_token_holders, ) -from src.redeem.os_token_converter import OsTokenConverter, create_os_token_converter -from src.redeem.typings import Allocator, LeverageStrategyPosition, RedeemablePosition +from src.redemptions.os_token_converter import ( + OsTokenConverter, + create_os_token_converter, +) +from src.redemptions.typings import ( + Allocator, + LeverageStrategyPosition, + RedeemablePosition, +) logger = logging.getLogger(__name__) diff --git a/src/commands/tests/test_internal/test_update_redeemable_positions.py b/src/commands/tests/test_internal/test_update_redeemable_positions.py index 0fed82dd..af980d05 100644 --- a/src/commands/tests/test_internal/test_update_redeemable_positions.py +++ b/src/commands/tests/test_internal/test_update_redeemable_positions.py @@ -16,8 +16,8 @@ ) from src.config.networks import MAINNET, NETWORKS from src.config.settings import settings -from src.redeem.os_token_converter import OsTokenConverter -from src.redeem.typings import ( +from src.redemptions.os_token_converter import OsTokenConverter +from src.redemptions.typings import ( Allocator, LeverageStrategyPosition, RedeemablePosition, @@ -355,7 +355,7 @@ async def test_update_redeemable_positions( patch_os_token_arbitrum_contract_address(), patch_os_token_contract_address(os_token_contract_address), mock.patch( - 'src.redeem.graph.graph_client.fetch_pages', + 'src.redemptions.graph.graph_client.fetch_pages', side_effect=[allocators, leverage_positions, os_token_holders], ), mock.patch( @@ -363,7 +363,7 @@ async def test_update_redeemable_positions( return_value=os_token_converter, ), mock.patch( - 'src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data + 'src.redemptions.api_client.APIClient._fetch_json', return_value=mock_protocol_data ), mock.patch( 'src.common.clients.IpfsMultiUploadClient.upload_json', diff --git a/src/redeem/__init__.py b/src/redemptions/__init__.py similarity index 100% rename from src/redeem/__init__.py rename to src/redemptions/__init__.py diff --git a/src/redeem/api_client.py b/src/redemptions/api_client.py similarity index 100% rename from src/redeem/api_client.py rename to src/redemptions/api_client.py diff --git a/src/redeem/graph.py b/src/redemptions/graph.py similarity index 97% rename from src/redeem/graph.py rename to src/redemptions/graph.py index 250f872b..46e152b6 100644 --- a/src/redeem/graph.py +++ b/src/redemptions/graph.py @@ -7,7 +7,7 @@ from web3.types import ChecksumAddress, Wei from src.common.clients import graph_client -from src.redeem.typings import Allocator, LeverageStrategyPosition, VaultShares +from src.redemptions.typings import Allocator, LeverageStrategyPosition, VaultShares logger = logging.getLogger(__name__) diff --git a/src/redeem/os_token_converter.py b/src/redemptions/os_token_converter.py similarity index 100% rename from src/redeem/os_token_converter.py rename to src/redemptions/os_token_converter.py diff --git a/src/redeem/tests/__init__.py b/src/redemptions/tests/__init__.py similarity index 100% rename from src/redeem/tests/__init__.py rename to src/redemptions/tests/__init__.py diff --git a/src/redeem/tests/api_samples/protocols.json b/src/redemptions/tests/api_samples/protocols.json similarity index 100% rename from src/redeem/tests/api_samples/protocols.json rename to src/redemptions/tests/api_samples/protocols.json diff --git a/src/redeem/tests/api_samples/with_boost.json b/src/redemptions/tests/api_samples/with_boost.json similarity index 100% rename from src/redeem/tests/api_samples/with_boost.json rename to src/redemptions/tests/api_samples/with_boost.json diff --git a/src/redeem/tests/test_api_client.py b/src/redemptions/tests/test_api_client.py similarity index 85% rename from src/redeem/tests/test_api_client.py rename to src/redemptions/tests/test_api_client.py index e167f4e9..c027a737 100644 --- a/src/redeem/tests/test_api_client.py +++ b/src/redemptions/tests/test_api_client.py @@ -6,14 +6,14 @@ from web3.types import Wei from src.config.settings import settings -from src.redeem.api_client import APIClient +from src.redemptions.api_client import APIClient class TestAPIClient: @pytest.mark.usefixtures('fake_settings') async def test_zero_when_no_protocol_data(self): client = APIClient() - with patch('src.redeem.api_client.APIClient._fetch_json', return_value=[]): + with patch('src.redemptions.api_client.APIClient._fetch_json', return_value=[]): result = await client.get_protocols_locked_os_token( Web3.to_checksum_address('0x1234567890abcdef1234567890abcdef12345678') ) @@ -55,7 +55,9 @@ async def test_excludes_stakewise_protocol_from_total(self): ], }, ] - with patch('src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data): + with patch( + 'src.redemptions.api_client.APIClient._fetch_json', return_value=mock_protocol_data + ): client = APIClient() result = await client.get_protocols_locked_os_token( Web3.to_checksum_address('0x1234567890abcdef1234567890abcdef12345678') @@ -64,10 +66,10 @@ async def test_excludes_stakewise_protocol_from_total(self): @pytest.mark.usefixtures('fake_settings') async def test_real_data(self): - with open('src/redeem/tests/api_samples/protocols.json', 'r') as f: + with open('src/redemptions/tests/api_samples/protocols.json', 'r') as f: mock_protocol_data = json.load(f) with patch( - 'src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data + 'src.redemptions.api_client.APIClient._fetch_json', return_value=mock_protocol_data ), patch.object( settings.network_config, 'OS_TOKEN_CONTRACT_ADDRESS', @@ -85,10 +87,10 @@ async def test_real_data(self): @pytest.mark.usefixtures('fake_settings') async def test_real_data_with_boost(self): - with open('src/redeem/tests/api_samples/with_boost.json', 'r') as f: + with open('src/redemptions/tests/api_samples/with_boost.json', 'r') as f: mock_protocol_data = json.load(f) with patch( - 'src.redeem.api_client.APIClient._fetch_json', return_value=mock_protocol_data + 'src.redemptions.api_client.APIClient._fetch_json', return_value=mock_protocol_data ), patch.object( settings.network_config, 'OS_TOKEN_CONTRACT_ADDRESS', diff --git a/src/redeem/tests/test_graph.py b/src/redemptions/tests/test_graph.py similarity index 81% rename from src/redeem/tests/test_graph.py rename to src/redemptions/tests/test_graph.py index 20c93555..6b93b10d 100644 --- a/src/redeem/tests/test_graph.py +++ b/src/redemptions/tests/test_graph.py @@ -6,8 +6,8 @@ from web3 import Web3 from web3.types import Wei -from src.redeem.graph import graph_get_allocators -from src.redeem.typings import Allocator, VaultShares +from src.redemptions.graph import graph_get_allocators +from src.redemptions.typings import Allocator, VaultShares @pytest.mark.usefixtures('fake_settings') @@ -17,7 +17,7 @@ async def test_graph_get_allocators(): vault_1 = faker.eth_address().lower() vault_2 = faker.eth_address().lower() - with patch('src.redeem.graph.graph_client.fetch_pages', return_value=[]): + with patch('src.redemptions.graph.graph_client.fetch_pages', return_value=[]): result = await graph_get_allocators(random.randint(1, 1000000)) assert result == [] @@ -25,7 +25,7 @@ async def test_graph_get_allocators(): {'address': address_1, 'vault': {'id': vault_1}, 'mintedOsTokenShares': '0'}, {'address': address_2, 'vault': {'id': vault_2}, 'mintedOsTokenShares': '1000'}, ] - with patch('src.redeem.graph.graph_client.fetch_pages', return_value=mock_response): + with patch('src.redemptions.graph.graph_client.fetch_pages', return_value=mock_response): result = await graph_get_allocators(random.randint(1, 1000000)) assert result == [ Allocator( @@ -40,7 +40,7 @@ async def test_graph_get_allocators(): {'address': address_1, 'vault': {'id': vault_1}, 'mintedOsTokenShares': '150'}, {'address': address_1, 'vault': {'id': vault_2}, 'mintedOsTokenShares': '1000'}, ] - with patch('src.redeem.graph.graph_client.fetch_pages', return_value=mock_response): + with patch('src.redemptions.graph.graph_client.fetch_pages', return_value=mock_response): result = await graph_get_allocators(random.randint(1, 1000000)) assert result == [ Allocator( diff --git a/src/redeem/typings.py b/src/redemptions/typings.py similarity index 100% rename from src/redeem/typings.py rename to src/redemptions/typings.py From 526af08ded66cdf5abf1875070997176c5104256 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 27 Jan 2026 19:33:52 +0300 Subject: [PATCH 30/65] Metavault handling Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 268 +++++++++++++--------- src/common/contracts.py | 27 ++- src/redemptions/typings.py | 7 + 3 files changed, 186 insertions(+), 116 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 1212285c..0f3cdf9f 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -6,13 +6,20 @@ from typing import cast import click -from eth_typing import ChecksumAddress -from multiproof.standard import standard_leaf_hash +from eth_typing import BlockNumber, ChecksumAddress, HexStr +from sw_utils import InterruptHandler from web3 import Web3 +from web3.exceptions import ContractLogicError, Web3RPCError from web3.types import Wei -from src.common.clients import execution_client, ipfs_fetch_client, setup_clients +from src.common.clients import ( + close_clients, + execution_client, + ipfs_fetch_client, + setup_clients, +) from src.common.contracts import ( + MetaVaultContract, VaultContract, multicall_contract, os_token_redeemer_contract, @@ -25,12 +32,14 @@ from src.common.wallet import wallet from src.config.networks import AVAILABLE_NETWORKS, ZERO_CHECKSUM_ADDRESS from src.config.settings import settings -from src.redeem.os_token_converter import create_os_token_converter -from src.redeem.typings import OsTokenPosition, RedeemablePosition +from src.redemptions.os_token_converter import create_os_token_converter +from src.redemptions.typings import OsTokenPosition, RedeemablePosition from src.validators.execution import get_withdrawable_assets logger = logging.getLogger(__name__) +SLEEP_INTERVAL = 60 # 1 minute + @click.option( '--wallet-password-file', @@ -93,7 +102,11 @@ case_sensitive=False, ), ) -@click.command(help='Updates redeemable positions') +@click.command( + help='Monitors the EthOsTokenRedeemer/GnoOsTokenRedeemer contracts' + ' and automatically processes OsToken position redemptions' + ' and exit queue checkpoints.' +) # pylint: disable-next=too-many-arguments def process_redeemer( execution_endpoints: str, @@ -127,109 +140,139 @@ def process_redeemer( # pylint: disable-next=too-many-locals async def main() -> None: """ - Fetch redeemable positions, calculate kept os token amounts and upload to IPFS. + Monitors the EthOsTokenRedeemer/GnoOsTokenRedeemer contracts + and automatically processes OsToken position redemptions + and exit queue checkpoints. """ setup_logging() await setup_clients() await _startup_check() + try: + with InterruptHandler() as interrupt_handler: + while not interrupt_handler.exit: + block_number = await execution_client.eth.block_number + await process( + block_number=block_number, + ) + await interrupt_handler.sleep(SLEEP_INTERVAL) - while True: - block_number = await execution_client.eth.block_number - # Check Exit Queue Processing - can_process_exit_queue = await os_token_redeemer_contract.can_process_exit_queue( - block_number - ) - if can_process_exit_queue: - logger.info('Exit queue can be processed. Calling processExitQueue...') - tx_hash = await os_token_redeemer_contract.process_exit_queue() - logger.info('ProcessExitQueue transaction sent. Tx Hash: %s', tx_hash.hex()) - - # Check Queued Shares for Redemption - queued_shares = await os_token_redeemer_contract.queued_shares(block_number) - if queued_shares == 0: - logger.info('No queued shares for redemption. Skipping to next interval.') - await asyncio.sleep(300) # Sleep for 5 minutes before next check - continue - - os_token_converter = await create_os_token_converter(block_number) - queued_assets = os_token_converter.to_assets(queued_shares) - - # Fetch Positions from IPFS - redeemable_positions_meta = await os_token_redeemer_contract.redeemable_positions( - block_number - ) - redeemable_positions = await fetch_redeemable_positions(redeemable_positions_meta.ipfs_hash) + finally: + await close_clients() - # Calculate Redeemable Shares Per Position - for redeemable_position in redeemable_positions: - # Compute leaf hash - leaf_hash = get_redeemable_position_leaf_hash( - redeemable_position=redeemable_position, nonce=redeemable_positions_meta.nonce - 1 - ) - # Get already processed shares - processed_shares = await os_token_redeemer_contract.leaf_to_processed_shares( - leaf_hash, block_number - ) +# pylint: disable-next=too-many-locals +async def process(block_number: BlockNumber) -> None: + """ + Monitors the EthOsTokenRedeemer/GnoOsTokenRedeemer contracts + and automatically processes OsToken position redemptions + and exit queue checkpoints. + """ + # Check Exit Queue Processing + await _process_exit_queue(block_number) + + # Check Queued Shares for Redemption + queued_shares = await os_token_redeemer_contract.queued_shares(block_number) + if queued_shares == 0: + logger.info('No queued shares for redemption. Skipping to next interval.') + return + + os_token_converter = await create_os_token_converter(block_number) + nonce = await os_token_redeemer_contract.nonce(block_number) + + queued_assets = os_token_converter.to_assets(queued_shares) + logger.info( + 'Queued Shares for Redemption: %s(~%s assets)', + queued_shares, + Web3.from_wei(queued_assets, 'ether'), + ) - # Calculate redeemable shares - redeemable_position.redeemable_shares = redeemable_position.amount - processed_shares + # Fetch Positions from IPFS + redeemable_positions_meta = await os_token_redeemer_contract.redeemable_positions(block_number) + redeemable_positions = await fetch_redeemable_positions(redeemable_positions_meta.ipfs_hash) + + # Calculate Redeemable Shares Per Position + for redeemable_position in redeemable_positions: + # Compute leaf hash + leaf_hash = redeemable_position.merkle_leaf_bytes(nonce - 1) + # Get already processed shares + leaf_processed_shares = await os_token_redeemer_contract.leaf_to_processed_shares( + leaf_hash, block_number + ) + # Calculate redeemable shares + redeemable_position.redeemable_shares = Wei( + redeemable_position.amount - leaf_processed_shares + ) - # Filter Positions by Vault Withdrawable Assets + # Group positions by vault + vault_to_positions: defaultdict[ChecksumAddress, list[RedeemablePosition]] = defaultdict(list) + for position in redeemable_positions: + vault_to_positions[position.vault].append(position) - # Group positions by vault - vault_to_positions: defaultdict[ChecksumAddress, list[RedeemablePosition]] = defaultdict( - list - ) - for position in redeemable_positions: - vault_to_positions[position.vault].append(position) - - processing_positions = [] - vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None] = {} - for vault_address, positions in vault_to_positions.items(): - # Check if state update is required - harvest_params = await get_harvest_params(vault_address, block_number) - vault_to_harvest_params[vault_address] = harvest_params - withdrawable_assets = await get_withdrawable_assets(vault_address, harvest_params) - - # Process each position in the vault - for position in positions: - # Convert redeemable shares to assets - redeemable_assets = os_token_converter.to_assets(position.redeemable_shares) - - if redeemable_assets <= withdrawable_assets: - shares_to_redeem = min(position.amount, queued_shares) - logger.info( - f"Position Owner: {position.owner}, " - f"Vault: {position.vault}, " - f"Shares to Redeem: {shares_to_redeem}" - ) - processing_positions.append( - OsTokenPosition( - vault=position.vault, - owner=position.owner, - leaf_shares=position.amount, - shares_to_redeem=shares_to_redeem, - ) - ) - withdrawable_assets -= redeemable_assets - queued_shares -= shares_to_redeem + positions_to_redeem = [] + vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None] = {} + for vault_address, positions in vault_to_positions.items(): + # Check if state update is required + harvest_params = await get_harvest_params(vault_address, block_number) + vault_to_harvest_params[vault_address] = harvest_params + withdrawable_assets = await get_withdrawable_assets(vault_address, harvest_params) # Handle Meta-Vaults with Insufficient Withdrawable Assets - # Execute Redemption with Multicall - await execute_redemption( - redeemed_positions=processing_positions, - vault_to_harvest_params=vault_to_harvest_params, - ) + vault_positions_shares = Wei(sum(position.redeemable_shares for position in positions)) + vault_positions_assets = os_token_converter.to_assets(vault_positions_shares) + if vault_positions_assets > withdrawable_assets and await _is_meta_vault(vault_address): + # Check if vault is a meta-vault + logger.info( + 'Vault %s is a meta-vault with insufficient withdrawable assets.', vault_address + ) + additional_assets_needed = Wei(vault_positions_assets - withdrawable_assets) + redeemed_assets = await os_token_redeemer_contract.redeem_sub_vaults_assets( + vault_address, additional_assets_needed + ) + withdrawable_assets = Wei(withdrawable_assets + redeemed_assets) + + # Process each position in the vault + for position in positions: + # Convert redeemable shares to assets + redeemable_assets = os_token_converter.to_assets(position.redeemable_shares) + + if redeemable_assets <= withdrawable_assets: + shares_to_redeem = min(position.amount, queued_shares) + logger.info( + 'Position Owner: %s, Vault: %s, Shares to Redeem: %s', + position.owner, + position.vault, + shares_to_redeem, + ) + positions_to_redeem.append( + OsTokenPosition( + vault=position.vault, + owner=position.owner, + leaf_shares=position.amount, + shares_to_redeem=shares_to_redeem, + ) + ) + withdrawable_assets = Wei(withdrawable_assets - redeemable_assets) + queued_shares = Wei(queued_assets - shares_to_redeem) + + # Execute Redemption with Multicall + tx_hash = await execute_redemption( + positions_to_redeem=positions_to_redeem, + vault_to_harvest_params=vault_to_harvest_params, + ) + logger.info( + 'Successfully redeemed %s OsToken positions. Transaction hash: %s', + len(positions_to_redeem), + tx_hash, + ) async def execute_redemption( - redeemed_positions: list[OsTokenPosition], - vault_to_harvest_params: HarvestParams | None, -) -> None: + positions_to_redeem: list[OsTokenPosition], + vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], +) -> HexStr | None: calls = [] - for vault in set(pos.vault for pos in redeemed_positions): + for vault in set(pos.vault for pos in positions_to_redeem): harvest_params = vault_to_harvest_params.get(vault) if harvest_params: vault_contract = VaultContract(vault) @@ -244,9 +287,9 @@ async def execute_redemption( redeem_os_token_positions_call = os_token_redeemer_contract.encode_abi( fn_name='redeemOsTokenPositions', - args=[redeemed_positions, proof, proofFlags], + args=[positions_to_redeem, proof, proofFlags], ) - calls.append((os_token_redeemer_contract.address, redeem_os_token_positions_call)) + calls.append((os_token_redeemer_contract.contract_address, redeem_os_token_positions_call)) try: tx_function = multicall_contract.functions.aggregate(calls) tx = await transaction_gas_wrapper(tx_function=tx_function) @@ -261,25 +304,12 @@ async def execute_redemption( tx, timeout=settings.execution_transaction_timeout ) if not tx_receipt['status']: - logger.error(logger.error('Failed to redeem os token positions...')) + logger.error('Failed to redeem os token positions...') return None return tx_hash -def get_redeemable_position_leaf_hash(redeemable_position: RedeemablePosition, nonce: int) -> bytes: - """Get the leaf hash for a redeemable position.""" - vault = redeemable_position.vault - owner = redeemable_position.owner - amount = redeemable_position.amount - - leaf = standard_leaf_hash( - values=(nonce, vault, amount, owner), - types=['uint256', 'address', 'uint256', 'address'], - ) - return leaf - - async def fetch_redeemable_positions(ipfs_hash: str) -> list[RedeemablePosition]: # Fetch redeemable positions data from IPFS data = cast(list[dict], await ipfs_fetch_client.fetch_json(ipfs_hash)) @@ -297,6 +327,28 @@ async def fetch_redeemable_positions(ipfs_hash: str) -> list[RedeemablePosition] ] +async def _process_exit_queue(block_number: BlockNumber) -> None: + """ + Call processExitQueue() on the redeemer contract if canProcessExitQueue + to create a new checkpoint that converts accumulated redeemed/swapped + shares into claimable assets for users in the exit queue. + """ + can_process_exit_queue = await os_token_redeemer_contract.can_process_exit_queue(block_number) + if can_process_exit_queue: + logger.info('Exit queue can be processed. Calling processExitQueue...') + tx_hash = await os_token_redeemer_contract.process_exit_queue() + logger.info('ProcessExitQueue transaction sent. Tx Hash: %s', tx_hash) + + +async def _is_meta_vault(vault_address: ChecksumAddress) -> bool: + meta_vault_contract = MetaVaultContract(vault_address) + try: + await meta_vault_contract.sub_vaults_rewards_nonce() + return True + except (Web3RPCError, ValueError, ContractLogicError): + return False + + async def _startup_check() -> None: positions_manager = await os_token_redeemer_contract.positions_manager() if positions_manager != wallet.account.address: diff --git a/src/common/contracts.py b/src/common/contracts.py index 4481b9b2..058ad2a9 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -27,7 +27,7 @@ settings, ) from src.meta_vault.typings import SubVaultExitRequest -from src.redeem.typings import RedeemablePositionsMeta +from src.redemptions.typings import RedeemablePositionsMeta from src.validators.typings import V2ValidatorEventData from src.withdrawals.typings import WithdrawalEvent @@ -408,6 +408,9 @@ async def withdrawable_assets(self) -> Wei: async def get_exit_queue_index(self, position_ticket: int) -> int: return await self.contract.functions.getExitQueueIndex(position_ticket).call() + async def sub_vaults_rewards_nonce(self) -> HexStr: + return await self.contract.functions.subVaultsRewardsNonce().call() + async def deposit_to_sub_vaults(self) -> HexStr: tx_function = self.contract.functions.depositToSubVaults() tx_hash = await transaction_gas_wrapper(tx_function) @@ -489,8 +492,10 @@ class OsTokenRedeemerContract(ContractWrapper): abi_path = 'abi/IOsTokenRedeemer.json' settings_key = 'OS_TOKEN_REDEEMER_CONTRACT_ADDRESS' - async def redeemable_positions(self) -> RedeemablePositionsMeta: - merkle_root, ipfs_hash = await self.contract.functions.redeemablePositions().call() + async def redeemable_positions(self, block_number: BlockNumber) -> RedeemablePositionsMeta: + merkle_root, ipfs_hash = await self.contract.functions.redeemablePositions().call( + block_identifier=block_number + ) return RedeemablePositionsMeta( merkle_root=Web3.to_hex(merkle_root), ipfs_hash=ipfs_hash, @@ -504,15 +509,14 @@ async def get_exit_queue_cumulative_tickets(self, block_number: BlockNumber) -> async def get_exit_queue_missing_assets(self, target_ticket: int) -> Wei: return await self.contract.functions.getExitQueueMissingAssets(target_ticket).call() - async def nonce(self) -> int: - return await self.contract.functions.nonce().call() + async def nonce(self, block_number: BlockNumber | None = None) -> int: + return await self.contract.functions.nonce().call(block_identifier=block_number) - ### async def positions_manager(self) -> ChecksumAddress: return await self.contract.functions.positionsManager().call() - async def queued_shares(self) -> Wei: - return await self.contract.functions.queuedShares().call() + async def queued_shares(self, block_number: BlockNumber | None = None) -> Wei: + return await self.contract.functions.queuedShares().call(block_identifier=block_number) async def can_process_exit_queue(self, block_number: BlockNumber | None = None) -> bool: return await self.contract.functions.canProcessExitQueue().call( @@ -531,6 +535,13 @@ async def process_exit_queue(self) -> HexStr: tx_hash = await transaction_gas_wrapper(tx_function) return Web3.to_hex(tx_hash) + async def redeem_sub_vaults_assets( + self, vault_address: ChecksumAddress, assets_to_redeem: Wei + ) -> Wei: + tx_function = self.contract.functions.redeemSubVaultsAssets(vault_address, assets_to_redeem) + tx = await transaction_gas_wrapper(tx_function) + return Wei(Web3.to_int(tx)) + class ValidatorsCheckerContract(ContractWrapper): abi_path = 'abi/IValidatorsChecker.json' diff --git a/src/redemptions/typings.py b/src/redemptions/typings.py index 23528f61..584fb252 100644 --- a/src/redemptions/typings.py +++ b/src/redemptions/typings.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from eth_typing import ChecksumAddress, HexStr +from multiproof.standard import standard_leaf_hash from web3 import Web3 from web3.types import Wei @@ -65,6 +66,12 @@ def as_dict(self) -> dict: 'amount': str(self.amount), } + def merkle_leaf_bytes(self, nonce: int) -> bytes: + return standard_leaf_hash( + values=(nonce, self.vault, self.amount, self.owner), + types=['uint256', 'address', 'uint256', 'address'], + ) + def merkle_leaf(self, nonce: int) -> tuple[int, ChecksumAddress, Wei, ChecksumAddress]: return nonce, self.vault, self.amount, self.owner From 06d7c71a1527c9ca0315819f6f56953ad1147ce7 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 27 Jan 2026 22:16:02 +0300 Subject: [PATCH 31/65] Update tests Signed-off-by: cyc60 --- .../test_update_redeemable_positions.py | 162 ++++++++++++++++-- 1 file changed, 144 insertions(+), 18 deletions(-) diff --git a/src/commands/tests/test_internal/test_update_redeemable_positions.py b/src/commands/tests/test_internal/test_update_redeemable_positions.py index af980d05..c8171ad6 100644 --- a/src/commands/tests/test_internal/test_update_redeemable_positions.py +++ b/src/commands/tests/test_internal/test_update_redeemable_positions.py @@ -1,6 +1,5 @@ import contextlib -from unittest import mock -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, patch import pytest from click.testing import CliRunner @@ -24,6 +23,8 @@ VaultShares, ) +os_token_contract_address = NETWORKS[MAINNET].OS_TOKEN_CONTRACT_ADDRESS + def test_create_redeemable_positions(): address_1 = faker.eth_address() @@ -235,7 +236,7 @@ def test_reduces_boosted_amount(): @pytest.mark.usefixtures('_init_config') class TestUpdateRedeemablePositions: @pytest.mark.usefixtures('fake_settings', 'setup_test_clients') - async def test_update_redeemable_positions( + async def test_basic_call( self, vault_address: str, execution_endpoints: str, @@ -246,7 +247,6 @@ async def test_update_redeemable_positions( address_2 = '0x24c8DBBC3d1C35C4159787b1f7a62bea1A814242' vault_1 = '0xEd735de172272C03CA6F60c1d90D83D9CFB46D22' vault_2 = '0xe8Ea1025b49D2B51C536cFBc0833F021ba4c6903' - os_token_contract_address = NETWORKS[MAINNET].OS_TOKEN_CONTRACT_ADDRESS allocators = [ { 'vault': { @@ -354,18 +354,13 @@ async def test_update_redeemable_positions( patch_os_token_redeemer_contract_nonce(6), patch_os_token_arbitrum_contract_address(), patch_os_token_contract_address(os_token_contract_address), - mock.patch( + patch_os_token_converter(os_token_converter), + patch_api_cliente(mock_protocol_data), + patch( 'src.redemptions.graph.graph_client.fetch_pages', side_effect=[allocators, leverage_positions, os_token_holders], ), - mock.patch( - 'src.commands.internal.update_redeemable_positions.create_os_token_converter', - return_value=os_token_converter, - ), - mock.patch( - 'src.redemptions.api_client.APIClient._fetch_json', return_value=mock_protocol_data - ), - mock.patch( + patch( 'src.common.clients.IpfsMultiUploadClient.upload_json', return_value=faker.ipfs_hash(), ) as ipfs_mock, @@ -380,19 +375,144 @@ async def test_update_redeemable_positions( in result.output.strip() ) + @pytest.mark.usefixtures('fake_settings', 'setup_test_clients') + async def test_full_position( + self, + vault_address: str, + execution_endpoints: str, + runner: CliRunner, + ): + # hardcoded to check merkle root + address_1 = '0x2242b8ab71521f6abEE4B4D83195E70AcB08727a' + vault_1 = '0xEd735de172272C03CA6F60c1d90D83D9CFB46D22' + allocators = [ + { + 'vault': { + 'id': vault_1.lower(), + }, + 'id': address_1.lower(), + 'address': address_1, + 'mintedOsTokenShares': Web3.to_wei(10, 'ether'), + }, + ] + leverage_positions = [] + os_token_holders = [] + mock_protocol_data = [] + os_token_converter = OsTokenConverter(110, 100) + args = [ + '--network', + MAINNET, + '--execution-endpoints', + execution_endpoints, + '--arbitrum-endpoint', + execution_endpoints, + '--verbose', + ] + with ( + patch_latest_block(11), + patch_get_erc_balance(Web3.to_wei(0, 'ether')), + patch_os_token_redeemer_contract_nonce(6), + patch_os_token_arbitrum_contract_address(), + patch_os_token_contract_address(os_token_contract_address), + patch_os_token_converter(os_token_converter), + patch_api_cliente(mock_protocol_data), + patch( + 'src.redemptions.graph.graph_client.fetch_pages', + side_effect=[allocators, leverage_positions, os_token_holders], + ), + patch( + 'src.common.clients.IpfsMultiUploadClient.upload_json', + return_value=faker.ipfs_hash(), + ) as ipfs_mock, + ): + result = runner.invoke(update_redeemable_positions, args, input='\n') + assert result.exit_code == 0 + ipfs_mock.assert_called_once_with( + [{'owner': address_1, 'vault': vault_1, 'amount': '10000000000000000000'}] + ) + assert ( + '0x9b4419ebea301ed07e591b477e69499f35e4c3cd69538c2f22a6a014b06e5bbd' + in result.output.strip() + ) + + @pytest.mark.usefixtures('fake_settings', 'setup_test_clients') + async def test_min_os_token_position_amount( + self, + vault_address: str, + execution_endpoints: str, + runner: CliRunner, + ): + # hardcoded to check merkle root + address_1 = '0x2242b8ab71521f6abEE4B4D83195E70AcB08727a' + vault_1 = '0xEd735de172272C03CA6F60c1d90D83D9CFB46D22' + allocators = [ + { + 'vault': { + 'id': vault_1.lower(), + }, + 'id': address_1.lower(), + 'address': address_1, + 'mintedOsTokenShares': Web3.to_wei(5, 'ether'), + }, + ] + leverage_positions = [] + os_token_holders = [] + mock_protocol_data = [] + os_token_converter = OsTokenConverter(110, 100) + args = [ + '--network', + MAINNET, + '--execution-endpoints', + execution_endpoints, + '--arbitrum-endpoint', + execution_endpoints, + '--verbose', + '--min-os-token-position-amount-gwei', + 6 * 10**9, # 6 ETH in Gwei + ] + with ( + patch_latest_block(11), + patch_get_erc_balance(Web3.to_wei(0, 'ether')), + patch_os_token_redeemer_contract_nonce(6), + patch_os_token_arbitrum_contract_address(), + patch_os_token_contract_address(os_token_contract_address), + patch_os_token_converter(os_token_converter), + patch_api_cliente(mock_protocol_data), + patch( + 'src.redemptions.graph.graph_client.fetch_pages', + side_effect=[allocators, leverage_positions, os_token_holders], + ), + patch( + 'src.common.clients.IpfsMultiUploadClient.upload_json', + return_value=faker.ipfs_hash(), + ) as ipfs_mock, + ): + result = runner.invoke(update_redeemable_positions, args, input='\n') + assert result.exit_code == 0 + ipfs_mock.assert_not_called() + @contextlib.contextmanager def patch_latest_block(block_number): - with mock.patch( + with patch( 'src.commands.internal.update_redeemable_positions.execution_client', new=AsyncMock() ) as execution_client_mock: execution_client_mock.eth.get_block_number.return_value = block_number yield +@contextlib.contextmanager +def patch_os_token_converter(os_token_converter: OsTokenConverter): + with patch( + 'src.commands.internal.update_redeemable_positions.create_os_token_converter', + return_value=os_token_converter, + ): + yield + + @contextlib.contextmanager def patch_get_erc_balance(balance): - with mock.patch( + with patch( 'src.commands.internal.update_redeemable_positions.Erc20Contract.get_balance', return_value=balance, ): @@ -401,7 +521,7 @@ def patch_get_erc_balance(balance): @contextlib.contextmanager def patch_os_token_arbitrum_contract_address(): - with mock.patch.object( + with patch.object( settings.network_config, 'OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS', NETWORKS[MAINNET].OS_TOKEN_ARBITRUM_CONTRACT_ADDRESS, @@ -411,7 +531,7 @@ def patch_os_token_arbitrum_contract_address(): @contextlib.contextmanager def patch_os_token_contract_address(address: ChecksumAddress): - with mock.patch.object( + with patch.object( settings.network_config, 'OS_TOKEN_CONTRACT_ADDRESS', address, @@ -421,8 +541,14 @@ def patch_os_token_contract_address(address: ChecksumAddress): @contextlib.contextmanager def patch_os_token_redeemer_contract_nonce(nonce): - with mock.patch( + with patch( 'src.commands.internal.update_redeemable_positions.os_token_redeemer_contract.nonce', return_value=nonce, ): yield + + +@contextlib.contextmanager +def patch_api_cliente(mock_protocol_data): + with patch('src.redemptions.api_client.APIClient._fetch_json', return_value=mock_protocol_data): + yield From 27cb7c543ca59083ff527f57855b20a8b48e095c Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 27 Jan 2026 23:46:21 +0300 Subject: [PATCH 32/65] Add proofs Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 49 +++++++++++++++++------ src/redemptions/typings.py | 8 ---- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 0f3cdf9f..648e45db 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -7,6 +7,8 @@ import click from eth_typing import BlockNumber, ChecksumAddress, HexStr +from multiproof import StandardMerkleTree +from multiproof.standard import MultiProof from sw_utils import InterruptHandler from web3 import Web3 from web3.exceptions import ContractLogicError, Web3RPCError @@ -33,7 +35,7 @@ from src.config.networks import AVAILABLE_NETWORKS, ZERO_CHECKSUM_ADDRESS from src.config.settings import settings from src.redemptions.os_token_converter import create_os_token_converter -from src.redemptions.typings import OsTokenPosition, RedeemablePosition +from src.redemptions.typings import RedeemablePosition from src.validators.execution import get_withdrawable_assets logger = logging.getLogger(__name__) @@ -244,11 +246,11 @@ async def process(block_number: BlockNumber) -> None: shares_to_redeem, ) positions_to_redeem.append( - OsTokenPosition( + RedeemablePosition( vault=position.vault, owner=position.owner, - leaf_shares=position.amount, - shares_to_redeem=shares_to_redeem, + amount=position.amount, + redeemable_shares=shares_to_redeem, ) ) withdrawable_assets = Wei(withdrawable_assets - redeemable_assets) @@ -258,6 +260,7 @@ async def process(block_number: BlockNumber) -> None: tx_hash = await execute_redemption( positions_to_redeem=positions_to_redeem, vault_to_harvest_params=vault_to_harvest_params, + nonce=nonce, ) logger.info( 'Successfully redeemed %s OsToken positions. Transaction hash: %s', @@ -267,9 +270,15 @@ async def process(block_number: BlockNumber) -> None: async def execute_redemption( - positions_to_redeem: list[OsTokenPosition], + positions_to_redeem: list[RedeemablePosition], vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], + nonce: int, ) -> HexStr | None: + + multiproof = _get_multi_proof( + positions_to_redeem=positions_to_redeem, + nonce=nonce, + ) calls = [] for vault in set(pos.vault for pos in positions_to_redeem): @@ -277,17 +286,15 @@ async def execute_redemption( if harvest_params: vault_contract = VaultContract(vault) calls.append( - [ - ( - vault_contract.contract_address, - vault_contract.get_update_state_call(harvest_params), - ) - ] + ( + vault_contract.contract_address, + vault_contract.get_update_state_call(harvest_params), + ) ) redeem_os_token_positions_call = os_token_redeemer_contract.encode_abi( fn_name='redeemOsTokenPositions', - args=[positions_to_redeem, proof, proofFlags], + args=[positions_to_redeem, multiproof.proof, multiproof.proof_flags], ) calls.append((os_token_redeemer_contract.contract_address, redeem_os_token_positions_call)) try: @@ -355,3 +362,21 @@ async def _startup_check() -> None: raise RuntimeError( f'The Position Manager role must be assigned to the address {wallet.account.address}.' ) + + +def _get_multi_proof( + nonce: int, + positions_to_redeem: list[RedeemablePosition], +) -> MultiProof[tuple[bytes, int]]: + leaves = [r.merkle_leaf(nonce) for r in positions_to_redeem] + tree = StandardMerkleTree.of( + leaves, + [ + 'uint256', + 'address', + 'uint256', + 'address', + ], + ) + multi_proof = tree.get_multi_proof(leaves) + return multi_proof diff --git a/src/redemptions/typings.py b/src/redemptions/typings.py index 584fb252..3f6a6dad 100644 --- a/src/redemptions/typings.py +++ b/src/redemptions/typings.py @@ -80,11 +80,3 @@ def merkle_leaf(self, nonce: int) -> tuple[int, ChecksumAddress, Wei, ChecksumAd class RedeemablePositionsMeta: merkle_root: HexStr ipfs_hash: str - - -@dataclass -class OsTokenPosition: - owner: ChecksumAddress - vault: ChecksumAddress - leaf_shares: Wei - shares_to_redeem: Wei From 69e0b73547bd9a81b01de26fb38182301b20c91d Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 28 Jan 2026 18:52:06 +0300 Subject: [PATCH 33/65] Fix IPFS mock Signed-off-by: cyc60 --- .../test_update_redeemable_positions.py | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/src/commands/tests/test_internal/test_update_redeemable_positions.py b/src/commands/tests/test_internal/test_update_redeemable_positions.py index c8171ad6..57880e87 100644 --- a/src/commands/tests/test_internal/test_update_redeemable_positions.py +++ b/src/commands/tests/test_internal/test_update_redeemable_positions.py @@ -1,5 +1,5 @@ import contextlib -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest from click.testing import CliRunner @@ -355,19 +355,16 @@ async def test_basic_call( patch_os_token_arbitrum_contract_address(), patch_os_token_contract_address(os_token_contract_address), patch_os_token_converter(os_token_converter), - patch_api_cliente(mock_protocol_data), + patch_api_client(mock_protocol_data), patch( 'src.redemptions.graph.graph_client.fetch_pages', side_effect=[allocators, leverage_positions, os_token_holders], ), - patch( - 'src.common.clients.IpfsMultiUploadClient.upload_json', - return_value=faker.ipfs_hash(), - ) as ipfs_mock, + patch_ipfs_client() as mock_upload_json, ): result = runner.invoke(update_redeemable_positions, args, input='\n') assert result.exit_code == 0 - ipfs_mock.assert_called_once_with( + mock_upload_json.assert_called_once_with( [{'owner': address_1, 'vault': vault_1, 'amount': '2563636363636363637'}] ) assert ( @@ -415,19 +412,16 @@ async def test_full_position( patch_os_token_arbitrum_contract_address(), patch_os_token_contract_address(os_token_contract_address), patch_os_token_converter(os_token_converter), - patch_api_cliente(mock_protocol_data), + patch_api_client(mock_protocol_data), patch( 'src.redemptions.graph.graph_client.fetch_pages', side_effect=[allocators, leverage_positions, os_token_holders], ), - patch( - 'src.common.clients.IpfsMultiUploadClient.upload_json', - return_value=faker.ipfs_hash(), - ) as ipfs_mock, + patch_ipfs_client() as mock_upload_json, ): result = runner.invoke(update_redeemable_positions, args, input='\n') assert result.exit_code == 0 - ipfs_mock.assert_called_once_with( + mock_upload_json.assert_called_once_with( [{'owner': address_1, 'vault': vault_1, 'amount': '10000000000000000000'}] ) assert ( @@ -477,19 +471,16 @@ async def test_min_os_token_position_amount( patch_os_token_arbitrum_contract_address(), patch_os_token_contract_address(os_token_contract_address), patch_os_token_converter(os_token_converter), - patch_api_cliente(mock_protocol_data), + patch_api_client(mock_protocol_data), patch( 'src.redemptions.graph.graph_client.fetch_pages', side_effect=[allocators, leverage_positions, os_token_holders], ), - patch( - 'src.common.clients.IpfsMultiUploadClient.upload_json', - return_value=faker.ipfs_hash(), - ) as ipfs_mock, + patch_ipfs_client() as mock_upload_json, ): result = runner.invoke(update_redeemable_positions, args, input='\n') assert result.exit_code == 0 - ipfs_mock.assert_not_called() + mock_upload_json.assert_not_called() @contextlib.contextmanager @@ -549,6 +540,20 @@ def patch_os_token_redeemer_contract_nonce(nonce): @contextlib.contextmanager -def patch_api_cliente(mock_protocol_data): +def patch_api_client(mock_protocol_data): with patch('src.redemptions.api_client.APIClient._fetch_json', return_value=mock_protocol_data): yield + + +@contextlib.contextmanager +def patch_ipfs_client(): + mock_upload_json = AsyncMock(return_value=faker.ipfs_hash()) + mock_ipfs_client = MagicMock() + mock_ipfs_client.upload_json = mock_upload_json + mock_build = MagicMock(return_value=mock_ipfs_client) + with patch( + 'src.commands.internal.update_redeemable_positions.build_ipfs_upload_clients', mock_build + ): + yield mock_upload_json + # with patch('src.redemptions.api_client.APIClient._fetch_json', return_value=mock_protocol_data): + # yield From 2dff8eb92a791983593ad371dcc635abe4402a6c Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 28 Jan 2026 19:10:24 +0300 Subject: [PATCH 34/65] Move log Signed-off-by: cyc60 --- src/commands/internal/update_redeemable_positions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 27bda8f1..72793440 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -202,12 +202,12 @@ async def process(arbitrum_endpoint: str | None, min_os_token_position_amount_gw logger.info('Fetched %s allocators from the subgraph', len(allocators)) # filter boost proxy positions + logger.info('Fetching boosted positions from the subgraph...') leverage_positions = await graph_get_leverage_positions(block_number) boost_proxies = {pos.proxy for pos in leverage_positions} logger.info('Found %s proxy positions to exclude', len(boost_proxies)) allocators = [a for a in allocators if a.address not in boost_proxies] # reduce boosted positions - logger.info('Fetching boosted positions from the subgraph...') os_token_converter = await create_os_token_converter(block_number) boost_ostoken_shares = await calculate_boost_ostoken_shares( users={a.address for a in allocators}, From 5d9a0ffc8622ae70f6d8c10c5d465b9fc9267bbc Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 28 Jan 2026 22:55:11 +0300 Subject: [PATCH 35/65] Copilot fixes Signed-off-by: cyc60 --- .../internal/update_redeemable_positions.py | 4 +- .../test_update_redeemable_positions.py | 42 +++++++++++++++++-- src/common/clients.py | 2 +- src/redemptions/api_client.py | 3 +- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 72793440..9e7680b7 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -374,7 +374,7 @@ def create_redeemable_positions( for allocator in allocators: allocator_kept_shares = kept_shares.get(allocator.address, Wei(0)) redeemable_amount = max(0, allocator.total_shares - allocator_kept_shares) - if redeemable_amount <= 0: + if redeemable_amount == 0: continue allocated_amount = 0 @@ -382,7 +382,7 @@ def create_redeemable_positions( for index, (vault_address, proportion) in enumerate(vaults_proportions): # dust handling if index == len(vaults_proportions) - 1: - vault_amount = int(redeemable_amount - allocated_amount) + vault_amount = max(0, int(redeemable_amount - allocated_amount)) else: vault_amount = int(redeemable_amount * proportion) diff --git a/src/commands/tests/test_internal/test_update_redeemable_positions.py b/src/commands/tests/test_internal/test_update_redeemable_positions.py index 57880e87..ceff1e71 100644 --- a/src/commands/tests/test_internal/test_update_redeemable_positions.py +++ b/src/commands/tests/test_internal/test_update_redeemable_positions.py @@ -86,7 +86,7 @@ def test_create_redeemable_positions(): result = create_redeemable_positions(allocators, kept_tokens) assert result == [RedeemablePosition(owner=address_1, vault=vault_1, amount=Wei(150))] - # test multiple vaults + # test multiple vaults #1 allocators = [ Allocator( address=Web3.to_checksum_address(address_1), @@ -96,13 +96,47 @@ def test_create_redeemable_positions(): ], ) ] + result = create_redeemable_positions(allocators, {}) + assert result == [ + RedeemablePosition(owner=address_1, vault=vault_1, amount=Wei(150)), + RedeemablePosition(owner=address_1, vault=vault_2, amount=Wei(150)), + ] + + allocators = [ + Allocator( + address=Web3.to_checksum_address(address_1), + vault_shares=[ + VaultShares(address=Web3.to_checksum_address(vault_1), minted_shares=Wei(333)), + VaultShares(address=Web3.to_checksum_address(vault_2), minted_shares=Wei(666)), + ], + ) + ] kept_tokens = { - address_1: Wei(0), + address_1: Wei(100), } result = create_redeemable_positions(allocators, kept_tokens) assert result == [ - RedeemablePosition(owner=address_1, vault=vault_1, amount=Wei(150)), - RedeemablePosition(owner=address_1, vault=vault_2, amount=Wei(150)), + RedeemablePosition(owner=address_1, vault=vault_1, amount=Wei(299)), + RedeemablePosition(owner=address_1, vault=vault_2, amount=Wei(600)), + ] + + # test multiple vaults #3 + allocators = [ + Allocator( + address=Web3.to_checksum_address(address_1), + vault_shares=[ + VaultShares(address=Web3.to_checksum_address(vault_1), minted_shares=Wei(333)), + VaultShares(address=Web3.to_checksum_address(vault_2), minted_shares=Wei(666)), + ], + ) + ] + kept_tokens = { + address_1: Wei(100), + } + result = create_redeemable_positions(allocators, kept_tokens) + assert result == [ + RedeemablePosition(owner=address_1, vault=vault_1, amount=Wei(299)), + RedeemablePosition(owner=address_1, vault=vault_2, amount=Wei(600)), ] diff --git a/src/common/clients.py b/src/common/clients.py index 2b9da771..29ac9c81 100644 --- a/src/common/clients.py +++ b/src/common/clients.py @@ -157,7 +157,7 @@ def build_ipfs_upload_clients() -> IpfsMultiUploadClient: if not clients: raise ValueError( - 'No IPFS clients settings. ' + 'No IPFS client settings configured. ' 'Please provide IPFS_LOCAL_CLIENT_ENDPOINT or third party IPFS services settings.' ) return IpfsMultiUploadClient(clients) diff --git a/src/redemptions/api_client.py b/src/redemptions/api_client.py index 9dcdad81..8b0a08e0 100644 --- a/src/redemptions/api_client.py +++ b/src/redemptions/api_client.py @@ -14,6 +14,7 @@ ) SUPPORTED_CHAINS = {'eth', 'arb'} API_SLEEP_TIMEOUT = 1 +STAKEWISE_DEBANK_PROTOCOL_IDS = ['stakewise', 'xdai_stakewise'] class APIClient: @@ -30,7 +31,7 @@ async def get_protocols_locked_os_token(self, address: ChecksumAddress) -> Wei: total_locked_os_token = Wei(0) for protocol in protocol_data: # boosted OsEth handled via graph separately - if protocol['id'] in ['stakewise', 'xdai_stakewise']: + if protocol['id'] in STAKEWISE_DEBANK_PROTOCOL_IDS: continue for portfolio_item in protocol.get('portfolio_item_list', []): supply_token_list = portfolio_item.get('detail', {}).get('supply_token_list', []) From bfb88663aa6168616b80b6b3d653022b80d862a3 Mon Sep 17 00:00:00 2001 From: Evgeny Gusarov Date: Sun, 25 Jan 2026 22:21:27 +0300 Subject: [PATCH 36/65] Add withdrawals to cover redemption assets --- poetry.lock | 74 +++------- pyproject.toml | 2 +- src/common/contracts.py | 43 +++++- src/common/decorators.py | 30 ++++ src/common/utils.py | 19 ++- src/common/withdrawals.py | 3 + src/config/networks.py | 6 +- src/meta_vault/service.py | 84 +++++++++++ src/meta_vault/typings.py | 6 + src/redemptions/os_token_converter.py | 5 + src/redemptions/tasks.py | 194 ++++++++++++++++++++++++++ src/redemptions/typings.py | 8 +- src/withdrawals/assets.py | 4 + src/withdrawals/tasks.py | 4 + 14 files changed, 420 insertions(+), 62 deletions(-) create mode 100644 src/common/decorators.py create mode 100644 src/meta_vault/service.py create mode 100644 src/redemptions/tasks.py diff --git a/poetry.lock b/poetry.lock index e4eabe64..ce3f71a6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -840,9 +840,6 @@ python-versions = "*" files = [ {file = "ckzg-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49ee4c830de89764bfd9e8188446f3020f14d32bd4486fcbc5a4a5afad775ac0"}, {file = "ckzg-2.1.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3b4f0c6c2f1a629d4d64e900c65633595c63d208001d588c61b6c8bc1b189dec"}, - {file = "ckzg-2.1.5-cp310-cp310-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:10c8bc524267a40fe7c4fabd4c23f131ea18fcabd6016cdc4ddcb95cc757faf5"}, - {file = "ckzg-2.1.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ea589e60db460ee9ebb678f20e74cc9289e912ccad66693b3263459933aaffc"}, - {file = "ckzg-2.1.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:97769b53f7d8c46e794d5c8aa609a4c00ec1fb050e69b6833b45dbb23a7b6501"}, {file = "ckzg-2.1.5-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a45aaea4a42babea48bb27e387fb209f2aaaaaa16abea25a4a92a056b616f9af"}, {file = "ckzg-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:060562273057911c39a1491e9b76055c095c10cfff1704ed70011e38b53f83d8"}, {file = "ckzg-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f12a90277b17e1cb5c326c5c261dad2ebb14a7136e754593e3a0a92c94799fc1"}, @@ -852,9 +849,6 @@ files = [ {file = "ckzg-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:2b7ef12896e2afff613f058e3bc8e3478ff626ae8a6f2d3200950304a536935f"}, {file = "ckzg-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cead4ba760a49eaa4d7a50a0483aad9727d6103fc00c408aef15f2cd8f8dec7b"}, {file = "ckzg-2.1.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3156983ba598fa05f0136325125e75197e4cf24ded255aaa6ace068cede92932"}, - {file = "ckzg-2.1.5-cp311-cp311-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:cac601a9690f133dd9d8e85f7a96578496427d42cdea771e0e07785b1cbbe9dc"}, - {file = "ckzg-2.1.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05860f1477880376106a6934becdcb3a2c6330fc2386fed0d7e8f3b0ce5df81c"}, - {file = "ckzg-2.1.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92b18b0ec177b9e2b4238936a8bffcfdaee7626a58f8d0c7c2ac554b8a05c9b6"}, {file = "ckzg-2.1.5-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d05e2c9466b2a4214dc19da35ea4cae636e033f3434768b982d37317a0f9c520"}, {file = "ckzg-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c754bbc253cfce8814d633f135be4891e6f83a50125f418fee01323ba306f59a"}, {file = "ckzg-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2b766d4aed52c8c717322f2af935da0b916bf59fbba771adb822499b45e491"}, @@ -864,9 +858,6 @@ files = [ {file = "ckzg-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:ce2047071353ee099d44aa6575974648663204eb9b42354bfa5ac6f9b8fb63e9"}, {file = "ckzg-2.1.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:edead535bd9afef27b8650bba09659debd4f52638aee5ec1ab7d2c9d7e86953c"}, {file = "ckzg-2.1.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:dc78622855de3d47767cdeecfdf58fd58911f43a0fa783524e414b7e75149020"}, - {file = "ckzg-2.1.5-cp312-cp312-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:e5639064b0dd147b73f2ce2c2506844b0c625b232396ac852dc52eced04bd529"}, - {file = "ckzg-2.1.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0864813902b96cde171e65334ce8d13c5ff5b6855f2e71a2272ae268fa07e8"}, - {file = "ckzg-2.1.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6f13f673a24c01e681eb66aed8f8e4ce191f009dd2149f3e1b9ad0dd59b4cd"}, {file = "ckzg-2.1.5-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:094add5f197a3d278924ec1480d258f3b8b0e9f8851ae409eec83a21a738bffe"}, {file = "ckzg-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b4b05f798784400e8c4dedaf1a1d57bbbc54de790855855add876fff3c9f629"}, {file = "ckzg-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64aef50a1cf599041b9af018bc885a3fad6a20bbaf443fc45f0457cb47914610"}, @@ -876,9 +867,6 @@ files = [ {file = "ckzg-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:827be2aeffc8a10bfb39b8dad45def82164dfcde735818c4053f5064474ae1b4"}, {file = "ckzg-2.1.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d955f4e18bb9a9b3a6f55114052edd41650c29edd5f81e417c8f01abace8207"}, {file = "ckzg-2.1.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0c0961a685761196264aa49b1cf06e8a2b2add4d57987853d7dd7a7240dc5de7"}, - {file = "ckzg-2.1.5-cp313-cp313-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:026ef3bba0637032c21f6bdb8e92aefeae7c67003bf631a4ee80c515a36a9dbd"}, - {file = "ckzg-2.1.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf031139a86e4ff00a717f9539331ef148ae9013b58848f2a7ac14596d812915"}, - {file = "ckzg-2.1.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f51339d58541ae450c78a509b32822eec643595d8b96949fb1963fba802dc78b"}, {file = "ckzg-2.1.5-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:badb1c7dc6b932bed2c3f7695e1ce3e4bcc9601706136957408ac2bde5dd0892"}, {file = "ckzg-2.1.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58d92816b9babaee87bd9f23be10c07d5d07c709be184aa7ea08ddb2bcf2541c"}, {file = "ckzg-2.1.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cf39f9abe8b3f1a71188fb601a8589672ee40eb0671fc36d8cdf4e78f00f43f"}, @@ -886,24 +874,6 @@ files = [ {file = "ckzg-2.1.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c39a1c7b32ac345cc44046076fd069ad6b7e6f7bef230ef9be414c712c4453b8"}, {file = "ckzg-2.1.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e4564765b0cc65929eca057241b9c030afac1dbae015f129cb60ca6abd6ff620"}, {file = "ckzg-2.1.5-cp313-cp313-win_amd64.whl", hash = "sha256:55013b36514b8176197655b929bc53f020aa51a144331720dead2efc3793ed85"}, - {file = "ckzg-2.1.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a0cab7deaed093898a92d3644d4ca8621b63cb49296833e2d8b3edac456656d5"}, - {file = "ckzg-2.1.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:caedc9eba3d28584be9b6051585f20745f6abfec0d0657cce3dd45edb7f28586"}, - {file = "ckzg-2.1.5-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:2f67e545d41ba960189b1011d078953311259674620c485e619c933494b88fd9"}, - {file = "ckzg-2.1.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6f65ff296033c259d0829093d2c55bb45651e001e0269b8b88d072fdc86ecc6"}, - {file = "ckzg-2.1.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d66d34ff33be94c8a1f0da86483cd5bfdc15842998f3654ed91b8fdbffa2a81"}, - {file = "ckzg-2.1.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:25cf954bae3e2b2db6fa5e811d9800f89199d3eb4fa906c96a1c03434d4893c9"}, - {file = "ckzg-2.1.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:34d7128735e0bcfcac876bff47d0f85e674f1e24f99014e326ec266abed7a82c"}, - {file = "ckzg-2.1.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1dec3efae8679f7b8e26263b8bb0d3061ef4c9c6fe395e55b71f8f0df90ca8a0"}, - {file = "ckzg-2.1.5-cp314-cp314-win_amd64.whl", hash = "sha256:ce37c0ee0effe55d4ceed1735a2d85a3556a86238f3c89b7b7d1ca4ce4e92358"}, - {file = "ckzg-2.1.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:db804d27f4b08e3aea440cdc6558af4ceb8256b18ea2b83681d80cc654a4085b"}, - {file = "ckzg-2.1.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d472e3beeb95a110275b4d27e51d1c2b26ab99ddb91ac1c5587d710080c39c5e"}, - {file = "ckzg-2.1.5-cp314-cp314t-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:4b44a018124a79138fab8fde25221083574c181c324519be51eab09b1e43ae27"}, - {file = "ckzg-2.1.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a91d7b444300cf8ecae4f55983726630530cdde15cab92023026230a30d094e"}, - {file = "ckzg-2.1.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8674c64efbf2a12edf6d776061847bbe182997737e7690a69af932ce61a9c2a"}, - {file = "ckzg-2.1.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4290aa17c6402c98f16017fd6ee0bff8aeb5c97be5c3cee7c72aea1b7d176f3a"}, - {file = "ckzg-2.1.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:a0f82b8958ea97df12e29094f0a672cbe7532399724ea61b2399545991ed6017"}, - {file = "ckzg-2.1.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:22300bf0d717a083c388de5cfafec08443c9938b3abde2e89f9d5d1fffde1c51"}, - {file = "ckzg-2.1.5-cp314-cp314t-win_amd64.whl", hash = "sha256:aa8228206c3e3729fc117ca38e27588c079b0928a5ab628ee4d9fccaa2b8467d"}, {file = "ckzg-2.1.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:44d585f756ab223e34ac80ae04be7969cb364ee250a91f9b2b1dae37e1f3020a"}, {file = "ckzg-2.1.5-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecade6a3aee63dffc8e8d4adba838460b40f9b29d46ffd9f4d4502261fbcddff"}, {file = "ckzg-2.1.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8548de14e6e53271b246c7dc0bf843030b7f2144edb9ea73c68f46174a2bacd6"}, @@ -922,9 +892,6 @@ files = [ {file = "ckzg-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:d2fed86e47399b06b564c8d3715a3ccec5d3a0a63326227a34e15515b8c514db"}, {file = "ckzg-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b1b52d359013b551b85fff538d2ef12763abd87efbc544d6f2808b9dd6bf0a4b"}, {file = "ckzg-2.1.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4cfe1cacea729c06196dcecec9c38f9b59bb7eadce51145e7ee27de10854dd59"}, - {file = "ckzg-2.1.5-cp38-cp38-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:86233ccbb0bcaf353990ce2a8e24f1aa37782272e64ca9b55dd45895829e4980"}, - {file = "ckzg-2.1.5-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3449126ee416b438c22cd7b7620e8f030c9ba7e030a80ebbd5924f04bc95905"}, - {file = "ckzg-2.1.5-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a687c0609de11d4eba5a982036fd77d21d35841effb468a41004c68ad13a7438"}, {file = "ckzg-2.1.5-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d22cef6551dee8d05151cc5184c37b190101b2027c0851301393561c559c669"}, {file = "ckzg-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2867e4a49f19248644206e82f5b8795e22096722dcd1e21acdad133e87632d5c"}, {file = "ckzg-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7e7fd0c4b4d2af5661e3d54648c0447d33f17cbafa5dd1b0576899864b5b7da"}, @@ -934,9 +901,6 @@ files = [ {file = "ckzg-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:15e0f7342a451569fa427c6ad3cb992975462c52c3ecdc2bd7c3ed35847bbb8b"}, {file = "ckzg-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3773ccdb3501ff3779988aa97e5b15629d58ac02281f186030f66d2fc2b4b7ec"}, {file = "ckzg-2.1.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2c9b798c6eb4db9cf82272e5a5c62be86f0d435206c6c49cc078cbb67ebd51bd"}, - {file = "ckzg-2.1.5-cp39-cp39-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:13c0630363a65182e99d064f7eb173195dcbdddc4048fd5b45cd0a3cd0c740f9"}, - {file = "ckzg-2.1.5-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:165efe1fac474ae58a26b742f910c0c90c01fc356aac8b680db2e02e44005adf"}, - {file = "ckzg-2.1.5-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fa231a1965be1a9c6fa50528132b71f1bc486335564baf6ab6d98aebedfb03d7"}, {file = "ckzg-2.1.5-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3ea32c21f71b786ea04b62cbe982b600da5e6f180b1d256fc9e397074041a6d"}, {file = "ckzg-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10455cc15e769a749c19fd3031dd0149eb92c2f9b4a054117cb20327242fd920"}, {file = "ckzg-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b21c38740aa5fcdc0cacfe9eda82cbf7bdffc743fa85344495bfecc18619d7d6"}, @@ -2574,13 +2538,13 @@ attrs = ">=19.2.0" [[package]] name = "packaging" -version = "25.0" +version = "26.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, - {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, + {file = "packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529"}, + {file = "packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4"}, ] [[package]] @@ -2975,13 +2939,13 @@ files = [ [[package]] name = "pycparser" -version = "2.23" +version = "3.0" description = "C parser in Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" files = [ - {file = "pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934"}, - {file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"}, + {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, + {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, ] [[package]] @@ -3715,13 +3679,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "rich" -version = "14.2.0" +version = "14.3.1" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.8.0" files = [ - {file = "rich-14.2.0-py3-none-any.whl", hash = "sha256:76bc51fe2e57d2b1be1f96c524b890b816e334ab4c1e45888799bfaab0021edd"}, - {file = "rich-14.2.0.tar.gz", hash = "sha256:73ff50c7c0c1c77c8243079283f4edb376f0f6442433aecb8ce7e6d0b92d1fe4"}, + {file = "rich-14.3.1-py3-none-any.whl", hash = "sha256:da750b1aebbff0b372557426fb3f35ba56de8ef954b3190315eb64076d6fb54e"}, + {file = "rich-14.3.1.tar.gz", hash = "sha256:b8c5f568a3a749f9290ec6bddedf835cec33696bfc1e48bcfecb276c7386e4b8"}, ] [package.dependencies] @@ -3800,13 +3764,13 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "80.9.0" +version = "80.10.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.9" files = [ - {file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"}, - {file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"}, + {file = "setuptools-80.10.1-py3-none-any.whl", hash = "sha256:fc30c51cbcb8199a219c12cc9c281b5925a4978d212f84229c909636d9f6984e"}, + {file = "setuptools-80.10.1.tar.gz", hash = "sha256:bf2e513eb8144c3298a3bd28ab1a5edb739131ec5c22e045ff93cd7f5319703a"}, ] [package.extras] @@ -3815,7 +3779,7 @@ core = ["importlib_metadata (>=6)", "jaraco.functools (>=4)", "jaraco.text (>=3. cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyobjc (<12)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] [[package]] @@ -3914,7 +3878,7 @@ test = ["py-ecc (==6.0.0)", "pytest (>=6.2.5)", "pytest-benchmark (>=3.2.3)"] [[package]] name = "sw-utils" -version = "v0.12.1" +version = "v0.12.4" description = "StakeWise Python utils" optional = false python-versions = "^3.10" @@ -3934,8 +3898,8 @@ web3 = "==7.13.0" [package.source] type = "git" url = "https://github.com/stakewise/sw-utils.git" -reference = "v0.12.1" -resolved_reference = "2db244e487825e2339c3d7d1d3cb1b78a2bf78ce" +reference = "v0.12.4" +resolved_reference = "14cc519fbc5374f469015a1b15813e9b34a432f0" [[package]] name = "tenacity" @@ -4455,4 +4419,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.12,<3.13" -content-hash = "505c74ff6ec6327eccebf589d0a7a0e376e585db84dbb96263a1dd367f268c62" +content-hash = "a166dc844b9343beb662b17ad343cd9ed0d90a577c878b8d17aadcc5b249e232" diff --git a/pyproject.toml b/pyproject.toml index 2742d001..21404552 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ python = ">=3.12,<3.13" python-decouple = "==3.8" sentry-sdk = "==1.45.1" py-ecc = "==8.0.0" -sw-utils = {git = "https://github.com/stakewise/sw-utils.git", rev = "v0.12.1"} +sw-utils = {git = "https://github.com/stakewise/sw-utils.git", rev = "v0.12.4"} staking-deposit = { git = "https://github.com/ethereum/staking-deposit-cli.git", rev = "v2.8.0" } multiproof = { git = "https://github.com/stakewise/multiproof.git", rev = "v0.1.10" } pycryptodomex = "3.19.1" diff --git a/src/common/contracts.py b/src/common/contracts.py index 39df8978..c4ef8fc2 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -26,7 +26,8 @@ EVENTS_CONCURRENCY_LIMIT, settings, ) -from src.meta_vault.typings import SubVaultExitRequest +from src.meta_vault.typings import SubVaultExitRequest, SubVaultRedemption +from src.redemptions.typings import RedeemablePositions from src.validators.typings import V2ValidatorEventData from src.withdrawals.typings import WithdrawalEvent @@ -142,6 +143,9 @@ def update_state(self, harvest_params: HarvestParams) -> HexStr: class VaultContract(ContractWrapper, VaultStateMixin): abi_path = 'abi/IEthVault.json' + async def vault_id(self) -> str: + return await self.contract.functions.vaultId().call() + def encoder(self) -> VaultEncoder: return VaultEncoder(self) @@ -407,6 +411,18 @@ async def withdrawable_assets(self) -> Wei: async def get_exit_queue_index(self, position_ticket: int) -> int: return await self.contract.functions.getExitQueueIndex(position_ticket).call() + async def calculate_sub_vaults_redemptions( + self, assets_to_redeem: Wei + ) -> list[SubVaultRedemption]: + res = await self.contract.functions.calculateSubVaultsRedemptions(assets_to_redeem).call() + return [ + SubVaultRedemption( + vault=Web3.to_checksum_address(entry[0]), + assets=Wei(entry[1]), + ) + for entry in res + ] + async def deposit_to_sub_vaults(self) -> HexStr: tx_function = self.contract.functions.depositToSubVaults() tx_hash = await transaction_gas_wrapper(tx_function) @@ -488,8 +504,29 @@ class OsTokenRedeemerContract(ContractWrapper): abi_path = 'abi/IOsTokenRedeemer.json' settings_key = 'OS_TOKEN_REDEEMER_CONTRACT_ADDRESS' - async def nonce(self) -> int: - return await self.contract.functions.nonce().call() + async def redeemable_positions(self) -> RedeemablePositions: + merkle_root, ipfs_hash = await self.contract.functions.redeemablePositions().call() + return RedeemablePositions( + merkle_root=Web3.to_hex(merkle_root), + ipfs_hash=ipfs_hash, + ) + + async def nonce(self, block_number: BlockNumber | None = None) -> int: + return await self.contract.functions.nonce().call(block_identifier=block_number) + + async def get_exit_queue_cumulative_tickets( + self, block_number: BlockNumber | None = None + ) -> int: + return await self.contract.functions.getExitQueueCumulativeTickets().call( + block_identifier=block_number + ) + + async def get_exit_queue_missing_assets( + self, target_ticket: int, block_number: BlockNumber | None = None + ) -> Wei: + return await self.contract.functions.getExitQueueMissingAssets(target_ticket).call( + block_identifier=block_number + ) class ValidatorsCheckerContract(ContractWrapper): diff --git a/src/common/decorators.py b/src/common/decorators.py new file mode 100644 index 00000000..6eef8ef0 --- /dev/null +++ b/src/common/decorators.py @@ -0,0 +1,30 @@ +import asyncio +from typing import Callable + + +def memoize(func: Callable) -> Callable: + """ + Helper to memoize both sync and async functions. + Main usage is for async functions because `functools.cache` won't work with them. + """ + cache: dict = {} + + async def memoized_async_func(*args, **kwargs): # type: ignore + key = (args, frozenset(sorted(kwargs.items()))) + if key in cache: + return cache[key] + result = await func(*args, **kwargs) + cache[key] = result + return result + + def memoized_sync_func(*args, **kwargs): # type: ignore + key = (args, frozenset(sorted(kwargs.items()))) + if key in cache: + return cache[key] + result = func(*args, **kwargs) + cache[key] = result + return result + + if asyncio.iscoroutinefunction(func): + return memoized_async_func + return memoized_sync_func diff --git a/src/common/utils.py b/src/common/utils.py index 2d7df7e3..35c792ea 100644 --- a/src/common/utils.py +++ b/src/common/utils.py @@ -5,7 +5,7 @@ from datetime import datetime, timezone from decimal import ROUND_FLOOR, Decimal, localcontext from pathlib import Path -from typing import Any +from typing import Any, AsyncGenerator, TypeVar import click import tenacity @@ -27,6 +27,9 @@ logger = logging.getLogger(__name__) +T = TypeVar('T') + + def get_build_version() -> str | None: path = Path(__file__).parents[1].joinpath('GIT_SHA') if not path.exists(): @@ -154,3 +157,17 @@ def calc_slot_by_block_timestamp(ts: Timestamp) -> int: return int( (ts - settings.network_config.GENESIS_TIMESTAMP) / settings.network_config.SECONDS_PER_SLOT ) + + +async def async_batched( + async_gen: AsyncGenerator[T, None], batch_size: int +) -> AsyncGenerator[list[T], None]: + """Batch items from an async iterator. Replacement for itertools.batched.""" + batch = [] + async for item in async_gen: + batch.append(item) + if len(batch) == batch_size: + yield batch + batch = [] + if batch: + yield batch diff --git a/src/common/withdrawals.py b/src/common/withdrawals.py index c4516f64..e7166fbf 100644 --- a/src/common/withdrawals.py +++ b/src/common/withdrawals.py @@ -13,6 +13,9 @@ async def get_pending_partial_withdrawals( chain_head: ChainHead, consensus_validators: list[ConsensusValidator] ) -> list[PendingPartialWithdrawal]: + """ + Get pending partial withdrawals from both consensus and execution layers. + """ consensus_withdrawals = await consensus_client.get_pending_partial_withdrawals( str(chain_head.slot) ) diff --git a/src/config/networks.py b/src/config/networks.py index d187bbd6..e9046a71 100644 --- a/src/config/networks.py +++ b/src/config/networks.py @@ -4,7 +4,7 @@ from datetime import timedelta from ens.constants import EMPTY_ADDR_HEX -from eth_typing import BlockNumber, ChecksumAddress +from eth_typing import BlockNumber, ChecksumAddress, HexStr from sw_utils.networks import GNOSIS, HOODI, MAINNET from sw_utils.networks import NETWORKS as BASE_NETWORKS from sw_utils.networks import BaseNetworkConfig @@ -51,6 +51,7 @@ class NetworkConfig(BaseNetworkConfig): TARGET_WITHDRAWAL_REQUESTS_PER_BLOCK: int TARGET_CONSOLIDATION_REQUESTS_PER_BLOCK: int NODE_CONFIG: NodeConfig + META_VAULT_ID: HexStr @dataclass @@ -137,6 +138,7 @@ def INITIAL_SYNC_ETA(self) -> int: 'MerkleExecute': timedelta(hours=1), }, ), + META_VAULT_ID=HexStr('0xcfece609e9557b5f0b085dadb8b7c99d43a9052d28ab3d68c9e3c1a9c3ab85c0'), ), HOODI: NetworkConfig( **asdict(BASE_NETWORKS[HOODI]), @@ -193,6 +195,7 @@ def INITIAL_SYNC_ETA(self) -> int: 'MerkleExecute': timedelta(minutes=30), }, ), + META_VAULT_ID=HexStr('0xcfece609e9557b5f0b085dadb8b7c99d43a9052d28ab3d68c9e3c1a9c3ab85c0'), ), GNOSIS: NetworkConfig( **asdict(BASE_NETWORKS[GNOSIS]), @@ -249,5 +252,6 @@ def INITIAL_SYNC_ETA(self) -> int: 'MerkleExecute': timedelta(hours=1), }, ), + META_VAULT_ID=HexStr('0xfb5cee5ecc2ff8d1a7a5ad55f89156551c040a926bec689720ab06063922454d'), ), } diff --git a/src/meta_vault/service.py b/src/meta_vault/service.py new file mode 100644 index 00000000..6c795c58 --- /dev/null +++ b/src/meta_vault/service.py @@ -0,0 +1,84 @@ +from collections import defaultdict +from typing import cast + +from eth_typing import ChecksumAddress +from web3.types import Wei + +from src.common.contracts import MetaVaultContract, VaultContract +from src.common.decorators import memoize +from src.config.settings import settings + + +async def distribute_meta_vault_redemption_assets( + vault_to_redemption_assets: defaultdict[ChecksumAddress, Wei], +) -> defaultdict[ChecksumAddress, Wei]: + """ + Parameters: + vault_to_redemption_assets: A mapping of vault addresses to their respective redemption assets, + which may include meta vaults. + + Distribute redemption assets from meta vaults to their underlying sub-vaults. + Returns a mapping of vault addresses to their respective redemption assets, + ensuring all assets are assigned to non-meta vaults. + """ + final_vault_to_redemption_assets: defaultdict[ChecksumAddress, int] = defaultdict(lambda: 0) + + for vault, assets in vault_to_redemption_assets.items(): + if await is_meta_vault(vault): + sub_vaults_redemptions = await get_meta_vault_redemption_assets( + meta_vault_address=vault, + assets_to_redeem=assets, + ) + for sub_vault, sub_assets in sub_vaults_redemptions.items(): + final_vault_to_redemption_assets[sub_vault] += sub_assets + else: + final_vault_to_redemption_assets[vault] += assets + + return cast(defaultdict[ChecksumAddress, Wei], final_vault_to_redemption_assets) + + +async def get_meta_vault_redemption_assets( + meta_vault_address: ChecksumAddress, + assets_to_redeem: Wei, +) -> defaultdict[ChecksumAddress, Wei]: + """ + This function distributes the specified assets to redeem from the meta vault + among its underlying sub-vaults. It handles both regular and nested meta vaults. + Finally every asset should be assigned to a non-meta vault. + + Returns a mapping of vault addresses to their respective redemption assets. + """ + vault_to_redemption_assets: defaultdict[ChecksumAddress, int] = defaultdict(lambda: 0) + meta_vault_contract = MetaVaultContract(meta_vault_address) + + sub_vaults_redemptions = await meta_vault_contract.calculate_sub_vaults_redemptions( + assets_to_redeem + ) + + for sub_vault_redemption in sub_vaults_redemptions: + if await is_meta_vault(sub_vault_redemption.vault): + sub_vault_assets = await get_meta_vault_redemption_assets( + sub_vault_redemption.vault, + sub_vault_redemption.assets, + ) + for vault, assets in sub_vault_assets.items(): + vault_to_redemption_assets[vault] += assets + else: + vault_to_redemption_assets[sub_vault_redemption.vault] += sub_vault_redemption.assets + + return cast(defaultdict[ChecksumAddress, Wei], vault_to_redemption_assets) + + +@memoize +async def is_meta_vault(vault_address: ChecksumAddress) -> bool: + """ + Checks if the given vault is a meta vault by comparing its vault ID + with the predefined META_VAULT_ID. + + Currently, meta vaults are not included in the vaults table. + Since adding a new field to the vaults table is not feasible, + memoization is used to minimize the number of EL calls. + """ + vault_contract = VaultContract(vault_address) + vault_id = await vault_contract.vault_id() + return vault_id == settings.network_config.META_VAULT_ID diff --git a/src/meta_vault/typings.py b/src/meta_vault/typings.py index 2818dc7e..57b46fd8 100644 --- a/src/meta_vault/typings.py +++ b/src/meta_vault/typings.py @@ -109,3 +109,9 @@ class ContractCall: address: ChecksumAddress data: HexStr description: str + + +@dataclass +class SubVaultRedemption: + vault: ChecksumAddress + assets: Wei diff --git a/src/redemptions/os_token_converter.py b/src/redemptions/os_token_converter.py index a669ea7a..01674490 100644 --- a/src/redemptions/os_token_converter.py +++ b/src/redemptions/os_token_converter.py @@ -18,6 +18,11 @@ def to_shares(self, assets: Wei) -> Wei: return Wei(0) return Wei((assets * self.total_shares) // self.total_assets) + def to_assets(self, shares: Wei) -> Wei: + if self.total_shares == 0: + return Wei(0) + return Wei((shares * self.total_assets) // self.total_shares) + async def create_os_token_converter(block_number: BlockNumber) -> OsTokenConverter: total_assets = await os_token_vault_controller_contract.total_assets(block_number) diff --git a/src/redemptions/tasks.py b/src/redemptions/tasks.py new file mode 100644 index 00000000..8158a1e9 --- /dev/null +++ b/src/redemptions/tasks.py @@ -0,0 +1,194 @@ +import logging +from collections import defaultdict +from typing import AsyncGenerator, cast + +from eth_typing import BlockNumber, ChecksumAddress, HexStr +from multiproof.standard import standard_leaf_hash +from sw_utils import convert_to_mgno +from sw_utils.networks import GNO_NETWORKS +from sw_utils.typings import ChainHead, ProtocolConfig +from web3 import Web3 +from web3.types import Gwei, Wei + +from src.common.clients import ipfs_fetch_client +from src.common.consensus import get_chain_latest_head +from src.common.contracts import multicall_contract, os_token_redeemer_contract +from src.common.protocol_config import get_protocol_config +from src.common.utils import async_batched +from src.config.settings import settings +from src.meta_vault.service import distribute_meta_vault_redemption_assets +from src.redemptions.os_token_converter import create_os_token_converter +from src.redemptions.typings import RedeemablePosition + +logger = logging.getLogger(__name__) + + +batch_size = 20 + + +async def get_redemption_assets() -> Gwei: + """ + Get redemption assets for operator's vault. + For Gno networks return value in mGNO-GWei. + """ + protocol_config = await get_protocol_config() + chain_head = await get_chain_latest_head() + + # Aggregate redemption assets per vault + vault_to_redemption_assets = await get_vault_to_redemption_assets( + chain_head=chain_head, protocol_config=protocol_config + ) + # Distribute redemption assets from meta vaults to their underlying vaults + vault_to_redemption_assets = await distribute_meta_vault_redemption_assets( + vault_to_redemption_assets=vault_to_redemption_assets + ) + # Filter by operator's vault + redemption_assets = vault_to_redemption_assets[settings.vault] + + if settings.network in GNO_NETWORKS: + # Convert GNO -> mGNO + redemption_assets = convert_to_mgno(redemption_assets) + + return Gwei(int(Web3.from_wei(redemption_assets, 'gwei'))) + + +async def get_vault_to_redemption_assets( + chain_head: ChainHead, protocol_config: ProtocolConfig +) -> defaultdict[ChecksumAddress, Wei]: + """ + Get redemption assets per vault. + For Gno networks return value in GNO-Wei. + """ + # todo: finalized head or latest head? + # or skip block argument? + ticket = await os_token_redeemer_contract.get_exit_queue_cumulative_tickets( + block_number=chain_head.block_number + ) + + total_redemption_assets = await os_token_redeemer_contract.get_exit_queue_missing_assets( + ticket, block_number=chain_head.block_number + ) + + # OsToken in-protocol rate may increase while vault assets are exiting. + # Ensure sufficient assets are allocated for redemption by applying + # a conservative APR adjustment. + total_redemption_assets = Wei( + int(total_redemption_assets * protocol_config.os_token_redeem_multiplier) + ) + + vault_to_redemption_assets = await aggregate_redemption_assets_by_vaults( + total_redemption_assets, + block_number=chain_head.block_number, + ) + return vault_to_redemption_assets + + +async def aggregate_redemption_assets_by_vaults( + total_redemption_assets: Wei, block_number: BlockNumber +) -> defaultdict[ChecksumAddress, Wei]: + """ + Iterate through redeemable positions until the total redemption assets are exhausted. + Aggregate unprocessed assets by vaults. + + :param total_redemption_assets: The total amount of assets available for redemption. + For Gno networks total_redemption_assets is in GNO-Wei. + + :return: A mapping of vault addresses to their corresponding unprocessed assets. + """ + # Convert total redemption assets to shares + os_token_converter = await create_os_token_converter(block_number) + total_redemption_shares = os_token_converter.to_shares(total_redemption_assets) + + nonce = await os_token_redeemer_contract.nonce() + vault_to_unprocessed_shares: defaultdict[ChecksumAddress, Wei] = defaultdict(lambda: Wei(0)) + + # Iterate through redeemable positions until total redemption shares are exhausted + async for redeemable_position_batch in async_batched(iter_redeemable_positions(), batch_size): + processed_shares_batch = await get_processed_shares_batch( + redeemable_positions_batch=redeemable_position_batch, nonce=nonce + ) + for redeemable_position, processed_shares in zip( + redeemable_position_batch, processed_shares_batch + ): + vault = redeemable_position.vault + leaf_shares = redeemable_position.amount + unprocessed_shares = leaf_shares - processed_shares + + # Skip if no unprocessed shares, handle rounding errors + if unprocessed_shares <= 1: + continue + + # Aggregate unprocessed shares by vault + unprocessed_shares = min(unprocessed_shares, total_redemption_shares) + vault_to_unprocessed_shares[vault] += unprocessed_shares # type: ignore + + total_redemption_shares -= unprocessed_shares # type: ignore + + # Stop iterating processed shares batch + if total_redemption_shares <= 0: + break + + # Stop iterating redeemable positions + if total_redemption_shares <= 0: + break + + # Convert shares to assets per vault + vault_to_unprocessed_assets = defaultdict(lambda: Wei(0)) + + for vault, shares in vault_to_unprocessed_shares.items(): + vault_to_unprocessed_assets[vault] = os_token_converter.to_assets(shares) + + return vault_to_unprocessed_assets + + +async def iter_redeemable_positions() -> AsyncGenerator[RedeemablePosition, None]: + redeemable_positions = await os_token_redeemer_contract.redeemable_positions() + + # Check whether redeemable positions are available + if not redeemable_positions.ipfs_hash: + return + + # Fetch redeemable positions data from IPFS + data = cast(list[dict], await ipfs_fetch_client.fetch_json(redeemable_positions.ipfs_hash)) + + # data structure example: + # [{"owner:" 0x01, "amount": 100000, "vault": 0x02}, ...] + + for item in data: + yield RedeemablePosition( + owner=Web3.to_checksum_address(item['owner']), + vault=Web3.to_checksum_address(item['vault']), + amount=Wei(int(item['amount'])), + ) + + +async def get_processed_shares_batch( + redeemable_positions_batch: list[RedeemablePosition], + nonce: int, + block_number: BlockNumber | None = None, +) -> list[Wei]: + calls: list[tuple[ChecksumAddress, HexStr]] = [] + + for redeemable_position in redeemable_positions_batch: + leaf_hash = get_redeemable_position_leaf_hash(redeemable_position, nonce) + call_data = os_token_redeemer_contract.encode_abi( + fn_name='leafToProcessedShares', + args=[leaf_hash], + ) + calls.append((os_token_redeemer_contract.contract_address, call_data)) + + _, results = await multicall_contract.aggregate(calls, block_number=block_number) + return [Wei(Web3.to_int(res)) for res in results] + + +def get_redeemable_position_leaf_hash(redeemable_position: RedeemablePosition, nonce: int) -> bytes: + """Get the leaf hash for a redeemable position.""" + vault = redeemable_position.vault + owner = redeemable_position.owner + amount = redeemable_position.amount + + leaf = standard_leaf_hash( + values=(nonce, vault, amount, owner), + types=['uint256', 'address', 'uint256', 'address'], + ) + return leaf diff --git a/src/redemptions/typings.py b/src/redemptions/typings.py index 1d5ada5e..a73db2b9 100644 --- a/src/redemptions/typings.py +++ b/src/redemptions/typings.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from eth_typing import ChecksumAddress +from eth_typing import ChecksumAddress, HexStr from web3 import Web3 from web3.types import Wei @@ -66,3 +66,9 @@ def as_dict(self) -> dict: def merkle_leaf(self, nonce: int) -> tuple[int, ChecksumAddress, Wei, ChecksumAddress]: return nonce, self.vault, self.amount, self.owner + + +@dataclass +class RedeemablePositions: + merkle_root: HexStr + ipfs_hash: str diff --git a/src/withdrawals/assets.py b/src/withdrawals/assets.py index 4a66544d..69e5fc90 100644 --- a/src/withdrawals/assets.py +++ b/src/withdrawals/assets.py @@ -44,6 +44,10 @@ async def get_queued_assets( pending_partial_withdrawals: list[PendingPartialWithdrawal], chain_head: ChainHead, ) -> Gwei: + """ + Get exit queue missing assets. + For Gno networks return value in mGNO-Gwei. + """ harvest_params = await get_harvest_params(chain_head.block_number) # Get exit queue cumulative tickets diff --git a/src/withdrawals/tasks.py b/src/withdrawals/tasks.py index 2f44ee42..f982425c 100644 --- a/src/withdrawals/tasks.py +++ b/src/withdrawals/tasks.py @@ -32,6 +32,7 @@ WITHDRAWALS_INTERVAL, settings, ) +from src.redemptions.tasks import get_redemption_assets from src.validators.consensus import fetch_consensus_validators from src.validators.database import VaultValidatorCrud from src.validators.exceptions import EmptyRelayerResponseException @@ -120,6 +121,9 @@ async def process(self) -> None: consolidations=consolidations, chain_head=chain_head, ) + redemption_assets = await get_redemption_assets() + queued_assets = Gwei(queued_assets + redemption_assets) + metrics.queued_assets.labels(network=settings.network).set(int(queued_assets)) if queued_assets < MIN_WITHDRAWAL_AMOUNT_GWEI: From 8c6f6d272162e2d6a70269cc1e218e783cad40e2 Mon Sep 17 00:00:00 2001 From: Evgeny Gusarov Date: Tue, 3 Feb 2026 18:29:22 +0300 Subject: [PATCH 37/65] Add tests --- src/common/tests/utils.py | 17 +- src/meta_vault/tests/test_service.py | 150 +++++++++++ src/redemptions/os_token_converter.py | 2 +- src/redemptions/tasks.py | 2 +- src/redemptions/tests/factories.py | 16 ++ .../tests/test_os_token_converter.py | 34 +++ src/redemptions/tests/test_tasks.py | 253 ++++++++++++++++++ 7 files changed, 471 insertions(+), 3 deletions(-) create mode 100644 src/meta_vault/tests/test_service.py create mode 100644 src/redemptions/tests/factories.py create mode 100644 src/redemptions/tests/test_os_token_converter.py create mode 100644 src/redemptions/tests/test_tasks.py diff --git a/src/common/tests/utils.py b/src/common/tests/utils.py index abe1544f..5a86035f 100644 --- a/src/common/tests/utils.py +++ b/src/common/tests/utils.py @@ -1,7 +1,22 @@ from decimal import Decimal -from web3.types import Gwei +from web3 import Web3 +from web3.types import Gwei, Wei def ether_to_gwei(value: int | float | Decimal) -> Gwei: return Gwei(int(value * 10**9)) + + +def parse_wei(value: str | list | dict) -> Wei: + if isinstance(value, str): + number, unit = value.split(' ') + return Web3.to_wei(number, unit) + + if isinstance(value, list): + return [parse_wei(value) for value in value] + + if isinstance(value, dict): + return {key: parse_wei(value) for key, value in value.items()} + + raise ValueError(f'Unsupported type for parse_wei: {type(value)}') diff --git a/src/meta_vault/tests/test_service.py b/src/meta_vault/tests/test_service.py new file mode 100644 index 00000000..08b40371 --- /dev/null +++ b/src/meta_vault/tests/test_service.py @@ -0,0 +1,150 @@ +from collections import defaultdict +from unittest.mock import AsyncMock, patch + +import pytest +from sw_utils.tests import faker + +from src.common.contracts import MetaVaultContract +from src.meta_vault.service import distribute_meta_vault_redemption_assets +from src.meta_vault.typings import SubVaultRedemption + + +@pytest.fixture +def vaults(): + return { + 'vault1': faker.eth_address(), + 'vault2': faker.eth_address(), + 'meta_vault': faker.eth_address(), + 'nested_meta_vault': faker.eth_address(), + 'sub_vault1': faker.eth_address(), + 'sub_vault2': faker.eth_address(), + } + + +class TestDistributeMetaVaultRedemptionAssets: + async def test_distribute_no_meta_vaults(self, vaults): + assets = defaultdict( + int, + { + vaults['vault1']: 100, + vaults['vault2']: 200, + }, + ) + + with patch('src.meta_vault.service.is_meta_vault', return_value=False): + result = await distribute_meta_vault_redemption_assets(assets) + assert dict(result) == { + vaults['vault1']: 100, + vaults['vault2']: 200, + } + + async def test_distribute_single_meta_vault(self, vaults): + assets = defaultdict( + int, + { + vaults['meta_vault']: 300, + vaults['vault1']: 100, + }, + ) + + def is_meta_vault_side_effect(addr): + return addr == vaults['meta_vault'] + + sub_vault_redemptions = [ + SubVaultRedemption(vault=vaults['sub_vault1'], assets=120), + SubVaultRedemption(vault=vaults['sub_vault2'], assets=180), + ] + + with patch( + 'src.meta_vault.service.is_meta_vault', + new=AsyncMock(side_effect=is_meta_vault_side_effect), + ), patch.object( + MetaVaultContract, + 'calculate_sub_vaults_redemptions', + return_value=sub_vault_redemptions, + ): + result = await distribute_meta_vault_redemption_assets(assets) + assert dict(result) == { + vaults['vault1']: 100, + vaults['sub_vault1']: 120, + vaults['sub_vault2']: 180, + } + + async def test_distribute_nested_meta_vaults(self, vaults): + assets = defaultdict( + int, + { + vaults['meta_vault']: 500, + }, + ) + + def is_meta_vault_side_effect(addr): + return addr in {vaults['meta_vault'], vaults['nested_meta_vault']} + + # meta_vault splits to nested_meta_vault and vault2 + meta_vault_redemptions = [ + SubVaultRedemption(vault=vaults['nested_meta_vault'], assets=300), + SubVaultRedemption(vault=vaults['vault2'], assets=200), + ] + # nested_meta_vault splits to sub_vault1 and sub_vault2 + nested_meta_vault_redemptions = [ + SubVaultRedemption(vault=vaults['sub_vault1'], assets=100), + SubVaultRedemption(vault=vaults['sub_vault2'], assets=200), + ] + + async def calculate_sub_vaults_redemptions_side_effect(self, assets): + if self.address == vaults['meta_vault']: + return meta_vault_redemptions + elif self.address == vaults['nested_meta_vault']: + return nested_meta_vault_redemptions + else: + return defaultdict(int) + + with patch( + 'src.meta_vault.service.is_meta_vault', + new=AsyncMock(side_effect=is_meta_vault_side_effect), + ), patch.object( + MetaVaultContract, + 'calculate_sub_vaults_redemptions', + new=calculate_sub_vaults_redemptions_side_effect, + ): + result = await distribute_meta_vault_redemption_assets(assets) + assert dict(result) == { + vaults['vault2']: 200, + vaults['sub_vault1']: 100, + vaults['sub_vault2']: 200, + } + + async def test_distribute_mixed_meta_and_non_meta(self, vaults): + assets = defaultdict( + int, + { + vaults['vault1']: 50, + vaults['meta_vault']: 150, + vaults['vault2']: 75, + }, + ) + + def is_meta_vault_side_effect(addr): + return addr == vaults['meta_vault'] + + sub_vault_redemptions = [ + SubVaultRedemption(vault=vaults['sub_vault1'], assets=60), + SubVaultRedemption(vault=vaults['sub_vault2'], assets=90), + ] + + with patch( + 'src.meta_vault.service.is_meta_vault', + new=AsyncMock(side_effect=is_meta_vault_side_effect), + ), patch.object( + MetaVaultContract, + 'calculate_sub_vaults_redemptions', + return_value=sub_vault_redemptions, + ): + result = await distribute_meta_vault_redemption_assets(assets) + assert dict(result) == { + vaults['vault1']: 50, + vaults['vault2']: 75, + vaults['sub_vault1']: 60, + vaults['sub_vault2']: 90, + } diff --git a/src/redemptions/os_token_converter.py b/src/redemptions/os_token_converter.py index 01674490..84230ec8 100644 --- a/src/redemptions/os_token_converter.py +++ b/src/redemptions/os_token_converter.py @@ -24,7 +24,7 @@ def to_assets(self, shares: Wei) -> Wei: return Wei((shares * self.total_assets) // self.total_shares) -async def create_os_token_converter(block_number: BlockNumber) -> OsTokenConverter: +async def create_os_token_converter(block_number: BlockNumber | None = None) -> OsTokenConverter: total_assets = await os_token_vault_controller_contract.total_assets(block_number) total_shares = await os_token_vault_controller_contract.total_shares(block_number) return OsTokenConverter(total_assets=total_assets, total_shares=total_shares) diff --git a/src/redemptions/tasks.py b/src/redemptions/tasks.py index 8158a1e9..9db81a10 100644 --- a/src/redemptions/tasks.py +++ b/src/redemptions/tasks.py @@ -84,7 +84,7 @@ async def get_vault_to_redemption_assets( async def aggregate_redemption_assets_by_vaults( - total_redemption_assets: Wei, block_number: BlockNumber + total_redemption_assets: Wei, block_number: BlockNumber | None = None ) -> defaultdict[ChecksumAddress, Wei]: """ Iterate through redeemable positions until the total redemption assets are exhausted. diff --git a/src/redemptions/tests/factories.py b/src/redemptions/tests/factories.py new file mode 100644 index 00000000..171c052b --- /dev/null +++ b/src/redemptions/tests/factories.py @@ -0,0 +1,16 @@ +from eth_typing import HexStr +from sw_utils.tests import faker + +from src.redemptions.typings import RedeemablePositions + + +def create_redeemable_positions( + merkle_root: HexStr | None = None, ipfs_hash: str | None = None +) -> RedeemablePositions: + if merkle_root is None: + merkle_root = faker.merkle_root() + + if ipfs_hash is None: + ipfs_hash = faker.ipfs_hash() + + return RedeemablePositions(merkle_root=merkle_root, ipfs_hash=ipfs_hash) diff --git a/src/redemptions/tests/test_os_token_converter.py b/src/redemptions/tests/test_os_token_converter.py new file mode 100644 index 00000000..96de4505 --- /dev/null +++ b/src/redemptions/tests/test_os_token_converter.py @@ -0,0 +1,34 @@ +import pytest +from web3.types import Wei + +from src.redemptions.os_token_converter import OsTokenConverter + + +class TestOsTokenConverter: + @pytest.mark.parametrize( + 'total_assets,total_shares,assets,expected_shares', + [ + (Wei(1000), Wei(100), Wei(500), Wei(50)), + (Wei(2000), Wei(200), Wei(1000), Wei(100)), + (Wei(0), Wei(100), Wei(500), Wei(0)), # Edge case: total_assets is 0 + (Wei(1000), Wei(0), Wei(500), Wei(0)), # Edge case: total_shares is 0 + ], + ) + def test_to_shares(self, total_assets, total_shares, assets, expected_shares): + converter = OsTokenConverter(total_assets=total_assets, total_shares=total_shares) + shares = converter.to_shares(assets) + assert shares == expected_shares + + @pytest.mark.parametrize( + 'total_assets,total_shares,shares,expected_assets', + [ + (Wei(1000), Wei(100), Wei(50), Wei(500)), + (Wei(2000), Wei(200), Wei(100), Wei(1000)), + (Wei(1000), Wei(0), Wei(50), Wei(0)), # Edge case: total_shares is 0 + (Wei(0), Wei(100), Wei(50), Wei(0)), # Edge case: total_assets is 0 + ], + ) + def test_to_assets(self, total_assets, total_shares, shares, expected_assets): + converter = OsTokenConverter(total_assets=total_assets, total_shares=total_shares) + assets = converter.to_assets(shares) + assert assets == expected_assets diff --git a/src/redemptions/tests/test_tasks.py b/src/redemptions/tests/test_tasks.py new file mode 100644 index 00000000..44a52bdf --- /dev/null +++ b/src/redemptions/tests/test_tasks.py @@ -0,0 +1,253 @@ +import itertools +import random +from contextlib import contextmanager +from decimal import Decimal +from unittest import mock + +import pytest +from eth_typing import HexStr +from sw_utils.tests import faker +from web3 import Web3 + +from src.redemptions.os_token_converter import os_token_vault_controller_contract +from src.common.tests.utils import parse_wei +from src.redemptions.tasks import ( + aggregate_redemption_assets_by_vaults, + batch_size, + ipfs_fetch_client, + os_token_redeemer_contract, +) +from src.redemptions.tests.factories import create_redeemable_positions +from src.redemptions.typings import RedeemablePositions + + +class TestAggregateRedemptionAssetsByVaults: + async def test_redeemable_positions_empty(self): + redeemable_positions = RedeemablePositions(merkle_root=HexStr('0x'), ipfs_hash='') + total_redemption_assets = Web3.to_wei(100, 'ether') + + with self.patch(redeemable_positions=redeemable_positions): + redemption_assets_by_vaults = await aggregate_redemption_assets_by_vaults( + total_redemption_assets + ) + assert redemption_assets_by_vaults == {} + + @pytest.mark.parametrize( + 'total_redemption_assets, leaf_shares, processed_shares, expected_redeemed_assets', + [ + # Case 1. Total redemption shares exceeds total leaf shares + parse_wei(['100 ether', '10 ether', '0 ether', '11 ether']), + parse_wei(['100 ether', '10 ether', '2 ether', '8.8 ether']), + parse_wei(['100 ether', '10 ether', '10 ether', '0 ether']), + parse_wei(['100 ether', '10 ether', '11 ether', '0 ether']), + # + # Case 2. Total redemption shares is less than total leaf shares + parse_wei(['1.1 ether', '10 ether', '0 ether', '1.1 ether']), + ], + ) + async def test_redeemable_positions_1_vault( + self, total_redemption_assets, leaf_shares, processed_shares, expected_redeemed_assets + ): + """ + The case of single vault. + """ + vault_1 = faker.eth_address() + redeemable_positions_ipfs_data = [ + { + 'owner': faker.eth_address(), + 'vault': vault_1, + 'amount': leaf_shares, + } + ] + processed_shares_batch = [processed_shares] + + with self.patch( + redeemable_positions_ipfs_data=redeemable_positions_ipfs_data, + processed_shares_batch=processed_shares_batch, + ): + redemption_assets_by_vaults = await aggregate_redemption_assets_by_vaults( + total_redemption_assets + ) + assert len(redemption_assets_by_vaults) <= 1 # length can be 0 if no assets to redeem + assert redemption_assets_by_vaults[vault_1] == expected_redeemed_assets + + @pytest.mark.parametrize( + 'total_redemption_assets, leaf_shares, processed_shares, expected_redeemed_assets', + [ + # Case 1. Total redemption shares exceeds total leaf shares + parse_wei( + [ + '100 ether', + {'vault_1': '10 ether', 'vault_2': '20 ether'}, # leaf shares + {'vault_1': '0 ether', 'vault_2': '0 ether'}, # processed shares + {'vault_1': '11 ether', 'vault_2': '22 ether'}, # expected redeemed assets + ] + ), + # + # Case 2. Total redemption shares is greater than vault 1 leaf shares + # but less than total leaf shares + parse_wei( + [ + '15 ether', + {'vault_1': '10 ether', 'vault_2': '20 ether'}, # leaf shares + {'vault_1': '0 ether', 'vault_2': '0 ether'}, # processed shares + {'vault_1': '11 ether', 'vault_2': '4 ether'}, # expected redeemed assets + ] + ), + # Case 3. Total redemption shares is less than vault 1 leaf shares + parse_wei( + [ + '5 ether', + {'vault_1': '10 ether', 'vault_2': '20 ether'}, # leaf shares + {'vault_1': '0 ether', 'vault_2': '0 ether'}, # processed shares + {'vault_1': '5 ether', 'vault_2': '0 ether'}, # expected redeemed assets + ] + ), + ], + ) + async def test_redeemable_positions_2_vaults( + self, total_redemption_assets, leaf_shares, processed_shares, expected_redeemed_assets + ): + """ + The case of two vaults. + """ + vault_1 = faker.eth_address() + vault_2 = faker.eth_address() + redeemable_positions_ipfs_data = [ + { + 'owner': faker.eth_address(), + 'vault': vault_1, + 'amount': leaf_shares['vault_1'], + }, + { + 'owner': faker.eth_address(), + 'vault': vault_2, + 'amount': leaf_shares['vault_2'], + }, + ] + processed_shares_batch = [processed_shares['vault_1'], processed_shares['vault_2']] + + with self.patch( + redeemable_positions_ipfs_data=redeemable_positions_ipfs_data, + processed_shares_batch=processed_shares_batch, + ): + redemption_assets_by_vaults = await aggregate_redemption_assets_by_vaults( + total_redemption_assets + ) + # length can be less than 2 if no assets to redeem + assert len(redemption_assets_by_vaults) <= 2 + # allow 1 wei difference due to rounding + assert ( + abs(redemption_assets_by_vaults[vault_1] - expected_redeemed_assets['vault_1']) <= 1 + ) + assert ( + abs(redemption_assets_by_vaults[vault_2] - expected_redeemed_assets['vault_2']) <= 1 + ) + + async def test_2_vaults_many_users(self): + """ + The case of two vaults with many users. + Check aggregation by vault and batching. + """ + vault_1 = faker.eth_address() + vault_2 = faker.eth_address() + redeemable_positions_ipfs_data = [] + processed_shares = [] + redemption_shares_vault_1 = 0 + redemption_shares_vault_2 = 0 + + redemption_users_count_per_vault = int(1.5 * batch_size) + processed_shares_max_index = 2 * batch_size + + # Create 50 users per vault + for index in range(50): + leaf_shares_1 = Web3.to_wei(random.randint(1, 5), 'ether') + redeemable_positions_ipfs_data.append( + { + 'owner': faker.eth_address(), + 'vault': vault_1, + 'amount': leaf_shares_1, + } + ) + if index < redemption_users_count_per_vault: + redemption_shares_vault_1 += Web3.to_wei(1, 'ether') + + if index < processed_shares_max_index: + processed_shares.append(leaf_shares_1 - Web3.to_wei(1, 'ether')) + + leaf_shares_2 = Web3.to_wei(random.randint(1, 5), 'ether') + redeemable_positions_ipfs_data.append( + { + 'owner': faker.eth_address(), + 'vault': vault_2, + 'amount': leaf_shares_2, + } + ) + + if index < redemption_users_count_per_vault: + redemption_shares_vault_2 += Web3.to_wei('0.5', 'ether') + + if index < processed_shares_max_index: + processed_shares.append(leaf_shares_2 - Web3.to_wei('0.5', 'ether')) + + total_redemption_shares = redemption_shares_vault_1 + redemption_shares_vault_2 + total_redemption_assets = total_redemption_shares * Decimal('1.1') + + processed_shares_batches = list(itertools.batched(processed_shares, batch_size)) + + with self.patch( + redeemable_positions_ipfs_data=redeemable_positions_ipfs_data, + processed_shares_batches=processed_shares_batches, + ): + redemption_assets_by_vaults = await aggregate_redemption_assets_by_vaults( + total_redemption_assets + ) + assert len(redemption_assets_by_vaults) == 2 + assert redemption_assets_by_vaults[vault_1] == redemption_shares_vault_1 * Decimal( + '1.1' + ) + assert redemption_assets_by_vaults[vault_2] == redemption_shares_vault_2 * Decimal( + '1.1' + ) + + @contextmanager + def patch( + self, + os_token_assets_to_shares_ratio: Decimal = Decimal('1.1'), + redeemable_positions: RedeemablePositions | None = None, + redeemable_positions_ipfs_data: list[dict] | None = None, + processed_shares_batch: list[int] | None = None, + processed_shares_batches: list[list[int]] | None = None, + ): + if redeemable_positions is None: + redeemable_positions = create_redeemable_positions() + + total_assets = Web3.to_wei(100 * os_token_assets_to_shares_ratio, 'ether') + total_shares = Web3.to_wei(100, 'ether') + + redeemable_positions_ipfs_data = redeemable_positions_ipfs_data or [] + + if processed_shares_batch is None: + processed_shares_batch = [0] * len(redeemable_positions_ipfs_data) + + if processed_shares_batches is None: + processed_shares_batches = [processed_shares_batch] + + with mock.patch.object( + os_token_redeemer_contract, 'redeemable_positions', return_value=redeemable_positions + ), mock.patch.object( + os_token_redeemer_contract, 'nonce', return_value=0 + ), mock.patch.object( + os_token_vault_controller_contract, 'total_assets', return_value=total_assets + ), mock.patch.object( + os_token_vault_controller_contract, 'total_shares', return_value=total_shares + ), mock.patch.object( + ipfs_fetch_client, 'fetch_json', return_value=redeemable_positions_ipfs_data + ), mock.patch( + 'src.redemptions.tasks.get_processed_shares_batch', side_effect=processed_shares_batches + ): + yield + + def get_processed_shares_batch(self, redeemable_positions_batch, nonce) -> list[int]: + """A placeholder for the patched method.""" + return [0] * len(redeemable_positions_batch) From 1838d6696c26eea3ecc7c2f4ae25e7cf6c31a5aa Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 4 Feb 2026 11:52:22 +0300 Subject: [PATCH 38/65] Unify is_meta_vault Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 14 ++------------ src/common/contracts.py | 3 --- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 648e45db..244aada3 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -11,7 +11,6 @@ from multiproof.standard import MultiProof from sw_utils import InterruptHandler from web3 import Web3 -from web3.exceptions import ContractLogicError, Web3RPCError from web3.types import Wei from src.common.clients import ( @@ -21,7 +20,6 @@ setup_clients, ) from src.common.contracts import ( - MetaVaultContract, VaultContract, multicall_contract, os_token_redeemer_contract, @@ -34,6 +32,7 @@ from src.common.wallet import wallet from src.config.networks import AVAILABLE_NETWORKS, ZERO_CHECKSUM_ADDRESS from src.config.settings import settings +from src.meta_vault.service import is_meta_vault from src.redemptions.os_token_converter import create_os_token_converter from src.redemptions.typings import RedeemablePosition from src.validators.execution import get_withdrawable_assets @@ -221,7 +220,7 @@ async def process(block_number: BlockNumber) -> None: # Handle Meta-Vaults with Insufficient Withdrawable Assets vault_positions_shares = Wei(sum(position.redeemable_shares for position in positions)) vault_positions_assets = os_token_converter.to_assets(vault_positions_shares) - if vault_positions_assets > withdrawable_assets and await _is_meta_vault(vault_address): + if vault_positions_assets > withdrawable_assets and await is_meta_vault(vault_address): # Check if vault is a meta-vault logger.info( 'Vault %s is a meta-vault with insufficient withdrawable assets.', vault_address @@ -347,15 +346,6 @@ async def _process_exit_queue(block_number: BlockNumber) -> None: logger.info('ProcessExitQueue transaction sent. Tx Hash: %s', tx_hash) -async def _is_meta_vault(vault_address: ChecksumAddress) -> bool: - meta_vault_contract = MetaVaultContract(vault_address) - try: - await meta_vault_contract.sub_vaults_rewards_nonce() - return True - except (Web3RPCError, ValueError, ContractLogicError): - return False - - async def _startup_check() -> None: positions_manager = await os_token_redeemer_contract.positions_manager() if positions_manager != wallet.account.address: diff --git a/src/common/contracts.py b/src/common/contracts.py index 119a730e..1633dcbd 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -423,9 +423,6 @@ async def calculate_sub_vaults_redemptions( for entry in res ] - async def sub_vaults_rewards_nonce(self) -> HexStr: - return await self.contract.functions.subVaultsRewardsNonce().call() - async def deposit_to_sub_vaults(self) -> HexStr: tx_function = self.contract.functions.depositToSubVaults() tx_hash = await transaction_gas_wrapper(tx_function) From 211e09e3252bc75b090e3cec0ea4be649f64eb69 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 18 Feb 2026 14:03:39 +0300 Subject: [PATCH 39/65] Unify redeemable positions fetch Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 64 ++++++++++------------- src/common/contracts.py | 7 --- 2 files changed, 27 insertions(+), 44 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 4b420590..7c72fde0 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -3,7 +3,6 @@ import sys from collections import defaultdict from pathlib import Path -from typing import cast import click from eth_typing import BlockNumber, ChecksumAddress, HexStr @@ -13,12 +12,7 @@ from web3 import Web3 from web3.types import Wei -from src.common.clients import ( - close_clients, - execution_client, - ipfs_fetch_client, - setup_clients, -) +from src.common.clients import close_clients, execution_client, setup_clients from src.common.contracts import ( VaultContract, multicall_contract, @@ -28,12 +22,17 @@ from src.common.harvest import get_harvest_params from src.common.logging import LOG_LEVELS, setup_logging from src.common.typings import HarvestParams -from src.common.utils import log_verbose +from src.common.utils import async_batched, log_verbose from src.common.wallet import wallet from src.config.networks import AVAILABLE_NETWORKS, ZERO_CHECKSUM_ADDRESS from src.config.settings import settings from src.meta_vault.service import is_meta_vault from src.redemptions.os_token_converter import create_os_token_converter +from src.redemptions.tasks import ( + batch_size, + get_processed_shares_batch, + iter_os_token_positions, +) from src.redemptions.typings import OsTokenPosition from src.validators.execution import get_withdrawable_assets @@ -188,21 +187,7 @@ async def process(block_number: BlockNumber) -> None: ) # Fetch Positions from IPFS - redeemable_positions_meta = await os_token_redeemer_contract.redeemable_positions(block_number) - redeemable_positions = await fetch_redeemable_positions(redeemable_positions_meta.ipfs_hash) - - # Calculate Redeemable Shares Per Position - for redeemable_position in redeemable_positions: - # Compute leaf hash - leaf_hash = redeemable_position.leaf_hash(nonce - 1) - # Get already processed shares - leaf_processed_shares = await os_token_redeemer_contract.leaf_to_processed_shares( - leaf_hash, block_number - ) - # Calculate redeemable shares - redeemable_position.redeemable_shares = Wei( - redeemable_position.amount - leaf_processed_shares - ) + redeemable_positions = await _fetch_redeemable_positions(nonce=nonce, block_number=block_number) # Group positions by vault vault_to_positions: defaultdict[ChecksumAddress, list[OsTokenPosition]] = defaultdict(list) @@ -316,21 +301,26 @@ async def execute_redemption( return tx_hash -async def fetch_redeemable_positions(ipfs_hash: str) -> list[OsTokenPosition]: - # Fetch redeemable positions data from IPFS - data = cast(list[dict], await ipfs_fetch_client.fetch_json(ipfs_hash)) - - # data structure example: - # [{"owner:" 0x01, "amount": 100000, "vault": 0x02}, ...] - - return [ - OsTokenPosition( - owner=Web3.to_checksum_address(item['owner']), - vault=Web3.to_checksum_address(item['vault']), - amount=Wei(int(item['amount'])), +async def _fetch_redeemable_positions( + nonce: int, block_number: BlockNumber +) -> list[OsTokenPosition]: + os_token_positions = [] + async for os_token_position_batch in async_batched( + iter_os_token_positions(block_number=block_number), batch_size + ): + processed_shares_batch = await get_processed_shares_batch( + os_token_positions_batch=os_token_position_batch, + nonce=nonce, + block_number=block_number, ) - for item in data - ] + for os_token_position, processed_shares in zip( + os_token_position_batch, processed_shares_batch + ): + # Calculate redeemable shares + os_token_position.redeemable_shares = Wei(os_token_position.amount - processed_shares) + os_token_positions.append(os_token_position) + + return os_token_positions async def _process_exit_queue(block_number: BlockNumber) -> None: diff --git a/src/common/contracts.py b/src/common/contracts.py index 0c18b762..c3f31049 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -569,13 +569,6 @@ async def can_process_exit_queue(self, block_number: BlockNumber | None = None) block_identifier=block_number ) - async def leaf_to_processed_shares( - self, leaf: bytes, block_number: BlockNumber | None = None - ) -> Wei: - return await self.contract.functions.leafToProcessedShares(leaf).call( - block_identifier=block_number - ) - async def process_exit_queue(self) -> HexStr: tx_function = self.contract.functions.processExitQueue() tx_hash = await transaction_gas_wrapper(tx_function) From 3d7a5094bb340d62bc1b55ec1ccfe4c4ef56a415 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 18 Feb 2026 14:30:03 +0300 Subject: [PATCH 40/65] Fix contract calls Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 31 +++++++++++++++++------ src/common/contracts.py | 12 +++++++-- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 7c72fde0..a745d910 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -178,6 +178,9 @@ async def process(block_number: BlockNumber) -> None: os_token_converter = await create_os_token_converter(block_number) nonce = await os_token_redeemer_contract.nonce(block_number) + # The contract increments nonce during setRedeemablePositions, + # but uses nonce - 1 for leaf hash computation during redemption. + tree_nonce = nonce - 1 queued_assets = os_token_converter.to_assets(queued_shares) logger.info( @@ -187,7 +190,9 @@ async def process(block_number: BlockNumber) -> None: ) # Fetch Positions from IPFS - redeemable_positions = await _fetch_redeemable_positions(nonce=nonce, block_number=block_number) + redeemable_positions = await _fetch_redeemable_positions( + nonce=tree_nonce, block_number=block_number + ) # Group positions by vault vault_to_positions: defaultdict[ChecksumAddress, list[OsTokenPosition]] = defaultdict(list) @@ -222,7 +227,7 @@ async def process(block_number: BlockNumber) -> None: redeemable_assets = os_token_converter.to_assets(position.redeemable_shares) if redeemable_assets <= withdrawable_assets: - shares_to_redeem = min(position.amount, queued_shares) + shares_to_redeem = min(position.redeemable_shares, queued_shares) logger.info( 'Position Owner: %s, Vault: %s, Shares to Redeem: %s', position.owner, @@ -238,13 +243,14 @@ async def process(block_number: BlockNumber) -> None: ) ) withdrawable_assets = Wei(withdrawable_assets - redeemable_assets) - queued_shares = Wei(queued_assets - shares_to_redeem) + queued_shares = Wei(queued_shares - shares_to_redeem) # Execute Redemption with Multicall tx_hash = await execute_redemption( + all_positions=redeemable_positions, positions_to_redeem=positions_to_redeem, vault_to_harvest_params=vault_to_harvest_params, - nonce=nonce, + nonce=tree_nonce, ) logger.info( 'Successfully redeemed %s OsToken positions. Transaction hash: %s', @@ -254,12 +260,14 @@ async def process(block_number: BlockNumber) -> None: async def execute_redemption( + all_positions: list[OsTokenPosition], positions_to_redeem: list[OsTokenPosition], vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], nonce: int, ) -> HexStr | None: multiproof = _get_multi_proof( + all_positions=all_positions, positions_to_redeem=positions_to_redeem, nonce=nonce, ) @@ -276,9 +284,14 @@ async def execute_redemption( ) ) + # Convert to tuples matching Solidity struct: + # OsTokenPosition(address vault, address owner, uint256 leafShares, uint256 sharesToRedeem) + positions_arg = [ + (pos.vault, pos.owner, pos.amount, pos.redeemable_shares) for pos in positions_to_redeem + ] redeem_os_token_positions_call = os_token_redeemer_contract.encode_abi( fn_name='redeemOsTokenPositions', - args=[positions_to_redeem, multiproof.proof, multiproof.proof_flags], + args=[positions_arg, multiproof.proof, multiproof.proof_flags], ) calls.append((os_token_redeemer_contract.contract_address, redeem_os_token_positions_call)) try: @@ -346,11 +359,12 @@ async def _startup_check() -> None: def _get_multi_proof( nonce: int, + all_positions: list[OsTokenPosition], positions_to_redeem: list[OsTokenPosition], ) -> MultiProof[tuple[bytes, int]]: - leaves = [r.merkle_leaf(nonce) for r in positions_to_redeem] + all_leaves = [r.merkle_leaf(nonce) for r in all_positions] tree = StandardMerkleTree.of( - leaves, + all_leaves, [ 'uint256', 'address', @@ -358,5 +372,6 @@ def _get_multi_proof( 'address', ], ) - multi_proof = tree.get_multi_proof(leaves) + redeem_leaves = [r.merkle_leaf(nonce) for r in positions_to_redeem] + multi_proof = tree.get_multi_proof(redeem_leaves) return multi_proof diff --git a/src/common/contracts.py b/src/common/contracts.py index c3f31049..c6beb316 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -578,8 +578,16 @@ async def redeem_sub_vaults_assets( self, vault_address: ChecksumAddress, assets_to_redeem: Wei ) -> Wei: tx_function = self.contract.functions.redeemSubVaultsAssets(vault_address, assets_to_redeem) - tx = await transaction_gas_wrapper(tx_function) - return Wei(Web3.to_int(tx)) + # Simulate the call first to get the return value (totalRedeemedAssets) + redeemed_assets: int = await tx_function.call() + # Send the actual transaction + tx_hash = await transaction_gas_wrapper(tx_function) + tx_receipt = await self.execution_client.eth.wait_for_transaction_receipt( + tx_hash, timeout=settings.execution_transaction_timeout + ) + if not tx_receipt['status']: + return Wei(0) + return Wei(redeemed_assets) class ValidatorsCheckerContract(ContractWrapper): From 8087439676649b05438c4dd99a015f72be1613a8 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 18 Feb 2026 14:53:57 +0300 Subject: [PATCH 41/65] Check empty merkle root Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 36 +++++++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index a745d910..20bd42f8 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -190,6 +190,12 @@ async def process(block_number: BlockNumber) -> None: ) # Fetch Positions from IPFS + redeemable_positions_meta = await os_token_redeemer_contract.redeemable_positions(block_number) + zero_merkle_root = redeemable_positions_meta.merkle_root == HexStr('0x' + '0' * 64) + if zero_merkle_root or not redeemable_positions_meta.ipfs_hash: + logger.info('No redeemable positions available. Skipping redemption processing.') + return + redeemable_positions = await _fetch_redeemable_positions( nonce=tree_nonce, block_number=block_number ) @@ -223,6 +229,9 @@ async def process(block_number: BlockNumber) -> None: # Process each position in the vault for position in positions: + if queued_shares <= 0: + break + # Convert redeemable shares to assets redeemable_assets = os_token_converter.to_assets(position.redeemable_shares) @@ -245,6 +254,13 @@ async def process(block_number: BlockNumber) -> None: withdrawable_assets = Wei(withdrawable_assets - redeemable_assets) queued_shares = Wei(queued_shares - shares_to_redeem) + if queued_shares <= 0: + break + + if not positions_to_redeem: + logger.info('No positions eligible for redemption.') + return + # Execute Redemption with Multicall tx_hash = await execute_redemption( all_positions=redeemable_positions, @@ -252,11 +268,12 @@ async def process(block_number: BlockNumber) -> None: vault_to_harvest_params=vault_to_harvest_params, nonce=tree_nonce, ) - logger.info( - 'Successfully redeemed %s OsToken positions. Transaction hash: %s', - len(positions_to_redeem), - tx_hash, - ) + if tx_hash: + logger.info( + 'Successfully redeemed %s OsToken positions. Transaction hash: %s', + len(positions_to_redeem), + tx_hash, + ) async def execute_redemption( @@ -346,7 +363,14 @@ async def _process_exit_queue(block_number: BlockNumber) -> None: if can_process_exit_queue: logger.info('Exit queue can be processed. Calling processExitQueue...') tx_hash = await os_token_redeemer_contract.process_exit_queue() - logger.info('ProcessExitQueue transaction sent. Tx Hash: %s', tx_hash) + logger.info('Waiting for processExitQueue transaction %s confirmation', tx_hash) + tx_receipt = await execution_client.eth.wait_for_transaction_receipt( + tx_hash, timeout=settings.execution_transaction_timeout + ) + if not tx_receipt['status']: + logger.error('processExitQueue transaction failed. Tx Hash: %s', tx_hash) + else: + logger.info('processExitQueue confirmed. Tx Hash: %s', tx_hash) async def _startup_check() -> None: From 0e8e1dbf8fed36073f8f722568f72b5d9f1c23c8 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 18 Feb 2026 14:56:19 +0300 Subject: [PATCH 42/65] Skip empty positions Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 20bd42f8..0d6d1a5e 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -346,8 +346,11 @@ async def _fetch_redeemable_positions( for os_token_position, processed_shares in zip( os_token_position_batch, processed_shares_batch ): - # Calculate redeemable shares - os_token_position.redeemable_shares = Wei(os_token_position.amount - processed_shares) + # Calculate redeemable shares, skip fully processed positions + redeemable_shares = os_token_position.amount - processed_shares + if redeemable_shares <= 0: + continue + os_token_position.redeemable_shares = Wei(redeemable_shares) os_token_positions.append(os_token_position) return os_token_positions From 9490f7494b69c2a1d223a45d6bffcf84ddb1bf55 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 18 Feb 2026 15:23:34 +0300 Subject: [PATCH 43/65] Refactoring Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 264 +++++++++++++--------- src/common/contracts.py | 11 +- 2 files changed, 162 insertions(+), 113 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 0d6d1a5e..3ce403da 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -2,6 +2,7 @@ import logging import sys from collections import defaultdict +from dataclasses import dataclass from pathlib import Path import click @@ -10,6 +11,7 @@ from multiproof.standard import MultiProof from sw_utils import InterruptHandler from web3 import Web3 +from web3.exceptions import Web3Exception from web3.types import Wei from src.common.clients import close_clients, execution_client, setup_clients @@ -27,7 +29,10 @@ from src.config.networks import AVAILABLE_NETWORKS, ZERO_CHECKSUM_ADDRESS from src.config.settings import settings from src.meta_vault.service import is_meta_vault -from src.redemptions.os_token_converter import create_os_token_converter +from src.redemptions.os_token_converter import ( + OsTokenConverter, + create_os_token_converter, +) from src.redemptions.tasks import ( batch_size, get_processed_shares_batch, @@ -39,6 +44,13 @@ logger = logging.getLogger(__name__) SLEEP_INTERVAL = 60 # 1 minute +ZERO_MERKLE_ROOT = HexStr('0x' + '0' * 64) + + +@dataclass +class PositionSelectionResult: + positions_to_redeem: list[OsTokenPosition] + vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None] @click.option( @@ -137,13 +149,7 @@ def process_redeemer( sys.exit(1) -# pylint: disable-next=too-many-locals async def main() -> None: - """ - Monitors the EthOsTokenRedeemer/GnoOsTokenRedeemer contracts - and automatically processes OsToken position redemptions - and exit queue checkpoints. - """ setup_logging() await setup_clients() await _startup_check() @@ -151,144 +157,190 @@ async def main() -> None: with InterruptHandler() as interrupt_handler: while not interrupt_handler.exit: block_number = await execution_client.eth.block_number - await process( - block_number=block_number, - ) + await process(block_number=block_number) await interrupt_handler.sleep(SLEEP_INTERVAL) finally: await close_clients() -# pylint: disable-next=too-many-locals async def process(block_number: BlockNumber) -> None: - """ - Monitors the EthOsTokenRedeemer/GnoOsTokenRedeemer contracts - and automatically processes OsToken position redemptions - and exit queue checkpoints. - """ - # Check Exit Queue Processing await _process_exit_queue(block_number) - # Check Queued Shares for Redemption queued_shares = await os_token_redeemer_contract.queued_shares(block_number) if queued_shares == 0: logger.info('No queued shares for redemption. Skipping to next interval.') return os_token_converter = await create_os_token_converter(block_number) - nonce = await os_token_redeemer_contract.nonce(block_number) + # The contract increments nonce during setRedeemablePositions, # but uses nonce - 1 for leaf hash computation during redemption. + nonce = await os_token_redeemer_contract.nonce(block_number) tree_nonce = nonce - 1 queued_assets = os_token_converter.to_assets(queued_shares) logger.info( - 'Queued Shares for Redemption: %s(~%s assets)', + 'Queued Shares for Redemption: %s (~%s assets)', queued_shares, Web3.from_wei(queued_assets, 'ether'), ) - # Fetch Positions from IPFS redeemable_positions_meta = await os_token_redeemer_contract.redeemable_positions(block_number) - zero_merkle_root = redeemable_positions_meta.merkle_root == HexStr('0x' + '0' * 64) - if zero_merkle_root or not redeemable_positions_meta.ipfs_hash: + if redeemable_positions_meta.merkle_root == ZERO_MERKLE_ROOT or ( + not redeemable_positions_meta.ipfs_hash + ): logger.info('No redeemable positions available. Skipping redemption processing.') return redeemable_positions = await _fetch_redeemable_positions( - nonce=tree_nonce, block_number=block_number + tree_nonce=tree_nonce, block_number=block_number ) - # Group positions by vault + result = await _select_positions_to_redeem( + redeemable_positions=redeemable_positions, + queued_shares=queued_shares, + os_token_converter=os_token_converter, + block_number=block_number, + ) + + if not result.positions_to_redeem: + logger.info('No positions eligible for redemption.') + return + + tx_hash = await _execute_redemption( + all_positions=redeemable_positions, + positions_to_redeem=result.positions_to_redeem, + vault_to_harvest_params=result.vault_to_harvest_params, + tree_nonce=tree_nonce, + ) + if tx_hash: + logger.info( + 'Successfully redeemed %s OsToken positions. Transaction hash: %s', + len(result.positions_to_redeem), + tx_hash, + ) + + +async def _select_positions_to_redeem( + redeemable_positions: list[OsTokenPosition], + queued_shares: int, + os_token_converter: OsTokenConverter, + block_number: BlockNumber, +) -> PositionSelectionResult: vault_to_positions: defaultdict[ChecksumAddress, list[OsTokenPosition]] = defaultdict(list) for position in redeemable_positions: vault_to_positions[position.vault].append(position) - positions_to_redeem = [] + positions_to_redeem: list[OsTokenPosition] = [] vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None] = {} + remaining_shares = queued_shares + for vault_address, positions in vault_to_positions.items(): - # Check if state update is required harvest_params = await get_harvest_params(vault_address, block_number) vault_to_harvest_params[vault_address] = harvest_params withdrawable_assets = await get_withdrawable_assets(vault_address, harvest_params) - # Handle Meta-Vaults with Insufficient Withdrawable Assets - vault_positions_shares = Wei(sum(position.redeemable_shares for position in positions)) - vault_positions_assets = os_token_converter.to_assets(vault_positions_shares) - if vault_positions_assets > withdrawable_assets and await is_meta_vault(vault_address): - # Check if vault is a meta-vault - logger.info( - 'Vault %s is a meta-vault with insufficient withdrawable assets.', vault_address - ) - additional_assets_needed = Wei(vault_positions_assets - withdrawable_assets) - redeemed_assets = await os_token_redeemer_contract.redeem_sub_vaults_assets( - vault_address, additional_assets_needed - ) - withdrawable_assets = Wei(withdrawable_assets + redeemed_assets) + withdrawable_assets = await _try_redeem_sub_vaults( + vault_address=vault_address, + positions=positions, + withdrawable_assets=withdrawable_assets, + harvest_params=harvest_params, + os_token_converter=os_token_converter, + ) - # Process each position in the vault for position in positions: - if queued_shares <= 0: + if remaining_shares <= 0: break - # Convert redeemable shares to assets redeemable_assets = os_token_converter.to_assets(position.redeemable_shares) + if redeemable_assets > withdrawable_assets: + continue - if redeemable_assets <= withdrawable_assets: - shares_to_redeem = min(position.redeemable_shares, queued_shares) - logger.info( - 'Position Owner: %s, Vault: %s, Shares to Redeem: %s', - position.owner, - position.vault, - shares_to_redeem, - ) - positions_to_redeem.append( - OsTokenPosition( - vault=position.vault, - owner=position.owner, - amount=position.amount, - redeemable_shares=shares_to_redeem, - ) + shares_to_redeem = Wei(min(position.redeemable_shares, remaining_shares)) + logger.info( + 'Position Owner: %s, Vault: %s, Shares to Redeem: %s', + position.owner, + position.vault, + shares_to_redeem, + ) + # redeemable_shares is overloaded here: in the source position it means + # "total shares available for redemption", in the redeem batch it means + # "shares to actually redeem in this transaction" (maps to sharesToRedeem + # in the Solidity struct). + positions_to_redeem.append( + OsTokenPosition( + vault=position.vault, + owner=position.owner, + amount=position.amount, + redeemable_shares=shares_to_redeem, ) - withdrawable_assets = Wei(withdrawable_assets - redeemable_assets) - queued_shares = Wei(queued_shares - shares_to_redeem) + ) + withdrawable_assets = Wei(withdrawable_assets - redeemable_assets) + remaining_shares -= shares_to_redeem - if queued_shares <= 0: + if remaining_shares <= 0: break - if not positions_to_redeem: - logger.info('No positions eligible for redemption.') - return - - # Execute Redemption with Multicall - tx_hash = await execute_redemption( - all_positions=redeemable_positions, + return PositionSelectionResult( positions_to_redeem=positions_to_redeem, vault_to_harvest_params=vault_to_harvest_params, - nonce=tree_nonce, ) - if tx_hash: + + +async def _try_redeem_sub_vaults( + vault_address: ChecksumAddress, + positions: list[OsTokenPosition], + withdrawable_assets: Wei, + harvest_params: HarvestParams | None, + os_token_converter: OsTokenConverter, +) -> Wei: + """If vault is a meta-vault with insufficient assets, redeem from sub-vaults. + + Returns the (possibly updated) withdrawable assets for the vault. + """ + vault_positions_shares = Wei(sum(p.redeemable_shares for p in positions)) + vault_positions_assets = os_token_converter.to_assets(vault_positions_shares) + + if vault_positions_assets <= withdrawable_assets or not await is_meta_vault(vault_address): + return withdrawable_assets + + logger.info('Vault %s is a meta-vault with insufficient withdrawable assets.', vault_address) + additional_assets_needed = Wei(vault_positions_assets - withdrawable_assets) + try: + tx_hash = await os_token_redeemer_contract.redeem_sub_vaults_assets( + vault_address, additional_assets_needed + ) logger.info( - 'Successfully redeemed %s OsToken positions. Transaction hash: %s', - len(positions_to_redeem), + 'redeemSubVaultsAssets confirmed for vault %s. Tx Hash: %s', + vault_address, tx_hash, ) + except RuntimeError: + logger.warning( + 'redeemSubVaultsAssets failed for vault %s. ' + 'Proceeding with current withdrawable assets.', + vault_address, + ) + return withdrawable_assets + + # Re-query actual withdrawable assets on-chain after sub-vault redemption + # to avoid TOCTOU issues with stale values + return await get_withdrawable_assets(vault_address, harvest_params) -async def execute_redemption( +async def _execute_redemption( all_positions: list[OsTokenPosition], positions_to_redeem: list[OsTokenPosition], vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], - nonce: int, + tree_nonce: int, ) -> HexStr | None: - multiproof = _get_multi_proof( all_positions=all_positions, positions_to_redeem=positions_to_redeem, - nonce=nonce, + tree_nonce=tree_nonce, ) - calls = [] + calls: list[tuple[ChecksumAddress, HexStr]] = [] for vault in set(pos.vault for pos in positions_to_redeem): harvest_params = vault_to_harvest_params.get(vault) @@ -301,22 +353,23 @@ async def execute_redemption( ) ) - # Convert to tuples matching Solidity struct: + # Maps to Solidity struct: # OsTokenPosition(address vault, address owner, uint256 leafShares, uint256 sharesToRedeem) + # amount -> leafShares, redeemable_shares -> sharesToRedeem positions_arg = [ (pos.vault, pos.owner, pos.amount, pos.redeemable_shares) for pos in positions_to_redeem ] - redeem_os_token_positions_call = os_token_redeemer_contract.encode_abi( + redeem_call = os_token_redeemer_contract.encode_abi( fn_name='redeemOsTokenPositions', args=[positions_arg, multiproof.proof, multiproof.proof_flags], ) - calls.append((os_token_redeemer_contract.contract_address, redeem_os_token_positions_call)) + calls.append((os_token_redeemer_contract.contract_address, redeem_call)) + try: tx_function = multicall_contract.functions.aggregate(calls) tx = await transaction_gas_wrapper(tx_function=tx_function) - except Exception as e: - logger.error('Failed to redeem os token positions: %s', e) - logger.exception(e) + except Web3Exception: + logger.exception('Failed to redeem os token positions') return None tx_hash = Web3.to_hex(tx) @@ -332,28 +385,31 @@ async def execute_redemption( async def _fetch_redeemable_positions( - nonce: int, block_number: BlockNumber + tree_nonce: int, block_number: BlockNumber ) -> list[OsTokenPosition]: - os_token_positions = [] - async for os_token_position_batch in async_batched( + positions: list[OsTokenPosition] = [] + async for batch in async_batched( iter_os_token_positions(block_number=block_number), batch_size ): processed_shares_batch = await get_processed_shares_batch( - os_token_positions_batch=os_token_position_batch, - nonce=nonce, + os_token_positions_batch=batch, + nonce=tree_nonce, block_number=block_number, ) - for os_token_position, processed_shares in zip( - os_token_position_batch, processed_shares_batch - ): - # Calculate redeemable shares, skip fully processed positions - redeemable_shares = os_token_position.amount - processed_shares + for position, processed_shares in zip(batch, processed_shares_batch): + redeemable_shares = position.amount - processed_shares if redeemable_shares <= 0: continue - os_token_position.redeemable_shares = Wei(redeemable_shares) - os_token_positions.append(os_token_position) + positions.append( + OsTokenPosition( + owner=position.owner, + vault=position.vault, + amount=position.amount, + redeemable_shares=Wei(redeemable_shares), + ) + ) - return os_token_positions + return positions async def _process_exit_queue(block_number: BlockNumber) -> None: @@ -385,20 +441,14 @@ async def _startup_check() -> None: def _get_multi_proof( - nonce: int, + tree_nonce: int, all_positions: list[OsTokenPosition], positions_to_redeem: list[OsTokenPosition], ) -> MultiProof[tuple[bytes, int]]: - all_leaves = [r.merkle_leaf(nonce) for r in all_positions] + all_leaves = [p.merkle_leaf(tree_nonce) for p in all_positions] tree = StandardMerkleTree.of( all_leaves, - [ - 'uint256', - 'address', - 'uint256', - 'address', - ], + ['uint256', 'address', 'uint256', 'address'], ) - redeem_leaves = [r.merkle_leaf(nonce) for r in positions_to_redeem] - multi_proof = tree.get_multi_proof(redeem_leaves) - return multi_proof + redeem_leaves = [p.merkle_leaf(tree_nonce) for p in positions_to_redeem] + return tree.get_multi_proof(redeem_leaves) diff --git a/src/common/contracts.py b/src/common/contracts.py index c6beb316..9a8c2914 100644 --- a/src/common/contracts.py +++ b/src/common/contracts.py @@ -576,18 +576,17 @@ async def process_exit_queue(self) -> HexStr: async def redeem_sub_vaults_assets( self, vault_address: ChecksumAddress, assets_to_redeem: Wei - ) -> Wei: + ) -> HexStr: tx_function = self.contract.functions.redeemSubVaultsAssets(vault_address, assets_to_redeem) - # Simulate the call first to get the return value (totalRedeemedAssets) - redeemed_assets: int = await tx_function.call() - # Send the actual transaction tx_hash = await transaction_gas_wrapper(tx_function) tx_receipt = await self.execution_client.eth.wait_for_transaction_receipt( tx_hash, timeout=settings.execution_transaction_timeout ) if not tx_receipt['status']: - return Wei(0) - return Wei(redeemed_assets) + raise RuntimeError( + f'redeemSubVaultsAssets transaction failed. Tx Hash: {Web3.to_hex(tx_hash)}' + ) + return Web3.to_hex(tx_hash) class ValidatorsCheckerContract(ContractWrapper): From 8b2014c012812fbef239d29f03dee3372b79cf96 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 18 Feb 2026 17:40:46 +0300 Subject: [PATCH 44/65] Add tests Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 26 +- .../test_internal/test_process_redeemer.py | 734 ++++++++++++++++++ 2 files changed, 758 insertions(+), 2 deletions(-) create mode 100644 src/commands/tests/test_internal/test_process_redeemer.py diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 3ce403da..a579646b 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -232,9 +232,8 @@ async def _select_positions_to_redeem( for position in redeemable_positions: vault_to_positions[position.vault].append(position) - positions_to_redeem: list[OsTokenPosition] = [] vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None] = {} - remaining_shares = queued_shares + vault_to_withdrawable_assets: dict[ChecksumAddress, Wei] = {} for vault_address, positions in vault_to_positions.items(): harvest_params = await get_harvest_params(vault_address, block_number) @@ -248,6 +247,29 @@ async def _select_positions_to_redeem( harvest_params=harvest_params, os_token_converter=os_token_converter, ) + vault_to_withdrawable_assets[vault_address] = withdrawable_assets + + return _filter_positions_to_redeem( + vault_to_positions=vault_to_positions, + vault_to_withdrawable_assets=vault_to_withdrawable_assets, + vault_to_harvest_params=vault_to_harvest_params, + queued_shares=queued_shares, + os_token_converter=os_token_converter, + ) + + +def _filter_positions_to_redeem( + vault_to_positions: dict[ChecksumAddress, list[OsTokenPosition]], + vault_to_withdrawable_assets: dict[ChecksumAddress, Wei], + vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], + queued_shares: int, + os_token_converter: OsTokenConverter, +) -> PositionSelectionResult: + positions_to_redeem: list[OsTokenPosition] = [] + remaining_shares = queued_shares + + for vault_address, positions in vault_to_positions.items(): + withdrawable_assets = vault_to_withdrawable_assets[vault_address] for position in positions: if remaining_shares <= 0: diff --git a/src/commands/tests/test_internal/test_process_redeemer.py b/src/commands/tests/test_internal/test_process_redeemer.py new file mode 100644 index 00000000..02fb5311 --- /dev/null +++ b/src/commands/tests/test_internal/test_process_redeemer.py @@ -0,0 +1,734 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from eth_typing import BlockNumber, ChecksumAddress, HexStr +from hexbytes import HexBytes +from sw_utils.tests import faker +from web3 import Web3 +from web3.exceptions import Web3Exception +from web3.types import Wei + +from src.commands.internal.process_redeemer import ( + ZERO_MERKLE_ROOT, + PositionSelectionResult, + _execute_redemption, + _fetch_redeemable_positions, + _filter_positions_to_redeem, + _get_multi_proof, + _process_exit_queue, + _select_positions_to_redeem, + _startup_check, + _try_redeem_sub_vaults, + process, +) +from src.common.typings import HarvestParams +from src.redemptions.os_token_converter import OsTokenConverter +from src.redemptions.typings import OsTokenPosition, RedeemablePositions + +MODULE = 'src.commands.internal.process_redeemer' + +VAULT_1 = Web3.to_checksum_address('0x' + '11' * 20) +VAULT_2 = Web3.to_checksum_address('0x' + '22' * 20) +OWNER_1 = Web3.to_checksum_address('0x' + '33' * 20) +OWNER_2 = Web3.to_checksum_address('0x' + '44' * 20) + + +# --- Pure function tests (no mocks) --- + + +class TestGetMultiProof: + def test_single_position(self) -> None: + position = make_position(amount=1000, redeemable_shares=500) + result = _get_multi_proof( + tree_nonce=5, + all_positions=[position], + positions_to_redeem=[position], + ) + assert len(result.leaves) == 1 + + def test_partial_redeem(self) -> None: + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, redeemable_shares=500) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, redeemable_shares=1000) + + result = _get_multi_proof( + tree_nonce=5, + all_positions=[pos1, pos2], + positions_to_redeem=[pos1], + ) + assert len(result.leaves) == 1 + assert len(result.proof) > 0 + + def test_all_positions_redeemed(self) -> None: + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, redeemable_shares=500) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, redeemable_shares=1000) + + result = _get_multi_proof( + tree_nonce=5, + all_positions=[pos1, pos2], + positions_to_redeem=[pos1, pos2], + ) + assert len(result.leaves) == 2 + + +class TestFilterPositionsToRedeem: + def test_empty_positions(self) -> None: + result = _filter_positions_to_redeem( + vault_to_positions={}, + vault_to_withdrawable_assets={}, + vault_to_harvest_params={}, + queued_shares=1000, + os_token_converter=make_converter(), + ) + assert result.positions_to_redeem == [] + assert result.vault_to_harvest_params == {} + + def test_single_position_sufficient_assets(self) -> None: + position = make_position(redeemable_shares=500) + # to_assets(500) = 500 * 110 / 100 = 550; withdrawable=1000 → selected + result = _filter_positions_to_redeem( + vault_to_positions={VAULT_1: [position]}, + vault_to_withdrawable_assets={VAULT_1: Wei(1000)}, + vault_to_harvest_params={VAULT_1: None}, + queued_shares=10000, + os_token_converter=make_converter(), + ) + assert len(result.positions_to_redeem) == 1 + assert result.positions_to_redeem[0].redeemable_shares == Wei(500) + + def test_single_position_insufficient_assets(self) -> None: + position = make_position(redeemable_shares=500) + # to_assets(500) = 550; withdrawable=100 → skipped + result = _filter_positions_to_redeem( + vault_to_positions={VAULT_1: [position]}, + vault_to_withdrawable_assets={VAULT_1: Wei(100)}, + vault_to_harvest_params={VAULT_1: None}, + queued_shares=10000, + os_token_converter=make_converter(), + ) + assert result.positions_to_redeem == [] + + def test_queued_shares_limits_redemption(self) -> None: + position = make_position(redeemable_shares=500) + result = _filter_positions_to_redeem( + vault_to_positions={VAULT_1: [position]}, + vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, + vault_to_harvest_params={VAULT_1: None}, + queued_shares=200, + os_token_converter=make_converter(), + ) + assert len(result.positions_to_redeem) == 1 + assert result.positions_to_redeem[0].redeemable_shares == Wei(200) + + def test_multiple_positions_limited_by_withdrawable_assets(self) -> None: + pos1 = make_position(owner=OWNER_1, redeemable_shares=500) + pos2 = make_position(owner=OWNER_2, redeemable_shares=1000) + # 1:1 converter; pos1=500 assets, pos2=1000 assets; withdrawable=700 + # pos1 fits (700-500=200 remaining), pos2 doesn't (1000>200) + result = _filter_positions_to_redeem( + vault_to_positions={VAULT_1: [pos1, pos2]}, + vault_to_withdrawable_assets={VAULT_1: Wei(700)}, + vault_to_harvest_params={VAULT_1: None}, + queued_shares=10000, + os_token_converter=make_converter(100, 100), + ) + assert len(result.positions_to_redeem) == 1 + assert result.positions_to_redeem[0].owner == OWNER_1 + + def test_multiple_vaults_both_selected(self) -> None: + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, redeemable_shares=500) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, redeemable_shares=800) + + vault_to_positions: dict[ChecksumAddress, list[OsTokenPosition]] = {} + vault_to_positions[VAULT_1] = [pos1] + vault_to_positions[VAULT_2] = [pos2] + + result = _filter_positions_to_redeem( + vault_to_positions=vault_to_positions, + vault_to_withdrawable_assets={VAULT_1: Wei(10000), VAULT_2: Wei(10000)}, + vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, + queued_shares=10000, + os_token_converter=make_converter(100, 100), + ) + assert len(result.positions_to_redeem) == 2 + + def test_stops_across_vaults_when_queued_shares_exhausted(self) -> None: + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, redeemable_shares=500) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, redeemable_shares=800) + + vault_to_positions: dict[ChecksumAddress, list[OsTokenPosition]] = {} + vault_to_positions[VAULT_1] = [pos1] + vault_to_positions[VAULT_2] = [pos2] + + result = _filter_positions_to_redeem( + vault_to_positions=vault_to_positions, + vault_to_withdrawable_assets={VAULT_1: Wei(10000), VAULT_2: Wei(10000)}, + vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, + queued_shares=500, + os_token_converter=make_converter(100, 100), + ) + assert len(result.positions_to_redeem) == 1 + assert result.positions_to_redeem[0].vault == VAULT_1 + + def test_preserves_harvest_params(self) -> None: + pos = make_position(redeemable_shares=500) + harvest_params = make_harvest_params() + + result = _filter_positions_to_redeem( + vault_to_positions={VAULT_1: [pos]}, + vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, + vault_to_harvest_params={VAULT_1: harvest_params}, + queued_shares=10000, + os_token_converter=make_converter(), + ) + assert result.vault_to_harvest_params[VAULT_1] is harvest_params + + def test_preserves_original_amount(self) -> None: + pos = make_position(amount=1000, redeemable_shares=500) + result = _filter_positions_to_redeem( + vault_to_positions={VAULT_1: [pos]}, + vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, + vault_to_harvest_params={VAULT_1: None}, + queued_shares=200, + os_token_converter=make_converter(), + ) + assert result.positions_to_redeem[0].amount == Wei(1000) + assert result.positions_to_redeem[0].redeemable_shares == Wei(200) + + def test_stops_within_vault_when_queued_shares_exhausted(self) -> None: + # Multiple positions in same vault, queued_shares runs out mid-vault + pos1 = make_position(owner=OWNER_1, redeemable_shares=400) + pos2 = make_position(owner=OWNER_2, redeemable_shares=300) + # 1:1 converter; pos1 consumes all 400 queued_shares, pos2 is skipped + result = _filter_positions_to_redeem( + vault_to_positions={VAULT_1: [pos1, pos2]}, + vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, + vault_to_harvest_params={VAULT_1: None}, + queued_shares=400, + os_token_converter=make_converter(100, 100), + ) + assert len(result.positions_to_redeem) == 1 + assert result.positions_to_redeem[0].owner == OWNER_1 + + def test_skips_position_then_selects_next(self) -> None: + # First position too expensive, second fits + pos1 = make_position(owner=OWNER_1, redeemable_shares=1000) + pos2 = make_position(owner=OWNER_2, redeemable_shares=100) + # 1:1 converter; pos1=1000 assets > 500 withdrawable; pos2=100 <= 500 + result = _filter_positions_to_redeem( + vault_to_positions={VAULT_1: [pos1, pos2]}, + vault_to_withdrawable_assets={VAULT_1: Wei(500)}, + vault_to_harvest_params={VAULT_1: None}, + queued_shares=10000, + os_token_converter=make_converter(100, 100), + ) + assert len(result.positions_to_redeem) == 1 + assert result.positions_to_redeem[0].owner == OWNER_2 + + +# --- Async function tests (with mocks) --- + + +class TestFetchRedeemablePositions: + async def test_empty_positions(self) -> None: + async def empty_gen(block_number: BlockNumber | None = None): # type: ignore[misc] + return + yield # noqa: unreachable + + with ( + patch(f'{MODULE}.iter_os_token_positions', side_effect=empty_gen), + patch(f'{MODULE}.get_processed_shares_batch', new=AsyncMock(return_value=[])), + ): + result = await _fetch_redeemable_positions(tree_nonce=5, block_number=BlockNumber(100)) + assert result == [] + + async def test_all_shares_processed(self) -> None: + pos = make_position(amount=1000) + + async def gen(block_number: BlockNumber | None = None): # type: ignore[misc] + yield pos + + with ( + patch(f'{MODULE}.iter_os_token_positions', side_effect=gen), + patch( + f'{MODULE}.get_processed_shares_batch', + new=AsyncMock(return_value=[Wei(1000)]), + ), + ): + result = await _fetch_redeemable_positions(tree_nonce=5, block_number=BlockNumber(100)) + assert result == [] + + async def test_partial_processed_shares(self) -> None: + pos = make_position(amount=1000) + + async def gen(block_number: BlockNumber | None = None): # type: ignore[misc] + yield pos + + with ( + patch(f'{MODULE}.iter_os_token_positions', side_effect=gen), + patch( + f'{MODULE}.get_processed_shares_batch', + new=AsyncMock(return_value=[Wei(300)]), + ), + ): + result = await _fetch_redeemable_positions(tree_nonce=5, block_number=BlockNumber(100)) + assert len(result) == 1 + assert result[0].redeemable_shares == Wei(700) + assert result[0].amount == Wei(1000) + + async def test_multiple_positions_mixed(self) -> None: + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000) + + async def gen(block_number: BlockNumber | None = None): # type: ignore[misc] + yield pos1 + yield pos2 + + # pos1 fully processed, pos2 partially processed + with ( + patch(f'{MODULE}.iter_os_token_positions', side_effect=gen), + patch( + f'{MODULE}.get_processed_shares_batch', + new=AsyncMock(return_value=[Wei(1000), Wei(500)]), + ), + ): + result = await _fetch_redeemable_positions(tree_nonce=5, block_number=BlockNumber(100)) + assert len(result) == 1 + assert result[0].owner == OWNER_2 + assert result[0].redeemable_shares == Wei(1500) + + +class TestTryRedeemSubVaults: + async def test_sufficient_withdrawable_assets(self) -> None: + positions = [make_position(redeemable_shares=500)] + # 1:1 converter; vault_positions_assets=500 <= withdrawable=1000 → return + result = await _try_redeem_sub_vaults( + vault_address=VAULT_1, + positions=positions, + withdrawable_assets=Wei(1000), + harvest_params=None, + os_token_converter=make_converter(100, 100), + ) + assert result == Wei(1000) + + async def test_insufficient_non_meta_vault(self) -> None: + positions = [make_position(redeemable_shares=500)] + # vault_positions_assets=500 > withdrawable=100, but not meta-vault + with patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)): + result = await _try_redeem_sub_vaults( + vault_address=VAULT_1, + positions=positions, + withdrawable_assets=Wei(100), + harvest_params=None, + os_token_converter=make_converter(100, 100), + ) + assert result == Wei(100) + + async def test_meta_vault_successful_redeem(self) -> None: + positions = [make_position(redeemable_shares=500)] + # vault_positions_assets=500 > withdrawable=100, meta-vault + with ( + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=True)), + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch( + f'{MODULE}.get_withdrawable_assets', + new=AsyncMock(return_value=Wei(600)), + ), + ): + mock_redeemer.redeem_sub_vaults_assets = AsyncMock(return_value='0xabc') + result = await _try_redeem_sub_vaults( + vault_address=VAULT_1, + positions=positions, + withdrawable_assets=Wei(100), + harvest_params=None, + os_token_converter=make_converter(100, 100), + ) + assert result == Wei(600) + mock_redeemer.redeem_sub_vaults_assets.assert_called_once_with(VAULT_1, Wei(400)) + + async def test_meta_vault_failed_redeem(self) -> None: + positions = [make_position(redeemable_shares=500)] + with ( + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=True)), + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + ): + mock_redeemer.redeem_sub_vaults_assets = AsyncMock(side_effect=RuntimeError('fail')) + result = await _try_redeem_sub_vaults( + vault_address=VAULT_1, + positions=positions, + withdrawable_assets=Wei(100), + harvest_params=None, + os_token_converter=make_converter(100, 100), + ) + assert result == Wei(100) + + +class TestProcessExitQueue: + async def test_cannot_process(self) -> None: + with patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer: + mock_redeemer.can_process_exit_queue = AsyncMock(return_value=False) + await _process_exit_queue(BlockNumber(100)) + mock_redeemer.process_exit_queue.assert_not_called() + + async def test_process_success(self) -> None: + mock_client = AsyncMock() + mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 1}) + with ( + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch(f'{MODULE}.execution_client', new=mock_client), + patch(f'{MODULE}.settings') as mock_settings, + ): + mock_settings.execution_transaction_timeout = 120 + mock_redeemer.can_process_exit_queue = AsyncMock(return_value=True) + mock_redeemer.process_exit_queue = AsyncMock(return_value='0xabc') + await _process_exit_queue(BlockNumber(100)) + mock_redeemer.process_exit_queue.assert_called_once() + + async def test_process_tx_fails(self) -> None: + mock_client = AsyncMock() + mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 0}) + with ( + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch(f'{MODULE}.execution_client', new=mock_client), + patch(f'{MODULE}.settings') as mock_settings, + ): + mock_settings.execution_transaction_timeout = 120 + mock_redeemer.can_process_exit_queue = AsyncMock(return_value=True) + mock_redeemer.process_exit_queue = AsyncMock(return_value='0xabc') + # Should not raise, just log error + await _process_exit_queue(BlockNumber(100)) + + +class TestStartupCheck: + async def test_authorized(self) -> None: + wallet_address = faker.eth_address() + mock_wallet = MagicMock() + mock_wallet.account.address = wallet_address + with ( + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch(f'{MODULE}.wallet', new=mock_wallet), + ): + mock_redeemer.positions_manager = AsyncMock(return_value=wallet_address) + await _startup_check() + + async def test_unauthorized(self) -> None: + mock_wallet = MagicMock() + mock_wallet.account.address = faker.eth_address() + with ( + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch(f'{MODULE}.wallet', new=mock_wallet), + ): + mock_redeemer.positions_manager = AsyncMock(return_value=faker.eth_address()) + with pytest.raises(RuntimeError, match='Position Manager role must be assigned'): + await _startup_check() + + +class TestSelectPositionsToRedeem: + async def test_empty_positions(self) -> None: + result = await _select_positions_to_redeem( + redeemable_positions=[], + queued_shares=10000, + os_token_converter=make_converter(), + block_number=BlockNumber(100), + ) + assert result.positions_to_redeem == [] + + async def test_calls_io_per_vault(self) -> None: + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, redeemable_shares=500) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, redeemable_shares=800) + + mock_harvest = AsyncMock(return_value=None) + mock_withdrawable = AsyncMock(return_value=Wei(10000)) + mock_sub_vaults = AsyncMock(return_value=Wei(10000)) + + with ( + patch(f'{MODULE}.get_harvest_params', mock_harvest), + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}._try_redeem_sub_vaults', mock_sub_vaults), + ): + result = await _select_positions_to_redeem( + redeemable_positions=[pos1, pos2], + queued_shares=10000, + os_token_converter=make_converter(100, 100), + block_number=BlockNumber(100), + ) + + assert mock_harvest.call_count == 2 + assert mock_withdrawable.call_count == 2 + assert mock_sub_vaults.call_count == 2 + assert len(result.positions_to_redeem) == 2 + + async def test_passes_harvest_params_to_get_withdrawable_assets(self) -> None: + pos = make_position(vault=VAULT_1, redeemable_shares=500) + harvest_params = make_harvest_params() + + mock_harvest = AsyncMock(return_value=harvest_params) + mock_withdrawable = AsyncMock(return_value=Wei(10000)) + mock_sub_vaults = AsyncMock(return_value=Wei(10000)) + + with ( + patch(f'{MODULE}.get_harvest_params', mock_harvest), + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}._try_redeem_sub_vaults', mock_sub_vaults), + ): + result = await _select_positions_to_redeem( + redeemable_positions=[pos], + queued_shares=10000, + os_token_converter=make_converter(100, 100), + block_number=BlockNumber(100), + ) + + mock_withdrawable.assert_called_once_with(VAULT_1, harvest_params) + assert result.vault_to_harvest_params[VAULT_1] is harvest_params + + +class TestExecuteRedemption: + async def test_successful_with_harvest_params(self) -> None: + pos = make_position(vault=VAULT_1, amount=1000, redeemable_shares=500) + harvest_params = make_harvest_params() + mock_client = AsyncMock() + mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 1}) + + with ( + patch(f'{MODULE}.VaultContract') as MockVaultContract, + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch(f'{MODULE}.multicall_contract'), + patch( + f'{MODULE}.transaction_gas_wrapper', + new=AsyncMock(return_value=b'\x01' * 32), + ), + patch(f'{MODULE}.execution_client', new=mock_client), + patch(f'{MODULE}.settings') as mock_settings, + ): + mock_settings.execution_transaction_timeout = 120 + mock_vault = MockVaultContract.return_value + mock_vault.contract_address = VAULT_1 + mock_vault.get_update_state_call.return_value = HexStr('0xupdate') + mock_redeemer.encode_abi.return_value = HexStr('0xredeem') + mock_redeemer.contract_address = VAULT_2 + + result = await _execute_redemption( + all_positions=[pos], + positions_to_redeem=[pos], + vault_to_harvest_params={VAULT_1: harvest_params}, + tree_nonce=5, + ) + + assert result is not None + mock_vault.get_update_state_call.assert_called_once_with(harvest_params) + + async def test_successful_without_harvest_params(self) -> None: + pos = make_position(vault=VAULT_1, amount=1000, redeemable_shares=500) + mock_client = AsyncMock() + mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 1}) + + with ( + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch(f'{MODULE}.multicall_contract'), + patch( + f'{MODULE}.transaction_gas_wrapper', + new=AsyncMock(return_value=b'\x01' * 32), + ), + patch(f'{MODULE}.execution_client', new=mock_client), + patch(f'{MODULE}.settings') as mock_settings, + ): + mock_settings.execution_transaction_timeout = 120 + mock_redeemer.encode_abi.return_value = HexStr('0xredeem') + mock_redeemer.contract_address = VAULT_2 + + result = await _execute_redemption( + all_positions=[pos], + positions_to_redeem=[pos], + vault_to_harvest_params={VAULT_1: None}, + tree_nonce=5, + ) + + assert result is not None + + async def test_web3_exception(self) -> None: + pos = make_position(amount=1000, redeemable_shares=500) + + with ( + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch(f'{MODULE}.multicall_contract'), + patch( + f'{MODULE}.transaction_gas_wrapper', + new=AsyncMock(side_effect=Web3Exception('fail')), + ), + ): + mock_redeemer.encode_abi.return_value = HexStr('0xredeem') + mock_redeemer.contract_address = VAULT_2 + + result = await _execute_redemption( + all_positions=[pos], + positions_to_redeem=[pos], + vault_to_harvest_params={VAULT_1: None}, + tree_nonce=5, + ) + + assert result is None + + async def test_tx_receipt_fails(self) -> None: + pos = make_position(amount=1000, redeemable_shares=500) + mock_client = AsyncMock() + mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 0}) + + with ( + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch(f'{MODULE}.multicall_contract'), + patch( + f'{MODULE}.transaction_gas_wrapper', + new=AsyncMock(return_value=b'\x01' * 32), + ), + patch(f'{MODULE}.execution_client', new=mock_client), + patch(f'{MODULE}.settings') as mock_settings, + ): + mock_settings.execution_transaction_timeout = 120 + mock_redeemer.encode_abi.return_value = HexStr('0xredeem') + mock_redeemer.contract_address = VAULT_2 + + result = await _execute_redemption( + all_positions=[pos], + positions_to_redeem=[pos], + vault_to_harvest_params={VAULT_1: None}, + tree_nonce=5, + ) + + assert result is None + + +class TestProcess: + async def test_no_queued_shares(self) -> None: + with ( + patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + ): + mock_redeemer.queued_shares = AsyncMock(return_value=Wei(0)) + await process(block_number=BlockNumber(100)) + + async def test_zero_merkle_root(self) -> None: + with ( + patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch( + f'{MODULE}.create_os_token_converter', + new=AsyncMock(return_value=make_converter()), + ), + ): + mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) + mock_redeemer.nonce = AsyncMock(return_value=5) + mock_redeemer.redeemable_positions = AsyncMock( + return_value=RedeemablePositions( + merkle_root=ZERO_MERKLE_ROOT, + ipfs_hash='QmTest', + ) + ) + await process(block_number=BlockNumber(100)) + + async def test_empty_ipfs_hash(self) -> None: + with ( + patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch( + f'{MODULE}.create_os_token_converter', + new=AsyncMock(return_value=make_converter()), + ), + ): + mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) + mock_redeemer.nonce = AsyncMock(return_value=5) + mock_redeemer.redeemable_positions = AsyncMock( + return_value=RedeemablePositions( + merkle_root=HexStr('0x' + 'ab' * 32), + ipfs_hash='', + ) + ) + await process(block_number=BlockNumber(100)) + + async def test_no_eligible_positions(self) -> None: + with ( + patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch( + f'{MODULE}.create_os_token_converter', + new=AsyncMock(return_value=make_converter()), + ), + patch( + f'{MODULE}._fetch_redeemable_positions', + new=AsyncMock(return_value=[]), + ), + patch(f'{MODULE}._execute_redemption') as mock_execute, + ): + mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) + mock_redeemer.nonce = AsyncMock(return_value=5) + mock_redeemer.redeemable_positions = AsyncMock( + return_value=RedeemablePositions( + merkle_root=HexStr('0x' + 'ab' * 32), + ipfs_hash='QmTest', + ) + ) + await process(block_number=BlockNumber(100)) + mock_execute.assert_not_called() + + async def test_successful_redemption(self) -> None: + positions = [make_position(amount=1000, redeemable_shares=500)] + selection = PositionSelectionResult( + positions_to_redeem=positions, + vault_to_harvest_params={VAULT_1: None}, + ) + + with ( + patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch( + f'{MODULE}.create_os_token_converter', + new=AsyncMock(return_value=make_converter()), + ), + patch( + f'{MODULE}._fetch_redeemable_positions', + new=AsyncMock(return_value=positions), + ), + patch( + f'{MODULE}._select_positions_to_redeem', + new=AsyncMock(return_value=selection), + ), + patch( + f'{MODULE}._execute_redemption', + new=AsyncMock(return_value='0xtxhash'), + ) as mock_execute, + ): + mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) + mock_redeemer.nonce = AsyncMock(return_value=5) + mock_redeemer.redeemable_positions = AsyncMock( + return_value=RedeemablePositions( + merkle_root=HexStr('0x' + 'ab' * 32), + ipfs_hash='QmTest', + ) + ) + await process(block_number=BlockNumber(100)) + mock_execute.assert_called_once() + + +def make_converter(total_assets: int = 110, total_shares: int = 100) -> OsTokenConverter: + return OsTokenConverter(Wei(total_assets), Wei(total_shares)) + + +def make_position( + vault: ChecksumAddress = VAULT_1, + owner: ChecksumAddress = OWNER_1, + amount: int = 1000, + redeemable_shares: int = 0, +) -> OsTokenPosition: + return OsTokenPosition( + vault=vault, + owner=owner, + amount=Wei(amount), + redeemable_shares=Wei(redeemable_shares), + ) + + +def make_harvest_params() -> HarvestParams: + return HarvestParams( + rewards_root=HexBytes(b'\x01' * 32), + reward=Wei(100), + unlocked_mev_reward=Wei(50), + proof=[HexBytes(b'\x02' * 32)], + ) From 2eb437820deefcec3138f2429ad205ab0d5b42aa Mon Sep 17 00:00:00 2001 From: cyc60 Date: Wed, 18 Feb 2026 19:20:05 +0300 Subject: [PATCH 45/65] Add interval option Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index a579646b..11282c4e 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -43,7 +43,7 @@ logger = logging.getLogger(__name__) -SLEEP_INTERVAL = 60 # 1 minute +DEFAULT_INTERVAL = 60 # 1 minute ZERO_MERKLE_ROOT = HexStr('0x' + '0' * 64) @@ -87,6 +87,13 @@ class PositionSelectionResult: envvar='GRAPH_ENDPOINT', help='API endpoint for graph node.', ) +@click.option( + '--interval', + type=int, + default=DEFAULT_INTERVAL, + envvar='INTERVAL', + help='Sleep interval in seconds between processing rounds.', +) @click.option( '--log-level', type=click.Choice( @@ -127,6 +134,7 @@ def process_redeemer( network: str, verbose: bool, log_level: str, + interval: int, wallet_file: str | None, wallet_password_file: str | None, ) -> None: @@ -143,13 +151,13 @@ def process_redeemer( log_level=log_level, ) try: - asyncio.run(main()) + asyncio.run(main(interval=interval)) except Exception as e: log_verbose(e) sys.exit(1) -async def main() -> None: +async def main(interval: int = DEFAULT_INTERVAL) -> None: setup_logging() await setup_clients() await _startup_check() @@ -158,7 +166,7 @@ async def main() -> None: while not interrupt_handler.exit: block_number = await execution_client.eth.block_number await process(block_number=block_number) - await interrupt_handler.sleep(SLEEP_INTERVAL) + await interrupt_handler.sleep(interval) finally: await close_clients() From 44dac790259d376016bf72c39fa77f666996d48c Mon Sep 17 00:00:00 2001 From: cyc60 Date: Thu, 19 Feb 2026 17:15:09 +0300 Subject: [PATCH 46/65] Remove unused options Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 19 ++----------------- src/redemptions/tasks.py | 4 +++- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 11282c4e..fec23135 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -44,7 +44,6 @@ logger = logging.getLogger(__name__) DEFAULT_INTERVAL = 60 # 1 minute -ZERO_MERKLE_ROOT = HexStr('0x' + '0' * 64) @dataclass @@ -81,12 +80,6 @@ class PositionSelectionResult: help='JWT secret key used for signing and verifying JSON Web Tokens' ' when connecting to execution nodes.', ) -@click.option( - '--graph-endpoint', - type=str, - envvar='GRAPH_ENDPOINT', - help='API endpoint for graph node.', -) @click.option( '--interval', type=int, @@ -130,7 +123,6 @@ class PositionSelectionResult: def process_redeemer( execution_endpoints: str, execution_jwt_secret: str | None, - graph_endpoint: str, network: str, verbose: bool, log_level: str, @@ -143,7 +135,6 @@ def process_redeemer( vault_dir=Path.home() / '.stakewise', execution_endpoints=execution_endpoints, execution_jwt_secret=execution_jwt_secret, - graph_endpoint=graph_endpoint, verbose=verbose, network=network, wallet_file=wallet_file, @@ -194,13 +185,6 @@ async def process(block_number: BlockNumber) -> None: Web3.from_wei(queued_assets, 'ether'), ) - redeemable_positions_meta = await os_token_redeemer_contract.redeemable_positions(block_number) - if redeemable_positions_meta.merkle_root == ZERO_MERKLE_ROOT or ( - not redeemable_positions_meta.ipfs_hash - ): - logger.info('No redeemable positions available. Skipping redemption processing.') - return - redeemable_positions = await _fetch_redeemable_positions( tree_nonce=tree_nonce, block_number=block_number ) @@ -236,6 +220,7 @@ async def _select_positions_to_redeem( os_token_converter: OsTokenConverter, block_number: BlockNumber, ) -> PositionSelectionResult: + # group positions by vault vault_to_positions: defaultdict[ChecksumAddress, list[OsTokenPosition]] = defaultdict(list) for position in redeemable_positions: vault_to_positions[position.vault].append(position) @@ -347,7 +332,7 @@ async def _try_redeem_sub_vaults( tx_hash, ) except RuntimeError: - logger.warning( + logger.error( 'redeemSubVaultsAssets failed for vault %s. ' 'Proceeding with current withdrawable assets.', vault_address, diff --git a/src/redemptions/tasks.py b/src/redemptions/tasks.py index f7ed2042..689740a0 100644 --- a/src/redemptions/tasks.py +++ b/src/redemptions/tasks.py @@ -22,6 +22,7 @@ batch_size = 20 +ZERO_MERKLE_ROOT = HexStr('0x' + '0' * 64) async def get_redemption_assets(chain_head: ChainHead) -> Gwei: @@ -151,7 +152,8 @@ async def iter_os_token_positions( # Check whether redeemable positions are available if not redeemable_positions.ipfs_hash: return - + if redeemable_positions.merkle_root == ZERO_MERKLE_ROOT: + return # Fetch redeemable positions data from IPFS data = cast(list[dict], await ipfs_fetch_client.fetch_json(redeemable_positions.ipfs_hash)) From 5c116cb7b624087e99365695bb35b37328ddd14a Mon Sep 17 00:00:00 2001 From: cyc60 Date: Thu, 19 Feb 2026 18:16:36 +0300 Subject: [PATCH 47/65] Split OsTokenPosition.redeemable_shares into available_shares and shares_to_redeem --- src/commands/internal/process_redeemer.py | 22 ++--- .../test_internal/test_process_redeemer.py | 82 ++++++++++--------- src/redemptions/typings.py | 3 +- 3 files changed, 53 insertions(+), 54 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index fec23135..e68c4416 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -268,27 +268,24 @@ def _filter_positions_to_redeem( if remaining_shares <= 0: break - redeemable_assets = os_token_converter.to_assets(position.redeemable_shares) + redeemable_assets = os_token_converter.to_assets(position.available_shares) if redeemable_assets > withdrawable_assets: continue - shares_to_redeem = Wei(min(position.redeemable_shares, remaining_shares)) + shares_to_redeem = Wei(min(position.available_shares, remaining_shares)) logger.info( 'Position Owner: %s, Vault: %s, Shares to Redeem: %s', position.owner, position.vault, shares_to_redeem, ) - # redeemable_shares is overloaded here: in the source position it means - # "total shares available for redemption", in the redeem batch it means - # "shares to actually redeem in this transaction" (maps to sharesToRedeem - # in the Solidity struct). positions_to_redeem.append( OsTokenPosition( vault=position.vault, owner=position.owner, amount=position.amount, - redeemable_shares=shares_to_redeem, + available_shares=position.available_shares, + shares_to_redeem=shares_to_redeem, ) ) withdrawable_assets = Wei(withdrawable_assets - redeemable_assets) @@ -314,7 +311,7 @@ async def _try_redeem_sub_vaults( Returns the (possibly updated) withdrawable assets for the vault. """ - vault_positions_shares = Wei(sum(p.redeemable_shares for p in positions)) + vault_positions_shares = Wei(sum(p.available_shares for p in positions)) vault_positions_assets = os_token_converter.to_assets(vault_positions_shares) if vault_positions_assets <= withdrawable_assets or not await is_meta_vault(vault_address): @@ -370,9 +367,8 @@ async def _execute_redemption( # Maps to Solidity struct: # OsTokenPosition(address vault, address owner, uint256 leafShares, uint256 sharesToRedeem) - # amount -> leafShares, redeemable_shares -> sharesToRedeem positions_arg = [ - (pos.vault, pos.owner, pos.amount, pos.redeemable_shares) for pos in positions_to_redeem + (pos.vault, pos.owner, pos.amount, pos.shares_to_redeem) for pos in positions_to_redeem ] redeem_call = os_token_redeemer_contract.encode_abi( fn_name='redeemOsTokenPositions', @@ -412,15 +408,15 @@ async def _fetch_redeemable_positions( block_number=block_number, ) for position, processed_shares in zip(batch, processed_shares_batch): - redeemable_shares = position.amount - processed_shares - if redeemable_shares <= 0: + available_shares = position.amount - processed_shares + if available_shares <= 0: continue positions.append( OsTokenPosition( owner=position.owner, vault=position.vault, amount=position.amount, - redeemable_shares=Wei(redeemable_shares), + available_shares=Wei(available_shares), ) ) diff --git a/src/commands/tests/test_internal/test_process_redeemer.py b/src/commands/tests/test_internal/test_process_redeemer.py index 02fb5311..ebcf9945 100644 --- a/src/commands/tests/test_internal/test_process_redeemer.py +++ b/src/commands/tests/test_internal/test_process_redeemer.py @@ -9,7 +9,6 @@ from web3.types import Wei from src.commands.internal.process_redeemer import ( - ZERO_MERKLE_ROOT, PositionSelectionResult, _execute_redemption, _fetch_redeemable_positions, @@ -23,6 +22,7 @@ ) from src.common.typings import HarvestParams from src.redemptions.os_token_converter import OsTokenConverter +from src.redemptions.tasks import ZERO_MERKLE_ROOT from src.redemptions.typings import OsTokenPosition, RedeemablePositions MODULE = 'src.commands.internal.process_redeemer' @@ -38,7 +38,7 @@ class TestGetMultiProof: def test_single_position(self) -> None: - position = make_position(amount=1000, redeemable_shares=500) + position = make_position(amount=1000, available_shares=500) result = _get_multi_proof( tree_nonce=5, all_positions=[position], @@ -47,8 +47,8 @@ def test_single_position(self) -> None: assert len(result.leaves) == 1 def test_partial_redeem(self) -> None: - pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, redeemable_shares=500) - pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, redeemable_shares=1000) + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, available_shares=500) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, available_shares=1000) result = _get_multi_proof( tree_nonce=5, @@ -59,8 +59,8 @@ def test_partial_redeem(self) -> None: assert len(result.proof) > 0 def test_all_positions_redeemed(self) -> None: - pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, redeemable_shares=500) - pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, redeemable_shares=1000) + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, available_shares=500) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, available_shares=1000) result = _get_multi_proof( tree_nonce=5, @@ -83,7 +83,7 @@ def test_empty_positions(self) -> None: assert result.vault_to_harvest_params == {} def test_single_position_sufficient_assets(self) -> None: - position = make_position(redeemable_shares=500) + position = make_position(available_shares=500) # to_assets(500) = 500 * 110 / 100 = 550; withdrawable=1000 → selected result = _filter_positions_to_redeem( vault_to_positions={VAULT_1: [position]}, @@ -93,10 +93,10 @@ def test_single_position_sufficient_assets(self) -> None: os_token_converter=make_converter(), ) assert len(result.positions_to_redeem) == 1 - assert result.positions_to_redeem[0].redeemable_shares == Wei(500) + assert result.positions_to_redeem[0].shares_to_redeem == Wei(500) def test_single_position_insufficient_assets(self) -> None: - position = make_position(redeemable_shares=500) + position = make_position(available_shares=500) # to_assets(500) = 550; withdrawable=100 → skipped result = _filter_positions_to_redeem( vault_to_positions={VAULT_1: [position]}, @@ -108,7 +108,7 @@ def test_single_position_insufficient_assets(self) -> None: assert result.positions_to_redeem == [] def test_queued_shares_limits_redemption(self) -> None: - position = make_position(redeemable_shares=500) + position = make_position(available_shares=500) result = _filter_positions_to_redeem( vault_to_positions={VAULT_1: [position]}, vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, @@ -117,11 +117,11 @@ def test_queued_shares_limits_redemption(self) -> None: os_token_converter=make_converter(), ) assert len(result.positions_to_redeem) == 1 - assert result.positions_to_redeem[0].redeemable_shares == Wei(200) + assert result.positions_to_redeem[0].shares_to_redeem == Wei(200) def test_multiple_positions_limited_by_withdrawable_assets(self) -> None: - pos1 = make_position(owner=OWNER_1, redeemable_shares=500) - pos2 = make_position(owner=OWNER_2, redeemable_shares=1000) + pos1 = make_position(owner=OWNER_1, available_shares=500) + pos2 = make_position(owner=OWNER_2, available_shares=1000) # 1:1 converter; pos1=500 assets, pos2=1000 assets; withdrawable=700 # pos1 fits (700-500=200 remaining), pos2 doesn't (1000>200) result = _filter_positions_to_redeem( @@ -135,8 +135,8 @@ def test_multiple_positions_limited_by_withdrawable_assets(self) -> None: assert result.positions_to_redeem[0].owner == OWNER_1 def test_multiple_vaults_both_selected(self) -> None: - pos1 = make_position(vault=VAULT_1, owner=OWNER_1, redeemable_shares=500) - pos2 = make_position(vault=VAULT_2, owner=OWNER_2, redeemable_shares=800) + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, available_shares=500) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, available_shares=800) vault_to_positions: dict[ChecksumAddress, list[OsTokenPosition]] = {} vault_to_positions[VAULT_1] = [pos1] @@ -152,8 +152,8 @@ def test_multiple_vaults_both_selected(self) -> None: assert len(result.positions_to_redeem) == 2 def test_stops_across_vaults_when_queued_shares_exhausted(self) -> None: - pos1 = make_position(vault=VAULT_1, owner=OWNER_1, redeemable_shares=500) - pos2 = make_position(vault=VAULT_2, owner=OWNER_2, redeemable_shares=800) + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, available_shares=500) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, available_shares=800) vault_to_positions: dict[ChecksumAddress, list[OsTokenPosition]] = {} vault_to_positions[VAULT_1] = [pos1] @@ -170,7 +170,7 @@ def test_stops_across_vaults_when_queued_shares_exhausted(self) -> None: assert result.positions_to_redeem[0].vault == VAULT_1 def test_preserves_harvest_params(self) -> None: - pos = make_position(redeemable_shares=500) + pos = make_position(available_shares=500) harvest_params = make_harvest_params() result = _filter_positions_to_redeem( @@ -183,7 +183,7 @@ def test_preserves_harvest_params(self) -> None: assert result.vault_to_harvest_params[VAULT_1] is harvest_params def test_preserves_original_amount(self) -> None: - pos = make_position(amount=1000, redeemable_shares=500) + pos = make_position(amount=1000, available_shares=500) result = _filter_positions_to_redeem( vault_to_positions={VAULT_1: [pos]}, vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, @@ -192,12 +192,12 @@ def test_preserves_original_amount(self) -> None: os_token_converter=make_converter(), ) assert result.positions_to_redeem[0].amount == Wei(1000) - assert result.positions_to_redeem[0].redeemable_shares == Wei(200) + assert result.positions_to_redeem[0].shares_to_redeem == Wei(200) def test_stops_within_vault_when_queued_shares_exhausted(self) -> None: # Multiple positions in same vault, queued_shares runs out mid-vault - pos1 = make_position(owner=OWNER_1, redeemable_shares=400) - pos2 = make_position(owner=OWNER_2, redeemable_shares=300) + pos1 = make_position(owner=OWNER_1, available_shares=400) + pos2 = make_position(owner=OWNER_2, available_shares=300) # 1:1 converter; pos1 consumes all 400 queued_shares, pos2 is skipped result = _filter_positions_to_redeem( vault_to_positions={VAULT_1: [pos1, pos2]}, @@ -211,8 +211,8 @@ def test_stops_within_vault_when_queued_shares_exhausted(self) -> None: def test_skips_position_then_selects_next(self) -> None: # First position too expensive, second fits - pos1 = make_position(owner=OWNER_1, redeemable_shares=1000) - pos2 = make_position(owner=OWNER_2, redeemable_shares=100) + pos1 = make_position(owner=OWNER_1, available_shares=1000) + pos2 = make_position(owner=OWNER_2, available_shares=100) # 1:1 converter; pos1=1000 assets > 500 withdrawable; pos2=100 <= 500 result = _filter_positions_to_redeem( vault_to_positions={VAULT_1: [pos1, pos2]}, @@ -272,7 +272,7 @@ async def gen(block_number: BlockNumber | None = None): # type: ignore[misc] ): result = await _fetch_redeemable_positions(tree_nonce=5, block_number=BlockNumber(100)) assert len(result) == 1 - assert result[0].redeemable_shares == Wei(700) + assert result[0].available_shares == Wei(700) assert result[0].amount == Wei(1000) async def test_multiple_positions_mixed(self) -> None: @@ -294,12 +294,12 @@ async def gen(block_number: BlockNumber | None = None): # type: ignore[misc] result = await _fetch_redeemable_positions(tree_nonce=5, block_number=BlockNumber(100)) assert len(result) == 1 assert result[0].owner == OWNER_2 - assert result[0].redeemable_shares == Wei(1500) + assert result[0].available_shares == Wei(1500) class TestTryRedeemSubVaults: async def test_sufficient_withdrawable_assets(self) -> None: - positions = [make_position(redeemable_shares=500)] + positions = [make_position(available_shares=500)] # 1:1 converter; vault_positions_assets=500 <= withdrawable=1000 → return result = await _try_redeem_sub_vaults( vault_address=VAULT_1, @@ -311,7 +311,7 @@ async def test_sufficient_withdrawable_assets(self) -> None: assert result == Wei(1000) async def test_insufficient_non_meta_vault(self) -> None: - positions = [make_position(redeemable_shares=500)] + positions = [make_position(available_shares=500)] # vault_positions_assets=500 > withdrawable=100, but not meta-vault with patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)): result = await _try_redeem_sub_vaults( @@ -324,7 +324,7 @@ async def test_insufficient_non_meta_vault(self) -> None: assert result == Wei(100) async def test_meta_vault_successful_redeem(self) -> None: - positions = [make_position(redeemable_shares=500)] + positions = [make_position(available_shares=500)] # vault_positions_assets=500 > withdrawable=100, meta-vault with ( patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=True)), @@ -346,7 +346,7 @@ async def test_meta_vault_successful_redeem(self) -> None: mock_redeemer.redeem_sub_vaults_assets.assert_called_once_with(VAULT_1, Wei(400)) async def test_meta_vault_failed_redeem(self) -> None: - positions = [make_position(redeemable_shares=500)] + positions = [make_position(available_shares=500)] with ( patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=True)), patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, @@ -433,8 +433,8 @@ async def test_empty_positions(self) -> None: assert result.positions_to_redeem == [] async def test_calls_io_per_vault(self) -> None: - pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, redeemable_shares=500) - pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, redeemable_shares=800) + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, available_shares=500) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, available_shares=800) mock_harvest = AsyncMock(return_value=None) mock_withdrawable = AsyncMock(return_value=Wei(10000)) @@ -458,7 +458,7 @@ async def test_calls_io_per_vault(self) -> None: assert len(result.positions_to_redeem) == 2 async def test_passes_harvest_params_to_get_withdrawable_assets(self) -> None: - pos = make_position(vault=VAULT_1, redeemable_shares=500) + pos = make_position(vault=VAULT_1, available_shares=500) harvest_params = make_harvest_params() mock_harvest = AsyncMock(return_value=harvest_params) @@ -483,7 +483,7 @@ async def test_passes_harvest_params_to_get_withdrawable_assets(self) -> None: class TestExecuteRedemption: async def test_successful_with_harvest_params(self) -> None: - pos = make_position(vault=VAULT_1, amount=1000, redeemable_shares=500) + pos = make_position(vault=VAULT_1, amount=1000, available_shares=500, shares_to_redeem=500) harvest_params = make_harvest_params() mock_client = AsyncMock() mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 1}) @@ -517,7 +517,7 @@ async def test_successful_with_harvest_params(self) -> None: mock_vault.get_update_state_call.assert_called_once_with(harvest_params) async def test_successful_without_harvest_params(self) -> None: - pos = make_position(vault=VAULT_1, amount=1000, redeemable_shares=500) + pos = make_position(vault=VAULT_1, amount=1000, available_shares=500, shares_to_redeem=500) mock_client = AsyncMock() mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 1}) @@ -545,7 +545,7 @@ async def test_successful_without_harvest_params(self) -> None: assert result is not None async def test_web3_exception(self) -> None: - pos = make_position(amount=1000, redeemable_shares=500) + pos = make_position(amount=1000, available_shares=500, shares_to_redeem=500) with ( patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, @@ -568,7 +568,7 @@ async def test_web3_exception(self) -> None: assert result is None async def test_tx_receipt_fails(self) -> None: - pos = make_position(amount=1000, redeemable_shares=500) + pos = make_position(amount=1000, available_shares=500, shares_to_redeem=500) mock_client = AsyncMock() mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 0}) @@ -669,7 +669,7 @@ async def test_no_eligible_positions(self) -> None: mock_execute.assert_not_called() async def test_successful_redemption(self) -> None: - positions = [make_position(amount=1000, redeemable_shares=500)] + positions = [make_position(amount=1000, available_shares=500, shares_to_redeem=500)] selection = PositionSelectionResult( positions_to_redeem=positions, vault_to_harvest_params={VAULT_1: None}, @@ -715,13 +715,15 @@ def make_position( vault: ChecksumAddress = VAULT_1, owner: ChecksumAddress = OWNER_1, amount: int = 1000, - redeemable_shares: int = 0, + available_shares: int = 0, + shares_to_redeem: int = 0, ) -> OsTokenPosition: return OsTokenPosition( vault=vault, owner=owner, amount=Wei(amount), - redeemable_shares=Wei(redeemable_shares), + available_shares=Wei(available_shares), + shares_to_redeem=Wei(shares_to_redeem), ) diff --git a/src/redemptions/typings.py b/src/redemptions/typings.py index 9058d27c..ff65e5a6 100644 --- a/src/redemptions/typings.py +++ b/src/redemptions/typings.py @@ -58,7 +58,8 @@ class OsTokenPosition: owner: ChecksumAddress vault: ChecksumAddress amount: Wei - redeemable_shares: Wei = Wei(0) + available_shares: Wei = Wei(0) + shares_to_redeem: Wei = Wei(0) def as_dict(self) -> dict: return { From 97b3b065c51079e647810fda34c1854e5b528dad Mon Sep 17 00:00:00 2001 From: cyc60 Date: Fri, 20 Feb 2026 14:34:15 +0300 Subject: [PATCH 48/65] Update tests Signed-off-by: cyc60 --- .../tests/test_internal/test_process_redeemer.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/commands/tests/test_internal/test_process_redeemer.py b/src/commands/tests/test_internal/test_process_redeemer.py index ebcf9945..e2d90c57 100644 --- a/src/commands/tests/test_internal/test_process_redeemer.py +++ b/src/commands/tests/test_internal/test_process_redeemer.py @@ -26,6 +26,7 @@ from src.redemptions.typings import OsTokenPosition, RedeemablePositions MODULE = 'src.commands.internal.process_redeemer' +TASKS_MODULE = 'src.redemptions.tasks' VAULT_1 = Web3.to_checksum_address('0x' + '11' * 20) VAULT_2 = Web3.to_checksum_address('0x' + '22' * 20) @@ -613,16 +614,19 @@ async def test_zero_merkle_root(self) -> None: f'{MODULE}.create_os_token_converter', new=AsyncMock(return_value=make_converter()), ), + patch(f'{TASKS_MODULE}.os_token_redeemer_contract') as mock_tasks_redeemer, + patch(f'{MODULE}._execute_redemption') as mock_execute, ): mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) mock_redeemer.nonce = AsyncMock(return_value=5) - mock_redeemer.redeemable_positions = AsyncMock( + mock_tasks_redeemer.redeemable_positions = AsyncMock( return_value=RedeemablePositions( merkle_root=ZERO_MERKLE_ROOT, ipfs_hash='QmTest', ) ) await process(block_number=BlockNumber(100)) + mock_execute.assert_not_called() async def test_empty_ipfs_hash(self) -> None: with ( @@ -632,16 +636,19 @@ async def test_empty_ipfs_hash(self) -> None: f'{MODULE}.create_os_token_converter', new=AsyncMock(return_value=make_converter()), ), + patch(f'{TASKS_MODULE}.os_token_redeemer_contract') as mock_tasks_redeemer, + patch(f'{MODULE}._execute_redemption') as mock_execute, ): mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) mock_redeemer.nonce = AsyncMock(return_value=5) - mock_redeemer.redeemable_positions = AsyncMock( + mock_tasks_redeemer.redeemable_positions = AsyncMock( return_value=RedeemablePositions( merkle_root=HexStr('0x' + 'ab' * 32), ipfs_hash='', ) ) await process(block_number=BlockNumber(100)) + mock_execute.assert_not_called() async def test_no_eligible_positions(self) -> None: with ( From 03102edf8e0cff4e278f70641af0cc77e2d1a3a5 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Thu, 5 Mar 2026 11:03:07 +0300 Subject: [PATCH 49/65] Use sw utils Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 30 ++++++++++++----------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index e68c4416..130cc510 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -9,7 +9,7 @@ from eth_typing import BlockNumber, ChecksumAddress, HexStr from multiproof import StandardMerkleTree from multiproof.standard import MultiProof -from sw_utils import InterruptHandler +from sw_utils import InterruptHandler, async_batched from web3 import Web3 from web3.exceptions import Web3Exception from web3.types import Wei @@ -24,7 +24,7 @@ from src.common.harvest import get_harvest_params from src.common.logging import LOG_LEVELS, setup_logging from src.common.typings import HarvestParams -from src.common.utils import async_batched, log_verbose +from src.common.utils import log_verbose from src.common.wallet import wallet from src.config.networks import AVAILABLE_NETWORKS, ZERO_CHECKSUM_ADDRESS from src.config.settings import settings @@ -180,9 +180,10 @@ async def process(block_number: BlockNumber) -> None: queued_assets = os_token_converter.to_assets(queued_shares) logger.info( - 'Queued Shares for Redemption: %s (~%s assets)', + 'Queued Shares for Redemption: %s (~%s %s)', queued_shares, Web3.from_wei(queued_assets, 'ether'), + settings.network_config.VAULT_BALANCE_SYMBOL, ) redeemable_positions = await _fetch_redeemable_positions( @@ -430,17 +431,18 @@ async def _process_exit_queue(block_number: BlockNumber) -> None: shares into claimable assets for users in the exit queue. """ can_process_exit_queue = await os_token_redeemer_contract.can_process_exit_queue(block_number) - if can_process_exit_queue: - logger.info('Exit queue can be processed. Calling processExitQueue...') - tx_hash = await os_token_redeemer_contract.process_exit_queue() - logger.info('Waiting for processExitQueue transaction %s confirmation', tx_hash) - tx_receipt = await execution_client.eth.wait_for_transaction_receipt( - tx_hash, timeout=settings.execution_transaction_timeout - ) - if not tx_receipt['status']: - logger.error('processExitQueue transaction failed. Tx Hash: %s', tx_hash) - else: - logger.info('processExitQueue confirmed. Tx Hash: %s', tx_hash) + if not can_process_exit_queue: + return + logger.info('Exit queue can be processed. Calling processExitQueue...') + tx_hash = await os_token_redeemer_contract.process_exit_queue() + logger.info('Waiting for processExitQueue transaction %s confirmation', tx_hash) + tx_receipt = await execution_client.eth.wait_for_transaction_receipt( + tx_hash, timeout=settings.execution_transaction_timeout + ) + if not tx_receipt['status']: + logger.error('processExitQueue transaction failed. Tx Hash: %s', tx_hash) + else: + logger.info('processExitQueue confirmed. Tx Hash: %s', tx_hash) async def _startup_check() -> None: From 774c62bb80683279b594803149d607dfeabd2158 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Thu, 5 Mar 2026 15:19:12 +0300 Subject: [PATCH 50/65] Fix redeemable_assets filter Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 130cc510..3b5c4b54 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -269,11 +269,11 @@ def _filter_positions_to_redeem( if remaining_shares <= 0: break - redeemable_assets = os_token_converter.to_assets(position.available_shares) + shares_to_redeem = Wei(min(position.available_shares, remaining_shares)) + redeemable_assets = os_token_converter.to_assets(shares_to_redeem) if redeemable_assets > withdrawable_assets: continue - shares_to_redeem = Wei(min(position.available_shares, remaining_shares)) logger.info( 'Position Owner: %s, Vault: %s, Shares to Redeem: %s', position.owner, From f28c9649ba66cc36b8fee1ad32fcfc7d7d4ccbfc Mon Sep 17 00:00:00 2001 From: cyc60 Date: Thu, 5 Mar 2026 16:31:36 +0300 Subject: [PATCH 51/65] Extract fetch_harvest_params_by_vault from select_positions --- src/commands/internal/process_redeemer.py | 285 +++++---- .../test_internal/test_process_redeemer.py | 592 +++++++++--------- 2 files changed, 451 insertions(+), 426 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 3b5c4b54..a76ffd16 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -2,14 +2,13 @@ import logging import sys from collections import defaultdict -from dataclasses import dataclass from pathlib import Path import click from eth_typing import BlockNumber, ChecksumAddress, HexStr from multiproof import StandardMerkleTree from multiproof.standard import MultiProof -from sw_utils import InterruptHandler, async_batched +from sw_utils import InterruptHandler from web3 import Web3 from web3.exceptions import Web3Exception from web3.types import Wei @@ -46,12 +45,6 @@ DEFAULT_INTERVAL = 60 # 1 minute -@dataclass -class PositionSelectionResult: - positions_to_redeem: list[OsTokenPosition] - vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None] - - @click.option( '--wallet-password-file', type=click.Path(exists=True, file_okay=True, dir_okay=False), @@ -164,8 +157,10 @@ async def main(interval: int = DEFAULT_INTERVAL) -> None: async def process(block_number: BlockNumber) -> None: + # Step 1: Process exit queue await _process_exit_queue(block_number) + # Step 2: Check queued shares queued_shares = await os_token_redeemer_contract.queued_shares(block_number) if queued_shares == 0: logger.info('No queued shares for redemption. Skipping to next interval.') @@ -186,93 +181,167 @@ async def process(block_number: BlockNumber) -> None: settings.network_config.VAULT_BALANCE_SYMBOL, ) - redeemable_positions = await _fetch_redeemable_positions( - tree_nonce=tree_nonce, block_number=block_number - ) + # Step 3: Fetch ALL positions from IPFS (needed for correct merkle tree) + all_positions = await fetch_positions_from_ipfs(block_number) + if not all_positions: + logger.info('No positions found. Skipping to next interval.') + return - result = await _select_positions_to_redeem( - redeemable_positions=redeemable_positions, + # Step 4: Calculate redeemable shares + redeemable = await calculate_redeemable_shares(all_positions, tree_nonce, block_number) + if not redeemable: + logger.info('No redeemable positions found. Skipping to next interval.') + return + + # Step 5: Fetch harvest params per vault + vault_to_harvest_params = await fetch_harvest_params_by_vault(redeemable, block_number) + + # Step 6: Select positions + positions_to_redeem = await select_positions( + redeemable=redeemable, queued_shares=queued_shares, - os_token_converter=os_token_converter, - block_number=block_number, + converter=os_token_converter, + vault_to_harvest_params=vault_to_harvest_params, ) - if not result.positions_to_redeem: + if not positions_to_redeem: logger.info('No positions eligible for redemption.') return - tx_hash = await _execute_redemption( - all_positions=redeemable_positions, - positions_to_redeem=result.positions_to_redeem, - vault_to_harvest_params=result.vault_to_harvest_params, + # Step 7: Execute redemption (uses all_positions for complete merkle tree) + tx_hash = await execute_redemption( + all_positions=all_positions, + positions_to_redeem=positions_to_redeem, + vault_to_harvest_params=vault_to_harvest_params, tree_nonce=tree_nonce, ) if tx_hash: logger.info( 'Successfully redeemed %s OsToken positions. Transaction hash: %s', - len(result.positions_to_redeem), + len(positions_to_redeem), tx_hash, ) -async def _select_positions_to_redeem( - redeemable_positions: list[OsTokenPosition], - queued_shares: int, - os_token_converter: OsTokenConverter, - block_number: BlockNumber, -) -> PositionSelectionResult: - # group positions by vault - vault_to_positions: defaultdict[ChecksumAddress, list[OsTokenPosition]] = defaultdict(list) - for position in redeemable_positions: - vault_to_positions[position.vault].append(position) +async def _process_exit_queue(block_number: BlockNumber) -> None: + """Call processExitQueue() on the redeemer contract if canProcessExitQueue.""" + can_process_exit_queue = await os_token_redeemer_contract.can_process_exit_queue(block_number) + if not can_process_exit_queue: + return + logger.info('Exit queue can be processed. Calling processExitQueue...') + tx_hash = await os_token_redeemer_contract.process_exit_queue() + logger.info('Waiting for processExitQueue transaction %s confirmation', tx_hash) + tx_receipt = await execution_client.eth.wait_for_transaction_receipt( + tx_hash, timeout=settings.execution_transaction_timeout + ) + if not tx_receipt['status']: + logger.error('processExitQueue transaction failed. Tx Hash: %s', tx_hash) + else: + logger.info('processExitQueue confirmed. Tx Hash: %s', tx_hash) - vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None] = {} - vault_to_withdrawable_assets: dict[ChecksumAddress, Wei] = {} - for vault_address, positions in vault_to_positions.items(): - harvest_params = await get_harvest_params(vault_address, block_number) - vault_to_harvest_params[vault_address] = harvest_params - withdrawable_assets = await get_withdrawable_assets(vault_address, harvest_params) - - withdrawable_assets = await _try_redeem_sub_vaults( - vault_address=vault_address, - positions=positions, - withdrawable_assets=withdrawable_assets, - harvest_params=harvest_params, - os_token_converter=os_token_converter, +async def fetch_positions_from_ipfs(block_number: BlockNumber) -> list[OsTokenPosition]: + """Collect ALL positions from IPFS. No filtering — needed for correct merkle tree.""" + positions: list[OsTokenPosition] = [] + async for position in iter_os_token_positions(block_number=block_number): + positions.append(position) + return positions + + +async def calculate_redeemable_shares( + all_positions: list[OsTokenPosition], + tree_nonce: int, + block_number: BlockNumber, +) -> list[OsTokenPosition]: + """Query processed shares and return positions with available_shares > 0.""" + redeemable: list[OsTokenPosition] = [] + + for i in range(0, len(all_positions), batch_size): + batch = all_positions[i : i + batch_size] + processed_shares_batch = await get_processed_shares_batch( + os_token_positions_batch=batch, + nonce=tree_nonce, + block_number=block_number, ) - vault_to_withdrawable_assets[vault_address] = withdrawable_assets + for position, processed_shares in zip(batch, processed_shares_batch): + available_shares = position.amount - processed_shares + if available_shares <= 0: + continue + redeemable.append( + OsTokenPosition( + owner=position.owner, + vault=position.vault, + amount=position.amount, + available_shares=Wei(available_shares), + ) + ) - return _filter_positions_to_redeem( - vault_to_positions=vault_to_positions, - vault_to_withdrawable_assets=vault_to_withdrawable_assets, - vault_to_harvest_params=vault_to_harvest_params, - queued_shares=queued_shares, - os_token_converter=os_token_converter, - ) + return redeemable -def _filter_positions_to_redeem( - vault_to_positions: dict[ChecksumAddress, list[OsTokenPosition]], - vault_to_withdrawable_assets: dict[ChecksumAddress, Wei], - vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], +async def fetch_harvest_params_by_vault( + redeemable: list[OsTokenPosition], + block_number: BlockNumber, +) -> dict[ChecksumAddress, HarvestParams | None]: + """Fetch harvest params for each unique vault in redeemable positions.""" + vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None] = {} + for vault_address in {pos.vault for pos in redeemable}: + vault_to_harvest_params[vault_address] = await get_harvest_params( + vault_address, block_number + ) + return vault_to_harvest_params + + +async def select_positions( + redeemable: list[OsTokenPosition], queued_shares: int, - os_token_converter: OsTokenConverter, -) -> PositionSelectionResult: + converter: OsTokenConverter, + vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], +) -> list[OsTokenPosition]: + """Select positions to redeem, capped by queued_shares and withdrawable assets. + + Per-position: if assets exceed withdrawable, attempt meta-vault sub-vault redemption + for just that position's deficit. + """ + # Group positions by vault + vault_to_positions: defaultdict[ChecksumAddress, list[OsTokenPosition]] = defaultdict(list) + for position in redeemable: + vault_to_positions[position.vault].append(position) + + vault_to_withdrawable: dict[ChecksumAddress, Wei] = {} + + # Fetch withdrawable assets per vault + for vault_address in vault_to_positions: + vault_to_withdrawable[vault_address] = await get_withdrawable_assets( + vault_address, vault_to_harvest_params.get(vault_address) + ) + positions_to_redeem: list[OsTokenPosition] = [] remaining_shares = queued_shares for vault_address, positions in vault_to_positions.items(): - withdrawable_assets = vault_to_withdrawable_assets[vault_address] - for position in positions: if remaining_shares <= 0: break shares_to_redeem = Wei(min(position.available_shares, remaining_shares)) - redeemable_assets = os_token_converter.to_assets(shares_to_redeem) - if redeemable_assets > withdrawable_assets: - continue + redeemable_assets = converter.to_assets(shares_to_redeem) + withdrawable = vault_to_withdrawable[vault_address] + + # If assets exceed withdrawable, try meta-vault sub-vault redemption + if redeemable_assets > withdrawable: + deficit = Wei(redeemable_assets - withdrawable) + withdrawable = await _try_redeem_meta_vault( + vault_address, + deficit, + withdrawable, + vault_to_harvest_params[vault_address], + ) + vault_to_withdrawable[vault_address] = withdrawable + + # Still insufficient after meta-vault attempt + if redeemable_assets > withdrawable: + continue logger.info( 'Position Owner: %s, Vault: %s, Shares to Redeem: %s', @@ -289,41 +358,31 @@ def _filter_positions_to_redeem( shares_to_redeem=shares_to_redeem, ) ) - withdrawable_assets = Wei(withdrawable_assets - redeemable_assets) + vault_to_withdrawable[vault_address] = Wei(withdrawable - redeemable_assets) remaining_shares -= shares_to_redeem if remaining_shares <= 0: break - return PositionSelectionResult( - positions_to_redeem=positions_to_redeem, - vault_to_harvest_params=vault_to_harvest_params, - ) + return positions_to_redeem -async def _try_redeem_sub_vaults( +async def _try_redeem_meta_vault( vault_address: ChecksumAddress, - positions: list[OsTokenPosition], - withdrawable_assets: Wei, + deficit: Wei, + current_withdrawable: Wei, harvest_params: HarvestParams | None, - os_token_converter: OsTokenConverter, ) -> Wei: - """If vault is a meta-vault with insufficient assets, redeem from sub-vaults. + """If vault is a meta-vault, redeem sub-vaults for the deficit. - Returns the (possibly updated) withdrawable assets for the vault. + Returns the (possibly updated) withdrawable assets. """ - vault_positions_shares = Wei(sum(p.available_shares for p in positions)) - vault_positions_assets = os_token_converter.to_assets(vault_positions_shares) - - if vault_positions_assets <= withdrawable_assets or not await is_meta_vault(vault_address): - return withdrawable_assets + if not await is_meta_vault(vault_address): + return current_withdrawable logger.info('Vault %s is a meta-vault with insufficient withdrawable assets.', vault_address) - additional_assets_needed = Wei(vault_positions_assets - withdrawable_assets) try: - tx_hash = await os_token_redeemer_contract.redeem_sub_vaults_assets( - vault_address, additional_assets_needed - ) + tx_hash = await os_token_redeemer_contract.redeem_sub_vaults_assets(vault_address, deficit) logger.info( 'redeemSubVaultsAssets confirmed for vault %s. Tx Hash: %s', vault_address, @@ -335,20 +394,20 @@ async def _try_redeem_sub_vaults( 'Proceeding with current withdrawable assets.', vault_address, ) - return withdrawable_assets + return current_withdrawable # Re-query actual withdrawable assets on-chain after sub-vault redemption - # to avoid TOCTOU issues with stale values return await get_withdrawable_assets(vault_address, harvest_params) -async def _execute_redemption( +async def execute_redemption( all_positions: list[OsTokenPosition], positions_to_redeem: list[OsTokenPosition], vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], tree_nonce: int, ) -> HexStr | None: - multiproof = _get_multi_proof( + """Build multiproof from all positions and execute the redemption transaction.""" + multiproof = build_multi_proof( all_positions=all_positions, positions_to_redeem=positions_to_redeem, tree_nonce=tree_nonce, @@ -396,55 +455,6 @@ async def _execute_redemption( return tx_hash -async def _fetch_redeemable_positions( - tree_nonce: int, block_number: BlockNumber -) -> list[OsTokenPosition]: - positions: list[OsTokenPosition] = [] - async for batch in async_batched( - iter_os_token_positions(block_number=block_number), batch_size - ): - processed_shares_batch = await get_processed_shares_batch( - os_token_positions_batch=batch, - nonce=tree_nonce, - block_number=block_number, - ) - for position, processed_shares in zip(batch, processed_shares_batch): - available_shares = position.amount - processed_shares - if available_shares <= 0: - continue - positions.append( - OsTokenPosition( - owner=position.owner, - vault=position.vault, - amount=position.amount, - available_shares=Wei(available_shares), - ) - ) - - return positions - - -async def _process_exit_queue(block_number: BlockNumber) -> None: - """ - Call processExitQueue() on the redeemer contract if canProcessExitQueue - to create a new checkpoint that converts accumulated redeemed/swapped - shares into claimable assets for users in the exit queue. - """ - can_process_exit_queue = await os_token_redeemer_contract.can_process_exit_queue(block_number) - if not can_process_exit_queue: - return - logger.info('Exit queue can be processed. Calling processExitQueue...') - tx_hash = await os_token_redeemer_contract.process_exit_queue() - logger.info('Waiting for processExitQueue transaction %s confirmation', tx_hash) - tx_receipt = await execution_client.eth.wait_for_transaction_receipt( - tx_hash, timeout=settings.execution_transaction_timeout - ) - if not tx_receipt['status']: - logger.error('processExitQueue transaction failed. Tx Hash: %s', tx_hash) - else: - logger.info('processExitQueue confirmed. Tx Hash: %s', tx_hash) - - async def _startup_check() -> None: positions_manager = await os_token_redeemer_contract.positions_manager() if positions_manager != wallet.account.address: @@ -453,11 +463,12 @@ async def _startup_check() -> None: ) -def _get_multi_proof( +def build_multi_proof( tree_nonce: int, all_positions: list[OsTokenPosition], positions_to_redeem: list[OsTokenPosition], ) -> MultiProof[tuple[bytes, int]]: + """Build a merkle multiproof from all positions, proving the positions to redeem.""" all_leaves = [p.merkle_leaf(tree_nonce) for p in all_positions] tree = StandardMerkleTree.of( all_leaves, diff --git a/src/commands/tests/test_internal/test_process_redeemer.py b/src/commands/tests/test_internal/test_process_redeemer.py index e2d90c57..405e39f3 100644 --- a/src/commands/tests/test_internal/test_process_redeemer.py +++ b/src/commands/tests/test_internal/test_process_redeemer.py @@ -9,16 +9,16 @@ from web3.types import Wei from src.commands.internal.process_redeemer import ( - PositionSelectionResult, - _execute_redemption, - _fetch_redeemable_positions, - _filter_positions_to_redeem, - _get_multi_proof, _process_exit_queue, - _select_positions_to_redeem, _startup_check, - _try_redeem_sub_vaults, + _try_redeem_meta_vault, + build_multi_proof, + calculate_redeemable_shares, + execute_redemption, + fetch_harvest_params_by_vault, + fetch_positions_from_ipfs, process, + select_positions, ) from src.common.typings import HarvestParams from src.redemptions.os_token_converter import OsTokenConverter @@ -37,10 +37,10 @@ # --- Pure function tests (no mocks) --- -class TestGetMultiProof: +class TestBuildMultiProof: def test_single_position(self) -> None: position = make_position(amount=1000, available_shares=500) - result = _get_multi_proof( + result = build_multi_proof( tree_nonce=5, all_positions=[position], positions_to_redeem=[position], @@ -51,7 +51,7 @@ def test_partial_redeem(self) -> None: pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, available_shares=500) pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, available_shares=1000) - result = _get_multi_proof( + result = build_multi_proof( tree_nonce=5, all_positions=[pos1, pos2], positions_to_redeem=[pos1], @@ -63,7 +63,7 @@ def test_all_positions_redeemed(self) -> None: pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, available_shares=500) pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, available_shares=1000) - result = _get_multi_proof( + result = build_multi_proof( tree_nonce=5, all_positions=[pos1, pos2], positions_to_redeem=[pos1, pos2], @@ -71,207 +71,305 @@ def test_all_positions_redeemed(self) -> None: assert len(result.leaves) == 2 -class TestFilterPositionsToRedeem: - def test_empty_positions(self) -> None: - result = _filter_positions_to_redeem( - vault_to_positions={}, - vault_to_withdrawable_assets={}, +class TestFetchHarvestParamsByVault: + async def test_empty_positions(self) -> None: + result = await fetch_harvest_params_by_vault([], BlockNumber(100)) + assert result == {} + + async def test_single_vault(self) -> None: + pos = make_position(vault=VAULT_1, available_shares=500) + hp = make_harvest_params() + mock_harvest = AsyncMock(return_value=hp) + + with patch(f'{MODULE}.get_harvest_params', mock_harvest): + result = await fetch_harvest_params_by_vault([pos], BlockNumber(100)) + + assert result[VAULT_1] is hp + mock_harvest.assert_called_once_with(VAULT_1, BlockNumber(100)) + + async def test_multiple_vaults(self) -> None: + pos1 = make_position(vault=VAULT_1, available_shares=500) + pos2 = make_position(vault=VAULT_2, available_shares=800) + hp = make_harvest_params() + mock_harvest = AsyncMock(return_value=hp) + + with patch(f'{MODULE}.get_harvest_params', mock_harvest): + result = await fetch_harvest_params_by_vault([pos1, pos2], BlockNumber(100)) + + assert len(result) == 2 + assert mock_harvest.call_count == 2 + + async def test_deduplicates_vaults(self) -> None: + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, available_shares=500) + pos2 = make_position(vault=VAULT_1, owner=OWNER_2, available_shares=800) + mock_harvest = AsyncMock(return_value=None) + + with patch(f'{MODULE}.get_harvest_params', mock_harvest): + result = await fetch_harvest_params_by_vault([pos1, pos2], BlockNumber(100)) + + assert len(result) == 1 + mock_harvest.assert_called_once_with(VAULT_1, BlockNumber(100)) + + +class TestSelectPositions: + async def test_empty_positions(self) -> None: + positions_to_redeem = await select_positions( + redeemable=[], + queued_shares=10000, + converter=make_converter(), vault_to_harvest_params={}, - queued_shares=1000, - os_token_converter=make_converter(), ) - assert result.positions_to_redeem == [] - assert result.vault_to_harvest_params == {} + assert positions_to_redeem == [] - def test_single_position_sufficient_assets(self) -> None: + async def test_single_position_sufficient_assets(self) -> None: position = make_position(available_shares=500) - # to_assets(500) = 500 * 110 / 100 = 550; withdrawable=1000 → selected - result = _filter_positions_to_redeem( - vault_to_positions={VAULT_1: [position]}, - vault_to_withdrawable_assets={VAULT_1: Wei(1000)}, - vault_to_harvest_params={VAULT_1: None}, - queued_shares=10000, - os_token_converter=make_converter(), - ) - assert len(result.positions_to_redeem) == 1 - assert result.positions_to_redeem[0].shares_to_redeem == Wei(500) + mock_withdrawable = AsyncMock(return_value=Wei(1000)) - def test_single_position_insufficient_assets(self) -> None: + with ( + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), + ): + positions_to_redeem = await select_positions( + redeemable=[position], + queued_shares=10000, + converter=make_converter(), + vault_to_harvest_params={VAULT_1: None}, + ) + assert len(positions_to_redeem) == 1 + assert positions_to_redeem[0].shares_to_redeem == Wei(500) + + async def test_single_position_insufficient_assets(self) -> None: position = make_position(available_shares=500) - # to_assets(500) = 550; withdrawable=100 → skipped - result = _filter_positions_to_redeem( - vault_to_positions={VAULT_1: [position]}, - vault_to_withdrawable_assets={VAULT_1: Wei(100)}, - vault_to_harvest_params={VAULT_1: None}, - queued_shares=10000, - os_token_converter=make_converter(), - ) - assert result.positions_to_redeem == [] + mock_withdrawable = AsyncMock(return_value=Wei(100)) - def test_queued_shares_limits_redemption(self) -> None: + with ( + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), + ): + positions_to_redeem = await select_positions( + redeemable=[position], + queued_shares=10000, + converter=make_converter(), + vault_to_harvest_params={VAULT_1: None}, + ) + assert positions_to_redeem == [] + + async def test_queued_shares_limits_redemption(self) -> None: position = make_position(available_shares=500) - result = _filter_positions_to_redeem( - vault_to_positions={VAULT_1: [position]}, - vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, - vault_to_harvest_params={VAULT_1: None}, - queued_shares=200, - os_token_converter=make_converter(), - ) - assert len(result.positions_to_redeem) == 1 - assert result.positions_to_redeem[0].shares_to_redeem == Wei(200) + mock_withdrawable = AsyncMock(return_value=Wei(10000)) + + with ( + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), + ): + positions_to_redeem = await select_positions( + redeemable=[position], + queued_shares=200, + converter=make_converter(), + vault_to_harvest_params={VAULT_1: None}, + ) + assert len(positions_to_redeem) == 1 + assert positions_to_redeem[0].shares_to_redeem == Wei(200) - def test_multiple_positions_limited_by_withdrawable_assets(self) -> None: + async def test_multiple_positions_limited_by_withdrawable_assets(self) -> None: pos1 = make_position(owner=OWNER_1, available_shares=500) pos2 = make_position(owner=OWNER_2, available_shares=1000) # 1:1 converter; pos1=500 assets, pos2=1000 assets; withdrawable=700 # pos1 fits (700-500=200 remaining), pos2 doesn't (1000>200) - result = _filter_positions_to_redeem( - vault_to_positions={VAULT_1: [pos1, pos2]}, - vault_to_withdrawable_assets={VAULT_1: Wei(700)}, - vault_to_harvest_params={VAULT_1: None}, - queued_shares=10000, - os_token_converter=make_converter(100, 100), - ) - assert len(result.positions_to_redeem) == 1 - assert result.positions_to_redeem[0].owner == OWNER_1 + mock_withdrawable = AsyncMock(return_value=Wei(700)) - def test_multiple_vaults_both_selected(self) -> None: + with ( + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), + ): + positions_to_redeem = await select_positions( + redeemable=[pos1, pos2], + queued_shares=10000, + converter=make_converter(100, 100), + vault_to_harvest_params={VAULT_1: None}, + ) + assert len(positions_to_redeem) == 1 + assert positions_to_redeem[0].owner == OWNER_1 + + async def test_multiple_vaults_both_selected(self) -> None: pos1 = make_position(vault=VAULT_1, owner=OWNER_1, available_shares=500) pos2 = make_position(vault=VAULT_2, owner=OWNER_2, available_shares=800) - vault_to_positions: dict[ChecksumAddress, list[OsTokenPosition]] = {} - vault_to_positions[VAULT_1] = [pos1] - vault_to_positions[VAULT_2] = [pos2] + mock_withdrawable = AsyncMock(return_value=Wei(10000)) - result = _filter_positions_to_redeem( - vault_to_positions=vault_to_positions, - vault_to_withdrawable_assets={VAULT_1: Wei(10000), VAULT_2: Wei(10000)}, - vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, - queued_shares=10000, - os_token_converter=make_converter(100, 100), - ) - assert len(result.positions_to_redeem) == 2 + with ( + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), + ): + positions_to_redeem = await select_positions( + redeemable=[pos1, pos2], + queued_shares=10000, + converter=make_converter(100, 100), + vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, + ) + assert len(positions_to_redeem) == 2 - def test_stops_across_vaults_when_queued_shares_exhausted(self) -> None: + async def test_stops_across_vaults_when_queued_shares_exhausted(self) -> None: pos1 = make_position(vault=VAULT_1, owner=OWNER_1, available_shares=500) pos2 = make_position(vault=VAULT_2, owner=OWNER_2, available_shares=800) - vault_to_positions: dict[ChecksumAddress, list[OsTokenPosition]] = {} - vault_to_positions[VAULT_1] = [pos1] - vault_to_positions[VAULT_2] = [pos2] - - result = _filter_positions_to_redeem( - vault_to_positions=vault_to_positions, - vault_to_withdrawable_assets={VAULT_1: Wei(10000), VAULT_2: Wei(10000)}, - vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, - queued_shares=500, - os_token_converter=make_converter(100, 100), - ) - assert len(result.positions_to_redeem) == 1 - assert result.positions_to_redeem[0].vault == VAULT_1 - - def test_preserves_harvest_params(self) -> None: - pos = make_position(available_shares=500) - harvest_params = make_harvest_params() + mock_withdrawable = AsyncMock(return_value=Wei(10000)) - result = _filter_positions_to_redeem( - vault_to_positions={VAULT_1: [pos]}, - vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, - vault_to_harvest_params={VAULT_1: harvest_params}, - queued_shares=10000, - os_token_converter=make_converter(), - ) - assert result.vault_to_harvest_params[VAULT_1] is harvest_params + with ( + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), + ): + positions_to_redeem = await select_positions( + redeemable=[pos1, pos2], + queued_shares=500, + converter=make_converter(100, 100), + vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, + ) + assert len(positions_to_redeem) == 1 + assert positions_to_redeem[0].vault == VAULT_1 - def test_preserves_original_amount(self) -> None: + async def test_preserves_original_amount(self) -> None: pos = make_position(amount=1000, available_shares=500) - result = _filter_positions_to_redeem( - vault_to_positions={VAULT_1: [pos]}, - vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, - vault_to_harvest_params={VAULT_1: None}, - queued_shares=200, - os_token_converter=make_converter(), - ) - assert result.positions_to_redeem[0].amount == Wei(1000) - assert result.positions_to_redeem[0].shares_to_redeem == Wei(200) + mock_withdrawable = AsyncMock(return_value=Wei(10000)) + + with ( + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), + ): + positions_to_redeem = await select_positions( + redeemable=[pos], + queued_shares=200, + converter=make_converter(), + vault_to_harvest_params={VAULT_1: None}, + ) + assert positions_to_redeem[0].amount == Wei(1000) + assert positions_to_redeem[0].shares_to_redeem == Wei(200) - def test_stops_within_vault_when_queued_shares_exhausted(self) -> None: - # Multiple positions in same vault, queued_shares runs out mid-vault + async def test_stops_within_vault_when_queued_shares_exhausted(self) -> None: pos1 = make_position(owner=OWNER_1, available_shares=400) pos2 = make_position(owner=OWNER_2, available_shares=300) - # 1:1 converter; pos1 consumes all 400 queued_shares, pos2 is skipped - result = _filter_positions_to_redeem( - vault_to_positions={VAULT_1: [pos1, pos2]}, - vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, - vault_to_harvest_params={VAULT_1: None}, - queued_shares=400, - os_token_converter=make_converter(100, 100), - ) - assert len(result.positions_to_redeem) == 1 - assert result.positions_to_redeem[0].owner == OWNER_1 + mock_withdrawable = AsyncMock(return_value=Wei(10000)) - def test_skips_position_then_selects_next(self) -> None: - # First position too expensive, second fits + with ( + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), + ): + positions_to_redeem = await select_positions( + redeemable=[pos1, pos2], + queued_shares=400, + converter=make_converter(100, 100), + vault_to_harvest_params={VAULT_1: None}, + ) + assert len(positions_to_redeem) == 1 + assert positions_to_redeem[0].owner == OWNER_1 + + async def test_skips_position_then_selects_next(self) -> None: pos1 = make_position(owner=OWNER_1, available_shares=1000) pos2 = make_position(owner=OWNER_2, available_shares=100) - # 1:1 converter; pos1=1000 assets > 500 withdrawable; pos2=100 <= 500 - result = _filter_positions_to_redeem( - vault_to_positions={VAULT_1: [pos1, pos2]}, - vault_to_withdrawable_assets={VAULT_1: Wei(500)}, - vault_to_harvest_params={VAULT_1: None}, - queued_shares=10000, - os_token_converter=make_converter(100, 100), - ) - assert len(result.positions_to_redeem) == 1 - assert result.positions_to_redeem[0].owner == OWNER_2 + mock_withdrawable = AsyncMock(return_value=Wei(500)) + + with ( + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), + ): + positions_to_redeem = await select_positions( + redeemable=[pos1, pos2], + queued_shares=10000, + converter=make_converter(100, 100), + vault_to_harvest_params={VAULT_1: None}, + ) + assert len(positions_to_redeem) == 1 + assert positions_to_redeem[0].owner == OWNER_2 + + async def test_calls_withdrawable_per_vault(self) -> None: + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, available_shares=500) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, available_shares=800) + + mock_withdrawable = AsyncMock(return_value=Wei(10000)) + + with ( + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), + ): + positions_to_redeem = await select_positions( + redeemable=[pos1, pos2], + queued_shares=10000, + converter=make_converter(100, 100), + vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, + ) + + assert mock_withdrawable.call_count == 2 + assert len(positions_to_redeem) == 2 + + async def test_passes_harvest_params_to_get_withdrawable_assets(self) -> None: + pos = make_position(vault=VAULT_1, available_shares=500) + hp = make_harvest_params() + + mock_withdrawable = AsyncMock(return_value=Wei(10000)) + + with ( + patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), + patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), + ): + await select_positions( + redeemable=[pos], + queued_shares=10000, + converter=make_converter(100, 100), + vault_to_harvest_params={VAULT_1: hp}, + ) + + mock_withdrawable.assert_called_once_with(VAULT_1, hp) # --- Async function tests (with mocks) --- -class TestFetchRedeemablePositions: +class TestFetchPositionsFromIpfs: async def test_empty_positions(self) -> None: async def empty_gen(block_number: BlockNumber | None = None): # type: ignore[misc] return yield # noqa: unreachable - with ( - patch(f'{MODULE}.iter_os_token_positions', side_effect=empty_gen), - patch(f'{MODULE}.get_processed_shares_batch', new=AsyncMock(return_value=[])), - ): - result = await _fetch_redeemable_positions(tree_nonce=5, block_number=BlockNumber(100)) + with patch(f'{MODULE}.iter_os_token_positions', side_effect=empty_gen): + result = await fetch_positions_from_ipfs(block_number=BlockNumber(100)) assert result == [] - async def test_all_shares_processed(self) -> None: - pos = make_position(amount=1000) + async def test_returns_all_positions(self) -> None: + pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000) + pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000) async def gen(block_number: BlockNumber | None = None): # type: ignore[misc] - yield pos + yield pos1 + yield pos2 - with ( - patch(f'{MODULE}.iter_os_token_positions', side_effect=gen), - patch( - f'{MODULE}.get_processed_shares_batch', - new=AsyncMock(return_value=[Wei(1000)]), - ), + with patch(f'{MODULE}.iter_os_token_positions', side_effect=gen): + result = await fetch_positions_from_ipfs(block_number=BlockNumber(100)) + assert len(result) == 2 + assert result[0] is pos1 + assert result[1] is pos2 + + +class TestCalculateRedeemableShares: + async def test_all_shares_processed(self) -> None: + pos = make_position(amount=1000) + with patch( + f'{MODULE}.get_processed_shares_batch', + new=AsyncMock(return_value=[Wei(1000)]), ): - result = await _fetch_redeemable_positions(tree_nonce=5, block_number=BlockNumber(100)) + result = await calculate_redeemable_shares( + [pos], tree_nonce=5, block_number=BlockNumber(100) + ) assert result == [] async def test_partial_processed_shares(self) -> None: pos = make_position(amount=1000) - - async def gen(block_number: BlockNumber | None = None): # type: ignore[misc] - yield pos - - with ( - patch(f'{MODULE}.iter_os_token_positions', side_effect=gen), - patch( - f'{MODULE}.get_processed_shares_batch', - new=AsyncMock(return_value=[Wei(300)]), - ), + with patch( + f'{MODULE}.get_processed_shares_batch', + new=AsyncMock(return_value=[Wei(300)]), ): - result = await _fetch_redeemable_positions(tree_nonce=5, block_number=BlockNumber(100)) + result = await calculate_redeemable_shares( + [pos], tree_nonce=5, block_number=BlockNumber(100) + ) assert len(result) == 1 assert result[0].available_shares == Wei(700) assert result[0].amount == Wei(1000) @@ -280,53 +378,31 @@ async def test_multiple_positions_mixed(self) -> None: pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000) pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000) - async def gen(block_number: BlockNumber | None = None): # type: ignore[misc] - yield pos1 - yield pos2 - # pos1 fully processed, pos2 partially processed - with ( - patch(f'{MODULE}.iter_os_token_positions', side_effect=gen), - patch( - f'{MODULE}.get_processed_shares_batch', - new=AsyncMock(return_value=[Wei(1000), Wei(500)]), - ), + with patch( + f'{MODULE}.get_processed_shares_batch', + new=AsyncMock(return_value=[Wei(1000), Wei(500)]), ): - result = await _fetch_redeemable_positions(tree_nonce=5, block_number=BlockNumber(100)) + result = await calculate_redeemable_shares( + [pos1, pos2], tree_nonce=5, block_number=BlockNumber(100) + ) assert len(result) == 1 assert result[0].owner == OWNER_2 assert result[0].available_shares == Wei(1500) -class TestTryRedeemSubVaults: - async def test_sufficient_withdrawable_assets(self) -> None: - positions = [make_position(available_shares=500)] - # 1:1 converter; vault_positions_assets=500 <= withdrawable=1000 → return - result = await _try_redeem_sub_vaults( - vault_address=VAULT_1, - positions=positions, - withdrawable_assets=Wei(1000), - harvest_params=None, - os_token_converter=make_converter(100, 100), - ) - assert result == Wei(1000) - - async def test_insufficient_non_meta_vault(self) -> None: - positions = [make_position(available_shares=500)] - # vault_positions_assets=500 > withdrawable=100, but not meta-vault +class TestTryRedeemMetaVault: + async def test_not_meta_vault(self) -> None: with patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)): - result = await _try_redeem_sub_vaults( + result = await _try_redeem_meta_vault( vault_address=VAULT_1, - positions=positions, - withdrawable_assets=Wei(100), + deficit=Wei(400), + current_withdrawable=Wei(100), harvest_params=None, - os_token_converter=make_converter(100, 100), ) assert result == Wei(100) async def test_meta_vault_successful_redeem(self) -> None: - positions = [make_position(available_shares=500)] - # vault_positions_assets=500 > withdrawable=100, meta-vault with ( patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=True)), patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, @@ -336,29 +412,26 @@ async def test_meta_vault_successful_redeem(self) -> None: ), ): mock_redeemer.redeem_sub_vaults_assets = AsyncMock(return_value='0xabc') - result = await _try_redeem_sub_vaults( + result = await _try_redeem_meta_vault( vault_address=VAULT_1, - positions=positions, - withdrawable_assets=Wei(100), + deficit=Wei(400), + current_withdrawable=Wei(100), harvest_params=None, - os_token_converter=make_converter(100, 100), ) assert result == Wei(600) mock_redeemer.redeem_sub_vaults_assets.assert_called_once_with(VAULT_1, Wei(400)) async def test_meta_vault_failed_redeem(self) -> None: - positions = [make_position(available_shares=500)] with ( patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=True)), patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, ): mock_redeemer.redeem_sub_vaults_assets = AsyncMock(side_effect=RuntimeError('fail')) - result = await _try_redeem_sub_vaults( + result = await _try_redeem_meta_vault( vault_address=VAULT_1, - positions=positions, - withdrawable_assets=Wei(100), + deficit=Wei(400), + current_withdrawable=Wei(100), harvest_params=None, - os_token_converter=make_converter(100, 100), ) assert result == Wei(100) @@ -423,65 +496,6 @@ async def test_unauthorized(self) -> None: await _startup_check() -class TestSelectPositionsToRedeem: - async def test_empty_positions(self) -> None: - result = await _select_positions_to_redeem( - redeemable_positions=[], - queued_shares=10000, - os_token_converter=make_converter(), - block_number=BlockNumber(100), - ) - assert result.positions_to_redeem == [] - - async def test_calls_io_per_vault(self) -> None: - pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, available_shares=500) - pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, available_shares=800) - - mock_harvest = AsyncMock(return_value=None) - mock_withdrawable = AsyncMock(return_value=Wei(10000)) - mock_sub_vaults = AsyncMock(return_value=Wei(10000)) - - with ( - patch(f'{MODULE}.get_harvest_params', mock_harvest), - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}._try_redeem_sub_vaults', mock_sub_vaults), - ): - result = await _select_positions_to_redeem( - redeemable_positions=[pos1, pos2], - queued_shares=10000, - os_token_converter=make_converter(100, 100), - block_number=BlockNumber(100), - ) - - assert mock_harvest.call_count == 2 - assert mock_withdrawable.call_count == 2 - assert mock_sub_vaults.call_count == 2 - assert len(result.positions_to_redeem) == 2 - - async def test_passes_harvest_params_to_get_withdrawable_assets(self) -> None: - pos = make_position(vault=VAULT_1, available_shares=500) - harvest_params = make_harvest_params() - - mock_harvest = AsyncMock(return_value=harvest_params) - mock_withdrawable = AsyncMock(return_value=Wei(10000)) - mock_sub_vaults = AsyncMock(return_value=Wei(10000)) - - with ( - patch(f'{MODULE}.get_harvest_params', mock_harvest), - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}._try_redeem_sub_vaults', mock_sub_vaults), - ): - result = await _select_positions_to_redeem( - redeemable_positions=[pos], - queued_shares=10000, - os_token_converter=make_converter(100, 100), - block_number=BlockNumber(100), - ) - - mock_withdrawable.assert_called_once_with(VAULT_1, harvest_params) - assert result.vault_to_harvest_params[VAULT_1] is harvest_params - - class TestExecuteRedemption: async def test_successful_with_harvest_params(self) -> None: pos = make_position(vault=VAULT_1, amount=1000, available_shares=500, shares_to_redeem=500) @@ -507,7 +521,7 @@ async def test_successful_with_harvest_params(self) -> None: mock_redeemer.encode_abi.return_value = HexStr('0xredeem') mock_redeemer.contract_address = VAULT_2 - result = await _execute_redemption( + result = await execute_redemption( all_positions=[pos], positions_to_redeem=[pos], vault_to_harvest_params={VAULT_1: harvest_params}, @@ -536,7 +550,7 @@ async def test_successful_without_harvest_params(self) -> None: mock_redeemer.encode_abi.return_value = HexStr('0xredeem') mock_redeemer.contract_address = VAULT_2 - result = await _execute_redemption( + result = await execute_redemption( all_positions=[pos], positions_to_redeem=[pos], vault_to_harvest_params={VAULT_1: None}, @@ -559,7 +573,7 @@ async def test_web3_exception(self) -> None: mock_redeemer.encode_abi.return_value = HexStr('0xredeem') mock_redeemer.contract_address = VAULT_2 - result = await _execute_redemption( + result = await execute_redemption( all_positions=[pos], positions_to_redeem=[pos], vault_to_harvest_params={VAULT_1: None}, @@ -587,7 +601,7 @@ async def test_tx_receipt_fails(self) -> None: mock_redeemer.encode_abi.return_value = HexStr('0xredeem') mock_redeemer.contract_address = VAULT_2 - result = await _execute_redemption( + result = await execute_redemption( all_positions=[pos], positions_to_redeem=[pos], vault_to_harvest_params={VAULT_1: None}, @@ -614,9 +628,11 @@ async def test_zero_merkle_root(self) -> None: f'{MODULE}.create_os_token_converter', new=AsyncMock(return_value=make_converter()), ), + patch(f'{MODULE}.settings') as mock_settings, patch(f'{TASKS_MODULE}.os_token_redeemer_contract') as mock_tasks_redeemer, - patch(f'{MODULE}._execute_redemption') as mock_execute, + patch(f'{MODULE}.execute_redemption') as mock_execute, ): + mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) mock_redeemer.nonce = AsyncMock(return_value=5) mock_tasks_redeemer.redeemable_positions = AsyncMock( @@ -636,9 +652,11 @@ async def test_empty_ipfs_hash(self) -> None: f'{MODULE}.create_os_token_converter', new=AsyncMock(return_value=make_converter()), ), + patch(f'{MODULE}.settings') as mock_settings, patch(f'{TASKS_MODULE}.os_token_redeemer_contract') as mock_tasks_redeemer, - patch(f'{MODULE}._execute_redemption') as mock_execute, + patch(f'{MODULE}.execute_redemption') as mock_execute, ): + mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) mock_redeemer.nonce = AsyncMock(return_value=5) mock_tasks_redeemer.redeemable_positions = AsyncMock( @@ -658,29 +676,21 @@ async def test_no_eligible_positions(self) -> None: f'{MODULE}.create_os_token_converter', new=AsyncMock(return_value=make_converter()), ), + patch(f'{MODULE}.settings') as mock_settings, patch( - f'{MODULE}._fetch_redeemable_positions', + f'{MODULE}.fetch_positions_from_ipfs', new=AsyncMock(return_value=[]), ), - patch(f'{MODULE}._execute_redemption') as mock_execute, + patch(f'{MODULE}.execute_redemption') as mock_execute, ): + mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) mock_redeemer.nonce = AsyncMock(return_value=5) - mock_redeemer.redeemable_positions = AsyncMock( - return_value=RedeemablePositions( - merkle_root=HexStr('0x' + 'ab' * 32), - ipfs_hash='QmTest', - ) - ) await process(block_number=BlockNumber(100)) mock_execute.assert_not_called() async def test_successful_redemption(self) -> None: positions = [make_position(amount=1000, available_shares=500, shares_to_redeem=500)] - selection = PositionSelectionResult( - positions_to_redeem=positions, - vault_to_harvest_params={VAULT_1: None}, - ) with ( patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), @@ -689,27 +699,31 @@ async def test_successful_redemption(self) -> None: f'{MODULE}.create_os_token_converter', new=AsyncMock(return_value=make_converter()), ), + patch(f'{MODULE}.settings') as mock_settings, patch( - f'{MODULE}._fetch_redeemable_positions', + f'{MODULE}.fetch_positions_from_ipfs', new=AsyncMock(return_value=positions), ), patch( - f'{MODULE}._select_positions_to_redeem', - new=AsyncMock(return_value=selection), + f'{MODULE}.calculate_redeemable_shares', + new=AsyncMock(return_value=positions), ), patch( - f'{MODULE}._execute_redemption', + f'{MODULE}.fetch_harvest_params_by_vault', + new=AsyncMock(return_value={VAULT_1: None}), + ), + patch( + f'{MODULE}.select_positions', + new=AsyncMock(return_value=positions), + ), + patch( + f'{MODULE}.execute_redemption', new=AsyncMock(return_value='0xtxhash'), ) as mock_execute, ): + mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) mock_redeemer.nonce = AsyncMock(return_value=5) - mock_redeemer.redeemable_positions = AsyncMock( - return_value=RedeemablePositions( - merkle_root=HexStr('0x' + 'ab' * 32), - ipfs_hash='QmTest', - ) - ) await process(block_number=BlockNumber(100)) mock_execute.assert_called_once() From f4874ac5489c77005c6c0b273937366172415588 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Fri, 6 Mar 2026 15:00:38 +0300 Subject: [PATCH 52/65] Refactoring Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 63 +++-- .../test_internal/test_process_redeemer.py | 242 +++++++++--------- 2 files changed, 163 insertions(+), 142 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index a76ffd16..1982ab46 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -160,6 +160,10 @@ async def process(block_number: BlockNumber) -> None: # Step 1: Process exit queue await _process_exit_queue(block_number) + # Re-fetch block number after exit queue processing + # to ensure we read the latest on-chain state + block_number = await execution_client.eth.block_number + # Step 2: Check queued shares queued_shares = await os_token_redeemer_contract.queued_shares(block_number) if queued_shares == 0: @@ -188,20 +192,25 @@ async def process(block_number: BlockNumber) -> None: return # Step 4: Calculate redeemable shares - redeemable = await calculate_redeemable_shares(all_positions, tree_nonce, block_number) - if not redeemable: + os_token_positions = await calculate_redeemable_shares(all_positions, tree_nonce, block_number) + if not os_token_positions: logger.info('No redeemable positions found. Skipping to next interval.') return # Step 5: Fetch harvest params per vault - vault_to_harvest_params = await fetch_harvest_params_by_vault(redeemable, block_number) + vault_to_harvest_params = await fetch_vault_harvest_params(os_token_positions, block_number) + vault_to_withdrawable_assets = await fetch_vault_withdrawable_assets( + vaults={p.vault for p in os_token_positions}, + vault_to_harvest_params=vault_to_harvest_params, + ) # Step 6: Select positions positions_to_redeem = await select_positions( - redeemable=redeemable, + os_token_positions=os_token_positions, queued_shares=queued_shares, converter=os_token_converter, vault_to_harvest_params=vault_to_harvest_params, + vault_to_withdrawable_assets=vault_to_withdrawable_assets, ) if not positions_to_redeem: @@ -279,7 +288,7 @@ async def calculate_redeemable_shares( return redeemable -async def fetch_harvest_params_by_vault( +async def fetch_vault_harvest_params( redeemable: list[OsTokenPosition], block_number: BlockNumber, ) -> dict[ChecksumAddress, HarvestParams | None]: @@ -292,11 +301,25 @@ async def fetch_harvest_params_by_vault( return vault_to_harvest_params +async def fetch_vault_withdrawable_assets( + vaults: set[ChecksumAddress], + vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], +) -> dict[ChecksumAddress, Wei]: + """Fetch harvest params for each unique vault in redeemable positions.""" + vault_to_withdrawable_assets: dict[ChecksumAddress, Wei] = {} + for vault in vaults: + vault_to_withdrawable_assets[vault] = await get_withdrawable_assets( + vault, vault_to_harvest_params.get(vault) + ) + return vault_to_withdrawable_assets + + async def select_positions( - redeemable: list[OsTokenPosition], + os_token_positions: list[OsTokenPosition], queued_shares: int, converter: OsTokenConverter, vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], + vault_to_withdrawable_assets: dict[ChecksumAddress, Wei], ) -> list[OsTokenPosition]: """Select positions to redeem, capped by queued_shares and withdrawable assets. @@ -305,17 +328,9 @@ async def select_positions( """ # Group positions by vault vault_to_positions: defaultdict[ChecksumAddress, list[OsTokenPosition]] = defaultdict(list) - for position in redeemable: + for position in os_token_positions: vault_to_positions[position.vault].append(position) - vault_to_withdrawable: dict[ChecksumAddress, Wei] = {} - - # Fetch withdrawable assets per vault - for vault_address in vault_to_positions: - vault_to_withdrawable[vault_address] = await get_withdrawable_assets( - vault_address, vault_to_harvest_params.get(vault_address) - ) - positions_to_redeem: list[OsTokenPosition] = [] remaining_shares = queued_shares @@ -326,7 +341,7 @@ async def select_positions( shares_to_redeem = Wei(min(position.available_shares, remaining_shares)) redeemable_assets = converter.to_assets(shares_to_redeem) - withdrawable = vault_to_withdrawable[vault_address] + withdrawable = vault_to_withdrawable_assets[vault_address] # If assets exceed withdrawable, try meta-vault sub-vault redemption if redeemable_assets > withdrawable: @@ -337,11 +352,14 @@ async def select_positions( withdrawable, vault_to_harvest_params[vault_address], ) - vault_to_withdrawable[vault_address] = withdrawable + vault_to_withdrawable_assets[vault_address] = withdrawable - # Still insufficient after meta-vault attempt + # Still insufficient after meta-vault attempt — try partial fill if redeemable_assets > withdrawable: - continue + shares_to_redeem = converter.to_shares(withdrawable) + if shares_to_redeem <= 0: + continue + redeemable_assets = converter.to_assets(shares_to_redeem) logger.info( 'Position Owner: %s, Vault: %s, Shares to Redeem: %s', @@ -358,12 +376,9 @@ async def select_positions( shares_to_redeem=shares_to_redeem, ) ) - vault_to_withdrawable[vault_address] = Wei(withdrawable - redeemable_assets) + vault_to_withdrawable_assets[vault_address] = Wei(withdrawable - redeemable_assets) remaining_shares -= shares_to_redeem - if remaining_shares <= 0: - break - return positions_to_redeem @@ -467,7 +482,7 @@ def build_multi_proof( tree_nonce: int, all_positions: list[OsTokenPosition], positions_to_redeem: list[OsTokenPosition], -) -> MultiProof[tuple[bytes, int]]: +) -> MultiProof[tuple[int, ChecksumAddress, int, ChecksumAddress]]: """Build a merkle multiproof from all positions, proving the positions to redeem.""" all_leaves = [p.merkle_leaf(tree_nonce) for p in all_positions] tree = StandardMerkleTree.of( diff --git a/src/commands/tests/test_internal/test_process_redeemer.py b/src/commands/tests/test_internal/test_process_redeemer.py index 405e39f3..d53d28f5 100644 --- a/src/commands/tests/test_internal/test_process_redeemer.py +++ b/src/commands/tests/test_internal/test_process_redeemer.py @@ -1,3 +1,4 @@ +import asyncio from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -15,8 +16,9 @@ build_multi_proof, calculate_redeemable_shares, execute_redemption, - fetch_harvest_params_by_vault, fetch_positions_from_ipfs, + fetch_vault_harvest_params, + fetch_vault_withdrawable_assets, process, select_positions, ) @@ -73,7 +75,7 @@ def test_all_positions_redeemed(self) -> None: class TestFetchHarvestParamsByVault: async def test_empty_positions(self) -> None: - result = await fetch_harvest_params_by_vault([], BlockNumber(100)) + result = await fetch_vault_harvest_params([], BlockNumber(100)) assert result == {} async def test_single_vault(self) -> None: @@ -82,7 +84,7 @@ async def test_single_vault(self) -> None: mock_harvest = AsyncMock(return_value=hp) with patch(f'{MODULE}.get_harvest_params', mock_harvest): - result = await fetch_harvest_params_by_vault([pos], BlockNumber(100)) + result = await fetch_vault_harvest_params([pos], BlockNumber(100)) assert result[VAULT_1] is hp mock_harvest.assert_called_once_with(VAULT_1, BlockNumber(100)) @@ -94,7 +96,7 @@ async def test_multiple_vaults(self) -> None: mock_harvest = AsyncMock(return_value=hp) with patch(f'{MODULE}.get_harvest_params', mock_harvest): - result = await fetch_harvest_params_by_vault([pos1, pos2], BlockNumber(100)) + result = await fetch_vault_harvest_params([pos1, pos2], BlockNumber(100)) assert len(result) == 2 assert mock_harvest.call_count == 2 @@ -105,7 +107,7 @@ async def test_deduplicates_vaults(self) -> None: mock_harvest = AsyncMock(return_value=None) with patch(f'{MODULE}.get_harvest_params', mock_harvest): - result = await fetch_harvest_params_by_vault([pos1, pos2], BlockNumber(100)) + result = await fetch_vault_harvest_params([pos1, pos2], BlockNumber(100)) assert len(result) == 1 mock_harvest.assert_called_once_with(VAULT_1, BlockNumber(100)) @@ -114,60 +116,66 @@ async def test_deduplicates_vaults(self) -> None: class TestSelectPositions: async def test_empty_positions(self) -> None: positions_to_redeem = await select_positions( - redeemable=[], + os_token_positions=[], queued_shares=10000, converter=make_converter(), vault_to_harvest_params={}, + vault_to_withdrawable_assets={}, ) assert positions_to_redeem == [] async def test_single_position_sufficient_assets(self) -> None: position = make_position(available_shares=500) - mock_withdrawable = AsyncMock(return_value=Wei(1000)) - with ( - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), - ): + positions_to_redeem = await select_positions( + os_token_positions=[position], + queued_shares=10000, + converter=make_converter(), + vault_to_harvest_params={VAULT_1: None}, + vault_to_withdrawable_assets={VAULT_1: Wei(1000)}, + ) + assert len(positions_to_redeem) == 1 + assert positions_to_redeem[0].shares_to_redeem == Wei(500) + + async def test_single_position_insufficient_assets_partial_fill(self) -> None: + position = make_position(available_shares=500) + # 1:1 converter; 500 shares = 500 assets, but only 100 withdrawable + # partial fill: to_shares(100) = 100, so shares_to_redeem = 100 + + with patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)): positions_to_redeem = await select_positions( - redeemable=[position], + os_token_positions=[position], queued_shares=10000, - converter=make_converter(), + converter=make_converter(100, 100), vault_to_harvest_params={VAULT_1: None}, + vault_to_withdrawable_assets={VAULT_1: Wei(100)}, ) assert len(positions_to_redeem) == 1 - assert positions_to_redeem[0].shares_to_redeem == Wei(500) + assert positions_to_redeem[0].shares_to_redeem == Wei(100) - async def test_single_position_insufficient_assets(self) -> None: + async def test_single_position_zero_withdrawable(self) -> None: position = make_position(available_shares=500) - mock_withdrawable = AsyncMock(return_value=Wei(100)) - with ( - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), - ): + with patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)): positions_to_redeem = await select_positions( - redeemable=[position], + os_token_positions=[position], queued_shares=10000, - converter=make_converter(), + converter=make_converter(100, 100), vault_to_harvest_params={VAULT_1: None}, + vault_to_withdrawable_assets={VAULT_1: Wei(0)}, ) assert positions_to_redeem == [] async def test_queued_shares_limits_redemption(self) -> None: position = make_position(available_shares=500) - mock_withdrawable = AsyncMock(return_value=Wei(10000)) - with ( - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), - ): - positions_to_redeem = await select_positions( - redeemable=[position], - queued_shares=200, - converter=make_converter(), - vault_to_harvest_params={VAULT_1: None}, - ) + positions_to_redeem = await select_positions( + os_token_positions=[position], + queued_shares=200, + converter=make_converter(), + vault_to_harvest_params={VAULT_1: None}, + vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, + ) assert len(positions_to_redeem) == 1 assert positions_to_redeem[0].shares_to_redeem == Wei(200) @@ -175,146 +183,115 @@ async def test_multiple_positions_limited_by_withdrawable_assets(self) -> None: pos1 = make_position(owner=OWNER_1, available_shares=500) pos2 = make_position(owner=OWNER_2, available_shares=1000) # 1:1 converter; pos1=500 assets, pos2=1000 assets; withdrawable=700 - # pos1 fits (700-500=200 remaining), pos2 doesn't (1000>200) - mock_withdrawable = AsyncMock(return_value=Wei(700)) + # pos1 fits fully (700-500=200 remaining), pos2 partial-fills to 200 shares - with ( - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), - ): + with patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)): positions_to_redeem = await select_positions( - redeemable=[pos1, pos2], + os_token_positions=[pos1, pos2], queued_shares=10000, converter=make_converter(100, 100), vault_to_harvest_params={VAULT_1: None}, + vault_to_withdrawable_assets={VAULT_1: Wei(700)}, ) - assert len(positions_to_redeem) == 1 + assert len(positions_to_redeem) == 2 assert positions_to_redeem[0].owner == OWNER_1 + assert positions_to_redeem[0].shares_to_redeem == Wei(500) + assert positions_to_redeem[1].owner == OWNER_2 + assert positions_to_redeem[1].shares_to_redeem == Wei(200) async def test_multiple_vaults_both_selected(self) -> None: pos1 = make_position(vault=VAULT_1, owner=OWNER_1, available_shares=500) pos2 = make_position(vault=VAULT_2, owner=OWNER_2, available_shares=800) - mock_withdrawable = AsyncMock(return_value=Wei(10000)) - - with ( - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), - ): - positions_to_redeem = await select_positions( - redeemable=[pos1, pos2], - queued_shares=10000, - converter=make_converter(100, 100), - vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, - ) + positions_to_redeem = await select_positions( + os_token_positions=[pos1, pos2], + queued_shares=10000, + converter=make_converter(100, 100), + vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, + vault_to_withdrawable_assets={VAULT_1: Wei(10000), VAULT_2: Wei(10000)}, + ) assert len(positions_to_redeem) == 2 async def test_stops_across_vaults_when_queued_shares_exhausted(self) -> None: pos1 = make_position(vault=VAULT_1, owner=OWNER_1, available_shares=500) pos2 = make_position(vault=VAULT_2, owner=OWNER_2, available_shares=800) - mock_withdrawable = AsyncMock(return_value=Wei(10000)) - - with ( - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), - ): - positions_to_redeem = await select_positions( - redeemable=[pos1, pos2], - queued_shares=500, - converter=make_converter(100, 100), - vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, - ) + positions_to_redeem = await select_positions( + os_token_positions=[pos1, pos2], + queued_shares=500, + converter=make_converter(100, 100), + vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, + vault_to_withdrawable_assets={VAULT_1: Wei(10000), VAULT_2: Wei(10000)}, + ) assert len(positions_to_redeem) == 1 assert positions_to_redeem[0].vault == VAULT_1 async def test_preserves_original_amount(self) -> None: pos = make_position(amount=1000, available_shares=500) - mock_withdrawable = AsyncMock(return_value=Wei(10000)) - with ( - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), - ): - positions_to_redeem = await select_positions( - redeemable=[pos], - queued_shares=200, - converter=make_converter(), - vault_to_harvest_params={VAULT_1: None}, - ) + positions_to_redeem = await select_positions( + os_token_positions=[pos], + queued_shares=200, + converter=make_converter(), + vault_to_harvest_params={VAULT_1: None}, + vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, + ) assert positions_to_redeem[0].amount == Wei(1000) assert positions_to_redeem[0].shares_to_redeem == Wei(200) async def test_stops_within_vault_when_queued_shares_exhausted(self) -> None: pos1 = make_position(owner=OWNER_1, available_shares=400) pos2 = make_position(owner=OWNER_2, available_shares=300) - mock_withdrawable = AsyncMock(return_value=Wei(10000)) - with ( - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), - ): - positions_to_redeem = await select_positions( - redeemable=[pos1, pos2], - queued_shares=400, - converter=make_converter(100, 100), - vault_to_harvest_params={VAULT_1: None}, - ) + positions_to_redeem = await select_positions( + os_token_positions=[pos1, pos2], + queued_shares=400, + converter=make_converter(100, 100), + vault_to_harvest_params={VAULT_1: None}, + vault_to_withdrawable_assets={VAULT_1: Wei(10000)}, + ) assert len(positions_to_redeem) == 1 assert positions_to_redeem[0].owner == OWNER_1 - async def test_skips_position_then_selects_next(self) -> None: + async def test_partial_fills_first_then_exhausts_withdrawable(self) -> None: pos1 = make_position(owner=OWNER_1, available_shares=1000) pos2 = make_position(owner=OWNER_2, available_shares=100) - mock_withdrawable = AsyncMock(return_value=Wei(500)) + # 1:1 converter; withdrawable=500 + # pos1 partial-fills to 500 shares (500 assets), withdrawable becomes 0 + # pos2 can't fill (0 withdrawable) - with ( - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), - ): + with patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)): positions_to_redeem = await select_positions( - redeemable=[pos1, pos2], + os_token_positions=[pos1, pos2], queued_shares=10000, converter=make_converter(100, 100), vault_to_harvest_params={VAULT_1: None}, + vault_to_withdrawable_assets={VAULT_1: Wei(500)}, ) assert len(positions_to_redeem) == 1 - assert positions_to_redeem[0].owner == OWNER_2 + assert positions_to_redeem[0].owner == OWNER_1 + assert positions_to_redeem[0].shares_to_redeem == Wei(500) async def test_calls_withdrawable_per_vault(self) -> None: - pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000, available_shares=500) - pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000, available_shares=800) - mock_withdrawable = AsyncMock(return_value=Wei(10000)) - with ( - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), - ): - positions_to_redeem = await select_positions( - redeemable=[pos1, pos2], - queued_shares=10000, - converter=make_converter(100, 100), + with patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable): + result = await fetch_vault_withdrawable_assets( + vaults={VAULT_1, VAULT_2}, vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, ) assert mock_withdrawable.call_count == 2 - assert len(positions_to_redeem) == 2 + assert result[VAULT_1] == Wei(10000) + assert result[VAULT_2] == Wei(10000) async def test_passes_harvest_params_to_get_withdrawable_assets(self) -> None: - pos = make_position(vault=VAULT_1, available_shares=500) hp = make_harvest_params() - mock_withdrawable = AsyncMock(return_value=Wei(10000)) - with ( - patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable), - patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)), - ): - await select_positions( - redeemable=[pos], - queued_shares=10000, - converter=make_converter(100, 100), + with patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable): + await fetch_vault_withdrawable_assets( + vaults={VAULT_1}, vault_to_harvest_params={VAULT_1: hp}, ) @@ -613,14 +590,23 @@ async def test_tx_receipt_fails(self) -> None: class TestProcess: async def test_no_queued_shares(self) -> None: + mock_client = MagicMock() + block_number_future: asyncio.Future[BlockNumber] = asyncio.Future() + block_number_future.set_result(BlockNumber(101)) + mock_client.eth.block_number = block_number_future with ( patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch(f'{MODULE}.execution_client', new=mock_client), ): mock_redeemer.queued_shares = AsyncMock(return_value=Wei(0)) await process(block_number=BlockNumber(100)) async def test_zero_merkle_root(self) -> None: + mock_client = MagicMock() + block_number_future: asyncio.Future[BlockNumber] = asyncio.Future() + block_number_future.set_result(BlockNumber(101)) + mock_client.eth.block_number = block_number_future with ( patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, @@ -631,6 +617,7 @@ async def test_zero_merkle_root(self) -> None: patch(f'{MODULE}.settings') as mock_settings, patch(f'{TASKS_MODULE}.os_token_redeemer_contract') as mock_tasks_redeemer, patch(f'{MODULE}.execute_redemption') as mock_execute, + patch(f'{MODULE}.execution_client', new=mock_client), ): mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) @@ -645,6 +632,10 @@ async def test_zero_merkle_root(self) -> None: mock_execute.assert_not_called() async def test_empty_ipfs_hash(self) -> None: + mock_client = MagicMock() + block_number_future: asyncio.Future[BlockNumber] = asyncio.Future() + block_number_future.set_result(BlockNumber(101)) + mock_client.eth.block_number = block_number_future with ( patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, @@ -655,6 +646,7 @@ async def test_empty_ipfs_hash(self) -> None: patch(f'{MODULE}.settings') as mock_settings, patch(f'{TASKS_MODULE}.os_token_redeemer_contract') as mock_tasks_redeemer, patch(f'{MODULE}.execute_redemption') as mock_execute, + patch(f'{MODULE}.execution_client', new=mock_client), ): mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) @@ -669,6 +661,10 @@ async def test_empty_ipfs_hash(self) -> None: mock_execute.assert_not_called() async def test_no_eligible_positions(self) -> None: + mock_client = MagicMock() + block_number_future: asyncio.Future[BlockNumber] = asyncio.Future() + block_number_future.set_result(BlockNumber(101)) + mock_client.eth.block_number = block_number_future with ( patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, @@ -682,6 +678,7 @@ async def test_no_eligible_positions(self) -> None: new=AsyncMock(return_value=[]), ), patch(f'{MODULE}.execute_redemption') as mock_execute, + patch(f'{MODULE}.execution_client', new=mock_client), ): mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) @@ -691,6 +688,10 @@ async def test_no_eligible_positions(self) -> None: async def test_successful_redemption(self) -> None: positions = [make_position(amount=1000, available_shares=500, shares_to_redeem=500)] + mock_client = MagicMock() + block_number_future: asyncio.Future[BlockNumber] = asyncio.Future() + block_number_future.set_result(BlockNumber(101)) + mock_client.eth.block_number = block_number_future with ( patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), @@ -709,9 +710,13 @@ async def test_successful_redemption(self) -> None: new=AsyncMock(return_value=positions), ), patch( - f'{MODULE}.fetch_harvest_params_by_vault', + f'{MODULE}.fetch_vault_harvest_params', new=AsyncMock(return_value={VAULT_1: None}), ), + patch( + f'{MODULE}.fetch_vault_withdrawable_assets', + new=AsyncMock(return_value={VAULT_1: Wei(10000)}), + ), patch( f'{MODULE}.select_positions', new=AsyncMock(return_value=positions), @@ -720,6 +725,7 @@ async def test_successful_redemption(self) -> None: f'{MODULE}.execute_redemption', new=AsyncMock(return_value='0xtxhash'), ) as mock_execute, + patch(f'{MODULE}.execution_client', new=mock_client), ): mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) From a5efdbea61a0962392f0b562297cbd974e4fb14b Mon Sep 17 00:00:00 2001 From: cyc60 Date: Fri, 6 Mar 2026 15:39:03 +0300 Subject: [PATCH 53/65] Add get_multiple_harvest_params Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 10 ++-- .../test_internal/test_process_redeemer.py | 18 +++---- src/common/harvest.py | 51 ++++++++++++++----- 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 1982ab46..fed2a55d 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -20,7 +20,7 @@ os_token_redeemer_contract, ) from src.common.execution import transaction_gas_wrapper -from src.common.harvest import get_harvest_params +from src.common.harvest import get_multiple_harvest_params from src.common.logging import LOG_LEVELS, setup_logging from src.common.typings import HarvestParams from src.common.utils import log_verbose @@ -293,12 +293,8 @@ async def fetch_vault_harvest_params( block_number: BlockNumber, ) -> dict[ChecksumAddress, HarvestParams | None]: """Fetch harvest params for each unique vault in redeemable positions.""" - vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None] = {} - for vault_address in {pos.vault for pos in redeemable}: - vault_to_harvest_params[vault_address] = await get_harvest_params( - vault_address, block_number - ) - return vault_to_harvest_params + vaults = list({pos.vault for pos in redeemable}) + return await get_multiple_harvest_params(vaults, block_number) async def fetch_vault_withdrawable_assets( diff --git a/src/commands/tests/test_internal/test_process_redeemer.py b/src/commands/tests/test_internal/test_process_redeemer.py index d53d28f5..bc73a2f2 100644 --- a/src/commands/tests/test_internal/test_process_redeemer.py +++ b/src/commands/tests/test_internal/test_process_redeemer.py @@ -81,36 +81,36 @@ async def test_empty_positions(self) -> None: async def test_single_vault(self) -> None: pos = make_position(vault=VAULT_1, available_shares=500) hp = make_harvest_params() - mock_harvest = AsyncMock(return_value=hp) + mock_batch = AsyncMock(return_value={VAULT_1: hp}) - with patch(f'{MODULE}.get_harvest_params', mock_harvest): + with patch(f'{MODULE}.get_multiple_harvest_params', mock_batch): result = await fetch_vault_harvest_params([pos], BlockNumber(100)) assert result[VAULT_1] is hp - mock_harvest.assert_called_once_with(VAULT_1, BlockNumber(100)) + mock_batch.assert_called_once_with([VAULT_1], BlockNumber(100)) async def test_multiple_vaults(self) -> None: pos1 = make_position(vault=VAULT_1, available_shares=500) pos2 = make_position(vault=VAULT_2, available_shares=800) hp = make_harvest_params() - mock_harvest = AsyncMock(return_value=hp) + mock_batch = AsyncMock(return_value={VAULT_1: hp, VAULT_2: hp}) - with patch(f'{MODULE}.get_harvest_params', mock_harvest): + with patch(f'{MODULE}.get_multiple_harvest_params', mock_batch): result = await fetch_vault_harvest_params([pos1, pos2], BlockNumber(100)) assert len(result) == 2 - assert mock_harvest.call_count == 2 + mock_batch.assert_called_once() async def test_deduplicates_vaults(self) -> None: pos1 = make_position(vault=VAULT_1, owner=OWNER_1, available_shares=500) pos2 = make_position(vault=VAULT_1, owner=OWNER_2, available_shares=800) - mock_harvest = AsyncMock(return_value=None) + mock_batch = AsyncMock(return_value={VAULT_1: None}) - with patch(f'{MODULE}.get_harvest_params', mock_harvest): + with patch(f'{MODULE}.get_multiple_harvest_params', mock_batch): result = await fetch_vault_harvest_params([pos1, pos2], BlockNumber(100)) assert len(result) == 1 - mock_harvest.assert_called_once_with(VAULT_1, BlockNumber(100)) + mock_batch.assert_called_once_with([VAULT_1], BlockNumber(100)) class TestSelectPositions: diff --git a/src/common/harvest.py b/src/common/harvest.py index 554f94c8..4903eb6a 100644 --- a/src/common/harvest.py +++ b/src/common/harvest.py @@ -1,3 +1,5 @@ +from typing import cast + from hexbytes import HexBytes from sw_utils.networks import GNO_NETWORKS from web3 import Web3 @@ -12,29 +14,50 @@ async def get_harvest_params( vault: ChecksumAddress, block_number: BlockNumber | None = None ) -> HarvestParams | None: - if not await keeper_contract.can_harvest(vault, block_number): - return None + """Get harvest params for a single vault.""" + result = await get_multiple_harvest_params([vault], block_number) + return result[vault] + + +async def get_multiple_harvest_params( + vaults: list[ChecksumAddress], block_number: BlockNumber | None = None +) -> dict[ChecksumAddress, HarvestParams | None]: + """Get harvest params for multiple vaults. + + IPFS data and last rewards are fetched once, then reused for all vaults. + """ + results: dict[ChecksumAddress, HarvestParams | None] = {} + if not vaults: + return results last_rewards = await keeper_contract.get_last_rewards_update(block_number) if last_rewards is None: - return None + return {vault: None for vault in vaults} + + ipfs_data = await ipfs_fetch_client.fetch_json(last_rewards.ipfs_hash) + + for vault in vaults: + if not await keeper_contract.can_harvest(vault, block_number): + results[vault] = None + continue + + vault_contract = VaultContract(vault) + results[vault] = await _extract_harvest_params( + vault_contract=vault_contract, + ipfs_data=cast(dict, ipfs_data), + rewards_root=last_rewards.rewards_root, + ) - vault_contract = VaultContract(vault) - harvest_params = await _fetch_harvest_params_from_ipfs( - vault_contract=vault_contract, - ipfs_hash=last_rewards.ipfs_hash, - rewards_root=last_rewards.rewards_root, - ) - return harvest_params + return results -async def _fetch_harvest_params_from_ipfs( - vault_contract: VaultContract, ipfs_hash: str, rewards_root: bytes +async def _extract_harvest_params( + vault_contract: VaultContract, ipfs_data: dict, rewards_root: bytes ) -> HarvestParams | None: - ipfs_data = await ipfs_fetch_client.fetch_json(ipfs_hash) + """Extract harvest params for a single vault from pre-fetched IPFS data.""" mev_escrow = await vault_contract.mev_escrow() - for vault_data in ipfs_data['vaults']: # type: ignore + for vault_data in ipfs_data['vaults']: if vault_contract.contract_address != Web3.to_checksum_address(vault_data['vault']): continue From bb35ee140fd7cf8f4fd48ef67e5bbfe734e59dc6 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Fri, 6 Mar 2026 15:42:19 +0300 Subject: [PATCH 54/65] Fix nonce Signed-off-by: cyc60 --- src/redemptions/tasks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/redemptions/tasks.py b/src/redemptions/tasks.py index 15a06092..bc928aff 100644 --- a/src/redemptions/tasks.py +++ b/src/redemptions/tasks.py @@ -88,7 +88,9 @@ async def aggregate_redemption_assets_by_vaults( os_token_converter = await create_os_token_converter(block_number) total_redemption_shares = os_token_converter.to_shares(total_redemption_assets) - nonce = await os_token_redeemer_contract.nonce(block_number=block_number) + # The contract increments nonce during setRedeemablePositions, + # but uses nonce - 1 for leaf hash computation during redemption. + tree_nonce = await os_token_redeemer_contract.nonce(block_number) - 1 vault_to_unprocessed_shares: defaultdict[ChecksumAddress, Wei] = defaultdict(lambda: Wei(0)) # Iterate through redeemable positions until total redemption shares are exhausted @@ -97,7 +99,7 @@ async def aggregate_redemption_assets_by_vaults( ): processed_shares_batch = await get_processed_shares_batch( os_token_positions_batch=os_token_position_batch, - nonce=nonce, + nonce=tree_nonce, block_number=block_number, ) for os_token_position, processed_shares in zip( From c4340298589ffe10c3eea00e5fadf13073a7487a Mon Sep 17 00:00:00 2001 From: cyc60 Date: Fri, 6 Mar 2026 15:50:36 +0300 Subject: [PATCH 55/65] Refactor Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index fed2a55d..f6221b6f 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -197,10 +197,11 @@ async def process(block_number: BlockNumber) -> None: logger.info('No redeemable positions found. Skipping to next interval.') return - # Step 5: Fetch harvest params per vault - vault_to_harvest_params = await fetch_vault_harvest_params(os_token_positions, block_number) + # Step 5: Fetch vault params + vaults = {p.vault for p in os_token_positions} + vault_to_harvest_params = await get_multiple_harvest_params(list(vaults), block_number) vault_to_withdrawable_assets = await fetch_vault_withdrawable_assets( - vaults={p.vault for p in os_token_positions}, + vaults=vaults, vault_to_harvest_params=vault_to_harvest_params, ) @@ -288,15 +289,6 @@ async def calculate_redeemable_shares( return redeemable -async def fetch_vault_harvest_params( - redeemable: list[OsTokenPosition], - block_number: BlockNumber, -) -> dict[ChecksumAddress, HarvestParams | None]: - """Fetch harvest params for each unique vault in redeemable positions.""" - vaults = list({pos.vault for pos in redeemable}) - return await get_multiple_harvest_params(vaults, block_number) - - async def fetch_vault_withdrawable_assets( vaults: set[ChecksumAddress], vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], From 37dae632eb07a0b24435d7624b3dc5d72b180530 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Fri, 6 Mar 2026 21:35:01 +0300 Subject: [PATCH 56/65] Add tests Signed-off-by: cyc60 --- .../test_internal/test_process_redeemer.py | 372 ++++++------------ src/common/tests/test_harvest.py | 38 ++ 2 files changed, 148 insertions(+), 262 deletions(-) create mode 100644 src/common/tests/test_harvest.py diff --git a/src/commands/tests/test_internal/test_process_redeemer.py b/src/commands/tests/test_internal/test_process_redeemer.py index bc73a2f2..bcca96f3 100644 --- a/src/commands/tests/test_internal/test_process_redeemer.py +++ b/src/commands/tests/test_internal/test_process_redeemer.py @@ -1,4 +1,6 @@ import asyncio +from contextlib import contextmanager +from typing import Iterator from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -17,18 +19,15 @@ calculate_redeemable_shares, execute_redemption, fetch_positions_from_ipfs, - fetch_vault_harvest_params, fetch_vault_withdrawable_assets, process, select_positions, ) from src.common.typings import HarvestParams from src.redemptions.os_token_converter import OsTokenConverter -from src.redemptions.tasks import ZERO_MERKLE_ROOT -from src.redemptions.typings import OsTokenPosition, RedeemablePositions +from src.redemptions.typings import OsTokenPosition MODULE = 'src.commands.internal.process_redeemer' -TASKS_MODULE = 'src.redemptions.tasks' VAULT_1 = Web3.to_checksum_address('0x' + '11' * 20) VAULT_2 = Web3.to_checksum_address('0x' + '22' * 20) @@ -73,46 +72,6 @@ def test_all_positions_redeemed(self) -> None: assert len(result.leaves) == 2 -class TestFetchHarvestParamsByVault: - async def test_empty_positions(self) -> None: - result = await fetch_vault_harvest_params([], BlockNumber(100)) - assert result == {} - - async def test_single_vault(self) -> None: - pos = make_position(vault=VAULT_1, available_shares=500) - hp = make_harvest_params() - mock_batch = AsyncMock(return_value={VAULT_1: hp}) - - with patch(f'{MODULE}.get_multiple_harvest_params', mock_batch): - result = await fetch_vault_harvest_params([pos], BlockNumber(100)) - - assert result[VAULT_1] is hp - mock_batch.assert_called_once_with([VAULT_1], BlockNumber(100)) - - async def test_multiple_vaults(self) -> None: - pos1 = make_position(vault=VAULT_1, available_shares=500) - pos2 = make_position(vault=VAULT_2, available_shares=800) - hp = make_harvest_params() - mock_batch = AsyncMock(return_value={VAULT_1: hp, VAULT_2: hp}) - - with patch(f'{MODULE}.get_multiple_harvest_params', mock_batch): - result = await fetch_vault_harvest_params([pos1, pos2], BlockNumber(100)) - - assert len(result) == 2 - mock_batch.assert_called_once() - - async def test_deduplicates_vaults(self) -> None: - pos1 = make_position(vault=VAULT_1, owner=OWNER_1, available_shares=500) - pos2 = make_position(vault=VAULT_1, owner=OWNER_2, available_shares=800) - mock_batch = AsyncMock(return_value={VAULT_1: None}) - - with patch(f'{MODULE}.get_multiple_harvest_params', mock_batch): - result = await fetch_vault_harvest_params([pos1, pos2], BlockNumber(100)) - - assert len(result) == 1 - mock_batch.assert_called_once_with([VAULT_1], BlockNumber(100)) - - class TestSelectPositions: async def test_empty_positions(self) -> None: positions_to_redeem = await select_positions( @@ -139,8 +98,6 @@ async def test_single_position_sufficient_assets(self) -> None: async def test_single_position_insufficient_assets_partial_fill(self) -> None: position = make_position(available_shares=500) - # 1:1 converter; 500 shares = 500 assets, but only 100 withdrawable - # partial fill: to_shares(100) = 100, so shares_to_redeem = 100 with patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)): positions_to_redeem = await select_positions( @@ -182,8 +139,6 @@ async def test_queued_shares_limits_redemption(self) -> None: async def test_multiple_positions_limited_by_withdrawable_assets(self) -> None: pos1 = make_position(owner=OWNER_1, available_shares=500) pos2 = make_position(owner=OWNER_2, available_shares=1000) - # 1:1 converter; pos1=500 assets, pos2=1000 assets; withdrawable=700 - # pos1 fits fully (700-500=200 remaining), pos2 partial-fills to 200 shares with patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)): positions_to_redeem = await select_positions( @@ -256,9 +211,6 @@ async def test_stops_within_vault_when_queued_shares_exhausted(self) -> None: async def test_partial_fills_first_then_exhausts_withdrawable(self) -> None: pos1 = make_position(owner=OWNER_1, available_shares=1000) pos2 = make_position(owner=OWNER_2, available_shares=100) - # 1:1 converter; withdrawable=500 - # pos1 partial-fills to 500 shares (500 assets), withdrawable becomes 0 - # pos2 can't fill (0 withdrawable) with patch(f'{MODULE}.is_meta_vault', new=AsyncMock(return_value=False)): positions_to_redeem = await select_positions( @@ -272,6 +224,8 @@ async def test_partial_fills_first_then_exhausts_withdrawable(self) -> None: assert positions_to_redeem[0].owner == OWNER_1 assert positions_to_redeem[0].shares_to_redeem == Wei(500) + +class TestFetchVaultWithdrawableAssets: async def test_calls_withdrawable_per_vault(self) -> None: mock_withdrawable = AsyncMock(return_value=Wei(10000)) @@ -285,7 +239,7 @@ async def test_calls_withdrawable_per_vault(self) -> None: assert result[VAULT_1] == Wei(10000) assert result[VAULT_2] == Wei(10000) - async def test_passes_harvest_params_to_get_withdrawable_assets(self) -> None: + async def test_passes_harvest_params(self) -> None: hp = make_harvest_params() mock_withdrawable = AsyncMock(return_value=Wei(10000)) @@ -355,7 +309,6 @@ async def test_multiple_positions_mixed(self) -> None: pos1 = make_position(vault=VAULT_1, owner=OWNER_1, amount=1000) pos2 = make_position(vault=VAULT_2, owner=OWNER_2, amount=2000) - # pos1 fully processed, pos2 partially processed with patch( f'{MODULE}.get_processed_shares_batch', new=AsyncMock(return_value=[Wei(1000), Wei(500)]), @@ -420,9 +373,10 @@ async def test_cannot_process(self) -> None: await _process_exit_queue(BlockNumber(100)) mock_redeemer.process_exit_queue.assert_not_called() - async def test_process_success(self) -> None: + @pytest.mark.parametrize('tx_status', [1, 0]) + async def test_process_exit_queue(self, tx_status: int) -> None: mock_client = AsyncMock() - mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 1}) + mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': tx_status}) with ( patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, patch(f'{MODULE}.execution_client', new=mock_client), @@ -434,20 +388,6 @@ async def test_process_success(self) -> None: await _process_exit_queue(BlockNumber(100)) mock_redeemer.process_exit_queue.assert_called_once() - async def test_process_tx_fails(self) -> None: - mock_client = AsyncMock() - mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 0}) - with ( - patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, - patch(f'{MODULE}.execution_client', new=mock_client), - patch(f'{MODULE}.settings') as mock_settings, - ): - mock_settings.execution_transaction_timeout = 120 - mock_redeemer.can_process_exit_queue = AsyncMock(return_value=True) - mock_redeemer.process_exit_queue = AsyncMock(return_value='0xabc') - # Should not raise, just log error - await _process_exit_queue(BlockNumber(100)) - class TestStartupCheck: async def test_authorized(self) -> None: @@ -477,26 +417,11 @@ class TestExecuteRedemption: async def test_successful_with_harvest_params(self) -> None: pos = make_position(vault=VAULT_1, amount=1000, available_shares=500, shares_to_redeem=500) harvest_params = make_harvest_params() - mock_client = AsyncMock() - mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 1}) - with ( - patch(f'{MODULE}.VaultContract') as MockVaultContract, - patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, - patch(f'{MODULE}.multicall_contract'), - patch( - f'{MODULE}.transaction_gas_wrapper', - new=AsyncMock(return_value=b'\x01' * 32), - ), - patch(f'{MODULE}.execution_client', new=mock_client), - patch(f'{MODULE}.settings') as mock_settings, - ): - mock_settings.execution_transaction_timeout = 120 - mock_vault = MockVaultContract.return_value + with _mock_execute_redemption(tx_status=1) as mocks: + mock_vault = mocks['MockVaultContract'].return_value mock_vault.contract_address = VAULT_1 mock_vault.get_update_state_call.return_value = HexStr('0xupdate') - mock_redeemer.encode_abi.return_value = HexStr('0xredeem') - mock_redeemer.contract_address = VAULT_2 result = await execute_redemption( all_positions=[pos], @@ -510,23 +435,8 @@ async def test_successful_with_harvest_params(self) -> None: async def test_successful_without_harvest_params(self) -> None: pos = make_position(vault=VAULT_1, amount=1000, available_shares=500, shares_to_redeem=500) - mock_client = AsyncMock() - mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 1}) - - with ( - patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, - patch(f'{MODULE}.multicall_contract'), - patch( - f'{MODULE}.transaction_gas_wrapper', - new=AsyncMock(return_value=b'\x01' * 32), - ), - patch(f'{MODULE}.execution_client', new=mock_client), - patch(f'{MODULE}.settings') as mock_settings, - ): - mock_settings.execution_transaction_timeout = 120 - mock_redeemer.encode_abi.return_value = HexStr('0xredeem') - mock_redeemer.contract_address = VAULT_2 + with _mock_execute_redemption(tx_status=1): result = await execute_redemption( all_positions=[pos], positions_to_redeem=[pos], @@ -539,17 +449,7 @@ async def test_successful_without_harvest_params(self) -> None: async def test_web3_exception(self) -> None: pos = make_position(amount=1000, available_shares=500, shares_to_redeem=500) - with ( - patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, - patch(f'{MODULE}.multicall_contract'), - patch( - f'{MODULE}.transaction_gas_wrapper', - new=AsyncMock(side_effect=Web3Exception('fail')), - ), - ): - mock_redeemer.encode_abi.return_value = HexStr('0xredeem') - mock_redeemer.contract_address = VAULT_2 - + with _mock_execute_redemption(tx_side_effect=Web3Exception('fail')): result = await execute_redemption( all_positions=[pos], positions_to_redeem=[pos], @@ -561,23 +461,8 @@ async def test_web3_exception(self) -> None: async def test_tx_receipt_fails(self) -> None: pos = make_position(amount=1000, available_shares=500, shares_to_redeem=500) - mock_client = AsyncMock() - mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': 0}) - - with ( - patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, - patch(f'{MODULE}.multicall_contract'), - patch( - f'{MODULE}.transaction_gas_wrapper', - new=AsyncMock(return_value=b'\x01' * 32), - ), - patch(f'{MODULE}.execution_client', new=mock_client), - patch(f'{MODULE}.settings') as mock_settings, - ): - mock_settings.execution_transaction_timeout = 120 - mock_redeemer.encode_abi.return_value = HexStr('0xredeem') - mock_redeemer.contract_address = VAULT_2 + with _mock_execute_redemption(tx_status=0): result = await execute_redemption( all_positions=[pos], positions_to_redeem=[pos], @@ -590,148 +475,111 @@ async def test_tx_receipt_fails(self) -> None: class TestProcess: async def test_no_queued_shares(self) -> None: - mock_client = MagicMock() - block_number_future: asyncio.Future[BlockNumber] = asyncio.Future() - block_number_future.set_result(BlockNumber(101)) - mock_client.eth.block_number = block_number_future - with ( - patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), - patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, - patch(f'{MODULE}.execution_client', new=mock_client), - ): - mock_redeemer.queued_shares = AsyncMock(return_value=Wei(0)) + with _mock_process() as mocks: + mocks['mock_redeemer'].queued_shares = AsyncMock(return_value=Wei(0)) await process(block_number=BlockNumber(100)) - async def test_zero_merkle_root(self) -> None: - mock_client = MagicMock() - block_number_future: asyncio.Future[BlockNumber] = asyncio.Future() - block_number_future.set_result(BlockNumber(101)) - mock_client.eth.block_number = block_number_future - with ( - patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), - patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, - patch( - f'{MODULE}.create_os_token_converter', - new=AsyncMock(return_value=make_converter()), - ), - patch(f'{MODULE}.settings') as mock_settings, - patch(f'{TASKS_MODULE}.os_token_redeemer_contract') as mock_tasks_redeemer, - patch(f'{MODULE}.execute_redemption') as mock_execute, - patch(f'{MODULE}.execution_client', new=mock_client), - ): - mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' - mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) - mock_redeemer.nonce = AsyncMock(return_value=5) - mock_tasks_redeemer.redeemable_positions = AsyncMock( - return_value=RedeemablePositions( - merkle_root=ZERO_MERKLE_ROOT, - ipfs_hash='QmTest', - ) - ) - await process(block_number=BlockNumber(100)) - mock_execute.assert_not_called() - - async def test_empty_ipfs_hash(self) -> None: - mock_client = MagicMock() - block_number_future: asyncio.Future[BlockNumber] = asyncio.Future() - block_number_future.set_result(BlockNumber(101)) - mock_client.eth.block_number = block_number_future - with ( - patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), - patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, - patch( - f'{MODULE}.create_os_token_converter', - new=AsyncMock(return_value=make_converter()), - ), - patch(f'{MODULE}.settings') as mock_settings, - patch(f'{TASKS_MODULE}.os_token_redeemer_contract') as mock_tasks_redeemer, - patch(f'{MODULE}.execute_redemption') as mock_execute, - patch(f'{MODULE}.execution_client', new=mock_client), - ): - mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' - mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) - mock_redeemer.nonce = AsyncMock(return_value=5) - mock_tasks_redeemer.redeemable_positions = AsyncMock( - return_value=RedeemablePositions( - merkle_root=HexStr('0x' + 'ab' * 32), - ipfs_hash='', - ) - ) - await process(block_number=BlockNumber(100)) - mock_execute.assert_not_called() - async def test_no_eligible_positions(self) -> None: - mock_client = MagicMock() - block_number_future: asyncio.Future[BlockNumber] = asyncio.Future() - block_number_future.set_result(BlockNumber(101)) - mock_client.eth.block_number = block_number_future - with ( - patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), - patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, - patch( - f'{MODULE}.create_os_token_converter', - new=AsyncMock(return_value=make_converter()), - ), - patch(f'{MODULE}.settings') as mock_settings, - patch( - f'{MODULE}.fetch_positions_from_ipfs', - new=AsyncMock(return_value=[]), - ), - patch(f'{MODULE}.execute_redemption') as mock_execute, - patch(f'{MODULE}.execution_client', new=mock_client), - ): - mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' - mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) - mock_redeemer.nonce = AsyncMock(return_value=5) + with _mock_process() as mocks: + mocks['mock_redeemer'].queued_shares = AsyncMock(return_value=Wei(1000)) + mocks['mock_redeemer'].nonce = AsyncMock(return_value=5) await process(block_number=BlockNumber(100)) - mock_execute.assert_not_called() + mocks['mock_execute'].assert_not_called() async def test_successful_redemption(self) -> None: positions = [make_position(amount=1000, available_shares=500, shares_to_redeem=500)] - mock_client = MagicMock() - block_number_future: asyncio.Future[BlockNumber] = asyncio.Future() - block_number_future.set_result(BlockNumber(101)) - mock_client.eth.block_number = block_number_future - with ( - patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), - patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, - patch( - f'{MODULE}.create_os_token_converter', - new=AsyncMock(return_value=make_converter()), - ), - patch(f'{MODULE}.settings') as mock_settings, - patch( - f'{MODULE}.fetch_positions_from_ipfs', - new=AsyncMock(return_value=positions), - ), - patch( - f'{MODULE}.calculate_redeemable_shares', - new=AsyncMock(return_value=positions), - ), - patch( - f'{MODULE}.fetch_vault_harvest_params', - new=AsyncMock(return_value={VAULT_1: None}), - ), - patch( - f'{MODULE}.fetch_vault_withdrawable_assets', - new=AsyncMock(return_value={VAULT_1: Wei(10000)}), - ), - patch( - f'{MODULE}.select_positions', - new=AsyncMock(return_value=positions), - ), - patch( - f'{MODULE}.execute_redemption', - new=AsyncMock(return_value='0xtxhash'), - ) as mock_execute, - patch(f'{MODULE}.execution_client', new=mock_client), - ): - mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' - mock_redeemer.queued_shares = AsyncMock(return_value=Wei(1000)) - mock_redeemer.nonce = AsyncMock(return_value=5) + with _mock_process(positions=positions) as mocks: + mocks['mock_redeemer'].queued_shares = AsyncMock(return_value=Wei(1000)) + mocks['mock_redeemer'].nonce = AsyncMock(return_value=5) await process(block_number=BlockNumber(100)) - mock_execute.assert_called_once() + mocks['mock_execute'].assert_called_once() + + +# --- Helpers --- + + +@contextmanager +def _mock_execute_redemption( + tx_status: int = 1, + tx_side_effect: Exception | None = None, +) -> Iterator[dict[str, MagicMock]]: + """Common mock setup for execute_redemption tests.""" + mock_client = AsyncMock() + mock_client.eth.wait_for_transaction_receipt = AsyncMock(return_value={'status': tx_status}) + + tx_mock = AsyncMock( + return_value=b'\x01' * 32, + side_effect=tx_side_effect, + ) + + with ( + patch(f'{MODULE}.VaultContract') as MockVaultContract, + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch(f'{MODULE}.multicall_contract'), + patch(f'{MODULE}.transaction_gas_wrapper', new=tx_mock), + patch(f'{MODULE}.execution_client', new=mock_client), + patch(f'{MODULE}.settings') as mock_settings, + ): + mock_settings.execution_transaction_timeout = 120 + mock_redeemer.encode_abi.return_value = HexStr('0xredeem') + mock_redeemer.contract_address = VAULT_2 + yield { + 'MockVaultContract': MockVaultContract, + 'mock_redeemer': mock_redeemer, + } + + +@contextmanager +def _mock_process( + positions: list[OsTokenPosition] | None = None, +) -> Iterator[dict[str, MagicMock]]: + """Common mock setup for process() tests.""" + positions = positions or [] + mock_client = MagicMock() + block_number_future: asyncio.Future[BlockNumber] = asyncio.Future() + block_number_future.set_result(BlockNumber(101)) + mock_client.eth.block_number = block_number_future + + with ( + patch(f'{MODULE}._process_exit_queue', new=AsyncMock()), + patch(f'{MODULE}.os_token_redeemer_contract') as mock_redeemer, + patch( + f'{MODULE}.create_os_token_converter', + new=AsyncMock(return_value=make_converter()), + ), + patch(f'{MODULE}.settings') as mock_settings, + patch( + f'{MODULE}.fetch_positions_from_ipfs', + new=AsyncMock(return_value=positions), + ), + patch( + f'{MODULE}.calculate_redeemable_shares', + new=AsyncMock(return_value=positions), + ), + patch( + f'{MODULE}.get_multiple_harvest_params', + new=AsyncMock(return_value={VAULT_1: None}), + ), + patch( + f'{MODULE}.fetch_vault_withdrawable_assets', + new=AsyncMock(return_value={VAULT_1: Wei(10000)}), + ), + patch( + f'{MODULE}.select_positions', + new=AsyncMock(return_value=positions), + ), + patch( + f'{MODULE}.execute_redemption', + new=AsyncMock(return_value='0xtxhash'), + ) as mock_execute, + patch(f'{MODULE}.execution_client', new=mock_client), + ): + mock_settings.network_config.VAULT_BALANCE_SYMBOL = 'ETH' + yield { + 'mock_redeemer': mock_redeemer, + 'mock_execute': mock_execute, + } def make_converter(total_assets: int = 110, total_shares: int = 100) -> OsTokenConverter: diff --git a/src/common/tests/test_harvest.py b/src/common/tests/test_harvest.py new file mode 100644 index 00000000..1bea35cb --- /dev/null +++ b/src/common/tests/test_harvest.py @@ -0,0 +1,38 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +from eth_typing import BlockNumber +from web3 import Web3 + +from src.common.harvest import get_multiple_harvest_params + +HARVEST_MODULE = 'src.common.harvest' + +VAULT_1 = Web3.to_checksum_address('0x' + '11' * 20) +VAULT_2 = Web3.to_checksum_address('0x' + '22' * 20) +OWNER_1 = Web3.to_checksum_address('0x' + '33' * 20) +OWNER_2 = Web3.to_checksum_address('0x' + '44' * 20) + + +class TestGetMultipleHarvestParams: + async def test_empty_vaults(self) -> None: + result = await get_multiple_harvest_params([], BlockNumber(100)) + assert result == {} + + async def test_no_last_rewards(self) -> None: + with patch(f'{HARVEST_MODULE}.keeper_contract') as mock_keeper: + mock_keeper.get_last_rewards_update = AsyncMock(return_value=None) + result = await get_multiple_harvest_params([VAULT_1], BlockNumber(100)) + assert result == {VAULT_1: None} + + async def test_cannot_harvest(self) -> None: + mock_last_rewards = MagicMock() + mock_last_rewards.ipfs_hash = 'QmTest' + with ( + patch(f'{HARVEST_MODULE}.keeper_contract') as mock_keeper, + patch(f'{HARVEST_MODULE}.ipfs_fetch_client') as mock_ipfs, + ): + mock_keeper.get_last_rewards_update = AsyncMock(return_value=mock_last_rewards) + mock_keeper.can_harvest = AsyncMock(return_value=False) + mock_ipfs.fetch_json = AsyncMock(return_value={}) + result = await get_multiple_harvest_params([VAULT_1, VAULT_2], BlockNumber(100)) + assert result == {VAULT_1: None, VAULT_2: None} From acf74dff15f8e602d179f0a6a159f0280a409c6f Mon Sep 17 00:00:00 2001 From: cyc60 Date: Fri, 6 Mar 2026 22:05:11 +0300 Subject: [PATCH 57/65] Refactor get_multiple_harvest_params Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 3 +-- src/common/harvest.py | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index f6221b6f..18f1f4bf 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -293,7 +293,6 @@ async def fetch_vault_withdrawable_assets( vaults: set[ChecksumAddress], vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], ) -> dict[ChecksumAddress, Wei]: - """Fetch harvest params for each unique vault in redeemable positions.""" vault_to_withdrawable_assets: dict[ChecksumAddress, Wei] = {} for vault in vaults: vault_to_withdrawable_assets[vault] = await get_withdrawable_assets( @@ -391,7 +390,7 @@ async def _try_redeem_meta_vault( vault_address, tx_hash, ) - except RuntimeError: + except (Web3Exception, RuntimeError): logger.error( 'redeemSubVaultsAssets failed for vault %s. ' 'Proceeding with current withdrawable assets.', diff --git a/src/common/harvest.py b/src/common/harvest.py index 4903eb6a..d221104a 100644 --- a/src/common/harvest.py +++ b/src/common/harvest.py @@ -24,27 +24,31 @@ async def get_multiple_harvest_params( ) -> dict[ChecksumAddress, HarvestParams | None]: """Get harvest params for multiple vaults. - IPFS data and last rewards are fetched once, then reused for all vaults. + Checks can_harvest for all vaults first, then fetches IPFS data only + if at least one vault is harvestable. """ - results: dict[ChecksumAddress, HarvestParams | None] = {} if not vaults: - return results + return {} last_rewards = await keeper_contract.get_last_rewards_update(block_number) if last_rewards is None: return {vault: None for vault in vaults} - ipfs_data = await ipfs_fetch_client.fetch_json(last_rewards.ipfs_hash) + harvestable_vaults: list[ChecksumAddress] = [ + vault for vault in vaults if await keeper_contract.can_harvest(vault, block_number) + ] - for vault in vaults: - if not await keeper_contract.can_harvest(vault, block_number): - results[vault] = None - continue + if not harvestable_vaults: + return {vault: None for vault in vaults} + + ipfs_data = cast(dict, await ipfs_fetch_client.fetch_json(last_rewards.ipfs_hash)) + results: dict[ChecksumAddress, HarvestParams | None] = {vault: None for vault in vaults} + for vault in harvestable_vaults: vault_contract = VaultContract(vault) results[vault] = await _extract_harvest_params( vault_contract=vault_contract, - ipfs_data=cast(dict, ipfs_data), + ipfs_data=ipfs_data, rewards_root=last_rewards.rewards_root, ) From 22e846c6cd80af67af3480b7a906d6abb32177d2 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 9 Mar 2026 11:39:13 +0300 Subject: [PATCH 58/65] Single _try_redeem_meta_vault call Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 56 ++++++++++++++--------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 18f1f4bf..3dcea1bc 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -251,7 +251,6 @@ async def _process_exit_queue(block_number: BlockNumber) -> None: async def fetch_positions_from_ipfs(block_number: BlockNumber) -> list[OsTokenPosition]: - """Collect ALL positions from IPFS. No filtering — needed for correct merkle tree.""" positions: list[OsTokenPosition] = [] async for position in iter_os_token_positions(block_number=block_number): positions.append(position) @@ -310,8 +309,8 @@ async def select_positions( ) -> list[OsTokenPosition]: """Select positions to redeem, capped by queued_shares and withdrawable assets. - Per-position: if assets exceed withdrawable, attempt meta-vault sub-vault redemption - for just that position's deficit. + Per-vault: if total assets needed exceed withdrawable, attempt meta-vault sub-vault + redemption once for the full deficit before processing individual positions. """ # Group positions by vault vault_to_positions: defaultdict[ChecksumAddress, list[OsTokenPosition]] = defaultdict(list) @@ -322,31 +321,44 @@ async def select_positions( remaining_shares = queued_shares for vault_address, positions in vault_to_positions.items(): + if remaining_shares <= 0: + break + + withdrawable = vault_to_withdrawable_assets[vault_address] + + # Calculate total potential assets needed for this vault + total_vault_shares = Wei( + min( + sum(p.available_shares for p in positions), + remaining_shares, + ) + ) + total_vault_assets = converter.to_assets(total_vault_shares) + + # Try meta-vault sub-vault redemption once per vault + if total_vault_assets > withdrawable: + deficit = Wei(total_vault_assets - withdrawable) + withdrawable = await _try_redeem_meta_vault( + vault_address, + deficit, + withdrawable, + vault_to_harvest_params[vault_address], + ) + vault_to_withdrawable_assets[vault_address] = withdrawable + for position in positions: if remaining_shares <= 0: break shares_to_redeem = Wei(min(position.available_shares, remaining_shares)) redeemable_assets = converter.to_assets(shares_to_redeem) - withdrawable = vault_to_withdrawable_assets[vault_address] - # If assets exceed withdrawable, try meta-vault sub-vault redemption + # Partial fill if assets exceed withdrawable if redeemable_assets > withdrawable: - deficit = Wei(redeemable_assets - withdrawable) - withdrawable = await _try_redeem_meta_vault( - vault_address, - deficit, - withdrawable, - vault_to_harvest_params[vault_address], - ) - vault_to_withdrawable_assets[vault_address] = withdrawable - - # Still insufficient after meta-vault attempt — try partial fill - if redeemable_assets > withdrawable: - shares_to_redeem = converter.to_shares(withdrawable) - if shares_to_redeem <= 0: - continue - redeemable_assets = converter.to_assets(shares_to_redeem) + shares_to_redeem = converter.to_shares(withdrawable) + if shares_to_redeem <= 0: + continue + redeemable_assets = converter.to_assets(shares_to_redeem) logger.info( 'Position Owner: %s, Vault: %s, Shares to Redeem: %s', @@ -363,9 +375,11 @@ async def select_positions( shares_to_redeem=shares_to_redeem, ) ) - vault_to_withdrawable_assets[vault_address] = Wei(withdrawable - redeemable_assets) + withdrawable = Wei(withdrawable - redeemable_assets) remaining_shares -= shares_to_redeem + vault_to_withdrawable_assets[vault_address] = withdrawable + return positions_to_redeem From 1899d73e6a0f9d766c4d1fd81e7555b3a2ae6351 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 9 Mar 2026 12:13:18 +0300 Subject: [PATCH 59/65] Refactoring Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 73 +++++++++++++---------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 3dcea1bc..ea56aa56 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -2,6 +2,7 @@ import logging import sys from collections import defaultdict +from dataclasses import replace from pathlib import Path import click @@ -312,7 +313,6 @@ async def select_positions( Per-vault: if total assets needed exceed withdrawable, attempt meta-vault sub-vault redemption once for the full deficit before processing individual positions. """ - # Group positions by vault vault_to_positions: defaultdict[ChecksumAddress, list[OsTokenPosition]] = defaultdict(list) for position in os_token_positions: vault_to_positions[position.vault].append(position) @@ -324,27 +324,14 @@ async def select_positions( if remaining_shares <= 0: break - withdrawable = vault_to_withdrawable_assets[vault_address] - - # Calculate total potential assets needed for this vault - total_vault_shares = Wei( - min( - sum(p.available_shares for p in positions), - remaining_shares, - ) + withdrawable = await _compute_vault_withdrawable_assets( + vault_address=vault_address, + withdrawable=vault_to_withdrawable_assets[vault_address], + positions=positions, + converter=converter, + remaining_shares=remaining_shares, + harvest_params=vault_to_harvest_params[vault_address], ) - total_vault_assets = converter.to_assets(total_vault_shares) - - # Try meta-vault sub-vault redemption once per vault - if total_vault_assets > withdrawable: - deficit = Wei(total_vault_assets - withdrawable) - withdrawable = await _try_redeem_meta_vault( - vault_address, - deficit, - withdrawable, - vault_to_harvest_params[vault_address], - ) - vault_to_withdrawable_assets[vault_address] = withdrawable for position in positions: if remaining_shares <= 0: @@ -353,7 +340,6 @@ async def select_positions( shares_to_redeem = Wei(min(position.available_shares, remaining_shares)) redeemable_assets = converter.to_assets(shares_to_redeem) - # Partial fill if assets exceed withdrawable if redeemable_assets > withdrawable: shares_to_redeem = converter.to_shares(withdrawable) if shares_to_redeem <= 0: @@ -366,23 +352,44 @@ async def select_positions( position.vault, shares_to_redeem, ) - positions_to_redeem.append( - OsTokenPosition( - vault=position.vault, - owner=position.owner, - amount=position.amount, - available_shares=position.available_shares, - shares_to_redeem=shares_to_redeem, - ) - ) + positions_to_redeem.append(replace(position, shares_to_redeem=shares_to_redeem)) withdrawable = Wei(withdrawable - redeemable_assets) remaining_shares -= shares_to_redeem - vault_to_withdrawable_assets[vault_address] = withdrawable - return positions_to_redeem +# pylint: disable-next=too-many-arguments +async def _compute_vault_withdrawable_assets( + withdrawable: Wei, + vault_address: ChecksumAddress, + harvest_params: HarvestParams | None, + positions: list[OsTokenPosition], + remaining_shares: int, + converter: OsTokenConverter, +) -> Wei: + """Compute withdrawable assets for a vault, attempting meta-vault redemption if needed.""" + # Pre-compute total assets needed across all positions in this vault + total_vault_assets = Wei(0) + for position in positions: + if remaining_shares <= 0: + break + shares = Wei(min(position.available_shares, remaining_shares)) + total_vault_assets = Wei(total_vault_assets + converter.to_assets(shares)) + remaining_shares -= shares + + # redeem meta vault + if total_vault_assets > withdrawable: + deficit = Wei(total_vault_assets - withdrawable) + withdrawable = await _try_redeem_meta_vault( + vault_address, + deficit, + withdrawable, + harvest_params, + ) + return withdrawable + + async def _try_redeem_meta_vault( vault_address: ChecksumAddress, deficit: Wei, From 1f32663277d9cbeff401a685f1274c91545a930a Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 9 Mar 2026 13:03:24 +0300 Subject: [PATCH 60/65] Handle zero nonce Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 5 ++++- src/redemptions/tasks.py | 17 +++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index ea56aa56..39a6c2e3 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -176,6 +176,9 @@ async def process(block_number: BlockNumber) -> None: # The contract increments nonce during setRedeemablePositions, # but uses nonce - 1 for leaf hash computation during redemption. nonce = await os_token_redeemer_contract.nonce(block_number) + if nonce == 0: + logger.info('Zero nonce for redemption. Skipping to next interval.') + return tree_nonce = nonce - 1 queued_assets = os_token_converter.to_assets(queued_shares) @@ -490,7 +493,7 @@ def build_multi_proof( tree_nonce: int, all_positions: list[OsTokenPosition], positions_to_redeem: list[OsTokenPosition], -) -> MultiProof[tuple[int, ChecksumAddress, int, ChecksumAddress]]: +) -> MultiProof[tuple[int, ChecksumAddress, Wei, ChecksumAddress]]: """Build a merkle multiproof from all positions, proving the positions to redeem.""" all_leaves = [p.merkle_leaf(tree_nonce) for p in all_positions] tree = StandardMerkleTree.of( diff --git a/src/redemptions/tasks.py b/src/redemptions/tasks.py index bc928aff..014d2fab 100644 --- a/src/redemptions/tasks.py +++ b/src/redemptions/tasks.py @@ -28,11 +28,18 @@ async def get_redemption_assets(chain_head: ChainHead) -> Wei: Get redemption assets for operator's vault. For Gno networks return value in GNO-Wei. """ + # The contract increments nonce during setRedeemablePositions, + # but uses nonce - 1 for leaf hash computation during redemption. + nonce = await os_token_redeemer_contract.nonce(chain_head.block_number) - 1 + if nonce == 0: + logger.info('Zero nonce for redemption. Skipping redemption assets.') + return Wei(0) + protocol_config = await get_protocol_config() # Aggregate redemption assets per vault vault_to_redemption_assets = await get_vault_to_redemption_assets( - chain_head=chain_head, protocol_config=protocol_config + chain_head=chain_head, tree_nonce=nonce - 1, protocol_config=protocol_config ) # Distribute redemption assets from meta vaults to their underlying vaults vault_to_redemption_assets = await distribute_meta_vault_redemption_assets( @@ -44,7 +51,7 @@ async def get_redemption_assets(chain_head: ChainHead) -> Wei: async def get_vault_to_redemption_assets( - chain_head: ChainHead, protocol_config: ProtocolConfig + chain_head: ChainHead, tree_nonce: int, protocol_config: ProtocolConfig ) -> defaultdict[ChecksumAddress, Wei]: """ Get redemption assets per vault. @@ -67,13 +74,14 @@ async def get_vault_to_redemption_assets( vault_to_redemption_assets = await aggregate_redemption_assets_by_vaults( total_redemption_assets, + tree_nonce=tree_nonce, block_number=chain_head.block_number, ) return vault_to_redemption_assets async def aggregate_redemption_assets_by_vaults( - total_redemption_assets: Wei, block_number: BlockNumber | None = None + total_redemption_assets: Wei, tree_nonce: int, block_number: BlockNumber | None = None ) -> defaultdict[ChecksumAddress, Wei]: """ Iterate through redeemable positions until the total redemption assets are exhausted. @@ -88,9 +96,6 @@ async def aggregate_redemption_assets_by_vaults( os_token_converter = await create_os_token_converter(block_number) total_redemption_shares = os_token_converter.to_shares(total_redemption_assets) - # The contract increments nonce during setRedeemablePositions, - # but uses nonce - 1 for leaf hash computation during redemption. - tree_nonce = await os_token_redeemer_contract.nonce(block_number) - 1 vault_to_unprocessed_shares: defaultdict[ChecksumAddress, Wei] = defaultdict(lambda: Wei(0)) # Iterate through redeemable positions until total redemption shares are exhausted From 8c97ae3353b26c4d67754df2e9db4755c82fb74c Mon Sep 17 00:00:00 2001 From: cyc60 Date: Mon, 9 Mar 2026 13:17:21 +0300 Subject: [PATCH 61/65] Fix tests Signed-off-by: cyc60 --- src/redemptions/tests/test_tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/redemptions/tests/test_tasks.py b/src/redemptions/tests/test_tasks.py index 16529ff7..3d901b24 100644 --- a/src/redemptions/tests/test_tasks.py +++ b/src/redemptions/tests/test_tasks.py @@ -28,7 +28,7 @@ async def test_redeemable_positions_empty(self): with self.patch(redeemable_positions=redeemable_positions): redemption_assets_by_vaults = await aggregate_redemption_assets_by_vaults( - total_redemption_assets + total_redemption_assets, tree_nonce=0 ) assert redemption_assets_by_vaults == {} @@ -66,7 +66,7 @@ async def test_redeemable_positions_1_vault( processed_shares_batch=processed_shares_batch, ): redemption_assets_by_vaults = await aggregate_redemption_assets_by_vaults( - total_redemption_assets + total_redemption_assets, tree_nonce=0 ) assert len(redemption_assets_by_vaults) <= 1 # length can be 0 if no assets to redeem assert redemption_assets_by_vaults[vault_1] == expected_redeemed_assets @@ -132,7 +132,7 @@ async def test_redeemable_positions_2_vaults( processed_shares_batch=processed_shares_batch, ): redemption_assets_by_vaults = await aggregate_redemption_assets_by_vaults( - total_redemption_assets + total_redemption_assets, tree_nonce=0 ) # length can be less than 2 if no assets to redeem assert len(redemption_assets_by_vaults) <= 2 @@ -200,7 +200,7 @@ async def test_2_vaults_many_users(self): processed_shares_batches=processed_shares_batches, ): redemption_assets_by_vaults = await aggregate_redemption_assets_by_vaults( - total_redemption_assets + total_redemption_assets, tree_nonce=0 ) assert len(redemption_assets_by_vaults) == 2 assert redemption_assets_by_vaults[vault_1] == redemption_shares_vault_1 * Decimal( From 88dcdfbadff396d0d4524cd2edc875f9cef3e4a3 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 10 Mar 2026 16:46:37 +0300 Subject: [PATCH 62/65] Review fixes #1 Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 66 +++++++++---------- .../internal/update_redeemable_positions.py | 1 + 2 files changed, 31 insertions(+), 36 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 39a6c2e3..2854e3a0 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -125,6 +125,7 @@ def process_redeemer( wallet_password_file: str | None, ) -> None: settings.set( + # No specific vault address is set — redemptions are processed across all vaults. vault=ZERO_CHECKSUM_ADDRESS, vault_dir=Path.home() / '.stakewise', execution_endpoints=execution_endpoints, @@ -173,13 +174,13 @@ async def process(block_number: BlockNumber) -> None: os_token_converter = await create_os_token_converter(block_number) - # The contract increments nonce during setRedeemablePositions, - # but uses nonce - 1 for leaf hash computation during redemption. + # The Merkle root was calculated before the nonce was incremented + # in setRedeemablePositions, so we use the previous nonce for Merkle proofs. nonce = await os_token_redeemer_contract.nonce(block_number) if nonce == 0: logger.info('Zero nonce for redemption. Skipping to next interval.') return - tree_nonce = nonce - 1 + prev_nonce = nonce - 1 queued_assets = os_token_converter.to_assets(queued_shares) logger.info( @@ -196,7 +197,7 @@ async def process(block_number: BlockNumber) -> None: return # Step 4: Calculate redeemable shares - os_token_positions = await calculate_redeemable_shares(all_positions, tree_nonce, block_number) + os_token_positions = await calculate_redeemable_shares(all_positions, prev_nonce, block_number) if not os_token_positions: logger.info('No redeemable positions found. Skipping to next interval.') return @@ -204,10 +205,15 @@ async def process(block_number: BlockNumber) -> None: # Step 5: Fetch vault params vaults = {p.vault for p in os_token_positions} vault_to_harvest_params = await get_multiple_harvest_params(list(vaults), block_number) - vault_to_withdrawable_assets = await fetch_vault_withdrawable_assets( - vaults=vaults, - vault_to_harvest_params=vault_to_harvest_params, - ) + # vault_to_withdrawable_assets = await fetch_vault_withdrawable_assets( + # vaults=vaults, + # vault_to_harvest_params=vault_to_harvest_params, + # ) + vault_to_withdrawable_assets: dict[ChecksumAddress, Wei] = {} + for vault in vaults: + vault_to_withdrawable_assets[vault] = await get_withdrawable_assets( + vault, vault_to_harvest_params.get(vault) + ) # Step 6: Select positions positions_to_redeem = await select_positions( @@ -227,7 +233,7 @@ async def process(block_number: BlockNumber) -> None: all_positions=all_positions, positions_to_redeem=positions_to_redeem, vault_to_harvest_params=vault_to_harvest_params, - tree_nonce=tree_nonce, + nonce=prev_nonce, ) if tx_hash: logger.info( @@ -263,7 +269,7 @@ async def fetch_positions_from_ipfs(block_number: BlockNumber) -> list[OsTokenPo async def calculate_redeemable_shares( all_positions: list[OsTokenPosition], - tree_nonce: int, + nonce: int, block_number: BlockNumber, ) -> list[OsTokenPosition]: """Query processed shares and return positions with available_shares > 0.""" @@ -273,7 +279,7 @@ async def calculate_redeemable_shares( batch = all_positions[i : i + batch_size] processed_shares_batch = await get_processed_shares_batch( os_token_positions_batch=batch, - nonce=tree_nonce, + nonce=nonce, block_number=block_number, ) for position, processed_shares in zip(batch, processed_shares_batch): @@ -292,18 +298,6 @@ async def calculate_redeemable_shares( return redeemable -async def fetch_vault_withdrawable_assets( - vaults: set[ChecksumAddress], - vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], -) -> dict[ChecksumAddress, Wei]: - vault_to_withdrawable_assets: dict[ChecksumAddress, Wei] = {} - for vault in vaults: - vault_to_withdrawable_assets[vault] = await get_withdrawable_assets( - vault, vault_to_harvest_params.get(vault) - ) - return vault_to_withdrawable_assets - - async def select_positions( os_token_positions: list[OsTokenPosition], queued_shares: int, @@ -327,9 +321,9 @@ async def select_positions( if remaining_shares <= 0: break - withdrawable = await _compute_vault_withdrawable_assets( + withdrawable_assets = await _compute_vault_withdrawable_assets( vault_address=vault_address, - withdrawable=vault_to_withdrawable_assets[vault_address], + withdrawable_assets=vault_to_withdrawable_assets[vault_address], positions=positions, converter=converter, remaining_shares=remaining_shares, @@ -343,8 +337,8 @@ async def select_positions( shares_to_redeem = Wei(min(position.available_shares, remaining_shares)) redeemable_assets = converter.to_assets(shares_to_redeem) - if redeemable_assets > withdrawable: - shares_to_redeem = converter.to_shares(withdrawable) + if redeemable_assets > withdrawable_assets: + shares_to_redeem = converter.to_shares(withdrawable_assets) if shares_to_redeem <= 0: continue redeemable_assets = converter.to_assets(shares_to_redeem) @@ -356,7 +350,7 @@ async def select_positions( shares_to_redeem, ) positions_to_redeem.append(replace(position, shares_to_redeem=shares_to_redeem)) - withdrawable = Wei(withdrawable - redeemable_assets) + withdrawable_assets = Wei(withdrawable_assets - redeemable_assets) remaining_shares -= shares_to_redeem return positions_to_redeem @@ -364,7 +358,7 @@ async def select_positions( # pylint: disable-next=too-many-arguments async def _compute_vault_withdrawable_assets( - withdrawable: Wei, + withdrawable_assets: Wei, vault_address: ChecksumAddress, harvest_params: HarvestParams | None, positions: list[OsTokenPosition], @@ -382,15 +376,15 @@ async def _compute_vault_withdrawable_assets( remaining_shares -= shares # redeem meta vault - if total_vault_assets > withdrawable: - deficit = Wei(total_vault_assets - withdrawable) - withdrawable = await _try_redeem_meta_vault( + if total_vault_assets > withdrawable_assets: + deficit = Wei(total_vault_assets - withdrawable_assets) + withdrawable_assets = await _try_redeem_meta_vault( vault_address, deficit, - withdrawable, + withdrawable_assets, harvest_params, ) - return withdrawable + return withdrawable_assets async def _try_redeem_meta_vault( @@ -430,13 +424,13 @@ async def execute_redemption( all_positions: list[OsTokenPosition], positions_to_redeem: list[OsTokenPosition], vault_to_harvest_params: dict[ChecksumAddress, HarvestParams | None], - tree_nonce: int, + nonce: int, ) -> HexStr | None: """Build multiproof from all positions and execute the redemption transaction.""" multiproof = build_multi_proof( all_positions=all_positions, positions_to_redeem=positions_to_redeem, - tree_nonce=tree_nonce, + tree_nonce=nonce, ) calls: list[tuple[ChecksumAddress, HexStr]] = [] diff --git a/src/commands/internal/update_redeemable_positions.py b/src/commands/internal/update_redeemable_positions.py index 03283cb2..8c892181 100644 --- a/src/commands/internal/update_redeemable_positions.py +++ b/src/commands/internal/update_redeemable_positions.py @@ -167,6 +167,7 @@ def update_redeemable_positions( access_key=api_access_key, ) settings.set( + # No specific vault address is set — redemptions are updated across all vaults. vault=ZERO_CHECKSUM_ADDRESS, vault_dir=Path.home() / '.stakewise', execution_endpoints=execution_endpoints, From 8c2a97c8f8b0fe91d3a111b437dce7877acc586e Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 10 Mar 2026 17:10:03 +0300 Subject: [PATCH 63/65] Add min-queued-assets option Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 40 +++++++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index 2854e3a0..af3b6a04 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -12,7 +12,7 @@ from sw_utils import InterruptHandler from web3 import Web3 from web3.exceptions import Web3Exception -from web3.types import Wei +from web3.types import Gwei, Wei from src.common.clients import close_clients, execution_client, setup_clients from src.common.contracts import ( @@ -44,6 +44,8 @@ logger = logging.getLogger(__name__) DEFAULT_INTERVAL = 60 # 1 minute +DEFAULT_MIN_QUEUED_ASSETS = Web3.to_wei(0.1, 'ether') +DEFAULT_MIN_QUEUED_ASSETS_GWEI = Web3.from_wei(DEFAULT_MIN_QUEUED_ASSETS, 'gwei') @click.option( @@ -81,6 +83,13 @@ envvar='INTERVAL', help='Sleep interval in seconds between processing rounds.', ) +@click.option( + '--min-queued-assets-gwei', + type=int, + default=DEFAULT_MIN_QUEUED_ASSETS_GWEI, + envvar='MIN_QUEUED_ASSETS_GWEI', + help='Minimum queued assets (in Gwei) to trigger redemption processing.', +) @click.option( '--log-level', type=click.Choice( @@ -121,6 +130,7 @@ def process_redeemer( verbose: bool, log_level: str, interval: int, + min_queued_assets_gwei: int, wallet_file: str | None, wallet_password_file: str | None, ) -> None: @@ -137,13 +147,16 @@ def process_redeemer( log_level=log_level, ) try: - asyncio.run(main(interval=interval)) + asyncio.run(main(interval=interval, min_queued_assets=Gwei(min_queued_assets_gwei))) except Exception as e: log_verbose(e) sys.exit(1) -async def main(interval: int = DEFAULT_INTERVAL) -> None: +async def main( + interval: int, + min_queued_assets: Gwei, +) -> None: setup_logging() await setup_clients() await _startup_check() @@ -151,14 +164,17 @@ async def main(interval: int = DEFAULT_INTERVAL) -> None: with InterruptHandler() as interrupt_handler: while not interrupt_handler.exit: block_number = await execution_client.eth.block_number - await process(block_number=block_number) + await process(block_number=block_number, min_queued_assets=min_queued_assets) await interrupt_handler.sleep(interval) finally: await close_clients() -async def process(block_number: BlockNumber) -> None: +async def process( + block_number: BlockNumber, + min_queued_assets: Gwei, +) -> None: # Step 1: Process exit queue await _process_exit_queue(block_number) @@ -168,8 +184,10 @@ async def process(block_number: BlockNumber) -> None: # Step 2: Check queued shares queued_shares = await os_token_redeemer_contract.queued_shares(block_number) - if queued_shares == 0: - logger.info('No queued shares for redemption. Skipping to next interval.') + if queued_shares < Web3.to_wei(min_queued_assets, 'gwei'): + logger.info( + 'Queued shares for redemption are below min-queued-assets. Skipping to next interval.' + ) return os_token_converter = await create_os_token_converter(block_number) @@ -190,6 +208,14 @@ async def process(block_number: BlockNumber) -> None: settings.network_config.VAULT_BALANCE_SYMBOL, ) + if queued_assets < min_queued_assets: + logger.info( + 'Queued assets %s below threshold %s. Skipping to next interval.', + Web3.from_wei(queued_assets, 'ether'), + Web3.from_wei(min_queued_assets, 'ether'), + ) + return + # Step 3: Fetch ALL positions from IPFS (needed for correct merkle tree) all_positions = await fetch_positions_from_ipfs(block_number) if not all_positions: From 1d8c5d9abc5727b6a167a21b08c5f65bd5d3cf3e Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 10 Mar 2026 17:35:44 +0300 Subject: [PATCH 64/65] Update tests Signed-off-by: cyc60 --- .../test_internal/test_process_redeemer.py | 61 ++++++------------- 1 file changed, 20 insertions(+), 41 deletions(-) diff --git a/src/commands/tests/test_internal/test_process_redeemer.py b/src/commands/tests/test_internal/test_process_redeemer.py index bcca96f3..0f9ebb36 100644 --- a/src/commands/tests/test_internal/test_process_redeemer.py +++ b/src/commands/tests/test_internal/test_process_redeemer.py @@ -9,7 +9,7 @@ from sw_utils.tests import faker from web3 import Web3 from web3.exceptions import Web3Exception -from web3.types import Wei +from web3.types import Gwei, Wei from src.commands.internal.process_redeemer import ( _process_exit_queue, @@ -19,7 +19,6 @@ calculate_redeemable_shares, execute_redemption, fetch_positions_from_ipfs, - fetch_vault_withdrawable_assets, process, select_positions, ) @@ -225,33 +224,6 @@ async def test_partial_fills_first_then_exhausts_withdrawable(self) -> None: assert positions_to_redeem[0].shares_to_redeem == Wei(500) -class TestFetchVaultWithdrawableAssets: - async def test_calls_withdrawable_per_vault(self) -> None: - mock_withdrawable = AsyncMock(return_value=Wei(10000)) - - with patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable): - result = await fetch_vault_withdrawable_assets( - vaults={VAULT_1, VAULT_2}, - vault_to_harvest_params={VAULT_1: None, VAULT_2: None}, - ) - - assert mock_withdrawable.call_count == 2 - assert result[VAULT_1] == Wei(10000) - assert result[VAULT_2] == Wei(10000) - - async def test_passes_harvest_params(self) -> None: - hp = make_harvest_params() - mock_withdrawable = AsyncMock(return_value=Wei(10000)) - - with patch(f'{MODULE}.get_withdrawable_assets', mock_withdrawable): - await fetch_vault_withdrawable_assets( - vaults={VAULT_1}, - vault_to_harvest_params={VAULT_1: hp}, - ) - - mock_withdrawable.assert_called_once_with(VAULT_1, hp) - - # --- Async function tests (with mocks) --- @@ -288,7 +260,7 @@ async def test_all_shares_processed(self) -> None: new=AsyncMock(return_value=[Wei(1000)]), ): result = await calculate_redeemable_shares( - [pos], tree_nonce=5, block_number=BlockNumber(100) + [pos], nonce=5, block_number=BlockNumber(100) ) assert result == [] @@ -299,7 +271,7 @@ async def test_partial_processed_shares(self) -> None: new=AsyncMock(return_value=[Wei(300)]), ): result = await calculate_redeemable_shares( - [pos], tree_nonce=5, block_number=BlockNumber(100) + [pos], nonce=5, block_number=BlockNumber(100) ) assert len(result) == 1 assert result[0].available_shares == Wei(700) @@ -314,7 +286,7 @@ async def test_multiple_positions_mixed(self) -> None: new=AsyncMock(return_value=[Wei(1000), Wei(500)]), ): result = await calculate_redeemable_shares( - [pos1, pos2], tree_nonce=5, block_number=BlockNumber(100) + [pos1, pos2], nonce=5, block_number=BlockNumber(100) ) assert len(result) == 1 assert result[0].owner == OWNER_2 @@ -427,7 +399,7 @@ async def test_successful_with_harvest_params(self) -> None: all_positions=[pos], positions_to_redeem=[pos], vault_to_harvest_params={VAULT_1: harvest_params}, - tree_nonce=5, + nonce=5, ) assert result is not None @@ -441,7 +413,7 @@ async def test_successful_without_harvest_params(self) -> None: all_positions=[pos], positions_to_redeem=[pos], vault_to_harvest_params={VAULT_1: None}, - tree_nonce=5, + nonce=5, ) assert result is not None @@ -454,7 +426,7 @@ async def test_web3_exception(self) -> None: all_positions=[pos], positions_to_redeem=[pos], vault_to_harvest_params={VAULT_1: None}, - tree_nonce=5, + nonce=5, ) assert result is None @@ -467,7 +439,7 @@ async def test_tx_receipt_fails(self) -> None: all_positions=[pos], positions_to_redeem=[pos], vault_to_harvest_params={VAULT_1: None}, - tree_nonce=5, + nonce=5, ) assert result is None @@ -477,13 +449,20 @@ class TestProcess: async def test_no_queued_shares(self) -> None: with _mock_process() as mocks: mocks['mock_redeemer'].queued_shares = AsyncMock(return_value=Wei(0)) - await process(block_number=BlockNumber(100)) + await process(block_number=BlockNumber(100), min_queued_assets=Gwei(1)) + + async def test_below_threshold(self) -> None: + with _mock_process() as mocks: + # 500 wei queued shares, threshold is 1000 Gwei + mocks['mock_redeemer'].queued_shares = AsyncMock(return_value=Wei(500)) + await process(block_number=BlockNumber(100), min_queued_assets=Gwei(1000)) + mocks['mock_execute'].assert_not_called() async def test_no_eligible_positions(self) -> None: with _mock_process() as mocks: mocks['mock_redeemer'].queued_shares = AsyncMock(return_value=Wei(1000)) mocks['mock_redeemer'].nonce = AsyncMock(return_value=5) - await process(block_number=BlockNumber(100)) + await process(block_number=BlockNumber(100), min_queued_assets=Gwei(0)) mocks['mock_execute'].assert_not_called() async def test_successful_redemption(self) -> None: @@ -492,7 +471,7 @@ async def test_successful_redemption(self) -> None: with _mock_process(positions=positions) as mocks: mocks['mock_redeemer'].queued_shares = AsyncMock(return_value=Wei(1000)) mocks['mock_redeemer'].nonce = AsyncMock(return_value=5) - await process(block_number=BlockNumber(100)) + await process(block_number=BlockNumber(100), min_queued_assets=Gwei(0)) mocks['mock_execute'].assert_called_once() @@ -562,8 +541,8 @@ def _mock_process( new=AsyncMock(return_value={VAULT_1: None}), ), patch( - f'{MODULE}.fetch_vault_withdrawable_assets', - new=AsyncMock(return_value={VAULT_1: Wei(10000)}), + f'{MODULE}.get_withdrawable_assets', + new=AsyncMock(return_value=Wei(10000)), ), patch( f'{MODULE}.select_positions', From a4d4608db3f4d4eb78ec729d8a7a8eb73ad07aa2 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 10 Mar 2026 18:53:46 +0300 Subject: [PATCH 65/65] Rm double comparison Signed-off-by: cyc60 --- src/commands/internal/process_redeemer.py | 34 +++++++++-------------- src/redemptions/tasks.py | 2 +- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/src/commands/internal/process_redeemer.py b/src/commands/internal/process_redeemer.py index af3b6a04..bd67e91e 100644 --- a/src/commands/internal/process_redeemer.py +++ b/src/commands/internal/process_redeemer.py @@ -44,8 +44,8 @@ logger = logging.getLogger(__name__) DEFAULT_INTERVAL = 60 # 1 minute -DEFAULT_MIN_QUEUED_ASSETS = Web3.to_wei(0.1, 'ether') -DEFAULT_MIN_QUEUED_ASSETS_GWEI = Web3.from_wei(DEFAULT_MIN_QUEUED_ASSETS, 'gwei') +DEFAULT_MIN_QUEUED_SHARES = Web3.to_wei(0.1, 'ether') +DEFAULT_MIN_QUEUED_SHARES_GWEI = Web3.from_wei(DEFAULT_MIN_QUEUED_SHARES, 'gwei') @click.option( @@ -84,11 +84,11 @@ help='Sleep interval in seconds between processing rounds.', ) @click.option( - '--min-queued-assets-gwei', + '--min-queued-shares-gwei', type=int, - default=DEFAULT_MIN_QUEUED_ASSETS_GWEI, - envvar='MIN_QUEUED_ASSETS_GWEI', - help='Minimum queued assets (in Gwei) to trigger redemption processing.', + default=DEFAULT_MIN_QUEUED_SHARES_GWEI, + envvar='MIN_QUEUED_SHARES_GWEI', + help='Minimum queued shares (in Gwei) to trigger redemption processing.', ) @click.option( '--log-level', @@ -184,14 +184,16 @@ async def process( # Step 2: Check queued shares queued_shares = await os_token_redeemer_contract.queued_shares(block_number) - if queued_shares < Web3.to_wei(min_queued_assets, 'gwei'): + os_token_converter = await create_os_token_converter(block_number) + queued_assets = os_token_converter.to_assets(queued_shares) + if queued_assets < Web3.to_wei(min_queued_assets, 'gwei'): logger.info( - 'Queued shares for redemption are below min-queued-assets. Skipping to next interval.' + 'Queued assets %s below threshold %s. Skipping to next interval.', + Web3.from_wei(queued_assets, 'ether'), + Web3.from_wei(Web3.to_wei(min_queued_assets, 'gwei'), 'ether'), ) return - os_token_converter = await create_os_token_converter(block_number) - # The Merkle root was calculated before the nonce was incremented # in setRedeemablePositions, so we use the previous nonce for Merkle proofs. nonce = await os_token_redeemer_contract.nonce(block_number) @@ -200,22 +202,12 @@ async def process( return prev_nonce = nonce - 1 - queued_assets = os_token_converter.to_assets(queued_shares) logger.info( - 'Queued Shares for Redemption: %s (~%s %s)', + 'Process queued shares for Redemption: %s (~%s %s)', queued_shares, Web3.from_wei(queued_assets, 'ether'), settings.network_config.VAULT_BALANCE_SYMBOL, ) - - if queued_assets < min_queued_assets: - logger.info( - 'Queued assets %s below threshold %s. Skipping to next interval.', - Web3.from_wei(queued_assets, 'ether'), - Web3.from_wei(min_queued_assets, 'ether'), - ) - return - # Step 3: Fetch ALL positions from IPFS (needed for correct merkle tree) all_positions = await fetch_positions_from_ipfs(block_number) if not all_positions: diff --git a/src/redemptions/tasks.py b/src/redemptions/tasks.py index 014d2fab..2f901caf 100644 --- a/src/redemptions/tasks.py +++ b/src/redemptions/tasks.py @@ -30,7 +30,7 @@ async def get_redemption_assets(chain_head: ChainHead) -> Wei: """ # The contract increments nonce during setRedeemablePositions, # but uses nonce - 1 for leaf hash computation during redemption. - nonce = await os_token_redeemer_contract.nonce(chain_head.block_number) - 1 + nonce = await os_token_redeemer_contract.nonce(chain_head.block_number) if nonce == 0: logger.info('Zero nonce for redemption. Skipping redemption assets.') return Wei(0)