diff --git a/.gitignore b/.gitignore index a0f6bce..0e90582 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ venv/ # Logs and save state logs/ +state/ # Python cache files __pycache__/ @@ -31,4 +32,10 @@ __pycache__/ *.pyo *.pyd -/redstone_script/node_modules \ No newline at end of file +/redstone_script/node_modules + +.flaskenv + +/state +/my_docs +temp.txt \ No newline at end of file diff --git a/README.md b/README.md index 5ebf82d..bfee9ab 100644 --- a/README.md +++ b/README.md @@ -159,4 +159,47 @@ To run the basic test script & broadcast to the configured RPC, modify the [Liqu forge script test/LiquidationSetupWithVaultCreated.sol --rpc-url $RPC_URL --broadcast --ffi -vvv --slow --evm-version shanghai ``` -This test is intended to create a position on an existing vault. To test a liquitation, you can either wait for price fluctuations to happen or manually change the LTV of the vault using the create.euler.finance UI if it is a governed vault that you control. \ No newline at end of file +This test is intended to create a position on an existing vault. To test a liquitation, you can either wait for price fluctuations to happen or manually change the LTV of the vault using the create.euler.finance UI if it is a governed vault that you control. + +## To run the liquidation bot as a service + +Define service in /etc/systemd/system/liquidation-bot.service +``` +[Unit] +Description=Liquidation Bot Service +After=network.target + +[Service] +User=admin +WorkingDirectory=/home/admin/liq-bot-v2 +Environment="PATH=/home/admin/liq-bot-v2/venv/bin" +Environment="FLASK_APP=application.py" +Environment="FLASK_RUN_HOST=0.0.0.0" +Environment="FLASK_RUN_PORT=8080" +Environment="FLASK_ENV=production" +Environment="FLASK_DEBUG=False" + +# Using flask instead of python3 +ExecStart=/home/admin/liq-bot-v2/venv/bin/flask run --host=0.0.0.0 --port=8080 + +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target +``` + +Reload systemd: +``` +sudo systemctl daemon-reload +``` + +Run service: +``` +sudo systemctl start liquidation-bot.service +``` + +Check service status: +``` +sudo systemctl status liquidation-bot.service +``` \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py index 9a49da0..6b364b9 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -15,7 +15,21 @@ def create_app(): def health_check(): return jsonify({"status": "healthy"}), 200 - chain_ids = [1] + chain_ids = [ + 1, # Ethereum mainnet + 80094, # Berachain + 60808, # BOB + # 10, # Optimism + # 56, # Binance Smart Chain + # 100, # Gnosis Chain + # 130, # Unichain + # 137, # Polygon + # 480, # World Chain + # 8453, # Base + # 42161, # Arbitrum One + # 43114, # Avalanche + # 57073, # Ink + ] monitor_thread = threading.Thread(target=start_monitor, args=(chain_ids,)) monitor_thread.start() diff --git a/app/config.yaml b/app/config.yaml index aea088d..f0ea966 100644 --- a/app/config.yaml +++ b/app/config.yaml @@ -45,8 +45,8 @@ global: ## REPORTING PARAMETERS ## # Interval for reporting low health accounts - LOW_HEALTH_REPORT_INTERVAL: 43200 # 12 hours - SLACK_REPORT_HEALTH_SCORE: 1.05 + LOW_HEALTH_REPORT_INTERVAL: 21600 # 30 minutes + SLACK_REPORT_HEALTH_SCORE: 1.03 BORROW_VALUE_THRESHOLD: 10000 # $1000 minimum for reporting # Threshold for excluding small positions from frequent notifications, in USD terms SMALL_POSITION_THRESHOLD: 1000000000000000000000 # 1000 USD @@ -77,7 +77,7 @@ global: # API Retry NUM_RETRIES: 3 RETRY_DELAY: 10 - # Acceptable amound of overswapping of collateral for 1INCH binary search + # Acceptable amount of overswapping of collateral for 1INCH binary search API_REQUEST_DELAY: .25 SWAP_SLIPPAGE: 1.0 # 1% SWAP_DEADLINE: 300 # 5 minutes @@ -114,7 +114,232 @@ chains: # PYTH address PYTH: "0x4305FB66699C3B2702D4d05CF36551390A4c69C6" + + # Berachain + 80094: + # Config + name: "Berachain" + EVC_DEPLOYMENT_BLOCK: 786266 + EXPLORER_URL: "https://berascan.com/" + RPC_NAME: "BERA_RPC_URL" + + # Deployed contract addresses + contracts: + LIQUIDATOR_CONTRACT: "0xe7CEB97B4E622E71505c850829B1feb0326D723b" + + #EVK Addresses + EVC: "0x45334608ECE7B2775136bC847EB92B5D332806A9" + SWAPPER: "0x4A35e6A872cf35623cd3fD07ebECEDFc0170D705" + SWAP_VERIFIER: "0x6fFf8Ac4AB123B62FF5e92aBb9fF702DCBD6C939" + + # Other relevant contracts & Units of Account + WETH: "0x0555E30da8f98308EdB960aa94C0Db47230d2B9c" + USD: "0x0000000000000000000000000000000000000348" + BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + + # PYTH address + PYTH: "0x2880aB155794e7179c9eE2e38200202908C17B43" + + # BOB + 60808: + # Config + name: "BOB" + EVC_DEPLOYMENT_BLOCK: 11481883 + EXPLORER_URL: "https://explorer.gobob.xyz/" + RPC_NAME: "BOB_RPC_URL" + + # Deployed contract addresses + contracts: + LIQUIDATOR_CONTRACT: "0xe7CEB97B4E622E71505c850829B1feb0326D723b" + + #EVK Addresses + EVC: "0x59f0FeEc4fA474Ad4ffC357cC8d8595B68abE47d" + SWAPPER: "0x697Ca30D765c1603890D88AAffBa3BeCCe72059d" + SWAP_VERIFIER: "0x296041DbdBC92171293F23c0a31e1574b791060d" + + # Other relevant contracts & Units of Account + WETH: "0x0555E30da8f98308EdB960aa94C0Db47230d2B9c" + USD: "0x0000000000000000000000000000000000000348" + BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + + # PYTH address + PYTH: "" + + # OP Mainnet + 10: + # Config + name: "Optimism" + EVC_DEPLOYMENT_BLOCK: 131522277 + EXPLORER_URL: "https://optimistic.etherscan.io/" + RPC_NAME: "OP_RPC_URL" + + # Deployed contract addresses + contracts: + LIQUIDATOR_CONTRACT: "0x29ee42D8849bc1aD341C49F0a8522AC57ae10b7B" + + #EVK Addresses + EVC: "0xbfB28650Cd13CE879E7D56569Ed4715c299823E4" + SWAPPER: "0x76B103bECa4459C9E0dd35a8E5ad48c8f93e768f" + SWAP_VERIFIER: "0x804C754ea602B54B28b0D3a10F8122e0a605dAD9" + + # Other relevant contracts & Units of Account + WETH: "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + USD: "0x0000000000000000000000000000000000000348" + BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + + # PYTH address + PYTH: "0xff1a0f4744e8582DF1aE09D5611b887B6a12925C" + + # BNB Smart Chain Mainnet + 56: + # Config + name: "BNB_Smart_Chain" + EVC_DEPLOYMENT_BLOCK: 46370642 + EXPLORER_URL: "https://bscscan.com/" + RPC_NAME: "BNBSC_RPC_URL" + + # Deployed contract addresses + contracts: + LIQUIDATOR_CONTRACT: "0x29ee42D8849bc1aD341C49F0a8522AC57ae10b7B" + + #EVK Addresses + EVC: "0xb2E5a73CeE08593d1a076a2AE7A6e02925a640ea" + SWAPPER: "0xAE4043937906975E95F885d8113D331133266Ee4" + SWAP_VERIFIER: "0xA8a4f96EC451f39Eb95913459901f39F5E1C068B" + + # Other relevant contracts & Units of Account + WETH: "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + USD: "0x0000000000000000000000000000000000000348" + BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + + # PYTH address + PYTH: "0x4D7E825f80bDf85e913E0DD2A2D54927e9dE1594" + + # Gnosis + 100: + # Config + name: "Gnosis" + EVC_DEPLOYMENT_BLOCK: 38384051 + EXPLORER_URL: "https://gnosisscan.io/" + RPC_NAME: "GNOSIS_RPC_URL" + + # Deployed contract addresses + contracts: + LIQUIDATOR_CONTRACT: "0x29ee42D8849bc1aD341C49F0a8522AC57ae10b7B" + + #EVK Addresses + EVC: "0xD1446CDaa29b342C04c6162023f3A645CB318736" + SWAPPER: "0x59072bf20763F311f560B26a887dca5a53E20922" + SWAP_VERIFIER: "0xf730e013eA862e91baD957Bc8F8837441259a4D2" + + # Other relevant contracts & Units of Account + WETH: "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + USD: "0x0000000000000000000000000000000000000348" + BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + + # PYTH address + PYTH: "0x2880aB155794e7179c9eE2e38200202908C17B43" + + # Unichain + 130: + # Config + name: "Unichain" + EVC_DEPLOYMENT_BLOCK: 8541493 + EXPLORER_URL: "https://unichain.blockscout.com/" + RPC_NAME: "UNICHAIN_RPC_URL" + + # Deployed contract addresses + contracts: + LIQUIDATOR_CONTRACT: "0x29ee42D8849bc1aD341C49F0a8522AC57ae10b7B" + + #EVK Addresses + EVC: "0x2A1176964F5D7caE5406B627Bf6166664FE83c60" + SWAPPER: "0x319E8ecd3BaB57fE684ca1aCfaB60c5603087B3A" + SWAP_VERIFIER: "0x7eaf8C22480129E5D7426e3A33880D7bE19B50a7" + + # Other relevant contracts & Units of Account + WETH: "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + USD: "0x0000000000000000000000000000000000000348" + BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + + # PYTH address + PYTH: "0x2880aB155794e7179c9eE2e38200202908C17B43" + + # Polygon + 137: + # Config + name: "Polygon" + EVC_DEPLOYMENT_BLOCK: 64475512 + EXPLORER_URL: "https://polygonscan.com/" + RPC_NAME: "POLYGON_RPC_URL" + + # Deployed contract addresses + contracts: + LIQUIDATOR_CONTRACT: "0x29ee42D8849bc1aD341C49F0a8522AC57ae10b7B" + + #EVK Addresses + EVC: "0xa1C13F5c4929521F0bf31cBE03025cb75C214DCB" + SWAPPER: "0xe6C04A97fdfAE75BBA82451Ae7296D8b48B8C0A4" + SWAP_VERIFIER: "0xD2c4D6831C6F7c2162015523b8105b972a3D2958" + + # Other relevant contracts & Units of Account + WETH: "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + USD: "0x0000000000000000000000000000000000000348" + BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + + # PYTH address + PYTH: "0xff1a0f4744e8582DF1aE09D5611b887B6a12925C" + # World Chain + 480: + # Config + name: "World_Chain" + EVC_DEPLOYMENT_BLOCK: 9860677 + EXPLORER_URL: "https://worldscan.org/" + RPC_NAME: "WORLD_CHAIN_RPC_URL" + + # Deployed contract addresses + contracts: + LIQUIDATOR_CONTRACT: "0x29ee42D8849bc1aD341C49F0a8522AC57ae10b7B" + + #EVK Addresses + EVC: "0x1384708aD4EEd6c264c196555422956306348359" + SWAPPER: "0x20C6fC317bE2a454E4b8c1A08B3E6A15Ee8cb38E" + SWAP_VERIFIER: "0x7aAA49a2eC3A7CCCeb88B6E4cB450175623F7230" + + # Other relevant contracts & Units of Account + WETH: "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2" + USD: "0x0000000000000000000000000000000000000348" + BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + + # PYTH address + PYTH: "" + + # Swell + 1923: + # Config + name: "Swell" + EVC_DEPLOYMENT_BLOCK: 485320 + EXPLORER_URL: "https://explorer.swellnetwork.io" + RPC_NAME: "SWELL_RPC_URL" + + # Deployed contract addresses + contracts: + LIQUIDATOR_CONTRACT: "0xA8A46596a7B17542d2cf6993FC61Ea0CBb4474c1" + + #EVK Addresses + EVC: "0x08739CBede6E28E387685ba20e6409bD16969Cde" + SWAPPER: "0x05Eb1A647265D974a1B0A57206048312604Ac6C3" + SWAP_VERIFIER: "0x392C1570b3Bf29B113944b759cAa9a9282DA12Fe" + + # Other relevant contracts & Units of Account + WETH: "0x4200000000000000000000000000000000000006" + USD: "0x0000000000000000000000000000000000000348" + BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + + # PYTH address + PYTH: "" + # Base 8453: # Config @@ -140,22 +365,22 @@ chains: # PYTH address PYTH: "0x8250f4aF4B972684F7b336503E2D6dFeDeB1487a" - # Swell - 1923: + # Arbitrum One + 42161: # Config - name: "Swell" - EVC_DEPLOYMENT_BLOCK: 485320 - EXPLORER_URL: "https://explorer.swellnetwork.io" - RPC_NAME: "SWELL_RPC_URL" + name: "Arbitrum_One" + EVC_DEPLOYMENT_BLOCK: 300690868 + EXPLORER_URL: "https://arbiscan.io/" + RPC_NAME: "ARBITRUM_ONE_RPC_URL" # Deployed contract addresses contracts: - LIQUIDATOR_CONTRACT: "0xA8A46596a7B17542d2cf6993FC61Ea0CBb4474c1" + LIQUIDATOR_CONTRACT: "0xdB940d3EcF986dDFcfC18D30D56Ad6C42428E537" #EVK Addresses - EVC: "0x08739CBede6E28E387685ba20e6409bD16969Cde" - SWAPPER: "0x05Eb1A647265D974a1B0A57206048312604Ac6C3" - SWAP_VERIFIER: "0x392C1570b3Bf29B113944b759cAa9a9282DA12Fe" + EVC: "0x6302ef0F34100CDDFb5489fbcB6eE1AA95CD1066" + SWAPPER: "0x6eE488A00A2ef1E2764cD7245F8a77C40060A7C7" + SWAP_VERIFIER: "0x7b16DAaFa76CfeC8C08D7a68aF31949B37ebfdF5" # Other relevant contracts & Units of Account WETH: "0x4200000000000000000000000000000000000006" @@ -163,5 +388,54 @@ chains: BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" # PYTH address - PYTH: "" + PYTH: "0xff1a0f4744e8582DF1aE09D5611b887B6a12925C" + + # Avalanche + 43114: + # Config + name: "Avalanche" + EVC_DEPLOYMENT_BLOCK: 56805692 + EXPLORER_URL: "https://subnets.avax.network/" + RPC_NAME: "AVALANCHE_RPC_URL" + + # Deployed contract addresses + contracts: + LIQUIDATOR_CONTRACT: "0xdB940d3EcF986dDFcfC18D30D56Ad6C42428E537" + + #EVK Addresses + EVC: "0xddcbe30A761Edd2e19bba930A977475265F36Fa1" + SWAPPER: "0x6E1C286e888Ab5911ca37aCeD81365d57eC29a06" + SWAP_VERIFIER: "0x0d7938D9c31Cd7dD693752074284af133c1142de" + + # Other relevant contracts & Units of Account + WETH: "0x4200000000000000000000000000000000000006" + USD: "0x0000000000000000000000000000000000000348" + BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + + # PYTH address + PYTH: "0x4305FB66699C3B2702D4d05CF36551390A4c69C6" + # Ink + 57073: + # Config + name: "Ink" + EVC_DEPLOYMENT_BLOCK: 402613 + EXPLORER_URL: "https://explorer.inkonchain.com/" + RPC_NAME: "INK_RPC_URL" + + # Deployed contract addresses + contracts: + LIQUIDATOR_CONTRACT: "0xdB940d3EcF986dDFcfC18D30D56Ad6C42428E537" + + #EVK Addresses + EVC: "0x59B8DfBcD3F004e7287eA021A67D54763fEAeE0C" + SWAPPER: "0x4A790B1Dd4b59f07Ac31841dCbB0707d0aa2e616" + SWAP_VERIFIER: "0x389d4Bfa6F869542b220E4C71c74D58333B9F000" + + # Other relevant contracts & Units of Account + WETH: "0x4200000000000000000000000000000000000006" + USD: "0x0000000000000000000000000000000000000348" + BTC: "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + + # PYTH address + PYTH: "0x2880aB155794e7179c9eE2e38200202908C17B43" \ No newline at end of file diff --git a/app/liquidation/bot_manager.py b/app/liquidation/bot_manager.py index 7e92487..0da224c 100644 --- a/app/liquidation/bot_manager.py +++ b/app/liquidation/bot_manager.py @@ -22,43 +22,64 @@ def __init__(self, chain_ids: List[int], notify: bool = True, execute_liquidatio def _initialize_chains(self): """Initialize components for each chain""" - print(self.chain_ids) + logger.info("Starting initialization for chain IDs: %s", self.chain_ids) for chain_id in self.chain_ids: - # Load chain-specific config - config = load_chain_config(chain_id) - self.configs[chain_id] = config - - # Create monitor instance - monitor = AccountMonitor( - chain_id=chain_id, - config=config, - notify=self.notify, - execute_liquidation=self.execute_liquidation - ) - monitor.load_state(config.SAVE_STATE_PATH) - self.monitors[chain_id] = monitor - - # Create listener instance - listener = EVCListener(monitor, config) - self.evc_listeners[chain_id] = listener + try: + logger.info("Initializing chain %s", chain_id) + # Load chain-specific config + config = load_chain_config(chain_id) + self.configs[chain_id] = config + logger.info("Successfully loaded config for chain %s (%s)", + chain_id, config.CHAIN_NAME) + + # Create monitor instance + monitor = AccountMonitor( + chain_id=chain_id, + config=config, + notify=self.notify, + execute_liquidation=self.execute_liquidation + ) + monitor.load_state(config.SAVE_STATE_PATH) + self.monitors[chain_id] = monitor + logger.info("Successfully initialized monitor for chain %s", chain_id) + + # Create listener instance + listener = EVCListener(monitor, config) + self.evc_listeners[chain_id] = listener + logger.info("Successfully initialized listener for chain %s", chain_id) + + except Exception as e: + logger.error("Failed to initialize chain %s: %s", chain_id, str(e), exc_info=True) + # Continue with other chains even if one fails + continue + + logger.info("Chain initialization complete. Successfully initialized chains: %s", + list(self.monitors.keys())) def start(self): """Start all chain monitors and evc_listeners""" - with ThreadPoolExecutor() as executor: + logger.info("Starting chain monitors and listeners for chains: %s", list(self.monitors.keys())) + + with ThreadPoolExecutor(max_workers=len(self.chain_ids) * 2) as executor: # First batch process historical logs - for chain_id in self.chain_ids: - self.evc_listeners[chain_id].batch_account_logs_on_startup() + for chain_id in self.monitors.keys(): + try: + logger.info("Starting batch processing for chain %s", chain_id) + self.evc_listeners[chain_id].batch_account_logs_on_startup() + except Exception as e: + logger.error("Failed to batch process logs for chain %s: %s", + chain_id, str(e), exc_info=True) # Start monitors monitor_futures = [ executor.submit(self._run_monitor, chain_id) - for chain_id in self.chain_ids + for chain_id in self.monitors.keys() ] # Start evc_listeners listener_futures = [ executor.submit(self._run_listener, chain_id) - for chain_id in self.chain_ids + for chain_id in self.monitors.keys() ] # Wait for all to complete (they shouldn't unless there's an error) @@ -70,13 +91,21 @@ def start(self): def _run_monitor(self, chain_id: int): """Run a single chain's monitor""" - monitor = self.monitors[chain_id] - monitor.start_queue_monitoring() + try: + logger.info("Starting monitor for chain %s", chain_id) + monitor = self.monitors[chain_id] + monitor.start_queue_monitoring() + except Exception as e: + logger.error("Monitor failed for chain %s: %s", chain_id, str(e), exc_info=True) def _run_listener(self, chain_id: int): """Run a single chain's listener""" - listener = self.evc_listeners[chain_id] - listener.start_event_monitoring() + try: + logger.info("Starting listener for chain %s", chain_id) + listener = self.evc_listeners[chain_id] + listener.start_event_monitoring() + except Exception as e: + logger.error("Listener failed for chain %s: %s", chain_id, str(e), exc_info=True) def stop(self): """Stop all chain instances""" diff --git a/app/liquidation/config_loader.py b/app/liquidation/config_loader.py index 20e47e1..8b62f8f 100644 --- a/app/liquidation/config_loader.py +++ b/app/liquidation/config_loader.py @@ -7,6 +7,7 @@ from typing import Dict, Any, Optional from dotenv import load_dotenv from web3 import Web3 +from .logging_setup import logger class Web3Singleton: @@ -61,12 +62,27 @@ def __init__(self, chain_id: int, global_config: Dict[str, Any], chain_config: D self.RISK_DASHBOARD_URL = os.getenv("RISK_DASHBOARD_URL") # Load chain-specific RPC from env using RPC_NAME from config - self.RPC_URL = os.getenv(self._chain["RPC_NAME"]) + rpc_name = self._chain["RPC_NAME"] + self.RPC_URL = os.getenv(rpc_name) + logger.info("[%s] Attempting to load RPC URL from env var %s: %s", + self.CHAIN_ID, rpc_name, + "Found" if self.RPC_URL else "Not found") + if not self.RPC_URL: - raise ValueError(f"Missing RPC URL for {self._chain["name"]}. " - f"Env var {self._chain["RPC_NAME"]} not found") + error_msg = f"Missing RPC URL for {self._chain["name"]}. Env var {rpc_name} not found" + logger.error("[%s] %s", self.CHAIN_ID, error_msg) + raise ValueError(error_msg) + + try: + self.w3 = setup_w3(self.RPC_URL) + # Test connection + block = self.w3.eth.block_number + logger.info("[%s] Successfully connected to RPC. Current block: %s", + self.CHAIN_ID, block) + except Exception as e: + logger.error("[%s] Failed to connect to RPC: %s", self.CHAIN_ID, str(e)) + raise - self.w3 = setup_w3(self.RPC_URL) self.mainnet_w3 = setup_w3(os.getenv("MAINNET_RPC_URL")) # Set chain-specific paths diff --git a/app/liquidation/liquidation_bot.py b/app/liquidation/liquidation_bot.py index f82e56b..2374962 100644 --- a/app/liquidation/liquidation_bot.py +++ b/app/liquidation/liquidation_bot.py @@ -17,9 +17,8 @@ from web3 import Web3 from web3.logs import DISCARD - -from app.liquidation.utils import (setup_logger, - create_contract_instance, +from .logging_setup import logger +from app.liquidation.utils import (create_contract_instance, make_api_request, global_exception_handler, post_liquidation_opportunity_on_slack, @@ -33,7 +32,6 @@ from app.liquidation.config_loader import ChainConfig ### ENVIRONMENT & CONFIG SETUP ### -logger = setup_logger() sys.excepthook = global_exception_handler @@ -58,6 +56,16 @@ def __init__(self, address, config: ChainConfig): self.unit_of_account = self.instance.functions.unitOfAccount().call() self.oracle_address = self.instance.functions.oracle().call() + self.creator = self.instance.functions.creator().call() + + self.total_borrowed = self.instance.functions.totalBorrows().call() + self.total_deposited = self.instance.functions.totalAssets().call() + + self.decimals = self.instance.functions.decimals().call() + self.total_borrowed_ui = self.total_borrowed / (10 ** self.decimals) + self.total_deposited_ui = self.total_deposited / (10 ** self.decimals) + self.utilization_ratio = round(self.total_borrowed / self.total_deposited * 100, 2) if self.total_deposited > 0 else 0 + self.pyth_feed_ids = [] self.redstone_feed_ids = [] @@ -75,39 +83,41 @@ def get_account_liquidity(self, account_address: str) -> Tuple[int, int]: balance = self.instance.functions.balanceOf( Web3.to_checksum_address(account_address)).call() except Exception as ex: # pylint: disable=broad-except - logger.error("Vault: Failed to get balance for account %s: %s", - account_address, ex, exc_info=True) + logger.error("[%s] Vault: Failed to get balance for account %s: %s", + self.config.CHAIN_ID, account_address, ex, exc_info=True) return (0, 0, 0) try: # Check if vault contains a Pyth oracle + # Slow_function (up to 1 min 14 seconds) self.pyth_feed_ids, self.redstone_feed_ids = PullOracleHandler.get_feed_ids(self, self.config) if len(self.pyth_feed_ids) > 0 and len(self.redstone_feed_ids) > 0: - logger.info("Vault: Pyth & Redstone oracle found for vault %s, " - "getting account liquidity through simulation", self.address) + logger.info("[%s] Vault: Pyth & Redstone oracle found for vault %s, " + "getting account liquidity through simulation", self.config.CHAIN_ID, self.address) collateral_value, liability_value = PullOracleHandler.get_account_values_with_pyth_and_redstone_simulation(self, account_address, self.pyth_feed_ids, self.redstone_feed_ids, self.config) elif len(self.pyth_feed_ids) > 0: - logger.info("Vault: Pyth Oracle found for vault %s, " - "getting account liquidity through simulation", self.address) + logger.info("[%s] Vault: Pyth Oracle found for vault %s, " + "getting account liquidity through simulation", self.config.CHAIN_ID, self.address) collateral_value, liability_value = PullOracleHandler.get_account_values_with_pyth_batch_simulation( self, account_address, self.pyth_feed_ids, self.config) elif len(self.redstone_feed_ids) > 0: - logger.info("Vault: Redstone Oracle found for vault %s, " - "getting account liquidity through simulation", self.address) + logger.info("[%s] Vault: Redstone Oracle found for vault %s, " + "getting account liquidity through simulation", self.config.CHAIN_ID, self.address) collateral_value, liability_value = PullOracleHandler.get_account_values_with_redstone_batch_simulation( self, account_address, self.redstone_feed_ids, self.config) else: - logger.info("Vault: Getting account liquidity normally for vault %s", self.address) + logger.info("[%s] Vault: Getting account liquidity normally for address %s in vault %s", + self.config.CHAIN_ID, account_address, self.address) (collateral_value, liability_value) = self.instance.functions.accountLiquidity( Web3.to_checksum_address(account_address), True ).call() except Exception as ex: # pylint: disable=broad-except if ex.args[0] != "0x43855d0f" and ex.args[0] != "0x6d588708": - logger.error("Vault: Failed to get account liquidity" + logger.error("[%s] Vault: Failed to get account liquidity" " for account %s: Contract error - %s", - account_address, ex) + self.config.CHAIN_ID, account_address, ex) # return (balance, 100, 100) return (balance, 0, 0) @@ -129,9 +139,9 @@ def check_liquidation(self, Returns: Tuple[int, int]: A tuple containing (max_repay, seized_collateral). """ - logger.info("Vault: Checking liquidation for account %s, collateral vault %s," + logger.info("[%s] Vault: Checking liquidation for account %s, collateral vault %s," " liquidator address %s, borrowed asset %s", - borower_address, collateral_address, + self.config.CHAIN_ID, borower_address, collateral_address, liquidator_address, self.underlying_asset_address) if len(self.pyth_feed_ids) > 0 and len(self.redstone_feed_ids) > 0: @@ -234,17 +244,19 @@ def get_health_score(self) -> float: self.value_borrowed = liability_value if self.controller.unit_of_account == self.config.WETH: - logger.info("Account: Getting a quote for %s WETH, unit of account %s", - liability_value, self.controller.unit_of_account) + logger.info("[%s] Account: Getting a quote for %s WETH, unit of account %s", + self.config.CHAIN_ID, liability_value, self.controller.unit_of_account) self.value_borrowed = get_eth_usd_quote(liability_value, self.config) - logger.info("Account: value borrowed: %s", self.value_borrowed) + logger.info("[%s] Account: value borrowed: %s", + self.config.CHAIN_ID, self.value_borrowed) elif self.controller.unit_of_account == self.config.BTC: - logger.info("Account: Getting a quote for %s BTC, unit of account %s", - liability_value, self.controller.unit_of_account) + logger.info("[%s] Account: Getting a quote for %s BTC, unit of account %s", + self.config.CHAIN_ID, liability_value, self.controller.unit_of_account) self.value_borrowed = get_btc_usd_quote(liability_value, self.config) - logger.info("Account: value borrowed: %s", self.value_borrowed) + logger.info("[%s] Account: value borrowed: %s", + self.config.CHAIN_ID, self.value_borrowed) # Special case for 0 values on balance or liability if liability_value == 0: @@ -254,8 +266,8 @@ def get_health_score(self) -> float: self.current_health_score = collateral_value / liability_value - logger.info("Account: %s health score: %s, Collateral Value: %s," - " Liability Value: %s", self.address, self.current_health_score, + logger.info("[%s] Account: %s health score: %s, Collateral Value: %s," + " Liability Value: %s", self.config.CHAIN_ID, self.address, self.current_health_score, collateral_value, liability_value) return self.current_health_score @@ -309,7 +321,7 @@ def get_time_of_next_update(self) -> float: if not(self.time_of_next_update < time_of_next_update and self.time_of_next_update > time.time()): self.time_of_next_update = time_of_next_update - logger.info("Account: %s next update scheduled for %s", self.address, + logger.info("[%s] Account: %s next update scheduled for %s", self.config.CHAIN_ID, self.address, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(self.time_of_next_update))) return self.time_of_next_update @@ -368,7 +380,7 @@ class AccountMonitor: updates, triggering liquidations, and managing the overall state of the monitored accounts. It also handles saving and loading the monitor's state. """ - def __init__(self, chain_id: int, config: ChainConfig, notify = False, execute_liquidation = False): + def __init__(self, chain_id: int, config: ChainConfig, notify = True, execute_liquidation = True): self.chain_id = chain_id self.w3 = config.w3, self.config = config @@ -393,18 +405,18 @@ def start_queue_monitoring(self) -> None: save_thread = threading.Thread(target=self.periodic_save) save_thread.start() - logger.info("AccountMonitor: Save thread started.") + logger.info("[%s] AccountMonitor: Save thread started.", self.chain_id) if self.notify: low_health_report_thread = threading.Thread(target= self.periodic_report_low_health_accounts) low_health_report_thread.start() - logger.info("AccountMonitor: Low health report thread started.") + logger.info("[%s] AccountMonitor: Low health report thread started.", self.chain_id) while self.running: with self.condition: while self.update_queue.empty(): - logger.info("AccountMonitor: Waiting for queue to be non-empty.") + logger.info("[%s] AccountMonitor: Waiting for queue to be non-empty.", self.chain_id) self.condition.wait() next_update_time, address = self.update_queue.get() @@ -412,8 +424,8 @@ def start_queue_monitoring(self) -> None: # check for special value that indicates # account should be skipped & removed from queue if next_update_time == -1: - logger.info("AccountMonitor: %s has no position," - " skipping and removing from queue", address) + logger.info("[%s] AccountMonitor: %s has no position," + " skipping and removing from queue", self.chain_id, address) continue current_time = time.time() @@ -437,7 +449,7 @@ def update_account_on_status_check_event(self, address: str, vault_address: str) # If the vault is not already tracked in the list, create it if vault_address not in self.vaults: self.vaults[vault_address] = Vault(vault_address, self.config) - logger.info("AccountMonitor: Vault %s added to vault list.", vault_address) + logger.info("[%s] AccountMonitor: Vault %s added to vault list.", self.chain_id, vault_address) vault = self.vaults[vault_address] @@ -447,11 +459,13 @@ def update_account_on_status_check_event(self, address: str, vault_address: str) account = Account(address, vault, self.config) self.accounts[address] = account - logger.info("AccountMonitor: Adding %s to account list with controller %s.", + logger.info("[%s] AccountMonitor: %s added to account list with controller %s.", + self.chain_id, address, vault.address) else: - logger.info("AccountMonitor: %s already in list with controller %s.", + logger.info("[%s] AccountMonitor: %s already in list with controller %s.", + self.chain_id, address, vault.address) @@ -468,11 +482,14 @@ def update_account_liquidity(self, address: str) -> None: account = self.accounts.get(address) if not account: - logger.error("AccountMonitor: %s not found in account list.", + logger.error("[%s] AccountMonitor: %s not found in account list.", + self.chain_id, address, exc_info=True) return - logger.info("AccountMonitor: Updating %s liquidity.", address) + logger.info("[%s] AccountMonitor: Updating %s liquidity.", + self.chain_id, + address) prev_scheduled_time = account.time_of_next_update health_score = account.update_liquidity() @@ -484,86 +501,89 @@ def update_account_liquidity(self, address: str) -> None: if (time.time() - self.recently_posted_low_value[account.address] < self.config.LOW_HEALTH_REPORT_INTERVAL and account.value_borrowed < self.config.SMALL_POSITION_THRESHOLD): - logger.info("Skipping posting notification " - "for account %s, recently posted", address) + logger.info("[%s] AccountMonitor: Skipping posting notification " + "for account %s, recently posted", self.chain_id, address) else: try: post_unhealthy_account_on_slack(address, account.controller.address, health_score, account.value_borrowed, self.config) - logger.info("Valut borrowed: %s", account.value_borrowed) + logger.info("[%s] AccountMonitor: Valut borrowed: %s", + self.chain_id, account.value_borrowed) if account.value_borrowed < self.config.SMALL_POSITION_THRESHOLD: self.recently_posted_low_value[account.address] = time.time() except Exception as ex: # pylint: disable=broad-except - logger.error("AccountMonitor: " + logger.error("[%s] AccountMonitor: " "Failed to post low health notification " "for account %s to slack: %s", - address, ex, exc_info=True) + self.chain_id, address, ex, exc_info=True) - logger.info("AccountMonitor: %s is unhealthy, " + logger.info("[%s] AccountMonitor: %s is unhealthy, " "checking liquidation profitability.", - address) + self.chain_id, address) (result, liquidation_data, params) = account.simulate_liquidation() if result: if self.notify: try: - logger.info("AccountMonitor: Posting liquidation notification " - "to slack for account %s.", address) + logger.info("[%s] AccountMonitor: Posting liquidation notification " + "to slack for account %s.", self.chain_id, address) post_liquidation_opportunity_on_slack(address, account.controller.address, liquidation_data, params, self.config) except Exception as ex: # pylint: disable=broad-except - logger.error("AccountMonitor: " + logger.error("[%s] AccountMonitor: " "Failed to post liquidation notification " " for account %s to slack: %s", - address, ex, exc_info=True) + self.chain_id, address, ex, exc_info=True) if self.execute_liquidation: try: tx_hash, tx_receipt = Liquidator.execute_liquidation( liquidation_data["tx"], self.config) if tx_hash and tx_receipt: - logger.info("AccountMonitor: %s liquidated " + logger.info("[%s] AccountMonitor: %s liquidated " "on collateral %s.", + self.chain_id, address, liquidation_data["collateral_address"]) if self.notify: try: - logger.info("AccountMonitor: Posting liquidation result" - " to slack for account %s.", address) + logger.info("[%s] AccountMonitor: Posting liquidation result" + " to slack for account %s.", + self.chain_id, address) post_liquidation_result_on_slack(address, account.controller.address, liquidation_data, tx_hash, self.config) except Exception as ex: # pylint: disable=broad-except - logger.error("AccountMonitor: " + logger.error("[%s] AccountMonitor: " "Failed to post liquidation result " " for account %s to slack: %s", - address, ex, exc_info=True) + self.chain_id, address, ex, exc_info=True) # Update account health score after liquidation # Need to know how healthy the account is after liquidation # and if we need to liquidate again account.update_liquidity() except Exception as ex: # pylint: disable=broad-except - logger.error("AccountMonitor: " + logger.error("[%s] AccountMonitor: " "Failed to execute liquidation for account %s: %s", - address, ex, exc_info=True) + self.chain_id, address, ex, exc_info=True) else: - logger.info("AccountMonitor: " + logger.info("[%s] AccountMonitor: " "Account %s is unhealthy but not profitable to liquidate.", - address) + self.chain_id, address) except Exception as ex: # pylint: disable=broad-except - logger.error("AccountMonitor: " + logger.error("[%s] AccountMonitor: " "Exception simulating liquidation for account %s: %s", - address, ex, exc_info=True) + self.chain_id, address, ex, exc_info=True) next_update_time = account.time_of_next_update # if next update hasn't changed, means we already have a check scheduled if next_update_time == prev_scheduled_time: - logger.info("AccountMonitor: %s next update already scheduled for %s", - address, time.strftime("%Y-%m-%d %H:%M:%S", + logger.info("[%s] AccountMonitor: %s next update already scheduled for %s", + self.chain_id, address, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(next_update_time))) return @@ -572,8 +592,8 @@ def update_account_liquidity(self, address: str) -> None: self.condition.notify() except Exception as ex: # pylint: disable=broad-except - logger.error("AccountMonitor: Exception updating account %s: %s", - address, ex, exc_info=True) + logger.error("[%s] AccountMonitor: Exception updating account %s: %s", + self.chain_id, address, ex, exc_info=True) def save_state(self, local_save: bool = True) -> None: """ @@ -600,11 +620,13 @@ def save_state(self, local_save: bool = True) -> None: self.last_saved_block = self.latest_block - logger.info("AccountMonitor: State saved at time %s up to block %s", + logger.info("[%s] AccountMonitor: State saved at time %s up to block %s", + self.chain_id, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), self.latest_block) except Exception as ex: # pylint: disable=broad-except - logger.error("AccountMonitor: Failed to save state: %s", ex, exc_info=True) + logger.error("[%s] AccountMonitor: Failed to save state: %s", + self.chain_id, ex, exc_info=True) def load_state(self, save_path: str, local_save: bool = True) -> None: """ @@ -620,16 +642,18 @@ def load_state(self, save_path: str, local_save: bool = True) -> None: state = json.load(f) self.vaults = {address: Vault(address, self.config) for address in state["vaults"]} - logger.info("Loaded %s vaults: %s", len(self.vaults), list(self.vaults.keys())) + + logger.info("[%s] Loaded %s vaults: %s", self.chain_id, len(self.vaults), list(self.vaults.keys())) self.accounts = {address: Account.from_dict(data, self.vaults, self.config) for address, data in state["accounts"].items()} - logger.info("Loaded %s accounts:", len(self.accounts)) + logger.info("[%s] Loaded %s accounts:", self.chain_id, len(self.accounts)) for address, account in self.accounts.items(): - logger.info(" Account %s: Controller: %s, " + logger.info("[%s] Account %s: Controller: %s, " "Health Score: %s, " "Next Update: %s", + self.chain_id, address, account.controller.address, account.current_health_score, @@ -640,8 +664,9 @@ def load_state(self, save_path: str, local_save: bool = True) -> None: self.last_saved_block = state["last_saved_block"] self.latest_block = self.last_saved_block - logger.info("AccountMonitor: State loaded from save" + logger.info("[%s] AccountMonitor: State loaded from save" " file %s from block %s to block %s", + self.chain_id, save_path, self.config.EVC_DEPLOYMENT_BLOCK, self.latest_block) @@ -649,36 +674,48 @@ def load_state(self, save_path: str, local_save: bool = True) -> None: # Load from remote location pass else: - logger.info("AccountMonitor: No saved state found.") + logger.info("[%s] AccountMonitor: No saved state found.", self.chain_id) except Exception as ex: # pylint: disable=broad-except - logger.error("AccountMonitor: Failed to load state: %s", ex, exc_info=True) + logger.error("[%s] AccountMonitor: Failed to load state: %s", self.chain_id, ex, exc_info=True) def rebuild_queue(self): """ Rebuild queue based on current account health """ - logger.info("Rebuilding queue based on current account health") + logger.info("[%s] Rebuilding queue based on current account health", self.chain_id) self.update_queue = queue.PriorityQueue() + idx = 0 + skipped_accounts = 0 + total_accounts = len(self.accounts) for address, account in self.accounts.items(): try: - health_score = account.update_liquidity() + idx += 1 + # To speed up the state loading, we skip the update if the health score is already good + if account.current_health_score < 1.05: + health_score = account.update_liquidity() + else: + skipped_accounts += 1 + logger.info("[%s] AccountMonitor: %s/%s (total skipped %s), %s has good health score %s, skipping initial update...", + self.chain_id, idx, total_accounts, skipped_accounts, address, account.current_health_score) + health_score = account.current_health_score if account.current_health_score == math.inf: - logger.info("AccountMonitor: %s has no borrow, skipping", address) + logger.info("[%s] AccountMonitor: %s/%s, %s has no borrow, skipping", + self.chain_id, idx, total_accounts, address) continue next_update_time = account.time_of_next_update self.update_queue.put((next_update_time, address)) - logger.info("AccountMonitor: %s added to queue" + logger.info("[%s] AccountMonitor: %s/%s %s added to queue" " with health score %s, next update at %s", - address, health_score, time.strftime("%Y-%m-%d %H:%M:%S", + self.chain_id, idx, total_accounts, address, health_score, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(next_update_time))) except Exception as ex: # pylint: disable=broad-except - logger.error("AccountMonitor: Failed to put account %s into rebuilt queue: %s", - address, ex, exc_info=True) + logger.error("[%s] AccountMonitor: Failed to put account %s/%s %s into rebuilt queue: %s", + self.chain_id, idx, total_accounts, address, ex, exc_info=True) - logger.info("AccountMonitor: Queue rebuilt with %s acccounts", self.update_queue.qsize()) + logger.info("[%s] AccountMonitor: Queue rebuilt with %s acccounts", self.chain_id, self.update_queue.qsize()) def get_accounts_by_health_score(self): """ @@ -694,7 +731,7 @@ def get_accounts_by_health_score(self): return [(account.address, account.owner, account.subaccount_number, account.current_health_score, account.value_borrowed, - account.controller.vault_name, account.controller.vault_symbol) + account.controller.vault_name, account.controller.vault_symbol, account.controller.address) for account in sorted_accounts] def periodic_report_low_health_accounts(self): @@ -754,7 +791,7 @@ def __init__(self): @staticmethod def get_account_values_with_pyth_and_redstone_simulation(vault, account_address, pyth_feed_ids, redstone_feed_ids, config: ChainConfig): - pyth_update_data = PullOracleHandler.get_pyth_update_data(pyth_feed_ids) + pyth_update_data = PullOracleHandler.get_pyth_update_data(pyth_feed_ids, config) pyth_update_fee = PullOracleHandler.get_pyth_update_fee(pyth_update_data, config) redstone_addresses, redstone_update_data = PullOracleHandler.get_redstone_update_payloads(redstone_feed_ids) @@ -770,7 +807,7 @@ def get_account_values_with_pyth_and_redstone_simulation(vault, account_address, @staticmethod def check_liquidation_with_pyth_and_redstone_simulation(vault, liquidator_address, borrower_address, collateral_address, pyth_feed_ids, redstone_feed_ids, config: ChainConfig): - pyth_update_data = PullOracleHandler.get_pyth_update_data(pyth_feed_ids) + pyth_update_data = PullOracleHandler.get_pyth_update_data(pyth_feed_ids, config) pyth_update_fee = PullOracleHandler.get_pyth_update_fee(pyth_update_data, config) redstone_addresses, redstone_update_data = PullOracleHandler.get_redstone_update_payloads(redstone_feed_ids) @@ -786,7 +823,7 @@ def check_liquidation_with_pyth_and_redstone_simulation(vault, liquidator_addres @staticmethod def get_account_values_with_pyth_batch_simulation(vault, account_address, feed_ids, config: ChainConfig): - update_data = PullOracleHandler.get_pyth_update_data(feed_ids) + update_data = PullOracleHandler.get_pyth_update_data(feed_ids, config) update_fee = PullOracleHandler.get_pyth_update_fee(update_data, config) liquidator = config.liquidator @@ -801,17 +838,24 @@ def get_account_values_with_pyth_batch_simulation(vault, account_address, feed_i @staticmethod def check_liquidation_with_pyth_batch_simulation(vault, liquidator_address, borrower_address, collateral_address, feed_ids, config: ChainConfig): - update_data = PullOracleHandler.get_pyth_update_data(feed_ids) + update_data = PullOracleHandler.get_pyth_update_data(feed_ids, config) update_fee = PullOracleHandler.get_pyth_update_fee(update_data, config) liquidator = config.liquidator + # Add logging to help troubleshoot the parameters + logger.info("[%s] PullOracleHandler: Pyth Simulation - Params: vault=%s, liquidator=%s, borrower=%s, collateral=%s", + config.CHAIN_ID, vault.address, liquidator_address, borrower_address, collateral_address) + logger.info("[%s] PullOracleHandler: Pyth Simulation - Update data length: %s, update fee: %s", + config.CHAIN_ID, len(update_data), update_fee) + + # Try the call with a higher gas limit to ensure it's not a gas estimation issue result = liquidator.functions.simulatePythUpdateAndCheckLiquidation( [update_data], update_fee, vault.address, liquidator_address, borrower_address, collateral_address ).call({ "value": update_fee - }) + }) return result[0], result[1] @staticmethod @@ -841,44 +885,59 @@ def check_liquidation_with_redstone_batch_simulation(vault, ).call() return result[0], result[1] + # Slow_function + # according to the logs it takes about 1 min 14 seconds + @staticmethod def get_feed_ids(vault, config: ChainConfig): try: + logger.info("[%s] get_feed_ids start for vault: %s", config.CHAIN_ID, vault.address) oracle_address = vault.oracle_address oracle = create_contract_instance(oracle_address, config.ORACLE_ABI_PATH, config) - + logger.info("[%s] get_feed_ids oracle: %s", config.CHAIN_ID, oracle_address) unit_of_account = vault.unit_of_account collateral_vault_list = vault.get_ltv_list() - asset_list = [Vault(collateral_vault, config).underlying_asset_address - for collateral_vault in collateral_vault_list] + logger.info("[%s] get_feed_ids collateral_vault_list length: %s", config.CHAIN_ID, len(collateral_vault_list)) + # Replace list comprehension with a loop to add delays + # This for takes about 1.6s/record, and some vaults can have 30+ collateral vaults, + # so this step can go to 1 min easily. + asset_list = [] + for collateral_vault in collateral_vault_list: + vault_instance = Vault(collateral_vault, config) + asset_list.append(vault_instance.underlying_asset_address) asset_list.append(vault.underlying_asset_address) + logger.info("[%s] get_feed_ids asset_list length: %s", config.CHAIN_ID, len(asset_list)) pyth_feed_ids = set() redstone_feed_ids = set() # logger.info("PullOracleHandler: Trying to get feed ids for oracle %s with assets %s and unit of account %s", oracle_address, collateral_vault_list, unit_of_account) - + # One iteration takes from 1 to 3 seconds, depending on the configured oracle (CrossAdapter, Chainlink or Redstone) + idx = 0 for asset in asset_list: + idx += 1 + # logger.info("get_feed_ids[%s]: asset %s", idx, asset) (_, _, _, configured_oracle_address) = oracle.functions.resolveOracle(0, asset, unit_of_account).call() configured_oracle = create_contract_instance(configured_oracle_address, config.ORACLE_ABI_PATH, config) - + # logger.info("get_feed_ids[%s]: configured_oracle %s", idx, configured_oracle_address) try: configured_oracle_name = configured_oracle.functions.name().call() + # logger.info("get_feed_ids[%s]: configured_oracle_name %s", idx, configured_oracle_name) except Exception as ex: # pylint: disable=broad-except - logger.info("PullOracleHandler: Error calling contract for oracle" - " at %s, asset %s: %s", configured_oracle_address, asset, ex) + logger.info("[%s] PullOracleHandler: Error calling contract for oracle" + " at %s, asset %s: %s", config.CHAIN_ID, configured_oracle_address, asset, ex) continue + if configured_oracle_name == "PythOracle": - logger.info("PullOracleHandler: Pyth oracle found for vault %s: " - "Address - %s", vault.address, configured_oracle_address) + logger.info("[%s] PullOracleHandler: Pyth oracle found for vault %s: " + "Address - %s", config.CHAIN_ID, vault.address, configured_oracle_address) pyth_feed_ids.add(configured_oracle.functions.feedId().call().hex()) elif configured_oracle_name == "RedstoneCoreOracle": - logger.info("PullOracleHandler: Redstone oracle found for" - " vault %s: Address - %s", - vault.address, configured_oracle_address) + logger.info("[%s] PullOracleHandler: Redstone oracle found for" + " vault %s: Address - %s", config.CHAIN_ID, vault.address, configured_oracle_address) redstone_feed_ids.add((configured_oracle_address, configured_oracle.functions.feedId().call().hex())) elif configured_oracle_name == "CrossAdapter": @@ -890,7 +949,7 @@ def get_feed_ids(vault, config: ChainConfig): return list(pyth_feed_ids), list(redstone_feed_ids) except Exception as ex: # pylint: disable=broad-except - logger.error("PullOracleHandler: Error calling contract: %s", ex, exc_info=True) + logger.error("[%s] PullOracleHandler: Error calling contract: %s", config.CHAIN_ID, ex, exc_info=True) @staticmethod def resolve_cross_oracle(cross_oracle, config): @@ -906,7 +965,7 @@ def resolve_cross_oracle(cross_oracle, config): elif oracle_base_name == "RedstoneCoreOracle": redstone_feed_ids.add((oracle_base_address, oracle_base.functions.feedId().call().hex())) - elif oracle_base_name == "CrossOracle": + elif oracle_base_name == "CrossAdapter": pyth_ids, redstone_ids = PullOracleHandler.resolve_cross_oracle(oracle_base, config) pyth_feed_ids.add(pyth_ids) redstone_feed_ids.add(redstone_ids) @@ -920,15 +979,15 @@ def resolve_cross_oracle(cross_oracle, config): elif oracle_quote_name == "RedstoneCoreOracle": redstone_feed_ids.add((oracle_quote_address, oracle_quote.functions.feedId().call().hex())) - elif oracle_quote_name == "CrossOracle": + elif oracle_quote_name == "CrossAdapter": pyth_ids, redstone_ids = PullOracleHandler.resolve_cross_oracle(oracle_quote, config) pyth_feed_ids.update(pyth_ids) redstone_feed_ids.update(redstone_ids) return pyth_feed_ids, redstone_feed_ids @staticmethod - def get_pyth_update_data(feed_ids): - logger.info("PullOracleHandler: Getting update data for feeds: %s", feed_ids) + def get_pyth_update_data(feed_ids, config: ChainConfig): + logger.info("[%s] PullOracleHandler: Getting update data for feeds: %s", config.CHAIN_ID, feed_ids) pyth_url = "https://hermes.pyth.network/v2/updates/price/latest?" for feed_id in feed_ids: pyth_url += "ids[]=" + feed_id + "&" @@ -939,7 +998,7 @@ def get_pyth_update_data(feed_ids): @staticmethod def get_pyth_update_fee(update_data, config): - logger.info("PullOracleHandler: Getting update fee for data: %s", update_data) + logger.info("[%s] PullOracleHandler: Getting update fee for data: %s", config.CHAIN_ID, update_data) pyth = create_contract_instance(config.PYTH, config.PYTH_ABI_PATH, config) return pyth.functions.getUpdateFee([update_data]).call() @@ -1008,8 +1067,8 @@ def start_event_monitoring(self) -> None: self.account_monitor.latest_block, current_block) except Exception as ex: # pylint: disable=broad-except - logger.error("EVCListener: Unexpected exception in event monitoring: %s", - ex, exc_info=True) + logger.error("[%s] EVCListener: Unexpected exception in event monitoring: %s", + self.config.CHAIN_ID, ex, exc_info=True) time.sleep(self.config.SCAN_INTERVAL) @@ -1018,7 +1077,8 @@ def scan_block_range_for_account_status_check(self, start_block: int, end_block: int, max_retries: int = 3, - seen_accounts: set = set()) -> None: + seen_accounts: set = set(), + startup_mode: bool = False) -> None: """ Scan a range of blocks for AccountStatusCheck events. @@ -1031,8 +1091,8 @@ def scan_block_range_for_account_status_check(self, """ for attempt in range(max_retries): try: - logger.info("EVCListener: Scanning blocks %s to %s for AccountStatusCheck events.", - start_block, end_block) + logger.info("[%s] EVCListener: Scanning blocks %s to %s for AccountStatusCheck events.", + self.config.CHAIN_ID, start_block, end_block) logs = self.evc_instance.events.AccountStatusCheck().get_logs( fromBlock=start_block, @@ -1045,43 +1105,49 @@ def scan_block_range_for_account_status_check(self, #if we've seen the account already and the status # check is not due to changing controller if account_address in seen_accounts: - same_controller = self.account_monitor.accounts.get( - account_address).controller.address == Web3.to_checksum_address( - vault_address) + existing_account = self.account_monitor.accounts.get(account_address) + if existing_account is None: + logger.info("[%s] EVCListener: Account %s not found in monitor, adding it", + self.config.CHAIN_ID, account_address) + seen_accounts.remove(account_address) # Remove from seen to allow re-processing + continue + + same_controller = existing_account.controller.address == Web3.to_checksum_address( + vault_address) - if same_controller: - logger.info("EVCListener: Account %s already seen with " - "controller %s, skipping", account_address, vault_address) + if same_controller and startup_mode: + logger.info("[%s] EVCListener: Account %s already seen with " + "controller %s, skipping", self.config.CHAIN_ID, account_address, vault_address) continue else: seen_accounts.add(account_address) - logger.info("EVCListener: AccountStatusCheck event found for account %s " + logger.info("[%s] EVCListener: AccountStatusCheck event found for account %s " "with controller %s, triggering monitor update.", - account_address, vault_address) + self.config.CHAIN_ID, account_address, vault_address) try: self.account_monitor.update_account_on_status_check_event( account_address, vault_address) except Exception as ex: # pylint: disable=broad-except - logger.error("EVCListener: Exception updating account %s " + logger.error("[%s] EVCListener: Exception updating account %s " "on AccountStatusCheck event: %s", - account_address, ex, exc_info=True) + self.config.CHAIN_ID, account_address, ex, exc_info=True) - logger.info("EVCListener: Finished scanning blocks %s to %s " - "for AccountStatusCheck events.", start_block, end_block) + logger.info("[%s] EVCListener: Finished scanning blocks %s to %s " + "for AccountStatusCheck events.", self.config.CHAIN_ID, start_block, end_block) self.account_monitor.latest_block = end_block return except Exception as ex: # pylint: disable=broad-except - logger.error("EVCListener: Exception scanning block range %s to %s " + logger.error("[%s] EVCListener: Exception scanning block range %s to %s " "(attempt %s/%s): %s", - start_block, end_block, attempt + 1, max_retries, ex, exc_info=True) + self.config.CHAIN_ID, start_block, end_block, attempt + 1, max_retries, ex, exc_info=True) if attempt == max_retries - 1: - logger.error("EVCListener: " + logger.error("[%s] EVCListener: " "Failed to scan block range %s to %s after %s attempts", - start_block, end_block, max_retries, exc_info=True) + self.config.CHAIN_ID, start_block, end_block, max_retries, exc_info=True) else: time.sleep(self.config.RETRY_DELAY) # cooldown between retries @@ -1096,14 +1162,17 @@ def batch_account_logs_on_startup(self) -> None: # assume it has been loaded from that and start from the last saved block start_block = max(int(self.config.EVC_DEPLOYMENT_BLOCK), self.account_monitor.last_saved_block) - + logger.info("[%s] EVCListener: Starting batch scan of AccountStatusCheck events from block %s", + self.config.CHAIN_ID, start_block) current_block = self.w3.eth.block_number - + logger.info("[%s] EVCListener: Current block is %s", + self.config.CHAIN_ID, current_block) batch_block_size = self.config.BATCH_SIZE - - logger.info("EVCListener: " + logger.info("[%s] EVCListener: Batch block size is %s", + self.config.CHAIN_ID, batch_block_size) + logger.info("[%s] EVCListener: " "Starting batch scan of AccountStatusCheck events from block %s to %s.", - start_block, current_block) + self.config.CHAIN_ID, start_block, current_block) seen_accounts = set() @@ -1111,21 +1180,22 @@ def batch_account_logs_on_startup(self) -> None: end_block = min(start_block + batch_block_size, current_block) self.scan_block_range_for_account_status_check(start_block, end_block, - seen_accounts=seen_accounts) + seen_accounts=seen_accounts, + startup_mode=True) self.account_monitor.save_state() start_block = end_block + 1 time.sleep(self.config.BATCH_INTERVAL) # Sleep in between batches to avoid rate limiting - logger.info("EVCListener: " + logger.info("[%s] EVCListener: " "Finished batch scan of AccountStatusCheck events from block %s to %s.", - start_block, current_block) + self.config.CHAIN_ID, start_block, current_block) except Exception as ex: # pylint: disable=broad-except - logger.error("EVCListener: " + logger.error("[%s] EVCListener: " "Unexpected exception in batch scanning account logs on startup: %s", - ex, exc_info=True) + self.config.CHAIN_ID, ex, exc_info=True) @staticmethod def get_account_owner_and_subaccount_number(account, config): @@ -1204,9 +1274,9 @@ def simulate_liquidation(vault: Vault, for collateral, collateral_vault in collateral_vaults.items(): try: - logger.info("Liquidator: Checking liquidation for " + logger.info("[%s] Liquidator: Checking liquidation for " "account %s, borrowed asset %s, collateral asset %s", - violator_address, borrowed_asset, collateral) + config.CHAIN_ID, violator_address, borrowed_asset, collateral) liquidation_results = Liquidator.calculate_liquidation_profit(vault, violator_address, @@ -1223,7 +1293,7 @@ def simulate_liquidation(vault: Vault, message = ("Exception simulating liquidation " f"for account {violator_address} with collateral {collateral}: {ex}") - logger.error("Liquidator: %s", message, exc_info=True) + logger.error("[%s] Liquidator: %s", config.CHAIN_ID, message, exc_info=True) time_of_last_post = liquidation_error_slack_cooldown.get(violator_address, 0) value_borrowed = violator_account.value_borrowed @@ -1241,11 +1311,11 @@ def simulate_liquidation(vault: Vault, if max_profit_data["tx"]: - logger.info("Liquidator: Profitable liquidation found for account %s. " + logger.info("[%s] Liquidator: Profitable liquidation found for account %s. " "Collateral: %s, Underlying Collateral Asset: %s, " "Remaining borrow asset after swap and repay: %s, " "Estimated profit in ETH: %s", - violator_address, max_profit_data["collateral_address"], + config.CHAIN_ID, violator_address, max_profit_data["collateral_address"], max_profit_data["collateral_asset"], max_profit_data["leftover_borrow"], max_profit_data["leftover_borrow_in_eth"]) return (True, max_profit_data, max_profit_params) @@ -1281,8 +1351,8 @@ def calculate_liquidation_profit(vault: Vault, seized_collateral_assets = collateral_vault.convert_to_assets(seized_collateral_shares) if max_repay == 0 or seized_collateral_shares == 0: - logger.info("Liquidator: Max Repay %s, Seized Collateral %s, liquidation not possible", - max_repay, seized_collateral_shares) + logger.info("[%s] Liquidator: Max Repay %s, Seized Collateral %s, liquidation not possible", + config.CHAIN_ID, max_repay, seized_collateral_shares) return ({"profit": 0}, None) swap_api_response = Quoter.get_swap_api_quote( @@ -1341,13 +1411,14 @@ def calculate_liquidation_profit(vault: Vault, continue swap_data.append(item["data"]) - logger.info("Liquidator: Seized collateral assets: %s, output amount: %s, " - "leftover_borrow: %s", seized_collateral_assets, amount_out, + logger.info("[%s] Liquidator: Seized collateral assets: %s, output amount: %s, " + "leftover_borrow: %s", config.CHAIN_ID, seized_collateral_assets, amount_out, leftover_borrow_in_eth) # leftover_borrow_in_eth = 1 if leftover_borrow_in_eth < 0: - logger.warning("Liquidator: Negative leftover borrow value, aborting liquidation") + logger.warning("[%s] Liquidator: Negative leftover borrow value, aborting liquidation", + config.CHAIN_ID) return ({"profit": 0}, None) @@ -1365,7 +1436,7 @@ def calculate_liquidation_profit(vault: Vault, ) - logger.info("Liquidator: Liquidation details: %s", params) + logger.info("[%s] Liquidator: Liquidation details: %s", config.CHAIN_ID, params) pyth_feed_ids = vault.pyth_feed_ids redstone_feed_ids = vault.redstone_feed_ids @@ -1409,8 +1480,8 @@ def calculate_liquidation_profit(vault: Vault, suggested_gas_price = int(config.w3.eth.gas_price * 1.2) if len(pyth_feed_ids)> 0: - logger.info("Liquidator: executing with pyth") - update_data = PullOracleHandler.get_pyth_update_data(pyth_feed_ids) + logger.info("[%s] Liquidator: executing with pyth", config.CHAIN_ID) + update_data = PullOracleHandler.get_pyth_update_data(pyth_feed_ids, config) update_fee = PullOracleHandler.get_pyth_update_fee(update_data, config) liquidation_tx = liquidator_contract.functions.liquidateSingleCollateralWithPythOracle( params, swap_data, [update_data] @@ -1422,7 +1493,7 @@ def calculate_liquidation_profit(vault: Vault, "gasPrice": suggested_gas_price }) elif len(redstone_feed_ids) > 0: - logger.info("Liquidator: executing with Redstone") + logger.info("[%s] Liquidator: executing with Redstone", config.CHAIN_ID) addresses, update_data = PullOracleHandler.get_redstone_update_payloads(redstone_feed_ids) liquidation_tx = liquidator_contract.functions.liquidateSingleCollateralWithRedstoneOracle( params, swap_data, update_data, addresses @@ -1433,7 +1504,7 @@ def calculate_liquidation_profit(vault: Vault, "nonce": config.w3.eth.get_transaction_count(config.LIQUIDATOR_EOA) }) else: - logger.info("Liquidator: executing normally") + logger.info("[%s] Liquidator: executing normally", config.CHAIN_ID) # liquidation_tx = liquidator_contract.functions.liquidateSingleCollateral( # params # ).build_transaction({ @@ -1453,13 +1524,13 @@ def calculate_liquidation_profit(vault: Vault, "from": config.LIQUIDATOR_EOA, "nonce": config.w3.eth.get_transaction_count(config.LIQUIDATOR_EOA) }) - logger.info("Leftover borrow in eth: %s", leftover_borrow_in_eth) - logger.info("Estimated gas: %s", config.w3.eth.estimate_gas(liquidation_tx)) - logger.info("Suggested gas price: %s", suggested_gas_price) + logger.info("[%s] Leftover borrow in eth: %s", config.CHAIN_ID, leftover_borrow_in_eth) + logger.info("[%s] Estimated gas: %s", config.CHAIN_ID, config.w3.eth.estimate_gas(liquidation_tx)) + logger.info("[%s] Suggested gas price: %s", config.CHAIN_ID, suggested_gas_price) net_profit = leftover_borrow_in_eth - ( config.w3.eth.estimate_gas(liquidation_tx) * suggested_gas_price) - logger.info("Net profit: %s", net_profit) + logger.info("[%s] Net profit: %s", config.CHAIN_ID, net_profit) return ({ "tx": liquidation_tx, @@ -1479,8 +1550,8 @@ def execute_liquidation(liquidation_transaction: Dict[str, Any], config: ChainCo liquidation_transaction (Dict[str, Any]): The liquidation transaction details. """ try: - logger.info("Liquidator: Executing liquidation transaction %s...", - liquidation_transaction) + logger.info("[%s] Liquidator: Executing liquidation transaction %s...", + config.CHAIN_ID, liquidation_transaction) # flashbots_provider = "https://rpc.flashbots.net" # flashbots_relay = "https://relay.flashbots.net" # flashbots_w3 = Web3(Web3.HTTPProvider(flashbots_provider)) @@ -1500,9 +1571,9 @@ def execute_liquidation(liquidation_transaction: Dict[str, Any], config: ChainCo result = liquidator_contract.events.Liquidation().process_receipt( tx_receipt, errors=DISCARD) - logger.info("Liquidator: Liquidation details: ") + logger.info("[%s] Liquidator: Liquidation details: ", config.CHAIN_ID) for event in result: - logger.info("Liquidator: %s", event["args"]) + logger.info("[%s] Liquidator: %s", config.CHAIN_ID, event["args"]) logger.info("Liquidator: Liquidation transaction executed successfully.") return tx_hash.hex(), tx_receipt @@ -1556,17 +1627,18 @@ def get_swap_api_quote( "currentDebt": str(current_debt), "targetDebt": str(target_debt) } - + logger.info("[%s] get_swap_api_quote: %s", config.CHAIN_ID, params) response = make_api_request(config.SWAP_API_URL, headers={}, params=params) - + logger.info("[%s] get_swap_api_quote: response %s", config.CHAIN_ID, response) + if not response or not response["success"]: - logger.error("Unable to get quote from swap api") + logger.error("[%s] Unable to get quote from swap api", config.CHAIN_ID) return None amount_out = int(response["data"]["amountOut"]) if amount_out < min_amount_out: - logger.error("Quote too low") + logger.error("[%s] Quote too low", config.CHAIN_ID) return None return response["data"] diff --git a/app/liquidation/logging_setup.py b/app/liquidation/logging_setup.py new file mode 100644 index 0000000..d7003eb --- /dev/null +++ b/app/liquidation/logging_setup.py @@ -0,0 +1,50 @@ +""" +Logging setup module +""" +import logging +import os +import sys +import traceback + +def setup_logger(): + """ + Set up and return a configured logger instance. + """ + logger = logging.getLogger('liquidation_bot') + logger.setLevel(logging.INFO) + + # Create formatters + detailed_formatter = logging.Formatter( + "%(asctime)s - %(levelname)s - %(message)s\n%(exc_info)s") + standard_formatter = logging.Formatter( + "%(asctime)s - %(levelname)s - %(message)s") + + class DetailedExceptionFormatter(logging.Formatter): + def format(self, record): + if record.levelno >= logging.ERROR: + record.exc_text = "".join( + traceback.format_exception(*record.exc_info)) if record.exc_info else "" + return detailed_formatter.format(record) + return standard_formatter.format(record) + + # Create handlers + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(DetailedExceptionFormatter()) + + # Ensure logs directory exists + os.makedirs("logs", exist_ok=True) + + # Add file handler + file_handler = logging.FileHandler("logs/account_monitor_logs.log", mode="a") + file_handler.setLevel(logging.INFO) + file_handler.setFormatter(DetailedExceptionFormatter()) + + # Add handlers to the logger + logger.addHandler(console_handler) + logger.addHandler(file_handler) + + return logger + +# Create a global logger instance +logger = setup_logger() \ No newline at end of file diff --git a/app/liquidation/routes.py b/app/liquidation/routes.py index 5035364..4496b5b 100644 --- a/app/liquidation/routes.py +++ b/app/liquidation/routes.py @@ -13,7 +13,7 @@ def start_monitor(chain_ids=None): """Start monitoring for specified chains, defaults to mainnet if none specified""" global chain_manager if chain_ids is None: - chain_ids = [1] # Default to Ethereum mainnet + chain_ids = [1] # Default Ethereum mainnet chain_manager = ChainManager(chain_ids, notify=True) @@ -33,7 +33,7 @@ def get_all_positions(): sorted_accounts = monitor.get_accounts_by_health_score() response = [] - for (address, owner, sub_account, health_score, value_borrowed, vault_name, vault_symbol) in sorted_accounts: + for (address, owner, sub_account, health_score, value_borrowed, vault_name, vault_symbol, vault_address) in sorted_accounts: if math.isinf(health_score): continue response.append({ @@ -43,7 +43,144 @@ def get_all_positions(): "health_score": health_score, "value_borrowed": value_borrowed, "vault_name": vault_name, + "vault_address": vault_address, "vault_symbol": vault_symbol }) return make_response(jsonify(response)) + +@liquidation.route("/account/
", methods=["GET"]) +def get_account_details(address): + """Get detailed information about a specific account""" + chain_id = int(request.args.get("chainId", 1)) # Default to mainnet if not specified + + if not chain_manager or chain_id not in chain_manager.monitors: + return jsonify({"error": f"Monitor not initialized for chain {chain_id}"}), 500 + + logger.info("API: Getting details for account %s on chain %s", address, chain_id) + monitor = chain_manager.monitors[chain_id] + + # Find the account in the monitor's accounts + account = monitor.accounts.get(address) + if not account: + return jsonify({"error": f"Account {address} not found on chain {chain_id}"}), 404 + + response = { + "address": address, + "owner": account.owner, + "sub_account": account.subaccount_number, + "health_score": account.current_health_score, + "value_borrowed": account.value_borrowed, + "vault_name": account.controller.vault_name, + "vault_symbol": account.controller.vault_symbol, + "vault_address": account.controller.address, + "vault_creator": account.controller.creator, + "vault_total_borrowed": account.controller.total_borrowed, + "vault_total_deposited": account.controller.total_deposited, + "vault_decimals": account.controller.decimals, + "vault_total_borrowed_ui": account.controller.total_borrowed_ui, + "vault_total_deposited_ui": account.controller.total_deposited_ui, + "vault_utilization_ratio": account.controller.utilization_ratio, + "underlying_asset": account.controller.underlying_asset_address, + "balance": account.balance, + "next_update_time": account.time_of_next_update + } + + return make_response(jsonify(response)) + +@liquidation.route("/vault/", methods=["GET"]) +def get_vault_details(address): + """Get detailed information about a specific vault""" + chain_id = int(request.args.get("chainId", 1)) # Default to mainnet if not specified + + if not chain_manager or chain_id not in chain_manager.monitors: + return jsonify({"error": f"Monitor not initialized for chain {chain_id}"}), 500 + + logger.info("API: Getting vault details for address %s on chain %s", address, chain_id) + monitor = chain_manager.monitors[chain_id] + + # Find the vault in the monitor's vaults + vault = monitor.vaults.get(address) + if not vault: + return jsonify({"error": f"Vault {address} not found on chain {chain_id}"}), 404 + + response = { + "address": vault.address, + "name": vault.vault_name, + "symbol": vault.vault_symbol, + "creator": vault.creator, + "total_borrowed": vault.total_borrowed, + "total_deposited": vault.total_deposited, + "decimals": vault.decimals, + "total_borrowed_ui": vault.total_borrowed_ui, + "total_deposited_ui": vault.total_deposited_ui, + "utilization_ratio": vault.utilization_ratio, + "underlying_asset": vault.underlying_asset_address + } + + return make_response(jsonify(response)) + +@liquidation.route("/vaults/creator/", methods=["GET"]) +def get_vaults_by_creator(address): + """Get all vaults created by a specific address""" + chain_id = int(request.args.get("chainId", 1)) # Default to mainnet if not specified + + if not chain_manager or chain_id not in chain_manager.monitors: + return jsonify({"error": f"Monitor not initialized for chain {chain_id}"}), 500 + + logger.info("API: Getting all vaults for creator %s on chain %s", address, chain_id) + monitor = chain_manager.monitors[chain_id] + + # Find all vaults by this creator + creator_vaults = [] + for vault in monitor.vaults.values(): + if vault.creator.lower() == address.lower(): + creator_vaults.append({ + "address": vault.address, + "name": vault.vault_name, + "symbol": vault.vault_symbol, + "total_borrowed": vault.total_borrowed, + "total_deposited": vault.total_deposited, + "decimals": vault.decimals, + "total_borrowed_ui": vault.total_borrowed_ui, + "total_deposited_ui": vault.total_deposited_ui, + "utilization_ratio": vault.utilization_ratio, + "underlying_asset": vault.underlying_asset_address + }) + + if not creator_vaults: + return jsonify({"message": f"No vaults found for creator {address} on chain {chain_id}", "vaults": []}), 200 + + return make_response(jsonify({"vaults": creator_vaults})) + +@liquidation.route("/vault//positions", methods=["GET"]) +def get_vault_positions(address): + """Get all positions (accounts) opened with a specific vault""" + chain_id = int(request.args.get("chainId", 1)) # Default to mainnet if not specified + + if not chain_manager or chain_id not in chain_manager.monitors: + return jsonify({"error": f"Monitor not initialized for chain {chain_id}"}), 500 + + logger.info("API: Getting all positions for vault %s on chain %s", address, chain_id) + monitor = chain_manager.monitors[chain_id] + + # Find all accounts that use this vault as their controller + vault_accounts = [] + for account in monitor.accounts.values(): + if account.controller.address.lower() == address.lower(): + if math.isinf(account.current_health_score): + continue + vault_accounts.append({ + "address": account.address, + "owner": account.owner, + "sub_account": account.subaccount_number, + "health_score": account.current_health_score, + "value_borrowed": account.value_borrowed, + "balance": account.balance, + "next_update_time": account.time_of_next_update + }) + + # Sort by health score ascending (most risky first) + vault_accounts.sort(key=lambda x: x["health_score"]) + + return make_response(jsonify(vault_accounts)) diff --git a/app/liquidation/utils.py b/app/liquidation/utils.py index f510354..58e26c9 100644 --- a/app/liquidation/utils.py +++ b/app/liquidation/utils.py @@ -1,12 +1,15 @@ """ Utility functions for the liquidation bot. """ -import logging +import os +import sys import json import functools import time import traceback +import threading import requests +import logging from typing import Any, Callable, Dict, Optional @@ -14,66 +17,31 @@ from web3.contract import Contract from urllib.parse import urlencode +from .logging_setup import logger from .config_loader import ChainConfig LOGS_PATH = "logs/account_monitor_logs.log" -def setup_logger() -> logging.Logger: - """ - Set up and configure a logger for the liquidation bot. - - Args: - logs_path (str): Path to the log output file. - - Returns: - logging.Logger: Configured logger instance. - """ - logger = logging.getLogger("liquidation_bot") - logger.setLevel(logging.DEBUG) - - console_handler = logging.StreamHandler() - file_handler = logging.FileHandler(LOGS_PATH, mode="a") - - detailed_formatter = logging.Formatter( - "%(asctime)s - %(levelname)s - %(message)s\n%(exc_info)s") - - # Create a standard formatter for other log levels - standard_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") - - class DetailedExceptionFormatter(logging.Formatter): - def format(self, record): - if record.levelno >= logging.ERROR: - record.exc_text = "".join( - traceback.format_exception(*record.exc_info)) if record.exc_info else "" - return detailed_formatter.format(record) - else: - return standard_formatter.format(record) - - console_handler.setFormatter(DetailedExceptionFormatter()) - file_handler.setFormatter(DetailedExceptionFormatter()) - - logger.addHandler(console_handler) - logger.addHandler(file_handler) - - - return logger - def create_contract_instance(address: str, abi_path: str, config: ChainConfig) -> Contract: """ - Create and return a contract instance. + Create a contract instance from an address and ABI file. Args: - address (str): The address of the contract. - abi_path (str): Path to the ABI JSON file. + address (str): The contract address. + abi_path (str): Path to the ABI file. + config (ChainConfig): Chain configuration object. Returns: - Contract: Web3 contract instance. + Contract: A Web3 contract instance. """ - with open(abi_path, "r", encoding="utf-8") as file: - interface = json.load(file) - abi = interface["abi"] - - return config.w3.eth.contract(address=address, abi=abi) + try: + with open(abi_path, "r", encoding="utf-8") as file: + interface = json.load(file) + abi = interface["abi"] + return config.w3.eth.contract(address=Web3.to_checksum_address(address), abi=abi) + except Exception as ex: # pylint: disable=broad-except + logger.error("Failed to create contract instance: %s", ex, exc_info=True) + raise def retry_request(logger: logging.Logger, max_retries: int = 3, @@ -110,6 +78,34 @@ def wrapper(*args, **kwargs): return wrapper return decorator +def rate_limit(logger: logging.Logger, + max_calls_per_second: int = 10): + """ + Decorator to limit the number of calls to a function to max_calls_per_second. + Args: + logger (logging.Logger): Logger instance to log retry attempts. + max_calls_per_second (int, optional): Maximum number of retry attempts. + Defaults to config.MAX_CALLS_PER_SECOND. + + Returns: + Callable: Decorated function. + """ + lock = threading.Lock() + last_called = [0.0] + def decorator(func: Callable) -> Callable: + @functools.wraps(func) + def wrapper(*args, **kwargs): + with lock: + elapsed = time.time() - last_called[0] + wait_time = max(0, 1.0 / max_calls_per_second - elapsed) + logger.info(f"RL: waiting for {wait_time} seconds") + time.sleep(wait_time) + result = func(*args, **kwargs) + last_called[0] = time.time() + return result + return wrapper + return decorator + def get_spy_link(account, config: ChainConfig): """ Get account owner from EVC @@ -124,7 +120,8 @@ def get_spy_link(account, config: ChainConfig): return spy_link -@retry_request(logging.getLogger("liquidation_bot")) +@retry_request(logger) +@rate_limit(logger) def make_api_request(url: str, headers: Dict[str, str], params: Dict[str, Any]) -> Optional[Dict[str, Any]]: @@ -160,13 +157,7 @@ def global_exception_handler(exctype: type, value: BaseException, tb: Any) -> No tb (Any): A traceback object encapsulating the call stack at the point where the exception occurred. """ - logger = logging.getLogger("liquidation_bot") - - # Get the full traceback as a string - trace_str = "".join(traceback.format_exception(exctype, value, tb)) - - # Log the full exception information - logger.critical("Uncaught exception:\n %s", trace_str) + logger.critical("Uncaught exception:\n %s", "".join(traceback.format_exception(exctype, value, tb))) def post_unhealthy_account_on_slack(account_address: str, vault_address: str, health_score: float, value_borrowed: int, config: ChainConfig) -> None: @@ -310,18 +301,18 @@ def post_low_health_account_report(sorted_accounts, config: ChainConfig) -> None """ # Filter accounts below the threshold low_health_accounts = [ - (address, owner, subaccount, score, value, _, _) for address, owner, subaccount, score, value, _, _ in sorted_accounts + (address, owner, subaccount, score, value, _, _, _) for address, owner, subaccount, score, value, _, _, _ in sorted_accounts if (score < config.SLACK_REPORT_HEALTH_SCORE and value > (config.BORROW_VALUE_THRESHOLD * 10**18)) or score < 1.0 ] - total_value = sum(value / 10**18 for _, _, _, _, value, _, _ in sorted_accounts) + total_value = sum(value / 10**18 for _, _, _, _, value, _, _, _ in sorted_accounts) message = ":warning: *Account Health Report* :warning:\n\n" if not low_health_accounts: message += f"No accounts with health score below `{config.SLACK_REPORT_HEALTH_SCORE}` detected.\n" else: - for i, (address, _, _, score, value, _, _) in enumerate(low_health_accounts, start=1): + for i, (address, _, _, score, value, _, _, _) in enumerate(low_health_accounts, start=1): # Format score to 4 decimal places formatted_score = f"{score:.4f}" diff --git a/application.py b/application.py index e2a5228..d54b922 100644 --- a/application.py +++ b/application.py @@ -8,4 +8,4 @@ application = create_app() if __name__ == "__main__": - application.run(host="0.0.0.0", port=8080, debug=True) + application.run(host="0.0.0.0", port=8282, debug=True) diff --git a/contracts/DeployLiquidator_BOB.sol b/contracts/DeployLiquidator_BOB.sol new file mode 100644 index 0000000..f12cedd --- /dev/null +++ b/contracts/DeployLiquidator_BOB.sol @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.24; + +import {Script} from "forge-std/Script.sol"; +import {Test} from "forge-std/Test.sol"; + +import {Liquidator} from "./Liquidator.sol"; + +import "forge-std/console2.sol"; + +contract DeployLiquidator is Script { + function run() public { + uint256 deployerPrivateKey = vm.envUint("LIQUIDATOR_PRIVATE_KEY"); + + address swapperAddress = 0x697Ca30D765c1603890D88AAffBa3BeCCe72059d; + address swapVerifierAddress = 0x296041DbdBC92171293F23c0a31e1574b791060d; + address evcAddress = 0x59f0FeEc4fA474Ad4ffC357cC8d8595B68abE47d; + address pyth = 0x2880aB155794e7179c9eE2e38200202908C17B43; + + address deployer = vm.addr(deployerPrivateKey); + vm.startBroadcast(deployerPrivateKey); + + uint256 beforeGas = gasleft(); + console2.log("Gas before: ", beforeGas); + console2.log("Gas price: ", tx.gasprice); + + Liquidator liquidator = new Liquidator(deployer, swapperAddress, swapVerifierAddress, evcAddress, pyth); + uint256 afterGas = gasleft(); + console2.log("Gas after: ", afterGas); + + console2.log("Total gas cost: ", (beforeGas - afterGas) * tx.gasprice); + + console2.log("Liquidator deployed at: ", address(liquidator)); + } +} diff --git a/contracts/DeployLiquidator_Bera.sol b/contracts/DeployLiquidator_Bera.sol new file mode 100644 index 0000000..0de1ef5 --- /dev/null +++ b/contracts/DeployLiquidator_Bera.sol @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.24; + +import {Script} from "forge-std/Script.sol"; +import {Test} from "forge-std/Test.sol"; + +import {Liquidator} from "./Liquidator.sol"; + +import "forge-std/console2.sol"; + +contract DeployLiquidator is Script { + function run() public { + uint256 deployerPrivateKey = vm.envUint("LIQUIDATOR_PRIVATE_KEY"); + + address swapperAddress = 0x4A35e6A872cf35623cd3fD07ebECEDFc0170D705; + address swapVerifierAddress = 0x6fFf8Ac4AB123B62FF5e92aBb9fF702DCBD6C939; + address evcAddress = 0x45334608ECE7B2775136bC847EB92B5D332806A9; + address pyth = 0x2880aB155794e7179c9eE2e38200202908C17B43; + + address deployer = vm.addr(deployerPrivateKey); + vm.startBroadcast(deployerPrivateKey); + + uint256 beforeGas = gasleft(); + console2.log("Gas before: ", beforeGas); + console2.log("Gas price: ", tx.gasprice); + + Liquidator liquidator = new Liquidator(deployer, swapperAddress, swapVerifierAddress, evcAddress, pyth); + uint256 afterGas = gasleft(); + console2.log("Gas after: ", afterGas); + + console2.log("Total gas cost: ", (beforeGas - afterGas) * tx.gasprice); + + console2.log("Liquidator deployed at: ", address(liquidator)); + } +}