diff --git a/docker-compose.yml b/docker-compose.yml index e6de0f3..b387da7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,187 +45,209 @@ services: - 'host.docker.internal:host-gateway' tty: true - shovel_extrinsics: - build: - context: ./scraper_service - dockerfile: ./shovel_extrinsics/Dockerfile - container_name: shovel_extrinsics - depends_on: - clickhouse: - condition: service_started - env_file: - - .env - logging: - driver: 'json-file' - options: - max-size: '10m' - max-file: '3' - networks: - - app_network - restart: on-failure - extra_hosts: - - 'host.docker.internal:host-gateway' - tty: true + # shovel_extrinsics: + # build: + # context: ./scraper_service + # dockerfile: ./shovel_extrinsics/Dockerfile + # container_name: shovel_extrinsics + # depends_on: + # clickhouse: + # condition: service_started + # env_file: + # - .env + # logging: + # driver: 'json-file' + # options: + # max-size: '10m' + # max-file: '3' + # networks: + # - app_network + # restart: on-failure + # extra_hosts: + # - 'host.docker.internal:host-gateway' + # tty: true - shovel_events: - build: - context: ./scraper_service - dockerfile: ./shovel_events/Dockerfile - container_name: shovel_events - depends_on: - clickhouse: - condition: service_started - env_file: - - .env - logging: - driver: 'json-file' - options: - max-size: '10m' - max-file: '3' - networks: - - app_network - restart: on-failure - extra_hosts: - - 'host.docker.internal:host-gateway' - tty: true + # shovel_events: + # build: + # context: ./scraper_service + # dockerfile: ./shovel_events/Dockerfile + # container_name: shovel_events + # depends_on: + # clickhouse: + # condition: service_started + # env_file: + # - .env + # logging: + # driver: 'json-file' + # options: + # max-size: '10m' + # max-file: '3' + # networks: + # - app_network + # restart: on-failure + # extra_hosts: + # - 'host.docker.internal:host-gateway' + # tty: true - shovel_stake_map: - build: - context: ./scraper_service - dockerfile: ./shovel_stake_map/Dockerfile - container_name: shovel_stake_map - depends_on: - clickhouse: - condition: service_started - env_file: - - .env - logging: - driver: 'json-file' - options: - max-size: '10m' - max-file: '3' - networks: - - app_network - restart: on-failure - extra_hosts: - - 'host.docker.internal:host-gateway' - tty: true + # shovel_stake_map: + # build: + # context: ./scraper_service + # dockerfile: ./shovel_stake_map/Dockerfile + # container_name: shovel_stake_map + # depends_on: + # clickhouse: + # condition: service_started + # env_file: + # - .env + # logging: + # driver: 'json-file' + # options: + # max-size: '10m' + # max-file: '3' + # networks: + # - app_network + # restart: on-failure + # extra_hosts: + # - 'host.docker.internal:host-gateway' + # tty: true - shovel_hotkey_owner_map: - build: - context: ./scraper_service - dockerfile: ./shovel_hotkey_owner_map/Dockerfile - container_name: shovel_hotkey_owner_map - depends_on: - clickhouse: - condition: service_started - env_file: - - .env - logging: - driver: 'json-file' - options: - max-size: '10m' - max-file: '3' - networks: - - app_network - restart: on-failure - extra_hosts: - - 'host.docker.internal:host-gateway' - tty: true + # shovel_hotkey_owner_map: + # build: + # context: ./scraper_service + # dockerfile: ./shovel_hotkey_owner_map/Dockerfile + # container_name: shovel_hotkey_owner_map + # depends_on: + # clickhouse: + # condition: service_started + # env_file: + # - .env + # logging: + # driver: 'json-file' + # options: + # max-size: '10m' + # max-file: '3' + # networks: + # - app_network + # restart: on-failure + # extra_hosts: + # - 'host.docker.internal:host-gateway' + # tty: true - shovel_subnets: - build: - context: ./scraper_service - dockerfile: ./shovel_subnets/Dockerfile - container_name: shovel_subnets - depends_on: - clickhouse: - condition: service_started - env_file: - - .env - logging: - driver: 'json-file' - options: - max-size: '10m' - max-file: '3' - networks: - - app_network - restart: always - extra_hosts: - - 'host.docker.internal:host-gateway' - tty: true + # shovel_subnets: + # build: + # context: ./scraper_service + # dockerfile: ./shovel_subnets/Dockerfile + # container_name: shovel_subnets + # depends_on: + # clickhouse: + # condition: service_started + # env_file: + # - .env + # logging: + # driver: 'json-file' + # options: + # max-size: '10m' + # max-file: '3' + # networks: + # - app_network + # restart: always + # extra_hosts: + # - 'host.docker.internal:host-gateway' + # tty: true - shovel_daily_stake: - build: - context: ./scraper_service - dockerfile: ./shovel_daily_stake/Dockerfile - container_name: shovel_daily_stake - depends_on: - clickhouse: - condition: service_started - env_file: - - .env - logging: - driver: 'json-file' - options: - max-size: '10m' - max-file: '3' - networks: - - app_network - restart: on-failure - extra_hosts: - - 'host.docker.internal:host-gateway' - tty: true + # shovel_daily_stake: + # build: + # context: ./scraper_service + # dockerfile: ./shovel_daily_stake/Dockerfile + # container_name: shovel_daily_stake + # depends_on: + # clickhouse: + # condition: service_started + # env_file: + # - .env + # logging: + # driver: 'json-file' + # options: + # max-size: '10m' + # max-file: '3' + # networks: + # - app_network + # restart: on-failure + # extra_hosts: + # - 'host.docker.internal:host-gateway' + # tty: true - shovel_daily_balance: - build: - context: ./scraper_service - dockerfile: ./shovel_daily_balance/Dockerfile - container_name: shovel_daily_balance - depends_on: - clickhouse: - condition: service_started - env_file: - - .env - logging: - driver: 'json-file' - options: - max-size: '10m' - max-file: '3' - networks: - - app_network - restart: on-failure - extra_hosts: - - 'host.docker.internal:host-gateway' - tty: true + # shovel_daily_balance: + # build: + # context: ./scraper_service + # dockerfile: ./shovel_daily_balance/Dockerfile + # container_name: shovel_daily_balance + # depends_on: + # clickhouse: + # condition: service_started + # env_file: + # - .env + # logging: + # driver: 'json-file' + # options: + # max-size: '10m' + # max-file: '3' + # networks: + # - app_network + # restart: on-failure + # extra_hosts: + # - 'host.docker.internal:host-gateway' + # tty: true - shovel_tao_price: - build: - context: ./scraper_service - dockerfile: ./shovel_tao_price/Dockerfile - container_name: shovel_tao_price - depends_on: - clickhouse: - condition: service_started - env_file: - - .env - logging: - driver: 'json-file' - options: - max-size: '10m' - max-file: '3' - networks: - - app_network - restart: on-failure - extra_hosts: - - 'host.docker.internal:host-gateway' - tty: true + # shovel_tao_price: + # build: + # context: ./scraper_service + # dockerfile: ./shovel_tao_price/Dockerfile + # container_name: shovel_tao_price + # depends_on: + # clickhouse: + # condition: service_started + # env_file: + # - .env + # logging: + # driver: 'json-file' + # options: + # max-size: '10m' + # max-file: '3' + # networks: + # - app_network + # restart: on-failure + # extra_hosts: + # - 'host.docker.internal:host-gateway' + # tty: true + + # shovel_alpha_to_tao: + # build: + # context: ./scraper_service + # dockerfile: ./shovel_alpha_to_tao/Dockerfile + # container_name: shovel_alpha_to_tao + # depends_on: + # clickhouse: + # condition: service_started + # env_file: + # - .env + # logging: + # driver: 'json-file' + # options: + # max-size: '10m' + # max-file: '3' + # networks: + # - app_network + # restart: on-failure + # extra_hosts: + # - 'host.docker.internal:host-gateway' + # tty: true - shovel_alpha_to_tao: + shovel_validators: build: context: ./scraper_service - dockerfile: ./shovel_alpha_to_tao/Dockerfile - container_name: shovel_alpha_to_tao + dockerfile: ./shovel_validators/Dockerfile + container_name: shovel_validators depends_on: clickhouse: condition: service_started diff --git a/scraper_service/requirements.txt b/scraper_service/requirements.txt index c8c4f49..375846c 100644 --- a/scraper_service/requirements.txt +++ b/scraper_service/requirements.txt @@ -1,3 +1,4 @@ +async-substrate-interface==1.0.7 base58==2.1.1 certifi==2024.7.4 cffi==1.16.0 @@ -5,10 +6,6 @@ charset-normalizer==3.3.2 clickhouse-driver==0.2.8 cytoolz==0.12.3 ecdsa==0.19.0 -eth-hash==0.7.0 -eth-keys==0.5.1 -eth-typing==4.3.1 -eth-utils==2.3.1 idna==3.7 maturin==1.7.1 mem-top==0.2.1 @@ -24,7 +21,6 @@ PyNaCl==1.5.0 python-dotenv==1.0.1 pytz==2024.1 requests==2.32.3 -scalecodec==1.2.10 six==1.16.0 substrate-interface==1.7.10 toolz==0.12.1 diff --git a/scraper_service/shared/substrate.py b/scraper_service/shared/substrate.py index ab8c187..afa0bfc 100644 --- a/scraper_service/shared/substrate.py +++ b/scraper_service/shared/substrate.py @@ -1,6 +1,6 @@ import os from functools import lru_cache -from substrateinterface import SubstrateInterface +from async_substrate_interface import SubstrateInterface import threading thread_local = threading.local() diff --git a/scraper_service/shovel_block_timestamp/Dockerfile b/scraper_service/shovel_block_timestamp/Dockerfile index eaa1f21..05c1d2d 100644 --- a/scraper_service/shovel_block_timestamp/Dockerfile +++ b/scraper_service/shovel_block_timestamp/Dockerfile @@ -2,6 +2,17 @@ FROM python:3.12-slim WORKDIR /app +# Install Rust and required build dependencies +RUN apt-get update && apt-get install -y \ + curl \ + build-essential \ + pkg-config \ + libssl-dev \ + && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +# Add .cargo/bin to PATH +ENV PATH="/root/.cargo/bin:${PATH}" + COPY ./requirements.txt /app/requirements.txt RUN pip install --no-cache-dir -r /app/requirements.txt diff --git a/scraper_service/shovel_block_timestamp/main.py b/scraper_service/shovel_block_timestamp/main.py index ffb7881..1a54a34 100644 --- a/scraper_service/shovel_block_timestamp/main.py +++ b/scraper_service/shovel_block_timestamp/main.py @@ -16,6 +16,10 @@ class BlockTimestampShovel(ShovelBaseClass): table_name = "shovel_block_timestamps" + def __init__(self, name): + super().__init__(name) + self.starting_block = 4920351 + def process_block(self, n): do_process_block(self, n) diff --git a/scraper_service/shovel_subnets/rust_bindings/Cargo.lock b/scraper_service/shovel_subnets/rust_bindings/Cargo.lock index 53e2ad7..b346ec7 100644 --- a/scraper_service/shovel_subnets/rust_bindings/Cargo.lock +++ b/scraper_service/shovel_subnets/rust_bindings/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -393,9 +393,9 @@ checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cc" -version = "1.1.15" +version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" dependencies = [ "shlex", ] @@ -2122,15 +2122,14 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.8" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", "getrandom", "libc", - "spin", "untrusted", "windows-sys 0.52.0", ] diff --git a/scraper_service/shovel_validators/Dockerfile b/scraper_service/shovel_validators/Dockerfile new file mode 100644 index 0000000..b166ab1 --- /dev/null +++ b/scraper_service/shovel_validators/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.12-slim + +WORKDIR /app + +# Install Rust and required build dependencies +RUN apt-get update && apt-get install -y \ + curl \ + build-essential \ + pkg-config \ + libssl-dev \ + && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +# Add .cargo/bin to PATH +ENV PATH="/root/.cargo/bin:${PATH}" + +COPY ./requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY ./shared /app/shared +COPY ./shovel_validators /app/shovel_validators + +ENV PYTHONPATH="/app:/app/shared" + +CMD ["python", "-u", "shovel_validators/main.py"] diff --git a/scraper_service/shovel_validators/__init__.py b/scraper_service/shovel_validators/__init__.py new file mode 100644 index 0000000..0519ecb --- /dev/null +++ b/scraper_service/shovel_validators/__init__.py @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/scraper_service/shovel_validators/main.py b/scraper_service/shovel_validators/main.py new file mode 100644 index 0000000..995c61a --- /dev/null +++ b/scraper_service/shovel_validators/main.py @@ -0,0 +1,248 @@ +from shared.block_metadata import get_block_metadata +from shared.clickhouse.batch_insert import buffer_insert +from shared.clickhouse.utils import ( + get_clickhouse_client, + table_exists, +) +from shared.shovel_base_class import ShovelBaseClass +from shared.substrate import get_substrate_client, reconnect_substrate +from shared.exceptions import DatabaseConnectionError, ShovelProcessingError +import logging +from typing import Dict, List, Any, Optional +from functools import lru_cache +from typing import Union +from scalecodec.utils.ss58 import ss58_encode + +SS58_FORMAT = 42 + +def decode_account_id(account_id_bytes: Union[tuple[int], tuple[tuple[int]]]): + if isinstance(account_id_bytes, tuple) and isinstance(account_id_bytes[0], tuple): + account_id_bytes = account_id_bytes[0] + return ss58_encode(bytes(account_id_bytes).hex(), SS58_FORMAT) + +def decode_string(string: Union[str, tuple[int]]): + if isinstance(string, str): + return string + return bytes(string).decode('utf-8') + +logging.basicConfig(level=logging.INFO, + format="%(asctime)s %(process)d %(message)s") + +COMPOUNDING_PERIODS_PER_DAY = 7200 + +def create_validators_table(table_name): + if not table_exists(table_name): + query = f""" + CREATE TABLE IF NOT EXISTS {table_name} ( + block_number UInt64, + timestamp DateTime, + name String, + address String, + image Nullable(String), + description Nullable(String), + owner Nullable(String), + url Nullable(String), + nominators UInt64, + daily_return Float64, + registrations Array(UInt64), + validator_permits Array(UInt64), + apy Nullable(Float64), + subnet_hotkey_alpha Map(UInt64, Float64) + ) ENGINE = ReplacingMergeTree() + ORDER BY (block_number, address) + """ + + get_clickhouse_client().execute(query) + +def calculate_apy_from_daily_return(return_per_1000: float, compounding_periods: int = COMPOUNDING_PERIODS_PER_DAY) -> float: + daily_return = return_per_1000 / 1000 + apy = ((1 + (daily_return / compounding_periods)) ** (compounding_periods * 365)) - 1 + return round(apy * 100, 3) + +def get_subnet_uids(substrate, block_hash: str) -> List[int]: + try: + result = substrate.runtime_call( + api="SubnetInfoRuntimeApi", + method="get_subnets_info", + params=[], + block_hash=block_hash + ) + subnet_info = result.value + + return [info['netuid'] for info in subnet_info if 'netuid' in info] + except Exception as e: + logging.error(f"Failed to get subnet UIDs: {str(e)}") + return [] + +def get_active_validators(substrate, block_hash: str, delegate_info) -> List[str]: + try: + return [decode_account_id(delegate['delegate_ss58']) for delegate in delegate_info if 'delegate_ss58' in delegate] + except Exception as e: + logging.error(f"Failed to get active validators: {str(e)}") + return [] + +def is_registered_in_subnet(substrate, net_uid: int, address: str, block_hash: str) -> bool: + try: + result = substrate.query("SubtensorModule", "Uids", [net_uid, address], block_hash=block_hash) + return bool(result) + except Exception as e: + logging.error(f"Failed to check subnet registration for {address} in subnet {net_uid}: {str(e)}") + return False + +def get_total_hotkey_alpha(substrate, address: str, net_uid: int, block_hash: str) -> float: + try: + result = substrate.query("SubtensorModule", "TotalHotkeyAlpha", [address, net_uid], block_hash=block_hash) + print(f"hotkey alpha {result}") + return float(str(result[0])) if result else 0.0 + except Exception as e: + logging.error(f"Failed to get total hotkey alpha for {address} in subnet {net_uid}: {str(e)}") + return 0.0 + +def fetch_validator_info(substrate, address: str, block_hash: str, delegate_info) -> Dict[str, Any]: + try: + chain_info = next((d for d in delegate_info if decode_account_id(d['delegate_ss58']) == address), None) + + if not chain_info: + return { + "name": address, + "image": None, + "description": None, + "owner": None, + "url": None + } + + owner = decode_account_id(chain_info.get('owner_ss58')) + + if owner: + identity= substrate.query("SubtensorModule", "IdentitiesV2", [owner], block_hash=block_hash) + print(f"identity bytes {identity}") + else: + identity = None + + return { + "name": decode_string(identity.get('name', address)) if identity else address, + "image": decode_string( identity.get('image', '')) if identity else None, + "description": decode_string(identity.get('description', '')) if identity else None, + "owner": owner, + "url": decode_string(identity.get('url', '')) if identity else None + } + except Exception as e: + logging.error(f"Failed to fetch validator info for {address}: {str(e)}") + return { + "name": address, + "image": None, + "description": None, + "owner": None, + "url": None + } + +def fetch_validator_stats(substrate, address: str, block_hash: str, delegate_info) -> Dict[str, Any]: + try: + info = next((d for d in delegate_info if decode_account_id(d['delegate_ss58']) == address), None) + + if not info: + return { + "nominators": 0, + "daily_return": 0.0, + "registrations": [], + "validator_permits": [], + "apy": None, + "subnet_hotkey_alpha": {} + } + + return_per_1000 = int(info['return_per_1000'], 16) if isinstance(info['return_per_1000'], str) else info['return_per_1000'] + apy = calculate_apy_from_daily_return(return_per_1000) + + subnet_uids = get_subnet_uids(substrate, block_hash) + subnet_hotkey_alpha = {} + + for net_uid in subnet_uids: + if is_registered_in_subnet(substrate, net_uid, address, block_hash): + alpha = get_total_hotkey_alpha(substrate, address, net_uid, block_hash) + if alpha > 0: + subnet_hotkey_alpha[net_uid] = alpha + + return { + "nominators": len(info.get('nominators', [])), + "daily_return": info.get('total_daily_return', 0.0), + "registrations": info.get('registrations', []), + "validator_permits": info.get('validator_permits', []), + "apy": apy, + "subnet_hotkey_alpha": subnet_hotkey_alpha + } + except Exception as e: + logging.error(f"Failed to fetch validator stats for {address}: {str(e)}") + return { + "nominators": 0, + "daily_return": 0.0, + "registrations": [], + "validator_permits": [], + "apy": None, + "subnet_hotkey_alpha": {} + } + +class ValidatorsShovel(ShovelBaseClass): + table_name = "shovel_validators" + + def __init__(self, name): + super().__init__(name) + self.starting_block = 4920351 + + def process_block(self, n): + if n % 7200 != 0: + return + try: + substrate = get_substrate_client() + (block_timestamp, block_hash) = get_block_metadata(n) + + create_validators_table(self.table_name) + + delegate_info = substrate.runtime_call( + api="DelegateInfoRuntimeApi", + method="get_delegates", + params=[], + block_hash=block_hash + ).value + + validators = get_active_validators(substrate, block_hash, delegate_info) + + for validator_address in validators: + try: + info = fetch_validator_info(substrate, validator_address, block_hash, delegate_info) + stats = fetch_validator_stats(substrate, validator_address, block_hash, delegate_info) + + values = [ + n, + block_timestamp, + info["name"], + validator_address, + info["image"], + info["description"], + info["owner"], + info["url"], + stats["nominators"], + stats["daily_return"], + stats["registrations"], + stats["validator_permits"], + stats["apy"], + stats["subnet_hotkey_alpha"] + ] + + print(values) + + buffer_insert(self.table_name, values) + + except Exception as e: + logging.error(f"Error processing validator {validator_address}: {str(e)}") + continue + + except DatabaseConnectionError: + raise + except Exception as e: + raise ShovelProcessingError(f"Failed to process block {n}: {str(e)}") + +def main(): + ValidatorsShovel(name="validators").start() + +if __name__ == "__main__": + main()