getting Subsquid endpoints from their API rather than hardcoding them

Author: simone1999

IceCreamSwapWeb3/Subsquid.py

@@ -4,107 +4,21 @@
 from tqdm import tqdm
 from web3.types import FilterParams, LogReceipt
 
-ENDPOINTS = {
-    16600: 'https://v2.archive.subsquid.io/network/0g-testnet',
-    41455: 'https://v2.archive.subsquid.io/network/aleph-zero-evm-mainnet',
-    42161: 'https://v2.archive.subsquid.io/network/arbitrum-one',
-    42170: 'https://v2.archive.subsquid.io/network/arbitrum-nova',
-    421614: 'https://v2.archive.subsquid.io/network/arbitrum-sepolia',
-    10242: 'https://v2.archive.subsquid.io/network/arthera-mainnet',
-    592: 'https://v2.archive.subsquid.io/network/astar-mainnet',
-    3776: 'https://v2.archive.subsquid.io/network/astar-zkevm-mainnet',
-    6038361: 'https://v2.archive.subsquid.io/network/astar-zkyoto',
-    43114: 'https://v2.archive.subsquid.io/network/avalanche-mainnet',
-    43113: 'https://v2.archive.subsquid.io/network/avalanche-testnet',
-    8333: 'https://v2.archive.subsquid.io/network/b3-mainnet',
-    1993: 'https://v2.archive.subsquid.io/network/b3-sepolia',
-    8453: 'https://v2.archive.subsquid.io/network/base-mainnet',
-    84532: 'https://v2.archive.subsquid.io/network/base-sepolia',
-    80084: 'https://v2.archive.subsquid.io/network/berachain-bartio',
-    56: 'https://v2.archive.subsquid.io/network/binance-mainnet',
-    97: 'https://v2.archive.subsquid.io/network/binance-testnet',
-    355110: 'https://v2.archive.subsquid.io/network/bitfinity-mainnet',
-    355113: 'https://v2.archive.subsquid.io/network/bitfinity-testnet',
-    32520: 'https://v2.archive.subsquid.io/network/bitgert-mainnet',
-    64668: 'https://v2.archive.subsquid.io/network/bitgert-testnet',
-    81457: 'https://v2.archive.subsquid.io/network/blast-l2-mainnet',
-    168587773: 'https://v2.archive.subsquid.io/network/blast-sepolia',
-    60808: 'https://v2.archive.subsquid.io/network/bob-mainnet',
-    7700: 'https://v2.archive.subsquid.io/network/canto',
-    7701: 'https://v2.archive.subsquid.io/network/canto-testnet',
-    44787: 'https://v2.archive.subsquid.io/network/celo-alfajores-testnet',
-    42220: 'https://v2.archive.subsquid.io/network/celo-mainnet',
-    1116: 'https://v2.archive.subsquid.io/network/core-mainnet',
-    4157: 'https://v2.archive.subsquid.io/network/crossfi-testnet',
-    7560: 'https://v2.archive.subsquid.io/network/cyber-mainnet',
-    111557560: 'https://v2.archive.subsquid.io/network/cyberconnect-l2-testnet',
-    666666666: 'https://v2.archive.subsquid.io/network/degen-chain',
-    53935: 'https://v2.archive.subsquid.io/network/dfk-chain',
-    2000: 'https://v2.archive.subsquid.io/network/dogechain-mainnet',
-    568: 'https://v2.archive.subsquid.io/network/dogechain-testnet',
-    17000: 'https://v2.archive.subsquid.io/network/ethereum-holesky',
-    1: 'https://v2.archive.subsquid.io/network/ethereum-mainnet',
-    11155111: 'https://v2.archive.subsquid.io/network/ethereum-sepolia',
-    42793: 'https://v2.archive.subsquid.io/network/etherlink-mainnet',
-    128123: 'https://v2.archive.subsquid.io/network/etherlink-testnet',
-    2109: 'https://v2.archive.subsquid.io/network/exosama',
-    250: 'https://v2.archive.subsquid.io/network/fantom-mainnet',
-    4002: 'https://v2.archive.subsquid.io/network/fantom-testnet',
-    14: 'https://v2.archive.subsquid.io/network/flare-mainnet',
-    1625: 'https://v2.archive.subsquid.io/network/galxe-gravity',
-    88153591557: 'https://v2.archive.subsquid.io/network/gelato-arbitrum-blueberry',
-    123420111: 'https://v2.archive.subsquid.io/network/gelato-opcelestia-raspberry',
-    100: 'https://v2.archive.subsquid.io/network/gnosis-mainnet',
-    13371: 'https://v2.archive.subsquid.io/network/immutable-zkevm-mainnet',
-    13473: 'https://v2.archive.subsquid.io/network/immutable-zkevm-testnet',
-    1998: 'https://v2.archive.subsquid.io/network/kyoto-testnet',
-    59144: 'https://v2.archive.subsquid.io/network/linea-mainnet',
-    169: 'https://v2.archive.subsquid.io/network/manta-pacific',
-    3441006: 'https://v2.archive.subsquid.io/network/manta-pacific-sepolia',
-    5000: 'https://v2.archive.subsquid.io/network/mantle-mainnet',
-    5003: 'https://v2.archive.subsquid.io/network/mantle-sepolia',
-    4200: 'https://v2.archive.subsquid.io/network/merlin-mainnet',
-    686868: 'https://v2.archive.subsquid.io/network/merlin-testnet',
-    1088: 'https://v2.archive.subsquid.io/network/metis-mainnet',
-    34443: 'https://v2.archive.subsquid.io/network/mode-mainnet',
-    1287: 'https://v2.archive.subsquid.io/network/moonbase-testnet',
-    1284: 'https://v2.archive.subsquid.io/network/moonbeam-mainnet',
-    1285: 'https://v2.archive.subsquid.io/network/moonriver-mainnet',
-    42225: 'https://v2.archive.subsquid.io/network/nakachain',
-    245022934: 'https://v2.archive.subsquid.io/network/neon-devnet',
-    245022926: 'https://v2.archive.subsquid.io/network/neon-mainnet',
-    204: 'https://v2.archive.subsquid.io/network/opbnb-mainnet',
-    5611: 'https://v2.archive.subsquid.io/network/opbnb-testnet',
-    10: 'https://v2.archive.subsquid.io/network/optimism-mainnet',
-    11155420: 'https://v2.archive.subsquid.io/network/optimism-sepolia',
-    3338: 'https://v2.archive.subsquid.io/network/peaq-mainnet',
-    98864: 'https://v2.archive.subsquid.io/network/plume-testnet',
-    137: 'https://v2.archive.subsquid.io/network/polygon-mainnet',
-    80002: 'https://v2.archive.subsquid.io/network/polygon-amoy-testnet',
-    1101: 'https://v2.archive.subsquid.io/network/polygon-zkevm-mainnet',
-    2442: 'https://v2.archive.subsquid.io/network/polygon-zkevm-cardona-testnet',
-    97072271: 'https://v2.archive.subsquid.io/network/prom-testnet',
-    584548796: 'https://v2.archive.subsquid.io/network/prom-testnet-v2',
-    157: 'https://v2.archive.subsquid.io/network/puppynet',
-    534352: 'https://v2.archive.subsquid.io/network/scroll-mainnet',
-    534351: 'https://v2.archive.subsquid.io/network/scroll-sepolia',
-    109: 'https://v2.archive.subsquid.io/network/shibarium',
-    81: 'https://v2.archive.subsquid.io/network/shibuya-testnet',
-    336: 'https://v2.archive.subsquid.io/network/shiden-mainnet',
-    1482601649: 'https://v2.archive.subsquid.io/network/skale-nebula',
-    64165: 'https://v2.archive.subsquid.io/network/sonic-testnet',
-    93747: 'https://v2.archive.subsquid.io/network/stratovm-sepolia',
-    53302: 'https://v2.archive.subsquid.io/network/superseed-sepolia',
-    167000: 'https://v2.archive.subsquid.io/network/taiko-mainnet',
-    5678: 'https://v2.archive.subsquid.io/network/tanssi',
-    196: 'https://v2.archive.subsquid.io/network/xlayer-mainnet',
-    195: 'https://v2.archive.subsquid.io/network/xlayer-testnet',
-    810180: 'https://v2.archive.subsquid.io/network/zklink-nova-mainnet',
-    324: 'https://v2.archive.subsquid.io/network/zksync-mainnet',
-    300: 'https://v2.archive.subsquid.io/network/zksync-sepolia',
-    7777777: 'https://v2.archive.subsquid.io/network/zora-mainnet',
-    999999999: 'https://v2.archive.subsquid.io/network/zora-sepolia',
-}
+
+endpoint_cache: dict[int, str] | None = None
+def get_endpoints() -> dict[int, str]:
+    global endpoint_cache
+    if endpoint_cache is not None:
+        return endpoint_cache
+    res = requests.get("https://cdn.subsquid.io/archives/evm.json")
+    res.raise_for_status()
+
+    endpoints: dict[int, str] = {}
+    for chain in res.json()["archives"]:
+        endpoints[chain["chainId"]] = chain["providers"][0]["dataSourceUrl"]
+
+    endpoint_cache = endpoints
+    return endpoints
 
 def get_text(url: str) -> str:
     res = requests.get(url)
@@ -117,10 +31,11 @@ def get_filter(
         partial_allowed=False,
         p_bar: tqdm = None
 ) -> tuple[int, list[LogReceipt]]:
-    if chain_id not in ENDPOINTS:
+    endpoints = get_endpoints()
+    if chain_id not in endpoints:
         raise ValueError(f"Subsquid does not support Chain ID {chain_id}")
 
-    gateway_url = ENDPOINTS[chain_id]
+    gateway_url = endpoints[chain_id]
 
     assert isinstance(filter_params['fromBlock'], int)
     assert isinstance(filter_params['toBlock'], int)