Skip to content

Commit 709db37

Browse files
simone1999simone1999
committed
integrated Subsquid data layer for fast and consistent log retrival.
1 parent b85f438 commit 709db37

File tree

2 files changed

+216
-3
lines changed

2 files changed

+216
-3
lines changed

‎IceCreamSwapWeb3/EthAdvanced.py

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from web3.types import FilterParams, LogReceipt, CallOverride, BlockIdentifier, TxParams, BlockData
88

99
from IceCreamSwapWeb3 import Web3Advanced
10+
from IceCreamSwapWeb3.Subsquid import get_filter
1011

1112

1213
def exponential_retry(func_name: str = None):
@@ -123,7 +124,8 @@ def get_logs(
123124
filter_params: FilterParams,
124125
show_progress_bar: bool = False,
125126
p_bar=None,
126-
no_retry: bool = False
127+
no_retry: bool = False,
128+
use_subsquid: bool = True,
127129
) -> list[LogReceipt]:
128130
filter_block_range = self.w3.filter_block_range
129131
if filter_block_range == 0:
@@ -135,8 +137,8 @@ def get_logs(
135137
return self.get_logs_inner(filter_params, no_retry=no_retry)
136138

137139
# sanitizing block numbers, could be strings like "latest"
138-
filter_params["fromBlock"] = from_block = self.get_block_number_from_identifier(filter_params["fromBlock"])
139-
filter_params["toBlock"] = to_block = self.get_block_number_from_identifier(filter_params["toBlock"])
140+
filter_params["fromBlock"] = from_block = self.get_block_number_from_identifier(filter_params.get("fromBlock", "earliest"))
141+
filter_params["toBlock"] = to_block = self.get_block_number_from_identifier(filter_params.get("toBlock", "latest"))
140142
assert to_block >= from_block, f"{from_block=}, {to_block=}"
141143

142144
# note: fromBlock and toBlock are both inclusive. e.g. 5 to 6 are 2 blocks
@@ -152,6 +154,7 @@ def get_logs(
152154
show_progress_bar=show_progress_bar,
153155
p_bar=p_bar,
154156
no_retry=no_retry,
157+
use_subsquid=use_subsquid,
155158
)
156159

157160
# the latest blocks might be available on some nodes but not at others.
@@ -177,6 +180,21 @@ def get_logs(
177180
if num_blocks == 1:
178181
return self.get_logs_inner(filter_params, no_retry=no_retry)
179182

183+
if use_subsquid and from_block < self.w3.latest_seen_block - 100:
184+
kwargs["use_subsquid"] = False # make sure we only try once with Subsquid
185+
try:
186+
# trying to get logs from SubSquid
187+
till_block, results = get_filter(
188+
chain_id=self.chain_id,
189+
filter_params=filter_params,
190+
partial_allowed=True,
191+
)
192+
if till_block >= to_block:
193+
return results
194+
return results + self.get_logs({**filter_params, "fromBlock": till_block+1}, **kwargs)
195+
except Exception as e:
196+
print(f"Getting logs from SubSquid threw exception {repr(e)}, falling back to RPC")
197+
180198
# if we already know that the filter range is too large, split it
181199
if num_blocks > filter_block_range:
182200
results = []

‎IceCreamSwapWeb3/Subsquid.py

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
import requests
2+
from eth_utils import to_checksum_address
3+
from hexbytes import HexBytes
4+
from web3.types import FilterParams, LogReceipt
5+
6+
ENDPOINTS = {
7+
16600: 'https://v2.archive.subsquid.io/network/0g-testnet',
8+
41455: 'https://v2.archive.subsquid.io/network/aleph-zero-evm-mainnet',
9+
42161: 'https://v2.archive.subsquid.io/network/arbitrum-one',
10+
42170: 'https://v2.archive.subsquid.io/network/arbitrum-nova',
11+
421614: 'https://v2.archive.subsquid.io/network/arbitrum-sepolia',
12+
10242: 'https://v2.archive.subsquid.io/network/arthera-mainnet',
13+
592: 'https://v2.archive.subsquid.io/network/astar-mainnet',
14+
3776: 'https://v2.archive.subsquid.io/network/astar-zkevm-mainnet',
15+
6038361: 'https://v2.archive.subsquid.io/network/astar-zkyoto',
16+
43114: 'https://v2.archive.subsquid.io/network/avalanche-mainnet',
17+
43113: 'https://v2.archive.subsquid.io/network/avalanche-testnet',
18+
8333: 'https://v2.archive.subsquid.io/network/b3-mainnet',
19+
1993: 'https://v2.archive.subsquid.io/network/b3-sepolia',
20+
8453: 'https://v2.archive.subsquid.io/network/base-mainnet',
21+
84532: 'https://v2.archive.subsquid.io/network/base-sepolia',
22+
80084: 'https://v2.archive.subsquid.io/network/berachain-bartio',
23+
56: 'https://v2.archive.subsquid.io/network/binance-mainnet',
24+
97: 'https://v2.archive.subsquid.io/network/binance-testnet',
25+
355110: 'https://v2.archive.subsquid.io/network/bitfinity-mainnet',
26+
355113: 'https://v2.archive.subsquid.io/network/bitfinity-testnet',
27+
32520: 'https://v2.archive.subsquid.io/network/bitgert-mainnet',
28+
64668: 'https://v2.archive.subsquid.io/network/bitgert-testnet',
29+
81457: 'https://v2.archive.subsquid.io/network/blast-l2-mainnet',
30+
168587773: 'https://v2.archive.subsquid.io/network/blast-sepolia',
31+
60808: 'https://v2.archive.subsquid.io/network/bob-mainnet',
32+
7700: 'https://v2.archive.subsquid.io/network/canto',
33+
7701: 'https://v2.archive.subsquid.io/network/canto-testnet',
34+
44787: 'https://v2.archive.subsquid.io/network/celo-alfajores-testnet',
35+
42220: 'https://v2.archive.subsquid.io/network/celo-mainnet',
36+
1116: 'https://v2.archive.subsquid.io/network/core-mainnet',
37+
4157: 'https://v2.archive.subsquid.io/network/crossfi-testnet',
38+
7560: 'https://v2.archive.subsquid.io/network/cyber-mainnet',
39+
111557560: 'https://v2.archive.subsquid.io/network/cyberconnect-l2-testnet',
40+
666666666: 'https://v2.archive.subsquid.io/network/degen-chain',
41+
53935: 'https://v2.archive.subsquid.io/network/dfk-chain',
42+
2000: 'https://v2.archive.subsquid.io/network/dogechain-mainnet',
43+
568: 'https://v2.archive.subsquid.io/network/dogechain-testnet',
44+
17000: 'https://v2.archive.subsquid.io/network/ethereum-holesky',
45+
1: 'https://v2.archive.subsquid.io/network/ethereum-mainnet',
46+
11155111: 'https://v2.archive.subsquid.io/network/ethereum-sepolia',
47+
42793: 'https://v2.archive.subsquid.io/network/etherlink-mainnet',
48+
128123: 'https://v2.archive.subsquid.io/network/etherlink-testnet',
49+
2109: 'https://v2.archive.subsquid.io/network/exosama',
50+
250: 'https://v2.archive.subsquid.io/network/fantom-mainnet',
51+
4002: 'https://v2.archive.subsquid.io/network/fantom-testnet',
52+
14: 'https://v2.archive.subsquid.io/network/flare-mainnet',
53+
1625: 'https://v2.archive.subsquid.io/network/galxe-gravity',
54+
88153591557: 'https://v2.archive.subsquid.io/network/gelato-arbitrum-blueberry',
55+
123420111: 'https://v2.archive.subsquid.io/network/gelato-opcelestia-raspberry',
56+
100: 'https://v2.archive.subsquid.io/network/gnosis-mainnet',
57+
13371: 'https://v2.archive.subsquid.io/network/immutable-zkevm-mainnet',
58+
13473: 'https://v2.archive.subsquid.io/network/immutable-zkevm-testnet',
59+
1998: 'https://v2.archive.subsquid.io/network/kyoto-testnet',
60+
59144: 'https://v2.archive.subsquid.io/network/linea-mainnet',
61+
169: 'https://v2.archive.subsquid.io/network/manta-pacific',
62+
3441006: 'https://v2.archive.subsquid.io/network/manta-pacific-sepolia',
63+
5000: 'https://v2.archive.subsquid.io/network/mantle-mainnet',
64+
5003: 'https://v2.archive.subsquid.io/network/mantle-sepolia',
65+
4200: 'https://v2.archive.subsquid.io/network/merlin-mainnet',
66+
686868: 'https://v2.archive.subsquid.io/network/merlin-testnet',
67+
1088: 'https://v2.archive.subsquid.io/network/metis-mainnet',
68+
34443: 'https://v2.archive.subsquid.io/network/mode-mainnet',
69+
1287: 'https://v2.archive.subsquid.io/network/moonbase-testnet',
70+
1284: 'https://v2.archive.subsquid.io/network/moonbeam-mainnet',
71+
1285: 'https://v2.archive.subsquid.io/network/moonriver-mainnet',
72+
42225: 'https://v2.archive.subsquid.io/network/nakachain',
73+
245022934: 'https://v2.archive.subsquid.io/network/neon-devnet',
74+
245022926: 'https://v2.archive.subsquid.io/network/neon-mainnet',
75+
204: 'https://v2.archive.subsquid.io/network/opbnb-mainnet',
76+
5611: 'https://v2.archive.subsquid.io/network/opbnb-testnet',
77+
10: 'https://v2.archive.subsquid.io/network/optimism-mainnet',
78+
11155420: 'https://v2.archive.subsquid.io/network/optimism-sepolia',
79+
3338: 'https://v2.archive.subsquid.io/network/peaq-mainnet',
80+
98864: 'https://v2.archive.subsquid.io/network/plume-testnet',
81+
137: 'https://v2.archive.subsquid.io/network/polygon-mainnet',
82+
80002: 'https://v2.archive.subsquid.io/network/polygon-amoy-testnet',
83+
1101: 'https://v2.archive.subsquid.io/network/polygon-zkevm-mainnet',
84+
2442: 'https://v2.archive.subsquid.io/network/polygon-zkevm-cardona-testnet',
85+
97072271: 'https://v2.archive.subsquid.io/network/prom-testnet',
86+
584548796: 'https://v2.archive.subsquid.io/network/prom-testnet-v2',
87+
157: 'https://v2.archive.subsquid.io/network/puppynet',
88+
534352: 'https://v2.archive.subsquid.io/network/scroll-mainnet',
89+
534351: 'https://v2.archive.subsquid.io/network/scroll-sepolia',
90+
109: 'https://v2.archive.subsquid.io/network/shibarium',
91+
81: 'https://v2.archive.subsquid.io/network/shibuya-testnet',
92+
336: 'https://v2.archive.subsquid.io/network/shiden-mainnet',
93+
1482601649: 'https://v2.archive.subsquid.io/network/skale-nebula',
94+
64165: 'https://v2.archive.subsquid.io/network/sonic-testnet',
95+
93747: 'https://v2.archive.subsquid.io/network/stratovm-sepolia',
96+
53302: 'https://v2.archive.subsquid.io/network/superseed-sepolia',
97+
167000: 'https://v2.archive.subsquid.io/network/taiko-mainnet',
98+
5678: 'https://v2.archive.subsquid.io/network/tanssi',
99+
196: 'https://v2.archive.subsquid.io/network/xlayer-mainnet',
100+
195: 'https://v2.archive.subsquid.io/network/xlayer-testnet',
101+
810180: 'https://v2.archive.subsquid.io/network/zklink-nova-mainnet',
102+
324: 'https://v2.archive.subsquid.io/network/zksync-mainnet',
103+
300: 'https://v2.archive.subsquid.io/network/zksync-sepolia',
104+
7777777: 'https://v2.archive.subsquid.io/network/zora-mainnet',
105+
999999999: 'https://v2.archive.subsquid.io/network/zora-sepolia',
106+
}
107+
108+
def get_text(url: str) -> str:
109+
res = requests.get(url)
110+
res.raise_for_status()
111+
return res.text
112+
113+
def get_filter(
114+
chain_id: int,
115+
filter_params: FilterParams,
116+
partial_allowed=False
117+
) -> tuple[int, list[LogReceipt]]:
118+
if chain_id not in ENDPOINTS:
119+
raise ValueError(f"Subsquid does not support Chain ID {chain_id}")
120+
121+
gateway_url = ENDPOINTS[chain_id]
122+
123+
assert isinstance(filter_params['fromBlock'], int)
124+
assert isinstance(filter_params['toBlock'], int)
125+
from_block: int = filter_params['fromBlock']
126+
to_block: int = filter_params['toBlock']
127+
128+
latest_block = int(get_text(f"{gateway_url}/height"))
129+
130+
if from_block > latest_block:
131+
raise ValueError(f"Subsquid has only indexed till block {latest_block}")
132+
133+
if to_block > latest_block:
134+
if partial_allowed:
135+
to_block = latest_block
136+
else:
137+
raise ValueError(f"Subsquid has only indexed till block {latest_block}")
138+
139+
query = {
140+
"toBlock": to_block,
141+
"logs": [{}],
142+
"fields": {
143+
"log": {
144+
"address": True,
145+
"topics": True,
146+
"data": True,
147+
"transactionHash": True,
148+
}
149+
}
150+
}
151+
if "address" in filter_params:
152+
addresses = filter_params['address']
153+
if isinstance(addresses, str):
154+
addresses = [addresses]
155+
query["logs"][0]["address"] = [address.lower() for address in addresses]
156+
if "topics" in filter_params:
157+
topics = filter_params["topics"]
158+
assert len(topics) <= 4
159+
for i in range(len(topics)):
160+
topic: str | list[str]
161+
if isinstance(topics[i], str):
162+
topic = [topics[i]]
163+
else:
164+
try:
165+
topic = [single_topic.hex() for single_topic in topics[i]]
166+
except Exception:
167+
topic = [topics[i].hex()]
168+
query["logs"][0][f"topic{i}"] = topic
169+
170+
logs: list[LogReceipt] = []
171+
while from_block <= to_block:
172+
worker_url = get_text(f'{gateway_url}/{from_block}/worker')
173+
174+
query['fromBlock'] = from_block
175+
res = requests.post(worker_url, json=query)
176+
res.raise_for_status()
177+
blocks = res.json()
178+
179+
last_processed_block = blocks[-1]['header']['number']
180+
from_block = last_processed_block + 1
181+
182+
for block in blocks:
183+
for log in block['logs']:
184+
logs.append(LogReceipt(
185+
address=to_checksum_address(log['address']),
186+
blockHash=block["header"]["hash"],
187+
blockNumber=block["header"]["number"],
188+
data=HexBytes(log["data"]),
189+
logIndex=log["logIndex"],
190+
topics=[HexBytes(topic) for topic in log["topics"]],
191+
transactionHash=HexBytes(log["transactionHash"]),
192+
transactionIndex=log["transactionIndex"],
193+
removed=False,
194+
))
195+
return from_block, logs

0 commit comments

Comments
 (0)