Skip to content

Commit 6b5e47c

Browse files
vrtndvrtnd
committed
imprve connection and cache
1 parent 24a5b32 commit 6b5e47c

File tree

6 files changed

+106
-90
lines changed

6 files changed

+106
-90
lines changed

‎src/adapters/retrobridge/index.ts‎

Lines changed: 45 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -113,80 +113,54 @@ const constructParams = (chain: SupportedChains) => {
113113
return async (fromBlock: number, toBlock: number) => {
114114
const eventLogData = await getTxDataFromEVMEventLogs("retrobridge", chain as Chain, fromBlock, toBlock, eventParams);
115115

116-
const nativeEvents = await Promise.all([
117-
...bridgeAddress.map(async (address: string, i: number) => {
118-
await wait(300 * i); // for etherscan
119-
let txs: any[] = [];
120-
if (chain === "merlin") {
121-
txs = await getTxsBlockRangeMerlinScan(address, fromBlock, toBlock, {
122-
includeSignatures: ["0x"],
123-
});
124-
} else if (chain === "btr") {
125-
txs = await getTxsBlockRangeBtrScan(address, fromBlock, toBlock, {
126-
includeSignatures: ["0x"],
127-
});
128-
} else {
129-
txs = await getTxsBlockRangeEtherscan(chain, address, fromBlock, toBlock, {
130-
includeSignatures: ["0x"],
131-
});
132-
}
133-
const eventsRes: EventData[] = txs.map((tx: any) => {
134-
const event: EventData = {
135-
txHash: tx.hash,
136-
blockNumber: +tx.blockNumber,
137-
from: tx.from,
138-
to: tx.to,
139-
token: nativeTokens[chain],
140-
amount: tx.value,
141-
isDeposit: address.toLowerCase() === tx.to,
142-
};
143-
return event;
144-
});
116+
const allAddresses = [...bridgeAddress, ...contractAddress];
117+
const nativeEvents: EventData[][] = [];
118+
119+
for (let i = 0; i < allAddresses.length; i++) {
120+
await wait(500);
121+
const address = allAddresses[i];
122+
const isBridgeAddress = i < bridgeAddress.length;
145123

146-
return eventsRes;
147-
}),
148-
...contractAddress.map(async (address: string, i: number) => {
149-
await wait(300 * i); // for etherscan
150-
let txs: any[] = [];
151-
if (chain === "merlin") {
152-
txs = await getTxsBlockRangeMerlinScan(address, fromBlock, toBlock, {
153-
includeSignatures: [
154-
"0x769254a71d2f67d8ac6cb44f2803c0d05cfbcf9effadb6a984f10ff9de3df6c3",
155-
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
156-
],
157-
});
158-
} else if (chain === "btr") {
159-
txs = await getTxsBlockRangeBtrScan(address, fromBlock, toBlock, {
160-
includeSignatures: [
161-
"0x769254a71d2f67d8ac6cb44f2803c0d05cfbcf9effadb6a984f10ff9de3df6c3",
162-
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
163-
],
164-
});
165-
} else {
166-
txs = await getTxsBlockRangeEtherscan(chain, address, fromBlock, toBlock, {
167-
includeSignatures: [
168-
"0x769254a71d2f67d8ac6cb44f2803c0d05cfbcf9effadb6a984f10ff9de3df6c3",
169-
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
170-
],
171-
});
172-
}
173-
const eventsRes: EventData[] = txs.filter((tx: any) => String(tx.value) != "0").map((tx: any) => {
174-
const event: EventData = {
175-
txHash: tx.hash,
176-
blockNumber: +tx.blockNumber,
177-
from: tx.from,
178-
to: tx.to,
179-
token: nativeTokens[chain],
180-
amount: ethers.BigNumber.from(tx.value),
181-
isDeposit: address.toLowerCase() === tx.to,
182-
};
183-
return event;
124+
let txs: any[] = [];
125+
if (chain === "merlin") {
126+
txs = await getTxsBlockRangeMerlinScan(address, fromBlock, toBlock, {
127+
includeSignatures: isBridgeAddress ? ["0x"] : [
128+
"0x769254a71d2f67d8ac6cb44f2803c0d05cfbcf9effadb6a984f10ff9de3df6c3",
129+
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
130+
],
131+
});
132+
} else if (chain === "btr") {
133+
txs = await getTxsBlockRangeBtrScan(address, fromBlock, toBlock, {
134+
includeSignatures: isBridgeAddress ? ["0x"] : [
135+
"0x769254a71d2f67d8ac6cb44f2803c0d05cfbcf9effadb6a984f10ff9de3df6c3",
136+
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
137+
],
184138
});
139+
} else {
140+
txs = await getTxsBlockRangeEtherscan(chain, address, fromBlock, toBlock, {
141+
includeSignatures: isBridgeAddress ? ["0x"] : [
142+
"0x769254a71d2f67d8ac6cb44f2803c0d05cfbcf9effadb6a984f10ff9de3df6c3",
143+
"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
144+
],
145+
});
146+
}
147+
148+
const filteredTxs = isBridgeAddress ? txs : txs.filter((tx: any) => String(tx.value) != "0");
149+
const eventsRes: EventData[] = filteredTxs.map((tx: any) => {
150+
const event: EventData = {
151+
txHash: tx.hash,
152+
blockNumber: +tx.blockNumber,
153+
from: tx.from,
154+
to: tx.to,
155+
token: nativeTokens[chain],
156+
amount: isBridgeAddress ? tx.value : ethers.BigNumber.from(tx.value),
157+
isDeposit: address.toLowerCase() === tx.to,
158+
};
159+
return event;
160+
});
185161

186-
return eventsRes;
187-
})
188-
]
189-
);
162+
nativeEvents.push(eventsRes);
163+
}
190164
const allEvents = [...eventLogData, ...nativeEvents.flat()];
191165
return allEvents;
192166
};

‎src/helpers/etherscan.ts‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -63,21 +63,43 @@ export const getTxsBlockRangeEtherscan = async (
6363
if (!apiKey) {
6464
res = (
6565
await retry(
66-
() =>
67-
axios.get(
66+
async () => {
67+
const response = await axios.get(
6868
`${endpoint}/api?module=account&action=txlist&address=${address}&startblock=${startBlock}&endblock=${endBlock}`
69-
),
70-
{ factor: 1, retries: 3 }
69+
);
70+
if (response.data?.message?.includes("rate limit")) {
71+
throw new Error("Rate limit exceeded");
72+
}
73+
return response;
74+
},
75+
{
76+
retries: 5,
77+
factor: 2,
78+
minTimeout: 1000,
79+
maxTimeout: 30000,
80+
randomize: true
81+
}
7182
)
7283
).data as any;
7384
} else {
7485
res = (
7586
await retry(
76-
() =>
77-
axios.get(
87+
async () => {
88+
const response = await axios.get(
7889
`${endpoint}/api?module=account&action=txlist&address=${address}&startblock=${startBlock}&endblock=${endBlock}&apikey=${apiKey}`
79-
),
80-
{ factor: 1, retries: 3 }
90+
);
91+
if (response.data?.message?.includes("rate limit")) {
92+
throw new Error("Rate limit exceeded");
93+
}
94+
return response;
95+
},
96+
{
97+
retries: 5,
98+
factor: 2,
99+
minTimeout: 1000,
100+
maxTimeout: 30000,
101+
randomize: true
102+
}
81103
)
82104
).data as any;
83105
}

‎src/utils/adapter.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,8 @@ export const runAdapterToCurrentBlock = async (
146146
}
147147

148148
const adapterPromises = Promise.all(
149-
Object.keys(adapter).map(async (chain) => {
149+
Object.keys(adapter).map(async (chain, i) => {
150+
await wait(200 * i);
150151
const chainContractsAreOn = chainMappings[chain as Chain] ? chainMappings[chain as Chain] : chain;
151152

152153
let bridgeID: string;
@@ -272,7 +273,7 @@ export const runAllAdaptersToCurrentBlock = async (
272273
await insertConfigEntriesForAdapter(adapter, bridgeDbName, bridgeNetwork?.destinationChain);
273274
const adapterPromises = Promise.all(
274275
Object.keys(adapter).map(async (chain, i) => {
275-
await wait(100 * i); // attempt to space out API calls
276+
await wait(200 * i);
276277
const chainContractsAreOn = chainMappings[chain as Chain] ? chainMappings[chain as Chain] : chain;
277278
const { startBlock, endBlock, useRecordedBlocks } = await getBlocksForRunningAdapter(
278279
bridgeDbName,
@@ -325,7 +326,7 @@ export const runAllAdaptersTimestampRange = async (
325326
await insertConfigEntriesForAdapter(adapter, bridgeDbName, bridgeNetwork?.destinationChain);
326327
const adapterPromises = Promise.all(
327328
Object.keys(adapter).map(async (chain, i) => {
328-
await wait(100 * i); // attempt to space out API calls
329+
await wait(200 * i);
329330
const chainContractsAreOn = chainMappings[chain as Chain] ? chainMappings[chain as Chain] : chain;
330331
if (chainContractsAreOn === "tron" || chainContractsAreOn === "sui" || chainContractsAreOn === "solana") {
331332
console.info(`Skipping running adapter ${bridgeDbName} on chain ${chainContractsAreOn}.`);

‎src/utils/aggregate.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,10 @@ export const aggregateData = async (
439439
Array.from(tokensWithNullPrices).map(async (token: string) => {
440440
try {
441441
const [chain, tokenAddress] = token.split(":");
442+
if (chain === "solana") {
443+
await insertOrUpdateTokenWithoutPrice(token, "SOLANA_TOKEN");
444+
return;
445+
}
442446
const tokenSymbol = (await sdk.api.erc20.symbol(tokenAddress, chain)).output;
443447
await insertOrUpdateTokenWithoutPrice(token, tokenSymbol);
444448
} catch (e) {

‎src/utils/cache.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,9 @@ export const DEFAULT_TTL = 60 * 70; // 70 minutes
4242

4343
export const needsWarming = async (cacheKey: string): Promise<boolean> => {
4444
const ttl = await redis.ttl(cacheKey);
45+
if (ttl === -2) return true;
4546
if (ttl === -1) return false;
46-
return true;
47+
return ttl * 1000 < CACHE_WARM_THRESHOLD;
4748
};
4849

4950
export const warmCache = async (cacheKey: string): Promise<void> => {
@@ -68,7 +69,11 @@ export const getCacheKey = (...parts: (string | undefined)[]) => parts.filter(Bo
6869

6970
export const getCache = async (key: string): Promise<any> => {
7071
const value = await redis.get(key);
71-
console.log("Cache HIT", key);
72+
if (value) {
73+
console.log("Cache HIT", key);
74+
} else {
75+
console.log("Cache MISS", key);
76+
}
7277
return value ? JSON.parse(value) : null;
7378
};
7479

‎src/utils/db.js‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,26 @@ const connectionString =
77
process.env.DB_URL ??
88
`postgresql://${process.env.PSQL_USERNAME}:${process.env.PSQL_PW}@${process.env.PSQL_URL}:5433/postgres`;
99

10-
const sql = postgres(connectionString, {
11-
idle_timeout: 20,
10+
let sql = postgres(connectionString, {
11+
idle_timeout: 120,
1212
max_lifetime: 60 * 30,
13-
max: 7,
13+
max: 10,
14+
connect_timeout: 30,
15+
keep_alive: true,
16+
onclose: async function (connId) {
17+
console.log(`[WARN] Connection ${connId} closed unexpectedly`);
18+
}
1419
});
1520

16-
const querySql = postgres(connectionString, {
17-
idle_timeout: 20,
21+
let querySql = postgres(connectionString, {
22+
idle_timeout: 120,
1823
max_lifetime: 60 * 30,
19-
max: 4,
24+
max: 6,
25+
connect_timeout: 30,
26+
keep_alive: true,
27+
onclose: async function (connId) {
28+
console.log(`[WARN] Query connection ${connId} closed unexpectedly`);
29+
}
2030
});
2131

2232
export { sql, querySql };

0 commit comments

Comments
 (0)