feat: chunkedQueryFilter

Author: meetmangukiya

index.ts

@@ -57,6 +57,14 @@ const readConfig = async (): Promise<
       new Option('--multicall-size <n>', 'max number of calls in a multicall')
         .default(100)
         .argParser((x) => +x)
+    )
+    .addOption(
+      new Option(
+        '--query-logs-size <n>',
+        'max block range to use for eth_queryLogs'
+      )
+        .default(50000)
+        .argParser((x) => +x)
     );
   program.parse();
 
@@ -69,6 +77,7 @@ const readConfig = async (): Promise<
     module: selectedModule,
     lookbackRange,
     multicallSize,
+    queryLogsSize,
   } = options;
   const network = selectedNetwork || 'mainnet';
 
@@ -124,6 +133,7 @@ const readConfig = async (): Promise<
       lookbackRange,
       targetSafe,
       multicallSize,
+      queryLogsSize,
     },
     provider,
   ];

ts/common.ts

@@ -34,6 +34,7 @@ export interface IConfig {
   appData: string;
   lookbackRange: number;
   multicallSize: number;
+  queryLogsSize: number;
 }
 
 const toChainId = (network: keyof typeof networkSpecificConfigs) => {
@@ -88,6 +89,26 @@ export const chunkedMulticall = async (
   return ret;
 };
 
+export const chunkedQueryFilter = async (
+  contract: ethers.Contract,
+  filter: ethers.EventFilter,
+  fromBlock: number,
+  toBlock: number,
+  chunkSize: number
+) => {
+  const ret = [];
+  for (let i = fromBlock; i <= toBlock; i += chunkSize) {
+    ret.push(
+      ...(await contract.queryFilter(
+        filter,
+        i,
+        Math.min(i + chunkSize - 1, toBlock)
+      ))
+    );
+  }
+  return ret;
+};
+
 const logger = pino();
 export const getLogger = (name: string) => {
   return logger.child({ name, level: process.env.LOG_LEVEL || 'info' });

ts/token-fetcher.ts

@@ -2,14 +2,15 @@ import axios from 'axios';
 import {
   IConfig,
   chunkedMulticall,
+  chunkedQueryFilter,
   getLogger,
   getMulticall3,
   getOrderbookApi,
 } from './common';
 import { BigNumber, ethers } from 'ethers';
 import { erc20Abi, settlementAbi } from './abi';
 
-const logger = getLogger('explorer-apis');
+const logger = getLogger('token-fetcher');
 
 interface ITokenInfo {
   address: string;
@@ -60,10 +61,12 @@ export const getTokenInfosFromChain = async (
   const currentBlock = await provider.getBlockNumber();
   const fromBlock = currentBlock - config.lookbackRange;
   logger.info(`querying trades from ${fromBlock} to ${currentBlock}`);
-  const logs = await settlement.queryFilter(
+  const logs = await chunkedQueryFilter(
+    settlement,
     tradeFilter,
     currentBlock - config.lookbackRange,
-    currentBlock
+    currentBlock,
+    config.queryLogsSize
   );
   logger.info(`Found ${logs.length} trades`);
   const allTokens = Array.from(