Support pagination on getLogs requests (#701)

By default mirror node returns only 25 entries on request and next link for the next entries.

- Use `ILimitOrderParams` by default if none are set. Default: `limit: 100, order: ASC`
- Can pass `ILimitOrderParams` on `eth_getLogs` method, which will override default ones.
- Use next links if there is one and until there is one.
- Add unit and acceptance tests for this functionality.

Signed-off-by: georgi-l95 <[email protected]>

Author: georgi-l95

.DS_Store

@@ -21,3 +21,4 @@ DEV_MODE =
 MIRROR_NODE_RETRIES =
 MIRROR_NODE_RETRY_DELAY =
 GAS_PRICE_TINY_BAR_BUFFER =
+MIRROR_NODE_LIMIT_PARAM =

.env.example

@@ -81,9 +81,10 @@ HBAR_RATE_LIMIT_TINYBAR = 6000000000
 HBAR_RATE_LIMIT_DURATION = 60000
 RATE_LIMIT_DISABLED = false
 DEV_MODE = false
+GAS_PRICE_TINY_BAR_BUFFER = 10000000000
 MIRROR_NODE_RETRIES = 3
 MIRROR_NODE_RETRY_DELAY = 500
-GAS_PRICE_TINY_BAR_BUFFER = 10000000000
+MIRROR_NODE_LIMIT_PARAM = 100
 ```
 
 Note: Read more about `DEV_MODE` [here](docs/dev-mode.md)

README.md

@@ -28,3 +28,6 @@ data:
   ETH_GET_LOGS_BLOCK_RANGE_LIMIT: {{ .Values.config.ETH_GET_LOGS_BLOCK_RANGE_LIMIT }}
   RATE_LIMIT_DISABLED: {{ .Values.config.RATE_LIMIT_DISABLED }}
   DEV_MODE: {{ .Values.config.DEV_MODE }}
+  MIRROR_NODE_RETRIES: {{ .Values.config.MIRROR_NODE_RETRIES }}
+  MIRROR_NODE_RETRY_DELAY: {{ .Values.config.MIRROR_NODE_RETRY_DELAY }}
+  MIRROR_NODE_LIMIT_PARAM: {{ .Values.config.MIRROR_NODE_LIMIT_PARAM }}

helm-chart/templates/configmap.yaml

@@ -155,6 +155,23 @@ spec:
               configMapKeyRef:
                 name: {{ include "json-rpc-relay.fullname" . }}
                 key: GAS_PRICE_TINY_BAR_BUFFER
+          - name: MIRROR_NODE_RETRIES
+            valueFrom:
+              configMapKeyRef:
+                name: {{ include "json-rpc-relay.fullname" . }}
+                key: MIRROR_NODE_RETRIES
+                optional: true
+          - name: MIRROR_NODE_RETRY_DELAY
+            valueFrom:
+              configMapKeyRef:
+                name: {{ include "json-rpc-relay.fullname" . }}
+                key: MIRROR_NODE_RETRY_DELAY
+                optional: true
+          - name: MIRROR_NODE_LIMIT_PARAM
+            valueFrom:
+              configMapKeyRef:
+                name: {{ include "json-rpc-relay.fullname" . }}
+                key: MIRROR_NODE_LIMIT_PARAM
                 optional: true
         ports:
           - containerPort: {{ .Values.ports.containerPort }}

helm-chart/templates/deployment.yaml

@@ -6,11 +6,11 @@ image:
   repository: ghcr.io/hashgraph/hedera-json-rpc-relay
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
-  tag: ""
+  tag: ''
 
 imagePullSecrets: []
-nameOverride: ""
-fullnameOverride: ""
+nameOverride: ''
+fullnameOverride: ''
 
 serviceAccount:
   # Specifies whether a service account should be created
@@ -19,14 +19,16 @@ serviceAccount:
   annotations: {}
   # The name of the service account to use.
   # If not set and create is true, a name is generated using the fullname template
-  name: ""
+  name: ''
 
 podAnnotations: {}
 
-podSecurityContext: {}
+podSecurityContext:
+  {}
   # fsGroup: 2000
 
-securityContext: {}
+securityContext:
+  {}
   # capabilities:
   #   drop:
   #   - ALL
@@ -42,8 +44,9 @@ service:
 
 ingress:
   enabled: false
-  className: ""
-  annotations: {}
+  className: ''
+  annotations:
+    {}
     # kubernetes.io/ingress.class: nginx
     # kubernetes.io/tls-acme: "true"
   hosts:
@@ -56,7 +59,8 @@ ingress:
   #    hosts:
   #      - chart-example.local
 
-resources: {}
+resources:
+  {}
   # It's recommended not to specify default resources and to leave this as a choice for the user.
   # This also increases chances charts run on environments with little resources, such as Minikube.
   #  If you do want to specify resources, uncomment use the following
@@ -87,9 +91,9 @@ ports:
 
 config:
   # This value should be in decimal, not hexadecimal format.
-  CHAIN_ID: ""
-  MIRROR_NODE_URL: ""
-  LOG_LEVEL: ""
+  CHAIN_ID: ''
+  MIRROR_NODE_URL: ''
+  LOG_LEVEL: ''
   # Use config.local.HEDERA_NETWORK when running against a locally hosted hedera network
   # local:
   #   config.local.HEDERA_NETWORK should configured as a json set - {"$IPv4_ADDR_1:$PORT_1":"$ACCOUNT_ID_1","$IPv4_ADDR_2:$PORT_2":"$ACCOUNT_ID_2"}
@@ -98,10 +102,10 @@ config:
   # Use config.hosted.HEDERA_NETWORK when running against a hosted Hedera network.  Valid options are `previewnet`, `testnet`, and `mainnet`
   # hosted:
   #   HEDERA_NETWORK: ""
-  OPERATOR_ID_MAIN: ""
-  OPERATOR_KEY_MAIN: ""
-  OPERATOR_ID_ETH_SENDRAWTRANSACTION: ""
-  OPERATOR_KEY_ETH_SENDRAWTRANSACTION: ""
+  OPERATOR_ID_MAIN: ''
+  OPERATOR_KEY_MAIN: ''
+  OPERATOR_ID_ETH_SENDRAWTRANSACTION: ''
+  OPERATOR_KEY_ETH_SENDRAWTRANSACTION: ''
   DEFAULT_RATE_LIMIT: 200
   TIER_1_RATE_LIMIT: 100
   TIER_2_RATE_LIMIT: 200
@@ -113,9 +117,11 @@ config:
   RATE_LIMIT_DISABLED: false
   DEV_MODE: false
   GAS_PRICE_TINY_BAR_BUFFER: 10000000000
+  MIRROR_NODE_RETRIES: 3
+  MIRROR_NODE_RETRY_DELAY: 500
+  MIRROR_NODE_LIMIT_PARAM: 100
 
 # Enable rolling_restarts if SDK calls fail this is usually due to stale connections that get cycled on restart
 rolling_restart:
   enabled: false
-  schedule: "@daily"
-
+  schedule: '@daily'

helm-chart/values.yaml

@@ -339,6 +339,36 @@ export class MirrorNodeClient {
             requestId);
     }
 
+    public async getContractResultsLogsByNextLink(
+        link: string,
+        requestId?: string
+    ) {
+        const nextLink = link.replace(constants.NEXT_LINK_PREFIX, '');
+        return this.request(`${nextLink}`,
+        MirrorNodeClient.GET_CONTRACT_RESULT_LOGS_ENDPOINT,
+        [400, 404],
+        requestId);
+    }
+
+    public async pageAllResults(
+        result: any,
+        requestId?: string
+    ) {
+        let unproccesedLogs = result.logs;
+        if (result.links && result.links.next) {
+            let nextLink = result.links.next;
+            while (nextLink) {
+                let nextResult = await this.getContractResultsLogsByNextLink(nextLink, requestId);
+                if (!nextResult || !nextResult.logs) {
+                  break;
+                }
+                unproccesedLogs = unproccesedLogs.concat(nextResult.logs);
+                nextLink = nextResult.links.next;
+              }
+        }
+        return unproccesedLogs;
+    }
+
     public async getLatestBlock(requestId?: string) {
         return this.getBlocks(undefined, undefined, this.getLimitOrderQueryParam(1, MirrorNodeClient.ORDER.DESC), requestId);
     }
@@ -423,6 +453,9 @@ export class MirrorNodeClient {
         if (limitOrderParams) {
             this.setQueryParam(queryParamObject, 'limit', limitOrderParams.limit);
             this.setQueryParam(queryParamObject, 'order', limitOrderParams.order);
+        } else {
+            this.setQueryParam(queryParamObject, 'limit', parseInt(process.env.MIRROR_NODE_LIMIT_PARAM!) || 100);
+            this.setQueryParam(queryParamObject, 'order', constants.ORDER.ASC);
         }
     }
 

packages/relay/src/lib/clients/mirrorNodeClient.ts

@@ -59,5 +59,7 @@ export default {
     TX_CREATE_EXTRA: 32_000,
     TX_DATA_ZERO_COST: 4,
     REQUEST_ID_STRING: `Request ID: `,
-    BALANCES_UPDATE_INTERVAL: 900   // 15 minutes
+    BALANCES_UPDATE_INTERVAL: 900,   // 15 minutes
+    MIRROR_NODE_QUERY_LIMIT: 100,
+    NEXT_LINK_PREFIX: '/api/v1/'
 };

packages/relay/src/lib/constants.ts

@@ -1359,8 +1359,10 @@ export class EthImpl implements Eth {
       return [];
     }
 
+    let unproccesedLogs = await this.mirrorNodeClient.pageAllResults(result, requestId);
+
     const logs: Log[] = [];
-    for(const log of result.logs) {
+    for(const log of unproccesedLogs) {
       logs.push(
         new Log({
           address: await this.getLogEvmAddress(log.address, requestId) || log.address,