Reapplying refactor/enhancement in a fresh branch off of main since t… (#1186)

Reapplying refactor/enhancement in a fresh branch off of main since t… (#1164)

This PR optimizes the getBalance method by removing unnecessary calls to the mirror node.

---------

Signed-off-by: ebadiere <[email protected]>
Signed-off-by: Nana Essilfie-Conduah <[email protected]>
Co-authored-by: Nana Essilfie-Conduah <[email protected]>

Author: ebadiere

.github/workflows/dapp.yml

@@ -11,7 +11,7 @@ jobs:
   setup-local-hedera:
     name: Dapp Tests
     runs-on: ubuntu-latest
-    timeout-minutes: 35
+    timeout-minutes: 35 # Set to 35 minutes for now
     permissions:
       contents: write
     steps:

packages/relay/src/lib/clients/mirrorNodeClient.ts

@@ -129,6 +129,7 @@ export class MirrorNodeClient {
     private mirrorResponseHistogram;
 
     private readonly cache;
+    static readonly EVM_ADDRESS_REGEX: RegExp = /\/accounts\/([\d\.]+)/;    
 
     protected createAxiosClient(
         baseUrl: string
@@ -342,6 +343,49 @@ export class MirrorNodeClient {
             MirrorNodeClient.GET_ACCOUNTS_ENDPOINT,
             requestId);
     }
+    /*******************************************************************************
+     * To be used to make paginated calls for the account information when the 
+     * transaction count exceeds the constant MIRROR_NODE_QUERY_LIMIT.
+     *******************************************************************************/
+    public async getAccountPaginated(url: string, requestId?: string) {
+        const queryParamObject = {};
+        const accountId = this.extractAccountIdFromUrl(url, requestId);
+        const params = new URLSearchParams(url.split('?')[1]);
+        
+        this.setQueryParam(queryParamObject, 'limit', constants.MIRROR_NODE_QUERY_LIMIT);
+        this.setQueryParam(queryParamObject, 'timestamp', params.get('timestamp'));
+        const queryParams = this.getQueryParams(queryParamObject);
+
+        return this.getPaginatedResults(
+            `${MirrorNodeClient.GET_ACCOUNTS_ENDPOINT}${accountId}${queryParams}`,
+            MirrorNodeClient.GET_ACCOUNTS_ENDPOINT,
+            'transactions',
+            requestId
+        );
+    }
+
+    public extractAccountIdFromUrl(url: string, requestId?: string): string | null {
+        const substringStartIndex = url.indexOf("/accounts/") + "/accounts/".length;
+        if (url.startsWith("0x", substringStartIndex)) {
+            // evm addresss
+            const regex = /\/accounts\/(0x[a-fA-F0-9]{40})/;
+            const match = url.match(regex);
+            const accountId = match ? match[1] : null;
+            if (!accountId) {
+                this.logger.error(`${formatRequestIdMessage(requestId)} Unable to extract evm address from url ${url}`);
+            }
+            return String(accountId);
+        } else {
+            // account id 
+            const match = url.match(MirrorNodeClient.EVM_ADDRESS_REGEX);
+            const accountId = match ? match[1] : null;
+            if (!accountId) {
+                this.logger.error(`${formatRequestIdMessage(requestId)} Unable to extract account ID from url ${url}`);
+            }
+            return String(accountId);
+        }
+    }
+
 
     public async getTransactionsForAccount(accountId: string, timestampFrom: string, timestampTo: string, requestId?: string) {
         const queryParamObject = {};

packages/relay/src/lib/eth.ts

@@ -2,7 +2,7 @@
  *
  * Hedera JSON RPC Relay
  *
- * Copyright (C) 2022 Hedera Hashgraph, LLC
+ * Copyright (C) 2023 Hedera Hashgraph, LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,6 +35,10 @@ import crypto from 'crypto';
 const LRU = require('lru-cache');
 const _ = require('lodash');
 const createHash = require('keccak');
+interface LatestBlockNumberTimestamp {
+  blockNumber: string;
+  timeStampTo: string;
+}
 
 /**
  * Implementation of the "eth_" methods from the Ethereum JSON-RPC API.
@@ -378,6 +382,35 @@ export class EthImpl implements Eth {
     throw predefined.COULD_NOT_RETRIEVE_LATEST_BLOCK;
   }
 
+  /**
+   * Gets the most recent block number and timestamp.to which represents the block finality.
+   */
+  async blockNumberTimestamp(requestId?: string): Promise<LatestBlockNumberTimestamp> {
+    const requestIdPrefix = formatRequestIdMessage(requestId);
+    this.logger.trace(`${requestIdPrefix} blockNumber()`);
+
+    const cacheKey = `${constants.CACHE_KEY.ETH_BLOCK_NUMBER}`;
+   
+    const blocksResponse = await this.mirrorNodeClient.getLatestBlock(requestId);
+    const blocks = blocksResponse !== null ? blocksResponse.blocks : null;
+    if (Array.isArray(blocks) && blocks.length > 0) {
+      const currentBlock = EthImpl.numberTo0x(blocks[0].number);
+      const timestamp = blocks[0].timestamp.to;
+      const blockTimeStamp: LatestBlockNumberTimestamp = { blockNumber: currentBlock, timeStampTo: timestamp };
+      // save the latest block number in cache
+      this.cache.set(cacheKey, currentBlock, { ttl: EthImpl.ethBlockNumberCacheTtlMs });
+      this.logger.trace(
+        `${requestIdPrefix} caching ${cacheKey}:${JSON.stringify(currentBlock)}:${JSON.stringify(timestamp)} for ${
+          EthImpl.ethBlockNumberCacheTtlMs
+        } ms`
+      );
+
+      return blockTimeStamp;
+    }
+
+    throw predefined.COULD_NOT_RETRIEVE_LATEST_BLOCK;
+  }
+  
   /**
    * Gets the chain ID. This is a static value, in that it always returns
    * the same value. This can be specified via an environment variable
@@ -624,15 +657,31 @@ export class EthImpl implements Eth {
     const latestBlockTolerance = 1;
     this.logger.trace(`${requestIdPrefix} getBalance(account=${account}, blockNumberOrTag=${blockNumberOrTag})`);
 
+    let latestBlock: LatestBlockNumberTimestamp | null | undefined;
     // this check is required, because some tools like Metamask pass for parameter latest block, with a number (ex 0x30ea)
     // tolerance is needed, because there is a small delay between requesting latest block from blockNumber and passing it here
     if (!EthImpl.blockTagIsLatestOrPending(blockNumberOrTag)) {
-      const latestBlock = await this.blockNumber(requestId);
-      const blockDiff = Number(latestBlock) - Number(blockNumberOrTag);
+      const cacheKey = `${constants.CACHE_KEY.ETH_BLOCK_NUMBER}`;
+      const blockNumberCached = this.cache.get(cacheKey);
+      
+      if(blockNumberCached) {
+        this.logger.trace(`${requestIdPrefix} returning cached value ${cacheKey}:${JSON.stringify(blockNumberCached)}`);
+        latestBlock = { blockNumber: blockNumberCached, timeStampTo: '0' };
+
+      } else {
+        latestBlock = await this.blockNumberTimestamp(requestId);
+      }
+      const blockDiff = Number(latestBlock.blockNumber) - Number(blockNumberOrTag);
 
       if (blockDiff <= latestBlockTolerance) {
         blockNumberOrTag = EthImpl.blockLatest;
       }
+
+      // If ever we get the latest block from cache, and blockNumberOrTag is not latest, then we need to get the block timestamp
+      // This should rarely happen.
+      if((blockNumberOrTag !== EthImpl.blockLatest) && (latestBlock.timeStampTo === "0")) {
+        latestBlock = await this.blockNumberTimestamp(requestId);
+      }
     }
 
     // check cache first
@@ -657,71 +706,52 @@ export class EthImpl implements Eth {
 
           // A blockNumberOrTag has been provided. If it is `latest` or `pending` retrieve the balance from /accounts/{account.id}
           if (mirrorAccount) {
-            const latestBlock = await this.getHistoricalBlockResponse(EthImpl.blockLatest, true, requestId);
-
             // If the parsed blockNumber is the same as the one from the latest block retrieve the balance from /accounts/{account.id}
-            if (latestBlock && block.number !== latestBlock.number) {
-              const latestTimestamp = Number(latestBlock.timestamp.from.split('.')[0]);
+            if (latestBlock && block.number !== latestBlock.blockNumber) {
+              const latestTimestamp = Number(latestBlock.timeStampTo.split('.')[0]);
               const blockTimestamp = Number(block.timestamp.from.split('.')[0]);
               const timeDiff = latestTimestamp - blockTimestamp;
               // The block is from the last 15 minutes, therefore the historical balance hasn't been imported in the Mirror Node yet
               if (timeDiff < constants.BALANCES_UPDATE_INTERVAL) {
                 let currentBalance = 0;
-                let currentTimestamp;
                 let balanceFromTxs = 0;
                 if (mirrorAccount.balance) {
                   currentBalance = mirrorAccount.balance.balance;
-                  currentTimestamp = mirrorAccount.balance.timestamp;
                 }
 
-                // Need to check if there are any transactions before the block.timestamp.to in the current account set returned from the inital
-                // call to getAccountPageLimit.  If there are we may need to paginate.
-                let lastTransactionOnPageTimestamp;
-                if(mirrorAccount.links.next !== null) {
-                  // Get the end of the page of transactions timestamp
-                  const params = new URLSearchParams(mirrorAccount.links.next.split('?')[1]);
-                  if((params === null) || (params === undefined)) {
-                    this.logger.debug(`${requestIdPrefix} Unable to find expected search parameters in account next page link ${mirrorAccount.links.next}), returning 0x0 balance`);
-                    return EthImpl.zeroHex;
-                  }
-
-                  const timestampParameters = params.getAll('timestamp'); 
-                  lastTransactionOnPageTimestamp = timestampParameters[0].split(':')[1];
-                  if((lastTransactionOnPageTimestamp === null) || (lastTransactionOnPageTimestamp === undefined)) {
-                    this.logger.debug(`${requestIdPrefix} Unable to find expected beginning (gte:) timestamp in account next page link ${mirrorAccount.links.next}), returning 0x0 balance`);
-                    return EthImpl.zeroHex;
-                  }
-                }
-
-                let transactionsInTimeWindow: any = [];
-                if((typeof lastTransactionOnPageTimestamp !== "undefined") && (mirrorAccount.transactions[mirrorAccount.transactions.length -1].consensus_timestamp >= lastTransactionOnPageTimestamp)) {
-                  transactionsInTimeWindow = await this.mirrorNodeClient.getTransactionsForAccount(
-                    mirrorAccount.account,
-                    block.timestamp.to,
-                    currentTimestamp,
-                    requestId
-                  );
-                } else {
-                    transactionsInTimeWindow = mirrorAccount.transactions.filter((tx: any) => {
-                    return tx.consensus_timestamp >= block.timestamp.to && tx.consensus_timestamp <= currentTimestamp;
-                  });
-                }
-
-                for(const tx of transactionsInTimeWindow) {
-                  for (const transfer of tx.transfers) {
-                    if (transfer.account === mirrorAccount.account && !transfer.is_approval) {
-                      balanceFromTxs += transfer.amount;
-                    }
+                // The balance in the account is real time, so we simply subtract the transactions to the block.timestamp.to to get a block relevant balance.
+                // needs to be updated below.
+                const nextPage: string = mirrorAccount.links.next;
+
+                if(nextPage) {
+                  // If we have a pagination link that falls within the block.timestamp.to, we need to paginate to get the transactions for the block.timestamp.to
+                  const nextPageParams = new URLSearchParams(nextPage.split('?')[1]);
+                  const nextPageTimeMarker = nextPageParams.get('timestamp');
+                  if (nextPageTimeMarker && nextPageTimeMarker?.split(':')[1] >= block.timestamp.to) {
+                    // If nextPageTimeMarker is greater than the block.timestamp.to, then we need to paginate to get the transactions for the block.timestamp.to
+                    const pagedTransactions = await this.mirrorNodeClient.getAccountPaginated(nextPage, requestId);
+                    mirrorAccount.transactions = mirrorAccount.transactions.concat(pagedTransactions);
                   }
+                  // If nextPageTimeMarker is less than the block.timestamp.to, then just run the getBalanceAtBlockTimestamp function in this case as well.
                 }
 
+                balanceFromTxs = this.getBalanceAtBlockTimestamp(
+                  mirrorAccount.account,
+                  mirrorAccount.transactions,
+                  block.timestamp.to
+                );
+                
                 balanceFound = true;
                 weibars = BigInt(currentBalance - balanceFromTxs) * BigInt(constants.TINYBAR_TO_WEIBAR_COEF);
               }
 
               // The block is NOT from the last 15 minutes, use /balances rest API
               else {
-                const balance = await this.mirrorNodeClient.getBalanceAtTimestamp(mirrorAccount.account, block.timestamp.from, requestId);
+                const balance = await this.mirrorNodeClient.getBalanceAtTimestamp(
+                  mirrorAccount.account,
+                  block.timestamp.from,
+                  requestId
+                );
                 balanceFound = true;
                 if (balance.balances?.length) {
                   weibars = BigInt(balance.balances[0].balance) * BigInt(constants.TINYBAR_TO_WEIBAR_COEF);
@@ -731,20 +761,28 @@ export class EthImpl implements Eth {
           }
         }
       }
-
+      
       if (!balanceFound && mirrorAccount?.balance) {
         balanceFound = true;
         weibars = BigInt(mirrorAccount.balance.balance) * BigInt(constants.TINYBAR_TO_WEIBAR_COEF);
       }
 
       if (!balanceFound) {
-        this.logger.debug(`${requestIdPrefix} Unable to find account ${account} in block ${JSON.stringify(blockNumber)}(${blockNumberOrTag}), returning 0x0 balance`);
+        this.logger.debug(
+          `${requestIdPrefix} Unable to find account ${account} in block ${JSON.stringify(
+            blockNumber
+          )}(${blockNumberOrTag}), returning 0x0 balance`
+        );
         return EthImpl.zeroHex;
       }
 
       // save in cache the current balance for the account and blockNumberOrTag
-      this.cache.set(cacheKey, EthImpl.numberTo0x(weibars), {ttl: EthImpl.ethGetBalanceCacheTtlMs});
-      this.logger.trace(`${requestIdPrefix} caching ${cacheKey}:${JSON.stringify(cachedBalance)} for ${EthImpl.ethGetBalanceCacheTtlMs} ms`);
+      this.cache.set(cacheKey, EthImpl.numberTo0x(weibars), { ttl: EthImpl.ethGetBalanceCacheTtlMs });
+      this.logger.trace(
+        `${requestIdPrefix} caching ${cacheKey}:${JSON.stringify(cachedBalance)} for ${
+          EthImpl.ethGetBalanceCacheTtlMs
+        } ms`
+      );
 
       return EthImpl.numberTo0x(weibars);
     } catch (error: any) {
@@ -1722,4 +1760,26 @@ export class EthImpl implements Eth {
     return predefined.INTERNAL_ERROR();
   }
 
+  /**************************************************
+   * Returns the difference between the balance of  *
+   * the account and the transactions summed up     *
+   * to the block number queried.                   *
+   *************************************************/
+  getBalanceAtBlockTimestamp(account: string, transactions: any[], blockTimestamp: number) {
+    return transactions
+      .filter((transaction) => {
+        return transaction.consensus_timestamp >= blockTimestamp;
+      })
+      .flatMap((transaction) => {
+        return transaction.transfers.filter((transfer) => {
+          return transfer.account === account && !transfer.is_approval;
+        });
+      })
+      .map((transfer) => {
+        return transfer.amount;
+      })
+      .reduce((total, amount) => {
+        return total + amount;
+      }, 0);
+  }
 }