update indexers to use block proxy

Author: bayulaksana479

apps/indexer-balance/package.json

@@ -24,7 +24,7 @@
     "@types/lodash-es": "~4.17",
     "@types/node": "~22.10",
     "eslint-config-custom-node": "*",
-    "nb-blocks": "*",
+    "nb-neardata": "*",
     "nb-knex": "*",
     "nb-logger": "*",
     "nb-tsconfig": "*",

apps/indexer-balance/src/config.ts

@@ -11,30 +11,25 @@ const env = cleanEnv(process.env, {
   DATABASE_KEY: str({ default: '' }),
   DATABASE_URL: str(),
   DATABASE_URL_READ: str({ default: '' }),
+  NEARDATA_URL: url(),
   NETWORK: str({
     choices: [Network.MAINNET, Network.TESTNET],
   }),
-  S3_ACCESS_KEY: str(),
-  S3_BUCKET: str({ default: 'nearblocks' }),
-  S3_ENDPOINT: url(),
-  S3_REGION: str({ default: '' }),
-  S3_SECRET_KEY: str(),
   SENTRY_DSN: str({ default: '' }),
 });
 
+const genesisHeight = env.NETWORK === Network.MAINNET ? 9_820_210 : 42_376_888;
+
 const config: Config = {
   dbCa: env.DATABASE_CA,
   dbCert: env.DATABASE_CERT,
   dbKey: env.DATABASE_KEY,
   dbUrl: env.DATABASE_URL,
   dbUrlRead: env.DATABASE_URL_READ,
+  genesisHeight,
   indexerKey: 'balance',
+  neardataUrl: env.NEARDATA_URL,
   network: env.NETWORK,
-  s3AccessKey: env.S3_ACCESS_KEY,
-  s3Bucket: env.S3_BUCKET,
-  s3Endpoint: env.S3_ENDPOINT,
-  s3Region: env.S3_REGION,
-  s3SecretKey: env.S3_SECRET_KEY,
   sentryDsn: env.SENTRY_DSN,
   startBlockHeight: env.BALANCE_START_BLOCK,
 };

apps/indexer-balance/src/libs/utils.ts

@@ -1,6 +1,6 @@
 import { snakeCase, toUpper } from 'lodash-es';
 
-import { ExecutionStatus } from 'nb-blocks';
+import { ExecutionStatus } from 'nb-neardata';
 import { EventStatus, ExecutionOutcomeStatus, JsonValue } from 'nb-types';
 import { sleep } from 'nb-utils';
 

apps/indexer-balance/src/services/balance.ts

@@ -1,12 +1,12 @@
+import { Knex } from 'nb-knex';
 import {
   BlockHeader,
   StateChange as BlockStateChange,
   ExecutionOutcomeWithReceipt,
   IndexerTransactionWithOutcome,
   Message,
   Shard,
-} from 'nb-blocks';
-import { Knex } from 'nb-knex';
+} from 'nb-neardata';
 import {
   BalanceEvent,
   EventStatus,

apps/indexer-balance/src/services/stream.ts

@@ -1,21 +1,12 @@
-import { Message, streamBlock } from 'nb-blocks';
 import { logger } from 'nb-logger';
+import { Message, streamBlock } from 'nb-neardata';
 
 import config from '#config';
-import { dbRead, dbWrite, streamConfig } from '#libs/knex';
+import { dbRead, dbWrite } from '#libs/knex';
 import sentry from '#libs/sentry';
 import { storeBalance } from '#services/balance';
 
 const indexerKey = config.indexerKey;
-const s3Config = {
-  credentials: {
-    accessKeyId: config.s3AccessKey,
-    secretAccessKey: config.s3SecretKey,
-  },
-  endpoint: config.s3Endpoint,
-  forcePathStyle: true,
-  region: config.s3Region,
-};
 
 export const syncData = async () => {
   const settings = await dbRead('settings').where({ key: indexerKey }).first();
@@ -26,29 +17,17 @@ export const syncData = async () => {
     startBlockHeight = +latestBlock;
   }
 
-  const startBlock = startBlockHeight || 0;
-
-  logger.info(`syncing from block: ${startBlock}`);
+  logger.info(`syncing from block: ${startBlockHeight}`);
 
   const stream = streamBlock({
-    dbConfig: streamConfig,
-    s3Bucket: config.s3Bucket,
-    s3Config,
-    start: startBlock,
+    network: config.network,
+    start: startBlockHeight || config.genesisHeight,
+    url: config.neardataUrl,
   });
 
   for await (const message of stream) {
     await onMessage(message);
   }
-
-  stream.on('end', () => {
-    logger.error('stream ended');
-    process.exit();
-  });
-  stream.on('error', (error: Error) => {
-    logger.error(error);
-    process.exit();
-  });
 };
 
 export const onMessage = async (message: Message) => {

apps/indexer-balance/src/types/types.ts

@@ -6,13 +6,10 @@ export type Config = {
   dbKey: string;
   dbUrl: string;
   dbUrlRead: string;
+  genesisHeight: number;
   indexerKey: string;
+  neardataUrl: string;
   network: Network;
-  s3AccessKey: string;
-  s3Bucket: string;
-  s3Endpoint: string;
-  s3Region: string;
-  s3SecretKey: string;
   sentryDsn?: string;
   startBlockHeight: number;
 };

apps/indexer-base/src/config.ts

@@ -1,36 +1,19 @@
-import { types } from '@near-lake/framework';
-import { bool, cleanEnv, num, str, url } from 'envalid';
+import { cleanEnv, num, str, url } from 'envalid';
 
 import { Network } from 'nb-types';
 
-import { DataSource } from '#types/enum';
 import { Config } from '#types/types';
 
 const env = cleanEnv(process.env, {
-  BASE_DATA_SOURCE: str({
-    choices: [DataSource.FAST_NEAR, DataSource.NEAR_LAKE],
-    default: DataSource.NEAR_LAKE,
-  }),
   BASE_START_BLOCK: num({ default: 0 }),
   DATABASE_CA: str({ default: '' }),
   DATABASE_CERT: str({ default: '' }),
   DATABASE_KEY: str({ default: '' }),
   DATABASE_URL: str(),
-  DISABLE_AUTO_SWITCH: bool({ default: false }),
-  DISABLE_S3_UPLOAD: bool({ default: false }),
-  FASTNEAR_API_KEY: str(),
-  FASTNEAR_ENDPOINT: str({ default: undefined }),
-  NEARLAKE_ACCESS_KEY: str(),
-  NEARLAKE_ENDPOINT: url({ default: '' }),
-  NEARLAKE_SECRET_KEY: str(),
+  NEARDATA_URL: url(),
   NETWORK: str({
     choices: [Network.MAINNET, Network.TESTNET],
   }),
-  S3_ACCESS_KEY: str(),
-  S3_BUCKET: str({ default: '' }),
-  S3_ENDPOINT: url(),
-  S3_REGION: str({ default: '' }),
-  S3_SECRET_KEY: str(),
   SENTRY_DSN: str({ default: '' }),
 });
 
@@ -40,49 +23,18 @@ const genesisTimestamp =
     ? '1595350551591948000'
     : '1596166782911378000';
 const genesisFile = `https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/${env.NETWORK}/genesis.json`;
-let nearlakeEndpoint: null | types.EndpointConfig = null;
-const nearlakeBucketName =
-  env.NETWORK === Network.MAINNET
-    ? 'near-lake-data-mainnet'
-    : 'near-lake-data-testnet';
-
-if (env.NEARLAKE_ENDPOINT) {
-  const endpoint = new URL(env.NEARLAKE_ENDPOINT);
-  nearlakeEndpoint = {
-    hostname: endpoint.hostname,
-    path: endpoint.pathname,
-    port: +endpoint.port || 80,
-    protocol: endpoint.protocol,
-  };
-}
 
 const config: Config = {
-  dataSource: env.BASE_DATA_SOURCE,
   dbCa: env.DATABASE_CA,
   dbCert: env.DATABASE_CERT,
   dbKey: env.DATABASE_KEY,
   dbUrl: env.DATABASE_URL,
-  delta: 10, // start from blocks earlier on sync interuption
-  disableAutoSwitch: env.DISABLE_AUTO_SWITCH,
-  disableS3Upload: env.DISABLE_S3_UPLOAD,
-  fastnearApiKey: env.FASTNEAR_API_KEY,
-  fastnearEndpoint: env.FASTNEAR_ENDPOINT,
   genesisFile: genesisFile, // url to download genesis data
   genesisHeight,
   genesisTimestamp,
   insertLimit: 2_500, // records to insert into the db at a time
-  nearlakeAccessKey: env.NEARLAKE_ACCESS_KEY,
-  nearlakeBucketName,
-  nearlakeEndpoint,
-  nearlakeRegionName: 'eu-central-1',
-  nearlakeSecretKey: env.NEARLAKE_SECRET_KEY,
+  neardataUrl: env.NEARDATA_URL,
   network: env.NETWORK,
-  preloadSize: 100, // blocks to preload in nearlake
-  s3AccessKey: env.S3_ACCESS_KEY,
-  s3Bucket: env.S3_BUCKET,
-  s3Endpoint: env.S3_ENDPOINT,
-  s3Region: env.S3_REGION,
-  s3SecretKey: env.S3_SECRET_KEY,
   sentryDsn: env.SENTRY_DSN,
   startBlockHeight: env.BASE_START_BLOCK,
 };