fix: optimize block endpoint (#1190)

* refactor: optimized block endpoint

* refactor: made suggested changes

* fix: removed unnecessary string to buffer conversions

* fix: made suggested changes

Author: MHassanTariq

src/api/controllers/db-controller.ts

@@ -412,6 +412,20 @@ export async function getMicroblocksFromDataStore(args: {
   };
 }
 
+export async function getBlocksWithMetadata(args: { limit: number; offset: number; db: PgStore }) {
+  const blocks = await args.db.getBlocksWithMetadata({ limit: args.limit, offset: args.offset });
+  const results = blocks.results.map(block =>
+    parseDbBlock(
+      block.block,
+      block.txs,
+      block.microblocks_accepted,
+      block.microblocks_streamed,
+      block.microblock_tx_count
+    )
+  );
+  return { results, total: blocks.total };
+}
+
 export async function getBlockFromDataStore({
   blockIdentifer,
   db,

src/api/routes/block.ts

@@ -2,7 +2,7 @@ import * as express from 'express';
 import * as Bluebird from 'bluebird';
 import { BlockListResponse } from '@stacks/stacks-blockchain-api-types';
 
-import { getBlockFromDataStore } from '../controllers/db-controller';
+import { getBlockFromDataStore, getBlocksWithMetadata } from '../controllers/db-controller';
 import { timeout, waiter, has0xPrefix } from '../../helpers';
 import { InvalidRequestError, InvalidRequestErrorType } from '../../errors';
 import { parseLimitQuery, parsePagingQueryInput } from '../pagination';
@@ -28,19 +28,7 @@ export function createBlockRouter(db: PgStore): express.Router {
       const limit = parseBlockQueryLimit(req.query.limit ?? 20);
       const offset = parsePagingQueryInput(req.query.offset ?? 0);
 
-      // TODO: use getBlockWithMetadata or similar to avoid transaction integrity issues from lazy resolving block tx data (primarily the contract-call ABI data)
-      const { results: blocks, total } = await db.getBlocks({ offset, limit });
-      // TODO: fix duplicate pg queries
-      const results = await Bluebird.mapSeries(blocks, async block => {
-        const blockQuery = await getBlockFromDataStore({
-          blockIdentifer: { hash: block.block_hash },
-          db,
-        });
-        if (!blockQuery.found) {
-          throw new Error('unexpected block not found -- fix block enumeration query');
-        }
-        return blockQuery.result;
-      });
+      const { results, total } = await getBlocksWithMetadata({ offset, limit, db });
       setETagCacheHeaders(res);
       // TODO: block schema validation
       const response: BlockListResponse = { limit, offset, total, results };

src/datastore/common.ts

@@ -761,6 +761,17 @@ export interface TransferQueryResult {
   amount: string;
 }
 
+export interface BlocksWithMetadata {
+  results: {
+    block: DbBlock;
+    txs: string[];
+    microblocks_accepted: string[];
+    microblocks_streamed: string[];
+    microblock_tx_count: Record<string, number>;
+  }[];
+  total: number;
+}
+
 export interface NonFungibleTokenMetadataQueryResult {
   token_uri: string;
   name: string;

src/datastore/pg-store.ts

@@ -7,17 +7,19 @@ import { ChainID, ClarityAbi } from '@stacks/transactions';
 import { getTxTypeId } from '../api/controllers/db-controller';
 import {
   assertNotNullish,
+  FoundOrNot,
+  hexToBuffer,
+  unwrapOptional,
   bnsHexValueToName,
   bnsNameCV,
-  FoundOrNot,
   getBnsSmartContractId,
-  unwrapOptional,
 } from '../helpers';
 import { PgStoreEventEmitter } from './pg-store-event-emitter';
 import {
   AddressNftEventIdentifier,
   BlockIdentifier,
   BlockQueryResult,
+  BlocksWithMetadata,
   ContractTxQueryResult,
   DbAssetEventTypeId,
   DbBlock,
@@ -403,6 +405,96 @@ export class PgStore {
     });
   }
 
+  /**
+   * Returns Block information with metadata, including accepted and streamed microblocks hash
+   * @returns `BlocksWithMetadata` object including list of Blocks with metadata and total count.
+   */
+  async getBlocksWithMetadata({
+    limit,
+    offset,
+  }: {
+    limit: number;
+    offset: number;
+  }): Promise<BlocksWithMetadata> {
+    return await this.sql.begin(async sql => {
+      // get block list
+      const { results: blocks, total: block_count } = await this.getBlocks({ limit, offset });
+      const blockHashValues: string[] = [];
+      const indexBlockHashValues: string[] = [];
+      blocks.forEach(block => {
+        const indexBytea = block.index_block_hash;
+        const parentBytea = block.parent_index_block_hash;
+        indexBlockHashValues.push(indexBytea, parentBytea);
+        blockHashValues.push(indexBytea);
+      });
+
+      // get txs in those blocks
+      const txs = await sql<{ tx_id: string; index_block_hash: string }[]>`
+        SELECT tx_id, index_block_hash
+        FROM txs
+        WHERE index_block_hash IN ${sql(blockHashValues)}
+          AND canonical = true AND microblock_canonical = true
+        ORDER BY microblock_sequence DESC, tx_index DESC
+      `;
+
+      // get microblocks in those blocks
+      const microblocksQuery = await sql<
+        {
+          parent_index_block_hash: string;
+          index_block_hash: string;
+          microblock_hash: string;
+          transaction_count: number;
+        }[]
+      >`
+          SELECT parent_index_block_hash, index_block_hash, microblock_hash, (
+            SELECT COUNT(tx_id)::integer as transaction_count
+            FROM txs
+            WHERE txs.microblock_hash = microblocks.microblock_hash
+            AND canonical = true AND microblock_canonical = true
+          )
+          FROM microblocks
+          WHERE parent_index_block_hash IN ${sql(indexBlockHashValues)}
+          AND microblock_canonical = true
+          ORDER BY microblock_sequence DESC
+        `;
+      // parse data to return
+      const blocksMetadata = blocks.map(block => {
+        const transactions = txs
+          .filter(tx => tx.index_block_hash === block.index_block_hash)
+          .map(tx => tx.tx_id);
+        const microblocksAccepted = microblocksQuery
+          .filter(
+            microblock => block.parent_index_block_hash === microblock.parent_index_block_hash
+          )
+          .map(mb => {
+            return {
+              microblock_hash: mb.microblock_hash,
+              transaction_count: mb.transaction_count,
+            };
+          });
+        const microblocksStreamed = microblocksQuery
+          .filter(microblock => block.parent_index_block_hash === microblock.index_block_hash)
+          .map(mb => mb.microblock_hash);
+        const microblock_tx_count: Record<string, number> = {};
+        microblocksAccepted.forEach(mb => {
+          microblock_tx_count[mb.microblock_hash] = mb.transaction_count;
+        });
+        return {
+          block,
+          txs: transactions,
+          microblocks_accepted: microblocksAccepted.map(mb => mb.microblock_hash),
+          microblocks_streamed: microblocksStreamed,
+          microblock_tx_count,
+        };
+      });
+      const results: BlocksWithMetadata = {
+        results: blocksMetadata,
+        total: block_count,
+      };
+      return results;
+    });
+  }
+
   async getBlockTxs(indexBlockHash: string) {
     const result = await this.sql<{ tx_id: string; tx_index: number }[]>`
       SELECT tx_id, tx_index

src/tests/api-tests.ts

@@ -10848,6 +10848,57 @@ describe('api tests', () => {
     expect(result.body.txs[0]).toEqual(tx_id);
   });
 
+  test('/block', async () => {
+    const block_hash = '0x1234',
+      index_block_hash = '0xabcd',
+      tx_id = '0x12ff';
+
+    const block1 = new TestBlockBuilder({
+      block_hash,
+      index_block_hash,
+      // parent_index_block_hash: genesis_index_block_hash,
+      block_height: 1,
+    })
+      .addTx({ block_hash, tx_id, index_block_hash })
+      .build();
+    const microblock = new TestMicroblockStreamBuilder()
+      .addMicroblock({ parent_index_block_hash: index_block_hash })
+      .build();
+    await db.update(block1);
+    await db.updateMicroblocks(microblock);
+    const expectedResp = {
+      limit: 20,
+      offset: 0,
+      total: 1,
+      results: [
+        {
+          canonical: true,
+          height: 1,
+          hash: block_hash,
+          parent_block_hash: '0x',
+          burn_block_time: 94869286,
+          burn_block_time_iso: '1973-01-03T00:34:46.000Z',
+          burn_block_hash: '0xf44f44',
+          burn_block_height: 713000,
+          miner_txid: '0x4321',
+          parent_microblock_hash: '0x00',
+          parent_microblock_sequence: 0,
+          txs: [tx_id],
+          microblocks_accepted: [],
+          microblocks_streamed: [microblock.microblocks[0].microblock_hash],
+          execution_cost_read_count: 0,
+          execution_cost_read_length: 0,
+          execution_cost_runtime: 0,
+          execution_cost_write_count: 0,
+          execution_cost_write_length: 0,
+          microblock_tx_count: {},
+        },
+      ],
+    };
+    const result = await supertest(api.server).get(`/extended/v1/block/`);
+    expect(result.body).toEqual(expectedResp);
+  });
+
   afterEach(async () => {
     await api.terminate();
     await db?.close();