Skip to content

feat: add cursor pagination to transaction list endpoints #2432

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Marvy247 wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

feat: add cursor pagination to transaction list endpoints #2432

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 20 additions & 4 deletions src/api/routes/tx.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ import {
TransactionSearchResponseSchema,
TransactionTypeSchema,
} from '../schemas/entities/transactions';
import { PaginatedResponse } from '../schemas/util';
import { PaginatedResponse, PaginatedCursorResponse } from '../schemas/util';
import {
ErrorResponseSchema,
MempoolStatsResponseSchema,
Expand Down Expand Up @@ -78,6 +78,15 @@ export const TxRoutes: FastifyPluginAsync<
querystring: Type.Object({
offset: OffsetParam(),
limit: LimitParam(ResourceType.Tx),
cursor: Type.Optional(
Type.String({
description:
'Cursor for pagination. Use the cursor values returned in the response to navigate between pages.',
examples: [
'0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef:2147483647:10',
],
})
),
type: Type.Optional(Type.Array(TransactionTypeSchema)),
unanchored: UnanchoredParamSchema,
order: Type.Optional(Type.Enum({ asc: 'asc', desc: 'desc' })),
Expand Down Expand Up @@ -137,7 +146,7 @@ export const TxRoutes: FastifyPluginAsync<
exclude_function_args: ExcludeFunctionArgsParamSchema,
}),
response: {
200: TransactionResultsSchema,
200: PaginatedCursorResponse(TransactionSchema),
},
},
},
Expand Down Expand Up @@ -181,9 +190,16 @@ export const TxRoutes: FastifyPluginAsync<
contractId = req.query.contract_id;
}

const { results: txResults, total } = await fastify.db.getTxList({
const {
results: txResults,
total,
next_cursor,
prev_cursor,
current_cursor: cursor,
} = await fastify.db.getTxList({
offset,
limit,
cursor: req.query.cursor,
txTypeFilter,
includeUnanchored: req.query.unanchored ?? false,
fromAddress,
Expand All @@ -197,7 +213,7 @@ export const TxRoutes: FastifyPluginAsync<
sortBy: req.query.sort_by,
});
const results = txResults.map(tx => parseDbTx(tx, excludeFunctionArgs));
await reply.send({ limit, offset, total, results });
await reply.send({ limit, offset, total, next_cursor, prev_cursor, cursor, results });
}
);

Expand Down
22 changes: 20 additions & 2 deletions src/api/routes/v2/blocks.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,16 @@ export const BlockRoutesV2: FastifyPluginAsync<
querystring: Type.Object({
limit: LimitParam(ResourceType.Tx),
offset: OffsetParam(),
cursor: Type.Optional(
Type.String({
description:
'Cursor for pagination. Use the cursor values returned in the response to navigate between pages.',
examples: ['2147483647:10'],
})
),
}),
response: {
200: PaginatedResponse(TransactionSchema),
200: PaginatedCursorResponse(TransactionSchema),
},
},
},
Expand All @@ -162,14 +169,25 @@ export const BlockRoutesV2: FastifyPluginAsync<
const query = req.query;

try {
const { limit, offset, results, total } = await fastify.db.v2.getBlockTransactions({
const {
limit,
offset,
results,
total,
next_cursor,
prev_cursor,
current_cursor: cursor,
} = await fastify.db.v2.getBlockTransactions({
block: params,
...query,
});
const response = {
limit,
offset,
total,
next_cursor,
prev_cursor,
cursor,
results: results.map(r => parseDbTx(r, false)),
};
await reply.send(response);
Expand Down
88 changes: 84 additions & 4 deletions src/datastore/pg-store-v2.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -333,10 +333,36 @@ export class PgStoreV2 extends BasePgStoreModule {
block: BlockIdParam;
limit?: number;
offset?: number;
}): Promise<DbPaginatedResult<DbTx>> {
cursor?: string;
}): Promise<DbCursorPaginatedResult<DbTx>> {
return await this.sqlTransaction(async sql => {
const limit = args.limit ?? TransactionLimitParamSchema.default;
const offset = args.offset ?? 0;

// Parse cursor if provided (format: "microblockSequence:txIndex")
let cursorFilter = sql``;
if (args.cursor) {
const parts = args.cursor.split(':');
if (parts.length !== 2) {
throw new InvalidRequestError(
'Invalid cursor format',
InvalidRequestErrorType.invalid_param
);
}
const [microblockSequenceStr, txIndexStr] = parts;
const microblockSequence = parseInt(microblockSequenceStr, 10);
const txIndex = parseInt(txIndexStr, 10);
if (isNaN(microblockSequence) || isNaN(txIndex)) {
throw new InvalidRequestError(
'Invalid cursor format',
InvalidRequestErrorType.invalid_param
);
}
cursorFilter = sql`
AND (microblock_sequence, tx_index) >= (${microblockSequence}, ${txIndex})
`;
}

const txsQuery = await sql<(TxQueryResult & { total: number })[]>`
WITH block_ptr AS (
SELECT index_block_hash FROM blocks
Expand All @@ -362,17 +388,71 @@ export class PgStoreV2 extends BasePgStoreModule {
WHERE canonical = true
AND microblock_canonical = true
AND index_block_hash = (SELECT index_block_hash FROM block_ptr)
${cursorFilter}
ORDER BY microblock_sequence ASC, tx_index ASC
LIMIT ${limit}
OFFSET ${offset}
LIMIT ${limit + 1}
`;
if (txsQuery.count === 0)
throw new InvalidRequestError(`Block not found`, InvalidRequestErrorType.invalid_param);

const hasNextPage = txsQuery.length > limit;
const results = hasNextPage ? txsQuery.slice(0, limit) : txsQuery;

// Generate cursors
const lastResult = txsQuery[txsQuery.length - 1];
const prevCursor =
hasNextPage && lastResult
? `${lastResult.microblock_sequence}:${lastResult.tx_index}`
: null;

const firstResult = results[0];
const currentCursor = firstResult
? `${firstResult.microblock_sequence}:${firstResult.tx_index}`
: null;

// Generate next cursor by looking for the first item of the previous page
let nextCursor: string | null = null;
if (firstResult && limit > 1) {
const prevQuery = await sql<
{ microblock_sequence: number; tx_index: number }[]
>`
SELECT microblock_sequence, tx_index
FROM txs
WHERE canonical = true
AND microblock_canonical = true
AND index_block_hash = (
SELECT index_block_hash FROM blocks
WHERE ${
args.block.type === 'latest'
? sql`canonical = TRUE ORDER BY block_height DESC`
: args.block.type === 'hash'
? sql`(
block_hash = ${normalizeHashString(args.block.hash)}
OR index_block_hash = ${normalizeHashString(args.block.hash)}
) AND canonical = TRUE`
: sql`block_height = ${args.block.height} AND canonical = TRUE`
}
LIMIT 1
)
AND (microblock_sequence, tx_index) < (${firstResult.microblock_sequence}, ${firstResult.tx_index})
ORDER BY microblock_sequence DESC, tx_index DESC
OFFSET ${limit - 1}
LIMIT 1
`;
if (prevQuery.length > 0) {
const prev = prevQuery[0];
nextCursor = `${prev.microblock_sequence}:${prev.tx_index}`;
}
}

return {
limit,
offset,
results: txsQuery.map(t => parseTxQueryResult(t)),
results: results.map(t => parseTxQueryResult(t)),
total: txsQuery[0].total,
next_cursor: nextCursor,
prev_cursor: prevCursor,
current_cursor: currentCursor,
};
});
}
Expand Down
132 changes: 127 additions & 5 deletions src/datastore/pg-store.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1409,6 +1409,7 @@ export class PgStore extends BasePgStore {
async getTxList({
limit,
offset,
cursor,
txTypeFilter,
includeUnanchored,
fromAddress,
Expand All @@ -1423,6 +1424,7 @@ export class PgStore extends BasePgStore {
}: {
limit: number;
offset: number;
cursor?: string;
txTypeFilter: TransactionType[];
includeUnanchored: boolean;
fromAddress?: string;
Expand All @@ -1434,7 +1436,13 @@ export class PgStore extends BasePgStore {
nonce?: number;
order?: 'desc' | 'asc';
sortBy?: 'block_height' | 'burn_block_time' | 'fee';
}): Promise<{ results: DbTx[]; total: number }> {
}): Promise<{
results: DbTx[];
total: number;
next_cursor: string | null;
prev_cursor: string | null;
current_cursor: string | null;
}> {
return await this.sqlTransaction(async sql => {
const maxHeight = await this.getMaxBlockHeight(sql, { includeUnanchored });
const orderSql = order === 'asc' ? sql`ASC` : sql`DESC`;
Expand All @@ -1455,6 +1463,43 @@ export class PgStore extends BasePgStore {
throw new Error(`Invalid sortBy param: ${sortBy}`);
}

// Parse cursor if provided (format: "indexBlockHash:microblockSequence:txIndex")
let cursorFilter = sql``;
if (cursor) {
const parts = cursor.split(':');
if (parts.length !== 3) {
throw new Error('Invalid cursor format');
}
const [indexBlockHash, microblockSequenceStr, txIndexStr] = parts;
const microblockSequence = parseInt(microblockSequenceStr, 10);
const txIndex = parseInt(txIndexStr, 10);
if (!indexBlockHash || isNaN(microblockSequence) || isNaN(txIndex)) {
throw new Error('Invalid cursor format');
}

// Look up block_height from index_block_hash for the row-value comparison
const blockHeightQuery = await sql<{ block_height: number }[]>`
SELECT block_height FROM blocks WHERE index_block_hash = ${indexBlockHash} LIMIT 1
`;
if (blockHeightQuery.length === 0) {
throw new Error('Invalid cursor: block not found');
}
const blockHeight = blockHeightQuery[0].block_height;

// Apply cursor filter based on sort direction
if (order === 'asc') {
cursorFilter = sql`
AND (block_height, microblock_sequence, tx_index)
>= (${blockHeight}, ${microblockSequence}, ${txIndex})
`;
} else {
cursorFilter = sql`
AND (block_height, microblock_sequence, tx_index)
<= (${blockHeight}, ${microblockSequence}, ${txIndex})
`;
}
}

const txTypeFilterSql =
txTypeFilter.length > 0
? sql`AND type_id IN ${sql(txTypeFilter.flatMap<number>(t => getTxTypeId(t)))}`
Expand Down Expand Up @@ -1513,13 +1558,90 @@ export class PgStore extends BasePgStore {
${contractIdFilterSql}
${contractFuncFilterSql}
${nonceFilterSql}
${cursorFilter}
${orderBySql}
LIMIT ${limit}
OFFSET ${offset}
LIMIT ${limit + 1}
`;

const parsed = resultQuery.map(r => parseTxQueryResult(r));
return { results: parsed, total: totalQuery[0].count };
const hasNextPage = resultQuery.length > limit;
const results = hasNextPage ? resultQuery.slice(0, limit) : resultQuery;

// Generate cursors
const lastResult = resultQuery[resultQuery.length - 1];
const prevCursor =
hasNextPage && lastResult
? `${lastResult.index_block_hash}:${lastResult.microblock_sequence}:${lastResult.tx_index}`
: null;

const firstResult = results[0];
const currentCursor = firstResult
? `${firstResult.index_block_hash}:${firstResult.microblock_sequence}:${firstResult.tx_index}`
: null;

// Generate next cursor by looking for the first item of the previous page
let nextCursor: string | null = null;
if (firstResult && order !== 'asc') {
// For DESC order, look for items "before" our current first result (greater in DESC order)
const prevQuery = await sql<
{ index_block_hash: string; microblock_sequence: number; tx_index: number }[]
>`
SELECT index_block_hash, microblock_sequence, tx_index
FROM txs
WHERE canonical = true AND microblock_canonical = true AND block_height <= ${maxHeight}
${txTypeFilterSql}
${fromAddressFilterSql}
${toAddressFilterSql}
${startTimeFilterSql}
${endTimeFilterSql}
${contractIdFilterSql}
${contractFuncFilterSql}
${nonceFilterSql}
AND (block_height, microblock_sequence, tx_index)
> (${firstResult.block_height}, ${firstResult.microblock_sequence}, ${firstResult.tx_index})
ORDER BY block_height ASC, microblock_sequence ASC, tx_index ASC
OFFSET ${limit - 1}
LIMIT 1
`;
if (prevQuery.length > 0) {
const prev = prevQuery[0];
nextCursor = `${prev.index_block_hash}:${prev.microblock_sequence}:${prev.tx_index}`;
}
} else if (firstResult && order === 'asc') {
// For ASC order, look for items "before" our current first result (lesser in ASC order)
const prevQuery = await sql<
{ index_block_hash: string; microblock_sequence: number; tx_index: number }[]
>`
SELECT index_block_hash, microblock_sequence, tx_index
FROM txs
WHERE canonical = true AND microblock_canonical = true AND block_height <= ${maxHeight}
${txTypeFilterSql}
${fromAddressFilterSql}
${toAddressFilterSql}
${startTimeFilterSql}
${endTimeFilterSql}
${contractIdFilterSql}
${contractFuncFilterSql}
${nonceFilterSql}
AND (block_height, microblock_sequence, tx_index)
< (${firstResult.block_height}, ${firstResult.microblock_sequence}, ${firstResult.tx_index})
ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC
OFFSET ${limit - 1}
LIMIT 1
`;
if (prevQuery.length > 0) {
const prev = prevQuery[0];
nextCursor = `${prev.index_block_hash}:${prev.microblock_sequence}:${prev.tx_index}`;
}
}

const parsed = results.map(r => parseTxQueryResult(r));
return {
results: parsed,
total: totalQuery[0].count,
next_cursor: nextCursor,
prev_cursor: prevCursor,
current_cursor: currentCursor,
};
});
}

Expand Down
Loading