Skip to content

Commit bfdcce1

Browse files
zone117xzone117x
authored
feat: cursor-based pagination on blocks endpoint (#2060)
* feat: cursor-based pagination for `/v2/blocks` endpoint * feat: support `offset` with cursor-based pagination for `/v2/blocks` * feat: support negative offsets for random-page access in either direction * chore: limit on `/v2/blocks` offset (10 pages max) * fix: offset working without cursor * chore: create CursorOffsetParam * chore: create PaginatedCursorResponse * chore: use index_block_hash for cursor value * chore: return 404 when cursor is re-orged
1 parent b68d798 commit bfdcce1

File tree

7 files changed

+348
-29
lines changed

7 files changed

+348
-29
lines changed

‎src/api/routes/v2/blocks.ts

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,12 @@ import { parseDbTx } from '../../../api/controllers/db-controller';
66
import { FastifyPluginAsync } from 'fastify';
77
import { Type, TypeBoxTypeProvider } from '@fastify/type-provider-typebox';
88
import { Server } from 'node:http';
9-
import { LimitParam, OffsetParam } from '../../schemas/params';
10-
import { ResourceType } from '../../pagination';
9+
import { CursorOffsetParam, LimitParam, OffsetParam } from '../../schemas/params';
10+
import { getPagingQueryLimit, pagingQueryLimits, ResourceType } from '../../pagination';
1111
import { PaginatedResponse } from '../../schemas/util';
1212
import { NakamotoBlock, NakamotoBlockSchema } from '../../schemas/entities/block';
1313
import { TransactionSchema } from '../../schemas/entities/transactions';
14+
import { BlockListV2ResponseSchema } from '../../schemas/responses/responses';
1415

1516
export const BlockRoutesV2: FastifyPluginAsync<
1617
Record<never, never>,
@@ -28,21 +29,29 @@ export const BlockRoutesV2: FastifyPluginAsync<
2829
tags: ['Blocks'],
2930
querystring: Type.Object({
3031
limit: LimitParam(ResourceType.Block),
31-
offset: OffsetParam(),
32+
offset: CursorOffsetParam({ resource: ResourceType.Block }),
33+
cursor: Type.Optional(Type.String({ description: 'Cursor for pagination' })),
3234
}),
3335
response: {
34-
200: PaginatedResponse(NakamotoBlockSchema),
36+
200: BlockListV2ResponseSchema,
3537
},
3638
},
3739
},
3840
async (req, reply) => {
3941
const query = req.query;
40-
const { limit, offset, results, total } = await fastify.db.v2.getBlocks(query);
41-
const blocks: NakamotoBlock[] = results.map(r => parseDbNakamotoBlock(r));
42+
const limit = getPagingQueryLimit(ResourceType.Block, req.query.limit);
43+
const blockQuery = await fastify.db.v2.getBlocks({ ...query, limit });
44+
if (query.cursor && !blockQuery.current_cursor) {
45+
throw new NotFoundError('Cursor not found');
46+
}
47+
const blocks: NakamotoBlock[] = blockQuery.results.map(r => parseDbNakamotoBlock(r));
4248
await reply.send({
43-
limit,
44-
offset,
45-
total,
49+
limit: blockQuery.limit,
50+
offset: blockQuery.offset,
51+
total: blockQuery.total,
52+
next_cursor: blockQuery.next_cursor,
53+
prev_cursor: blockQuery.prev_cursor,
54+
cursor: blockQuery.current_cursor,
4655
results: blocks,
4756
});
4857
}

‎src/api/schemas/params.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,23 @@ export const LimitParam = (
2828
})
2929
);
3030

31+
export const CursorOffsetParam = (args: {
32+
resource: ResourceType;
33+
title?: string;
34+
description?: string;
35+
limitOverride?: number;
36+
maxPages?: number;
37+
}) =>
38+
Type.Optional(
39+
Type.Integer({
40+
default: 0,
41+
maximum: pagingQueryLimits[args.resource].maxLimit * (args.maxPages ?? 10),
42+
minimum: -pagingQueryLimits[args.resource].maxLimit * (args.maxPages ?? 10),
43+
title: args.title ?? 'Offset',
44+
description: args.description ?? 'Result offset',
45+
})
46+
);
47+
3148
export const UnanchoredParamSchema = Type.Optional(
3249
Type.Boolean({
3350
default: false,

‎src/api/schemas/responses/responses.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { Static, Type } from '@sinclair/typebox';
2-
import { OptionalNullable, PaginatedResponse } from '../util';
2+
import { Nullable, OptionalNullable, PaginatedCursorResponse, PaginatedResponse } from '../util';
33
import { MempoolStatsSchema } from '../entities/mempool-transactions';
44
import { MempoolTransactionSchema, TransactionSchema } from '../entities/transactions';
55
import { MicroblockSchema } from '../entities/microblock';
@@ -12,6 +12,7 @@ import {
1212
BurnchainRewardSchema,
1313
BurnchainRewardSlotHolderSchema,
1414
} from '../entities/burnchain-rewards';
15+
import { NakamotoBlockSchema } from '../entities/block';
1516

1617
export const ErrorResponseSchema = Type.Object(
1718
{
@@ -178,3 +179,6 @@ export const RunFaucetResponseSchema = Type.Object(
178179
}
179180
);
180181
export type RunFaucetResponse = Static<typeof RunFaucetResponseSchema>;
182+
183+
export const BlockListV2ResponseSchema = PaginatedCursorResponse(NakamotoBlockSchema);
184+
export type BlockListV2Response = Static<typeof BlockListV2ResponseSchema>;

‎src/api/schemas/util.ts

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,17 @@ export const PaginatedResponse = <T extends TSchema>(type: T, options?: ObjectOp
1212
},
1313
options
1414
);
15+
16+
export const PaginatedCursorResponse = <T extends TSchema>(type: T, options?: ObjectOptions) =>
17+
Type.Object(
18+
{
19+
limit: Type.Integer({ examples: [20] }),
20+
offset: Type.Integer({ examples: [0] }),
21+
total: Type.Integer({ examples: [1] }),
22+
next_cursor: Nullable(Type.String({ description: 'Next page cursor' })),
23+
prev_cursor: Nullable(Type.String({ description: 'Previous page cursor' })),
24+
cursor: Nullable(Type.String({ description: 'Current page cursor' })),
25+
results: Type.Array(type),
26+
},
27+
options
28+
);

‎src/datastore/common.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,6 +1143,16 @@ export type DbPaginatedResult<T> = {
11431143
results: T[];
11441144
};
11451145

1146+
export type DbCursorPaginatedResult<T> = {
1147+
limit: number;
1148+
offset: number;
1149+
next_cursor: string | null;
1150+
prev_cursor: string | null;
1151+
current_cursor: string | null;
1152+
total: number;
1153+
results: T[];
1154+
};
1155+
11461156
export interface BlocksWithMetadata {
11471157
results: {
11481158
block: DbBlock;

‎src/datastore/pg-store-v2.ts

Lines changed: 80 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import {
3636
PoxCycleQueryResult,
3737
DbPoxCycleSigner,
3838
DbPoxCycleSignerStacker,
39+
DbCursorPaginatedResult,
3940
} from './common';
4041
import {
4142
BLOCK_COLUMNS,
@@ -59,37 +60,97 @@ async function assertTxIdExists(sql: PgSqlClient, tx_id: string) {
5960
}
6061

6162
export class PgStoreV2 extends BasePgStoreModule {
62-
async getBlocks(args: BlockPaginationQueryParams): Promise<DbPaginatedResult<DbBlock>> {
63+
async getBlocks(args: {
64+
limit: number;
65+
offset?: number;
66+
cursor?: string;
67+
}): Promise<DbCursorPaginatedResult<DbBlock>> {
6368
return await this.sqlTransaction(async sql => {
64-
const limit = args.limit ?? BlockLimitParamSchema.default;
69+
const limit = args.limit;
6570
const offset = args.offset ?? 0;
66-
const blocksQuery = await sql<(BlockQueryResult & { total: number })[]>`
67-
WITH block_count AS (
68-
SELECT block_count AS count FROM chain_tip
71+
const cursor = args.cursor ?? null;
72+
73+
const blocksQuery = await sql<
74+
(BlockQueryResult & { total: number; next_block_hash: string; prev_block_hash: string })[]
75+
>`
76+
WITH cursor_block AS (
77+
WITH ordered_blocks AS (
78+
SELECT *, LEAD(block_height, ${offset}) OVER (ORDER BY block_height DESC) offset_block_height
79+
FROM blocks
80+
WHERE canonical = true
81+
ORDER BY block_height DESC
6982
)
70-
SELECT
71-
${sql(BLOCK_COLUMNS)},
72-
(SELECT count FROM block_count)::int AS total
83+
SELECT offset_block_height as block_height
84+
FROM ordered_blocks
85+
WHERE index_block_hash = ${cursor ?? sql`(SELECT index_block_hash FROM chain_tip LIMIT 1)`}
86+
LIMIT 1
87+
),
88+
selected_blocks AS (
89+
SELECT ${sql(BLOCK_COLUMNS)}
7390
FROM blocks
7491
WHERE canonical = true
92+
AND block_height <= (SELECT block_height FROM cursor_block)
7593
ORDER BY block_height DESC
7694
LIMIT ${limit}
77-
OFFSET ${offset}
95+
),
96+
prev_page AS (
97+
SELECT index_block_hash as prev_block_hash
98+
FROM blocks
99+
WHERE canonical = true
100+
AND block_height < (
101+
SELECT block_height
102+
FROM selected_blocks
103+
ORDER BY block_height DESC
104+
LIMIT 1
105+
)
106+
ORDER BY block_height DESC
107+
OFFSET ${limit - 1}
108+
LIMIT 1
109+
),
110+
next_page AS (
111+
SELECT index_block_hash as next_block_hash
112+
FROM blocks
113+
WHERE canonical = true
114+
AND block_height > (
115+
SELECT block_height
116+
FROM selected_blocks
117+
ORDER BY block_height DESC
118+
LIMIT 1
119+
)
120+
ORDER BY block_height ASC
121+
OFFSET ${limit - 1}
122+
LIMIT 1
123+
)
124+
SELECT
125+
(SELECT block_count FROM chain_tip)::int AS total,
126+
sb.*,
127+
nb.next_block_hash,
128+
pb.prev_block_hash
129+
FROM selected_blocks sb
130+
LEFT JOIN next_page nb ON true
131+
LEFT JOIN prev_page pb ON true
132+
ORDER BY sb.block_height DESC
78133
`;
79-
if (blocksQuery.count === 0)
80-
return {
81-
limit,
82-
offset,
83-
results: [],
84-
total: 0,
85-
};
134+
135+
// Parse blocks
86136
const blocks = blocksQuery.map(b => parseBlockQueryResult(b));
87-
return {
137+
const total = blocksQuery[0]?.total ?? 0;
138+
139+
// Determine cursors
140+
const nextCursor = blocksQuery[0]?.next_block_hash ?? null;
141+
const prevCursor = blocksQuery[0]?.prev_block_hash ?? null;
142+
const currentCursor = blocksQuery[0]?.index_block_hash ?? null;
143+
144+
const result: DbCursorPaginatedResult<DbBlock> = {
88145
limit,
89-
offset,
146+
offset: offset,
90147
results: blocks,
91-
total: blocksQuery[0].total,
148+
total: total,
149+
next_cursor: nextCursor,
150+
prev_cursor: prevCursor,
151+
current_cursor: currentCursor,
92152
};
153+
return result;
93154
});
94155
}
95156

0 commit comments

Comments
 (0)