Skip to content

Commit 5505d35

Browse files
rafaelcrrafaelcr
authored
fix: optimize re-org queries and indexes (#1821)
1 parent c2bc6a6 commit 5505d35

File tree

5 files changed

+113
-46
lines changed

5 files changed

+113
-46
lines changed

‎migrations/1705013096459_update-re-org-indexes.js

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/* eslint-disable camelcase */
2+
3+
exports.shorthands = undefined;
4+
5+
exports.up = pgm => {
6+
pgm.dropIndex('txs', 'index_block_hash');
7+
pgm.createIndex('txs', ['index_block_hash', 'canonical']);
8+
9+
pgm.dropIndex('miner_rewards', 'index_block_hash');
10+
pgm.createIndex('miner_rewards', ['index_block_hash', 'canonical']);
11+
12+
pgm.dropIndex('stx_lock_events', 'index_block_hash');
13+
pgm.createIndex('stx_lock_events', ['index_block_hash', 'canonical']);
14+
15+
pgm.dropIndex('stx_events', 'index_block_hash');
16+
pgm.createIndex('stx_events', ['index_block_hash', 'canonical']);
17+
18+
pgm.dropIndex('ft_events', 'index_block_hash');
19+
pgm.createIndex('ft_events', ['index_block_hash', 'canonical']);
20+
21+
pgm.dropIndex('nft_events', 'index_block_hash');
22+
pgm.createIndex('nft_events', ['index_block_hash', 'canonical']);
23+
24+
pgm.dropIndex('pox2_events', 'index_block_hash');
25+
pgm.createIndex('pox2_events', ['index_block_hash', 'canonical']);
26+
27+
pgm.dropIndex('pox3_events', 'index_block_hash');
28+
pgm.createIndex('pox3_events', ['index_block_hash', 'canonical']);
29+
30+
pgm.dropIndex('pox4_events', 'index_block_hash');
31+
pgm.createIndex('pox4_events', ['index_block_hash', 'canonical']);
32+
33+
pgm.dropIndex('contract_logs', 'index_block_hash');
34+
pgm.createIndex('contract_logs', ['index_block_hash', 'canonical']);
35+
36+
pgm.dropIndex('smart_contracts', 'index_block_hash');
37+
pgm.createIndex('smart_contracts', ['index_block_hash', 'canonical']);
38+
39+
pgm.dropIndex('names', 'index_block_hash');
40+
pgm.createIndex('names', ['index_block_hash', 'canonical']);
41+
42+
pgm.dropIndex('namespaces', 'index_block_hash');
43+
pgm.createIndex('namespaces', ['index_block_hash', 'canonical']);
44+
45+
pgm.dropIndex('subdomains', 'index_block_hash');
46+
pgm.createIndex('subdomains', ['index_block_hash', 'canonical']);
47+
};
48+
49+
exports.down = pgm => {
50+
pgm.dropIndex('txs', ['index_block_hash', 'canonical']);
51+
pgm.createIndex('txs', 'index_block_hash', { method: 'hash' });
52+
53+
pgm.dropIndex('miner_rewards', ['index_block_hash', 'canonical']);
54+
pgm.createIndex('miner_rewards', 'index_block_hash', { method: 'hash' });
55+
56+
pgm.dropIndex('stx_lock_events', ['index_block_hash', 'canonical']);
57+
pgm.createIndex('stx_lock_events', 'index_block_hash', { method: 'hash' });
58+
59+
pgm.dropIndex('stx_events', ['index_block_hash', 'canonical']);
60+
pgm.createIndex('stx_events', 'index_block_hash', { method: 'hash' });
61+
62+
pgm.dropIndex('ft_events', ['index_block_hash', 'canonical']);
63+
pgm.createIndex('ft_events', 'index_block_hash', { method: 'hash' });
64+
65+
pgm.dropIndex('nft_events', ['index_block_hash', 'canonical']);
66+
pgm.createIndex('nft_events', 'index_block_hash', { method: 'hash' });
67+
68+
pgm.dropIndex('pox2_events', ['index_block_hash', 'canonical']);
69+
pgm.createIndex('pox2_events', 'index_block_hash', { method: 'hash' });
70+
71+
pgm.dropIndex('pox3_events', ['index_block_hash', 'canonical']);
72+
pgm.createIndex('pox3_events', 'index_block_hash', { method: 'hash' });
73+
74+
pgm.dropIndex('pox4_events', ['index_block_hash', 'canonical']);
75+
pgm.createIndex('pox4_events', 'index_block_hash', { method: 'hash' });
76+
77+
pgm.dropIndex('contract_logs', ['index_block_hash', 'canonical']);
78+
pgm.createIndex('contract_logs', 'index_block_hash', { method: 'hash' });
79+
80+
pgm.dropIndex('smart_contracts', ['index_block_hash', 'canonical']);
81+
pgm.createIndex('smart_contracts', 'index_block_hash', { method: 'hash' });
82+
83+
pgm.dropIndex('names', ['index_block_hash', 'canonical']);
84+
pgm.createIndex('names', 'index_block_hash', { method: 'hash' });
85+
86+
pgm.dropIndex('namespaces', ['index_block_hash', 'canonical']);
87+
pgm.createIndex('namespaces', 'index_block_hash', { method: 'hash' });
88+
89+
pgm.dropIndex('subdomains', ['index_block_hash', 'canonical']);
90+
pgm.createIndex('subdomains', 'index_block_hash', { method: 'hash' });
91+
};

‎src/datastore/pg-write-store.ts

Lines changed: 18 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1134,29 +1134,6 @@ export class PgWriteStore extends PgStore {
11341134
});
11351135
}
11361136

1137-
async updateStxEvent(sql: PgSqlClient, tx: DbTx, event: DbStxEvent) {
1138-
const values: StxEventInsertValues = {
1139-
event_index: event.event_index,
1140-
tx_id: event.tx_id,
1141-
tx_index: event.tx_index,
1142-
block_height: event.block_height,
1143-
index_block_hash: tx.index_block_hash,
1144-
parent_index_block_hash: tx.parent_index_block_hash,
1145-
microblock_hash: tx.microblock_hash,
1146-
microblock_sequence: tx.microblock_sequence,
1147-
microblock_canonical: tx.microblock_canonical,
1148-
canonical: event.canonical,
1149-
asset_event_type_id: event.asset_event_type_id,
1150-
sender: event.sender ?? null,
1151-
recipient: event.recipient ?? null,
1152-
amount: event.amount,
1153-
memo: event.memo ?? null,
1154-
};
1155-
await sql`
1156-
INSERT INTO stx_events ${sql(values)}
1157-
`;
1158-
}
1159-
11601137
async updateFtEvents(sql: PgSqlClient, entries: { tx: DbTx; ftEvents: DbFtEvent[] }[]) {
11611138
const values: FtEventInsertValues[] = [];
11621139
for (const { tx, ftEvents } of entries) {
@@ -1438,8 +1415,10 @@ export class PgWriteStore extends PgStore {
14381415
acceptedMicroblocks: string[];
14391416
orphanedMicroblocks: string[];
14401417
}> {
1441-
// Find the parent microblock if this anchor block points to one. If not, perform a sanity check for expected block headers in this case:
1442-
// > Anchored blocks that do not have parent microblock streams will have their parent microblock header hashes set to all 0's, and the parent microblock sequence number set to 0.
1418+
// Find the parent microblock if this anchor block points to one. If not, perform a sanity check
1419+
// for expected block headers in this case: Anchored blocks that do not have parent microblock
1420+
// streams will have their parent microblock header hashes set to all 0's, and the parent
1421+
// microblock sequence number set to 0.
14431422
let acceptedMicroblockTip: DbMicroblock | undefined;
14441423
if (BigInt(blockData.parentMicroblockHash) === 0n) {
14451424
if (blockData.parentMicroblockSequence !== 0) {
@@ -2521,29 +2500,25 @@ export class PgWriteStore extends PgStore {
25212500
canonical: boolean,
25222501
updatedEntities: ReOrgUpdatedEntities
25232502
): Promise<{ txsMarkedCanonical: string[]; txsMarkedNonCanonical: string[] }> {
2524-
const txResult = await sql<TxQueryResult[]>`
2503+
const txResult = await sql<{ tx_id: string }[]>`
25252504
UPDATE txs
25262505
SET canonical = ${canonical}
25272506
WHERE index_block_hash = ${indexBlockHash} AND canonical != ${canonical}
2528-
RETURNING ${sql(TX_COLUMNS)}
2507+
RETURNING tx_id
25292508
`;
2530-
const txIds = txResult.map(row => parseTxQueryResult(row));
2509+
const txIds = txResult.map(row => row.tx_id);
25312510
if (canonical) {
2532-
updatedEntities.markedCanonical.txs += txResult.length;
2511+
updatedEntities.markedCanonical.txs += txResult.count;
25332512
} else {
2534-
updatedEntities.markedNonCanonical.txs += txResult.length;
2513+
updatedEntities.markedNonCanonical.txs += txResult.count;
25352514
}
2536-
for (const txId of txIds) {
2537-
logger.debug(`Marked tx as ${canonical ? 'canonical' : 'non-canonical'}: ${txId.tx_id}`);
2538-
}
2539-
if (txIds.length) {
2515+
if (txResult.count)
25402516
await sql`
25412517
UPDATE principal_stx_txs
25422518
SET canonical = ${canonical}
2543-
WHERE tx_id IN ${sql(txIds.map(tx => tx.tx_id))}
2519+
WHERE tx_id IN ${sql(txIds)}
25442520
AND index_block_hash = ${indexBlockHash} AND canonical != ${canonical}
25452521
`;
2546-
}
25472522

25482523
const minerRewardResults = await sql`
25492524
UPDATE miner_rewards
@@ -2599,10 +2574,11 @@ export class PgWriteStore extends PgStore {
25992574
} else {
26002575
updatedEntities.markedNonCanonical.nftEvents += nftResult.count;
26012576
}
2602-
await this.updateNftCustodyFromReOrg(sql, {
2603-
index_block_hash: indexBlockHash,
2604-
microblocks: [],
2605-
});
2577+
if (nftResult.count)
2578+
await this.updateNftCustodyFromReOrg(sql, {
2579+
index_block_hash: indexBlockHash,
2580+
microblocks: [],
2581+
});
26062582

26072583
const pox2Result = await sql`
26082584
UPDATE pox2_events
@@ -2693,8 +2669,8 @@ export class PgWriteStore extends PgStore {
26932669
}
26942670

26952671
return {
2696-
txsMarkedCanonical: canonical ? txIds.map(t => t.tx_id) : [],
2697-
txsMarkedNonCanonical: canonical ? [] : txIds.map(t => t.tx_id),
2672+
txsMarkedCanonical: canonical ? txIds : [],
2673+
txsMarkedNonCanonical: canonical ? [] : txIds,
26982674
};
26992675
}
27002676

‎src/tests/datastore-tests.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ describe('postgres datastore', () => {
167167
createStxEvent('addrA', 'addrC', 35),
168168
];
169169
for (const event of events) {
170-
await db.updateStxEvent(client, tx, event);
170+
await db.updateStxEvents(client, [{ tx, stxEvents: [event] }]);
171171
}
172172

173173
const createStxLockEvent = (

‎src/tests/other-tests.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ describe('other tests', () => {
157157
event_type: DbEventTypeId.StxAsset,
158158
amount: 10_000_000_000_000n,
159159
};
160-
await db.updateStxEvent(client, tx, stxBurnEvent1);
160+
await db.updateStxEvents(client, [{ tx, stxEvents: [stxBurnEvent1] }]);
161161
const expectedTotalStx2 = stxMintEvent1.amount + stxMintEvent2.amount - stxBurnEvent1.amount;
162162
const result2 = await supertest(api.server).get(`/extended/v1/stx_supply`);
163163
expect(result2.status).toBe(200);

‎src/tests/search-tests.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -717,7 +717,7 @@ describe('search tests', () => {
717717
recipient: addr3,
718718
sender: 'none',
719719
};
720-
await db.updateStxEvent(client, stxTx1, stxEvent1);
720+
await db.updateStxEvents(client, [{ tx: stxTx1, stxEvents: [stxEvent1] }]);
721721

722722
// test address as a stx event recipient
723723
const searchResult3 = await supertest(api.server).get(`/extended/v1/search/${addr3}`);
@@ -745,7 +745,7 @@ describe('search tests', () => {
745745
sender: addr4,
746746
};
747747

748-
await db.updateStxEvent(client, stxTx1, stxEvent2);
748+
await db.updateStxEvents(client, [{ tx: stxTx1, stxEvents: [stxEvent2] }]);
749749

750750
// test address as a stx event sender
751751
const searchResult4 = await supertest(api.server).get(`/extended/v1/search/${addr4}`);

0 commit comments

Comments
 (0)