fix: indexes to optimize principal-based etag db lookups (#2157)

* fix: indexes to optimize principal-based etag db lookups

* chore: fix mempool_txs index and query

* fix: force usage of idx_nft_events_optimized index

Author: zone117x

migrations/1730724278636_principal_activity_txs_idx.js

@@ -0,0 +1,59 @@
+/** @param { import("node-pg-migrate").MigrationBuilder } pgm */
+exports.up = pgm => {
+  // Indexes used to speed up queries in the `getPrincipalLastActivityTxIds` function:
+  // https://github.com/hirosystems/stacks-blockchain-api/blob/e3c30c6e0cb14843d5f089b64010d738b0b27763/src/datastore/pg-store.ts#L4440-L4492
+  // See issue https://github.com/hirosystems/stacks-blockchain-api/issues/2147
+
+  pgm.createIndex(
+    'ft_events', 
+    [
+      'sender',
+      'recipient',
+      { name: 'block_height', order: 'DESC' },
+      { name: 'microblock_sequence', order: 'DESC' },
+      { name: 'tx_index', order: 'DESC' },
+      { name: 'event_index', order: 'DESC' }
+    ],
+    {
+      name: 'idx_ft_events_optimized',
+      where: 'canonical = TRUE AND microblock_canonical = TRUE',
+    }
+  );
+
+  pgm.createIndex(
+    'nft_events', 
+    [
+      'sender',
+      'recipient',
+      { name: 'block_height', order: 'DESC' },
+      { name: 'microblock_sequence', order: 'DESC' },
+      { name: 'tx_index', order: 'DESC' },
+      { name: 'event_index', order: 'DESC' }
+    ],
+    {
+      name: 'idx_nft_events_optimized',
+      where: 'canonical = TRUE AND microblock_canonical = TRUE',
+    }
+  );
+
+  pgm.createIndex(
+    'mempool_txs', 
+    [
+      'sender_address',
+      'sponsor_address',
+      'token_transfer_recipient_address',
+      { name: 'receipt_time', order: 'DESC' }
+    ],
+    {
+      name: 'idx_mempool_txs_optimized',
+      where: 'pruned = FALSE',
+    }
+  );
+};
+
+/** @param { import("node-pg-migrate").MigrationBuilder } pgm */
+exports.down = pgm => {
+  pgm.dropIndex('ft_events', ['sender', 'recipient', 'block_height', 'microblock_sequence', 'tx_index', 'event_index'], { name: 'idx_ft_events_optimized' });
+  pgm.dropIndex('nft_events', ['sender', 'recipient', 'block_height', 'microblock_sequence', 'tx_index', 'event_index'], { name: 'idx_nft_events_optimized' });
+  pgm.dropIndex('mempool_txs', ['sender_address', 'sponsor_address', 'token_transfer_recipient_address', 'receipt_time'], { name: 'idx_mempool_txs_optimized' });
+};

src/datastore/pg-store.ts

@@ -4453,10 +4453,27 @@ export class PgStore extends BasePgStore {
         UNION
         (
           SELECT '0x' || encode(tx_id, 'hex') AS tx_id
-          FROM ft_events
-          WHERE (sender = ${principal} OR recipient = ${principal})
-            AND canonical = true
-            AND microblock_canonical = true
+          FROM (
+            (
+              SELECT tx_id, block_height, microblock_sequence, tx_index, event_index
+              FROM ft_events
+              WHERE sender = ${principal}
+                AND canonical = true
+                AND microblock_canonical = true
+              ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC
+              LIMIT 1
+            )
+            UNION ALL
+            (
+              SELECT tx_id, block_height, microblock_sequence, tx_index, event_index
+              FROM ft_events
+              WHERE recipient = ${principal}
+                AND canonical = true
+                AND microblock_canonical = true
+              ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC
+              LIMIT 1
+            )
+          ) AS combined
           ORDER BY block_height DESC, microblock_sequence DESC, tx_index DESC, event_index DESC
           LIMIT 1
         )
@@ -4480,7 +4497,7 @@ export class PgStore extends BasePgStore {
                 (sender_address = ${principal}
                 OR sponsor_address = ${principal}
                 OR token_transfer_recipient_address = ${principal})
-              ORDER BY receipt_time DESC, sender_address DESC, nonce DESC
+              ORDER BY receipt_time DESC
               LIMIT 1
             )`
             : this.sql``