solana: backport new block normalization

solana/solana-ingest/src/ingest.ts

@@ -1,7 +1,9 @@
-import {addErrorContext} from '@subsquid/util-internal'
+import {addErrorContext, def} from '@subsquid/util-internal'
 import {Command, Ingest, IngestOptions, Range} from '@subsquid/util-internal-ingest-cli'
 import {toJSON} from '@subsquid/util-internal-json'
-import {mapRawBlock, RawBlock} from './mapping'
+import {RawBlock, removeVoteTransactions} from './mapping'
+import {assertValidity} from '@subsquid/util-internal-validation'
+import {mapRpcBlock} from '@subsquid/solana-normalization'
 
 
 interface Options extends IngestOptions {
@@ -28,12 +30,37 @@ export class SolanaIngest extends Ingest<Options> {
         program.option('--no-votes', 'Exclude vote transactions')
     }
 
-    protected async *getBlocks(range: Range): AsyncIterable<object[]> {
+    @def
+    private mapping(): (raw: unknown) => object {
         let votes = this.options().votes
+
+        return function mapRawBlock(raw: unknown): object {
+            assertValidity(RawBlock, raw)
+
+            if (!votes) {
+                removeVoteTransactions(raw.block)
+            }
+
+            let normalized = mapRpcBlock(raw, {
+                warn: function(props: any, msg: string): void {
+                    throw addErrorContext(new Error(msg), props)
+                },
+                error: function(props: any, msg: string): void {
+                    throw addErrorContext(new Error(msg), props)
+                }
+            })
+
+            return normalized
+        }
+    }
+
+    protected async *getBlocks(range: Range): AsyncIterable<object[]> {
+        let mapping = this.mapping()
+
         for await (let blocks of this.archive().getRawBlocks<RawBlock>(range)) {
             yield blocks.map(raw => {
                 try {
-                    let block = mapRawBlock(raw, !votes)
+                    let block = mapping(raw)
                     return toJSON(block)
                 } catch(err: any) {
                     throw addErrorContext(err, {

solana/solana-ingest/src/mapping.ts

@@ -1,9 +1,8 @@
-import {Block, mapRpcBlock} from '@subsquid/solana-normalization'
 import {Reward, Transaction} from '@subsquid/solana-rpc-data'
-import {array, assertValidity, B58, GetSrcType, NAT, object} from '@subsquid/util-internal-validation'
+import {array, B58, GetSrcType, NAT, object} from '@subsquid/util-internal-validation'
 
 
-const GetBlock = object({
+export const GetBlock = object({
     blockHeight: NAT,
     blockTime: NAT,
     blockhash: B58,
@@ -14,7 +13,10 @@ const GetBlock = object({
 })
 
 
-const RawBlock = object({
+export type GetBlock = GetSrcType<typeof GetBlock>
+
+
+export const RawBlock = object({
     hash: B58,
     height: NAT,
     slot: NAT,
@@ -25,32 +27,17 @@ const RawBlock = object({
 export type RawBlock = GetSrcType<typeof RawBlock>
 
 
-export function mapRawBlock(raw: unknown, noVotes: boolean): Block {
-    assertValidity(RawBlock, raw)
-    let block = mapRpcBlock(raw)
-    if (noVotes) {
-        removeVotes(block)
-    }
-    return block
+export function isVoteTransaction(tx: Transaction): boolean {
+    if (tx.transaction.message.instructions.length != 1) return false
+    let ins = tx.transaction.message.instructions[0]
+    return tx.transaction.message.accountKeys[ins.programIdIndex] === 'Vote111111111111111111111111111111111111111'
 }
 
 
-function removeVotes(block: Block): void {
-    let removed = new Set<number>()
-
-    for (let i of block.instructions) {
-        if (i.programId == 'Vote111111111111111111111111111111111111111') {
-            removed.add(i.transactionIndex)
-        }
-    }
-
-    function kept(item: {transactionIndex: number}): boolean {
-        return !removed.has(item.transactionIndex)
-    }
-
-    block.transactions = block.transactions.filter(kept)
-    block.instructions = block.instructions.filter(kept)
-    block.logs = block.logs.filter(kept)
-    block.balances = block.balances.filter(kept)
-    block.tokenBalances = block.tokenBalances.filter(kept)
+export function removeVoteTransactions(block: GetBlock): void {
+    if (!block.transactions) return
+    block.transactions = block.transactions.filter((tx: Transaction, index) => {
+        tx._index = tx._index ?? index
+        return !isVoteTransaction(tx)
+    })
 }