feat: PoC implementation of scanning for nBTC txs logic

packages/btcindexer/db/migrations/0001_initial_schema.sql

@@ -1,7 +1,7 @@
 -- This table tracks the blocks received from the relayer (queue for cron job)
 CREATE TABLE processed_blocks (
     height INTEGER PRIMARY KEY,
-    block_id TEXT NOT NULL UNIQUE,
+    hash TEXT NOT NULL UNIQUE,
     processed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
 );
 
@@ -11,7 +11,6 @@ CREATE TABLE nbtc_txs (
     block_hash TEXT NOT NULL,
     block_height INTEGER NOT NULL,
     vout INTEGER NOT NULL,
-    sender_address TEXT,
     sui_recipient TEXT NOT NULL,
     amount_sats INTEGER NOT NULL,
     status TEXT NOT NULL, -- 'broadcasting' | 'confirming' | 'finalized' | 'minting' | 'minted'

packages/btcindexer/src/btcblock.ts

@@ -16,24 +16,24 @@
 	rawBlockHex: string;
 }
 
-async function parseBlockPayload(body: ReadableStream | null): Promise<BlockPayload[]> {
-	if (!body) {
-		return [];
-	}
-
-	const reader = body.getReader();
-	const decoder = new TextDecoder();
-	let jsonString = ``;
-	while (true) {
-		const { done, value } = await reader.read();
-		if (done) break;
-		jsonString += decoder.decode(value, { stream: true });
+class BlockPayloadParser {
+	public static async fromStream(body: ReadableStream | null): Promise<BlockPayload[]> {
+		if (!body) {
+			return [];
+		}
+		const reader = body.getReader();
+		const decoder = new TextDecoder();
+		let jsonString = "";
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			jsonString += decoder.decode(value, { stream: true });
+		}
+		return JSON.parse(jsonString);
 	}
-
-	return JSON.parse(jsonString);
 }
 
-function parseBlocksFromPayload(payload: BlockPayload[]): ExtBlock[] {
+function parseBlockPayloadEntries(payload: BlockPayload[]): ExtBlock[] {
 	const blocks: ExtBlock[] = [];
 	for (const entry of payload)
 		try {
@@ -61,10 +61,10 @@
  * @returns A promise that resolves to an array of successfully parsed ExtendedBlock's.
  */
 export async function parseBlocksFromStream(body: ReadableStream | null): Promise<ExtBlock[]> {
-	const payload = await parseBlockPayload(body);
+	const payload = await BlockPayloadParser.fromStream(body);
 	if (payload.length === 0) {
 		return [];
 	}
-	return parseBlocksFromPayload(payload);
+	return parseBlockPayloadEntries(payload);
 }
-export type { Block, Transaction, TxInput, TxOutput } from "bitcoinjs-lib";
+export { Block, Transaction } from "bitcoinjs-lib";

packages/btcindexer/src/btcindexer-http.ts

@@ -1,17 +1,22 @@
 import type { IRequest } from "itty-router";
 import { parseBlocksFromStream } from "./btcblock";
 import { Indexer } from "./btcindexer";
+import { networks } from "bitcoinjs-lib";
 
 export class HIndexer {
 	public nbtcAddr: string;
+	public suiFallbackAddr: string;
+	public network: networks.Network;
 
 	constructor() {
 		// TODO: need to provide through env variable
 		this.nbtcAddr = "TODO";
+		this.suiFallbackAddr = "TODO";
+		this.network = networks.regtest;
 	}
 
 	newIndexer(env: Env): Indexer {
-		return new Indexer(env, this.nbtcAddr);
+		return new Indexer(env, this.nbtcAddr, this.suiFallbackAddr, this.network);
 	}
 
 	// NOTE: we may need to put this to a separate worker

packages/btcindexer/src/btcindexer.test.ts

@@ -1,8 +1,51 @@
-import { expect, test } from "vitest";
+import { describe, it, assert } from "vitest";
+import { Indexer } from "../src/btcindexer";
+import { Block, networks } from "bitcoinjs-lib";
 
-import { Indexer } from "./btcindexer";
+// generated using bitcoin-cli --regtest
+const REGTEST_DATA = {
+	DEPOSIT_ADDR: "bcrt1qfnyeg7dd5vqs2mtc4rekwm8mgpxkj647p39zhw",
+	SUI_ADDR: "0x123456789",
+	BLOCK_HEIGHT: 303,
+	BLOCK_ID: "39d7c49ae129865f3aca615bc222b185fdff0ff61385b838bbbf00da8cbbea9d",
+	TX_ID: "2060dfd3cdbffb7db6c968357f3c9df91b52a4cef5c02fad0b0836b0f25cc4ca",
+	DEPOSIT_AMOUNT_SATS: 50000000,
+	RAW_BLOCK_HEX:
+		"000000305c2f30d99ad69f247638613dcca7f455159e252878ea6fe10bbc4574a0076914d98ada655d950ac6507d14584fa679d12fb5203c384e52ef292d063fe29b1b645c4b6d68ffff7f200200000002020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff04022f0100ffffffff02de82814a0000000016001477174bfb906c0e52d750eac4b40fd86746ad50550000000000000000266a24aa21a9ed27611410788b35b819e10a0227f40f9bbc70df824e3a30879552f004b48451210120000000000000000000000000000000000000000000000000000000000000000000000000020000000001017fec0755f3524b89ad45383343f992a0d5cb797a695b59e30f2fc80794f001050000000000fdffffff032202089200000000160014b125723e78c2d779e3e299dfd95d72e9a067a0b780f0fa02000000001600144cc99479ada301056d78a8f3676cfb404d696abe00000000000000000d6a0b3078313233343536373839024730440220755610ff6b6fdea530c20d11b7765816beb75e16ce78fa200a7da25e251a7eb9022078e78bed1cc38822cd5dce3982e23c3cd401415ae72050b00c5f8b3441a2c178012103ef55b72bddf4960ddbb12a9a04f61f91fb613aa99b472115f25a5f8686e6c3f200000000",
+};
+const SUI_FALLBACK_ADDRESS = "0xFALLBACK";
 
-test("nbtc add tx", async () => {
-	const i = new Indexer({} as Env, "todo");
-	expect(await i.putNbtcTx()).toBe(true);
+const mkMockEnv = () => ({
+	DB: {} as D1Database,
+	btc_blocks: {} as KVNamespace,
+	nbtc_txs: {} as KVNamespace,
+});
+
+describe("Indexer.findNbtcDeposits", () => {
+	it("should correctly parse the real regtest transaction", () => {
+		const mockEnv = mkMockEnv();
+		const indexer = new Indexer(
+			mockEnv,
+			REGTEST_DATA.DEPOSIT_ADDR,
+			SUI_FALLBACK_ADDRESS,
+			networks.regtest,
+		);
+
+		const block = Block.fromHex(REGTEST_DATA.RAW_BLOCK_HEX);
+		const targetTx = block.transactions?.find((tx) => tx.getId() === REGTEST_DATA.TX_ID);
+
+		assert(targetTx, "Setup error");
+
+		const deposits = indexer.findNbtcDeposits(targetTx);
+		assert.equal(deposits.length, 1);
+		assert.equal(deposits[0].amountSats, REGTEST_DATA.DEPOSIT_AMOUNT_SATS);
+		assert.equal(deposits[0].suiRecipient, REGTEST_DATA.SUI_ADDR);
+		assert.equal(deposits[0].vout, 1);
+	});
+});
+
+describe.skip("Indexer.scanNewBlocks", () => {
+	it("should be tested later", () => {
+		// TODO: add a test for the scanNewBlocks using the same data
+	});
 });

packages/btcindexer/src/btcindexer.ts

@@ -1,16 +1,26 @@
-import { ExtBlock, Transaction } from "./btcblock";
+import { ExtBlock, Transaction, Block } from "./btcblock";
+import { address, networks } from "bitcoinjs-lib";
+import { OP_RETURN } from "./opcodes";
+
+interface Deposit {
+	vout: number;
+	amountSats: number;
+	suiRecipient: string;
+}
 
 export class Indexer {
 	d1: D1Database; // SQL DB
 	blocksDB: KVNamespace;
 	nbtcTxDB: KVNamespace;
-	nbtcAddr: string;
+	nbtcScriptHex: string;
+	suiFallbackAddr: string;
 
-	constructor(env: Env, nbtcAddr: string) {
+	constructor(env: Env, nbtcAddr: string, fallbackAddr: string, network: networks.Network) {
 		this.d1 = env.DB;
 		this.blocksDB = env.btc_blocks;
 		this.nbtcTxDB = env.nbtc_txs;
-		this.nbtcAddr = nbtcAddr;
+		this.suiFallbackAddr = fallbackAddr;
+		this.nbtcScriptHex = address.toOutputScript(nbtcAddr, network).toString("hex");
 	}
 
 	// returns number of processed and add blocks
@@ -19,23 +29,18 @@ export class Indexer {
 			return 0;
 		}
 		const insertBlockStmt = this.d1.prepare(
-			`INSERT INTO processed_blocks (height, block_id) VALUES (?, ?)`,
+			`INSERT INTO processed_blocks (height, hash) VALUES (?, ?)`,
 		);
 		const putKVs = blocks.map((b) => this.blocksDB.put(b.getId(), b.raw));
-		// TODO: the height is not part of the block itself. Probably we will need to send it from the relayer, sending blocks {height, raw}
-		const putD1s = blocks.map((b) => insertBlockStmt.bind(0, b.getHash()));
+		const putD1s = blocks.map((b) => insertBlockStmt.bind(b.height, b.getHash()));
+
 		try {
 			await Promise.all([...putKVs, this.d1.batch(putD1s)]);
 		} catch (e) {
 			console.error(`Failed to store one or more blocks in KV or D1:`, e);
 			// TODO: decide what to do in the case where some blocks were saved and some not, prolly we need more granular error
 			throw new Error(`Could not save all blocks data`);
 		}
-		// TODO: parse the raw blocks and scan them for NBTC transactions, then insert them into the nBTC txs table.
-		// TODO: index nBTC txs
-		// TODO: save light blocks in d1
-		// TODO: index nBTC txs in d1
-		// TODO: save raw nBTC txs in DB
 		return blocks.length;
 	}
 
@@ -54,4 +59,80 @@ export class Indexer {
 
 		return true;
 	}
+
+	async scanNewBlocks(): Promise<void> {
+		const blocksToProcess = await this.d1
+			.prepare("SELECT height, hash FROM processed_blocks ORDER BY height ASC LIMIT 10")
+			.all<{ height: number; hash: string }>();
+
+		if (!blocksToProcess.results || blocksToProcess.results.length === 0) {
+			return;
+		}
+
+		const nbtcTxStatements: D1PreparedStatement[] = [];
+
+		const insertNbtcTxStmt = this.d1.prepare(
+			"INSERT INTO nbtc_txs (tx_id, block_hash, block_height, vout, sui_recipient, amount_sats, status) VALUES (?, ?, ?, ?, ?, ?, ?)",
+		);
+
+		for (const blockInfo of blocksToProcess.results) {
+			const rawBlockBuffer = await this.blocksDB.get(blockInfo.hash, {
+				type: "arrayBuffer",
+			});
+			if (!rawBlockBuffer) {
+				continue;
+			}
+			const block = Block.fromBuffer(Buffer.from(rawBlockBuffer));
+
+			for (const tx of block.transactions ?? []) {
+				const deposits = this.findNbtcDeposits(tx);
+				for (const deposit of deposits) {
+					nbtcTxStatements.push(
+						insertNbtcTxStmt.bind(
+							tx.getId(),
+							blockInfo.hash,
+							blockInfo.height,
+							deposit.vout,
+							deposit.suiRecipient,
+							deposit.amountSats,
+							"confirming",
+						),
+					);
+				}
+			}
+		}
+
+		if (nbtcTxStatements.length > 0) await this.d1.batch(nbtcTxStatements);
+
+		const heightsToDelete = blocksToProcess.results.map((r) => r.height);
+		const heights = heightsToDelete.join(",");
+		const deleteQuery = `DELETE FROM processed_blocks WHERE height IN (${heights})`;
+		await this.d1.prepare(deleteQuery).run();
+	}
+
+	findNbtcDeposits(tx: Transaction): Deposit[] {
+		const deposits: Deposit[] = [];
+		let suiRecipient: string | null = null;
+
+		for (const vout of tx.outs) {
+			if (vout.script[0] === OP_RETURN) {
+				suiRecipient = vout.script.subarray(2).toString();
+				break; // valid tx should have only one OP_RETURN
+			}
+		}
+		// TODO: add more sophisticated validation for Sui address
+		if (!suiRecipient) suiRecipient = this.suiFallbackAddr;
+
+		for (let i = 0; i < tx.outs.length; i++) {
+			const vout = tx.outs[i];
+			if (vout.script.toString("hex") === this.nbtcScriptHex) {
+				deposits.push({
+					vout: i,
+					amountSats: Number(vout.value),
+					suiRecipient,
+				});
+			}
+		}
+		return deposits;
+	}
 }

packages/btcindexer/src/opcodes.ts

@@ -0,0 +1 @@
+export const OP_RETURN = 0x6a;