feat(btcindexer): add msgpack serialization and refactore router

Signed-off-by: Robert Zaremba <robert@zaremba.ch>

Author: robert-zaremba

packages/btcindexer/package.json

@@ -13,7 +13,8 @@
 		"bitcoinjs-lib": "^6.1.7",
 		"crypto-js": "^4.2.0",
 		"itty-router": "^5.0.18",
-		"merkletreejs": "^0.5.2"
+		"merkletreejs": "^0.5.2",
+		"msgpackr": "^1.11.5"
 	},
 	"devDependencies": {
 		"@types/crypto-js": "^4.2.2"

packages/btcindexer/src/btcblock.test.ts

@@ -7,10 +7,12 @@
  * `Env` object can be regenerated with `pnpm run typegen`.
  */
 
-import router from "./router";
+import HttpServer from "./server";
+
+const server = new HttpServer();
 
 export default {
-	fetch: router.fetch,
+	fetch: server.router.fetch,
 
 	// The scheduled handler is invoked at the interval set in our wrangler.jsonc's
 	// [[triggers]] configuration.

packages/btcindexer/src/examples.ts

@@ -0,0 +1,32 @@
+import { PutBlocks, PutBlocksReq } from "./put-blocks";
+
+export enum RestPath {
+	blocks = "/bitcoin/blocks",
+	nbtcTx = "/nbtc",
+}
+
+export enum ContentType {
+	msgpack = "application/vnd.msgpacvk",
+}
+
+const msgPackHeaders = {
+	"Content-Type": ContentType.msgpack,
+};
+
+export default class Client {
+	baseUrl: string;
+
+	constructor(baseUrl: string) {
+		if (baseUrl.endsWith("/")) baseUrl = baseUrl.slice(0, -1);
+
+		this.baseUrl = baseUrl;
+	}
+
+	async putBlocks(putBlocks: PutBlocks[]) {
+		return fetch(this.baseUrl + RestPath.blocks, {
+			method: "PUT",
+			headers: msgPackHeaders,
+			body: PutBlocksReq.encode(putBlocks),
+		});
+	}
+}

packages/btcindexer/src/index.ts

@@ -0,0 +1,56 @@
+import { describe, it, assert } from "vitest";
+import { newPutBlock, PutBlocksReq } from "./put-blocks";
+import { Block } from "bitcoinjs-lib";
+
+function bufferToHex(buffer: Buffer) {
+	// Convert to Uint8Array if it's an ArrayBuffer
+	const bytes = buffer instanceof ArrayBuffer ? new Uint8Array(buffer) : buffer;
+
+	return Array.from(bytes)
+		.map((b) => b.toString(16).padStart(2, "0"))
+		.join("");
+}
+
+describe("encode PutBlocks", () => {
+	// mainnet genesis block
+	const blockHex =
+		"0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000";
+	const block = Block.fromHex(blockHex);
+
+	it("should decode a single block", async () => {
+		assert(block.getId() == "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f");
+
+		const expected = [newPutBlock(156, block)];
+		const reqBytes = PutBlocksReq.encode(expected);
+		const got = PutBlocksReq.decode(reqBytes);
+		assert.lengthOf(got, 1);
+
+		const pb0 = got[0];
+		assert.equal(pb0.block.getId(), block.getId());
+		assert.equal(pb0.height, expected[0].height);
+		assert.include(bufferToHex(reqBytes), blockHex);
+	});
+
+	it("should decode multiple blocks", async () => {
+		// for simplicity we encode 2 genesis blocks
+		const expected = [newPutBlock(10, block), newPutBlock(11, block)];
+		const reqBytes = PutBlocksReq.encode(expected);
+		const got = PutBlocksReq.decode(reqBytes);
+		assert.lengthOf(got, 2);
+		assert.deepEqual(got, expected);
+	});
+
+	it("should handle empty array", async () => {
+		const reqBytes = PutBlocksReq.encode([]);
+		const got = PutBlocksReq.decode(reqBytes);
+		assert.deepEqual(got, []);
+	});
+
+	it("should abort on an invalid block", async () => {
+		const pbq = new PutBlocksReq(2, block);
+		const buffer = Buffer.from("invaliddddd", "utf8");
+		pbq.block = new Uint8Array(buffer);
+		const reqBytes = PutBlocksReq.msgpack([pbq]);
+		assert.throws(() => PutBlocksReq.decode(reqBytes), "Buffer too small");
+	});
+});

packages/btcindexer/src/router.ts

@@ -0,0 +1,38 @@
+import { Block } from "bitcoinjs-lib";
+import { pack, unpack } from "msgpackr";
+
+export interface PutBlocks {
+	height: number;
+	block: Block; // bitcoin core encoding of Block
+}
+
+export function newPutBlock(height: number, block: Block): PutBlocks {
+	return { height, block };
+}
+
+export class PutBlocksReq {
+	height: number;
+	block: Uint8Array; // bitcoin core encoding of Block
+
+	constructor(height: number, block: Block) {
+		this.height = height;
+		this.block = block.toBuffer();
+	}
+
+	static decode(req: ArrayBuffer): PutBlocks[] {
+		const putReq: PutBlocksReq[] = unpack(new Uint8Array(req));
+		return putReq.map((r): PutBlocks => {
+			return { height: r.height, block: Block.fromBuffer(Buffer.from(r.block)) };
+		});
+	}
+
+	static encode(putBlocks: PutBlocks[]): Buffer {
+		const req = putBlocks.map((r) => new PutBlocksReq(r.height, r.block));
+		return pack(req);
+	}
+
+	// directly encode this struct to msgpack
+	static msgpack(putBlocks: PutBlocksReq[]) {
+		return pack(putBlocks);
+	}
+}

packages/btcindexer/src/rpc/client.ts

@@ -1,23 +1,64 @@
-import type { IRequest } from "itty-router";
+import { IRequest, Router, error, json } from "itty-router";
+import { networks } from "bitcoinjs-lib";
+
 import { parseBlocksFromStream } from "./btcblock";
 import { Indexer } from "./btcindexer";
-import { networks } from "bitcoinjs-lib";
 import SuiClient from "./sui_client";
+import { RestPath } from "./rpc/client";
+
+import type { AppRouter, CFArgs } from "./routertype";
 
 const NBTC_MODULE = "nbtc";
 
-export class HIndexer {
-	public nbtcAddr: string;
-	public suiFallbackAddr: string;
-	public btcNetwork: networks.Network;
+export default class HttpServer {
+	nbtcAddr: string;
+	suiFallbackAddr: string;
+	btcNetwork: networks.Network;
+
+	router: AppRouter;
 
 	constructor() {
 		// TODO: need to provide through env variable
 		this.nbtcAddr = "TODO";
 		this.suiFallbackAddr = "TODO";
 		this.btcNetwork = networks.regtest;
+
+		this.router = this.createRouter();
 	}
 
+	createRouter() {
+		const r = Router<IRequest, CFArgs>({
+			catch: error,
+			// convert non `Response` objects to JSON Responses. If a handler returns `Response`
+			// object then it will be directly returned.
+			finally: [json],
+		});
+
+		r.put(RestPath.blocks, this.putBlocks);
+		r.put(RestPath.nbtcTx, this.putNbtcTx);
+
+		//
+		// TESTING
+		// we can return Response object directly, to avoid JSON serialization
+		r.get("/test/user/:id", (req) => new Response(`User ID: ${req.params.id}`));
+		// curl http://localhost:8787/test/kv/ -X PUT -d '{"key": "k102", "val": "v1"}'
+		r.put("/test/kv", this.putTestKV);
+		// curl "http://localhost:8787/test/kv/1" -i
+		r.get("/test/kv/:key", this.getTestKV);
+		r.get("/test", (req: Request) => {
+			const url = new URL(req.url);
+			url.pathname = "/__scheduled";
+			url.searchParams.append("cron", "* * * * *");
+			return new Response(
+				`To test the scheduled handler, ensure you have used the "--test-scheduled" then try running "curl ${url.href}".`,
+			);
+		});
+
+		r.all("/*", () => error(404, "Wrong Endpoint"));
+		return r;
+	}
+
+	// TODO: should be dependency or we should move it somewhere
 	newIndexer(env: Env): Indexer {
 		const suiClient = new SuiClient({
 			network: env.SUI_NETWORK,
@@ -45,7 +86,9 @@ export class HIndexer {
 		return { inserted: await i.putNbtcTx() };
 	};
 
+	//
 	// TODO: remove this
+	//
 	putTestKV = async (req: IRequest, env: Env) => {
 		const kv = env.btc_blocks;
 		const data = await req.json<{ key: string; val: string }>();
@@ -57,4 +100,9 @@ export class HIndexer {
 		return allKeys;
 		// return 1;
 	};
+	getTestKV = async (req: IRequest, env: Env) => {
+		const kv = env.btc_blocks;
+		const key = req.params.key;
+		return kv.get(key);
+	};
 }