feat: [CPU optimizations] use native rust module for decoding Clarity values, binary transaction blobs, post-condition binary blobs, Stacks addresses (#1094)

* feat: use native nodejs addon module for Stacks binary encoding and decoding (Clarity values, binary wire formats, addresses, etc)

* refactor: move transaction, post-condition, and clarity value decoding from js to native addon

* chore: fix rosetta pox contract-call parsing

* chore: use npm registry for stacks-encoding-native-js

* chore: fix unused export

* chore: commit dev file to be converted to unit tests

* refactor: use stacks-encoding-native lib for stacks-address operations

* chore: bump stacks-encoding-native lib

* chore: stacks-native-encoding version bump with fix for decoding multisig tx sender address

* chore: stacks-native-encoding version bump with fix for decoding zero-length prefixed contract call args

* test: introduce tests for on-btc-chain originating stx tx processing

* chore: prettier ignore json payload sample files

* chore: clear up type import syntax

* test: tests for on-btc-chain originating stx-lock tx processing

* feat: expand cpu profiling capabilties with distinct start and stop endpoints easier to use in scripts

* chore: fix compilation error in /utils

* feat: support for performing cpu profiling with the native addon

* fix: avoid re-injecting sourcemap support when running with ts-node

* chore: updates for latest stacks-native-encoding library function usage, only decode clarity value type signatures when needed

* chore: decode only clarity value repr string rather than deep value in several areas

* chore: fix package-lock.json

* feat: commit load testing script with sampled endpoint list

* chore: fix synthetic-stx-txs-tests

* chore: update to latest stacks-encoding-native types (using hex strings rather than js buffers)

* chore: bump stacks-encoding-native version

* chore: fix regular integration tests

* chore: fix bug with rosetta principal repr parsing

* chore: bump stacks-encoding-native version

* chore: add several TODO(perf) comments where buffers should be hex strings

* chore: remove stacks-native-encoding debug code that have since been converted into unit tests

* chore: remove debug code

* fix: bug fix for reverse file stream bug if read chunk starts with line-ending

* chore: wrap synthetic stx txs tests

Author: zone117x

.eslintrc.js

@@ -30,5 +30,6 @@ module.exports = {
 
     // TODO: temporarily disable this until the express async handler is typed correctly
     '@typescript-eslint/no-misused-promises': 'off',
+    '@typescript-eslint/no-unsafe-argument': 'off',
   },
 };

.prettierignore

@@ -3,3 +3,4 @@ src/migrations/*
 src/tests-rosetta/
 src/tests-rosetta-cli/
 src/tests-bns/
+src/tests/synthetic-tx-payloads/

.vscode/launch.json

@@ -235,7 +235,7 @@
       "runtimeArgs": ["-r", "ts-node/register/transpile-only"],
       "cwd": "${workspaceFolder}/utils",
       "args": ["${workspaceFolder}/utils/src/chaintip-cache-control-test.ts"]
-    }
+    },
   ],
   "compounds": [
     {

package-lock.json

@@ -27,9 +27,9 @@
     "start": "node ./lib/index.js",
     "lint": "npm run lint:eslint && npm run lint:prettier",
     "lint:unused-exports": "ts-unused-exports tsconfig.json --ignoreFiles=src/migrations/*",
-    "lint:eslint": "eslint . --ext .js,.jsx,.ts,.tsx -f codeframe",
+    "lint:eslint": "eslint . --ext .js,.jsx,.ts,.tsx -f unix",
     "lint:prettier": "prettier --check src/**/*.{ts,json}",
-    "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx -f codeframe --fix && prettier --write --check src/**/*.{ts,json}",
+    "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx -f unix --fix && prettier --write --check src/**/*.{ts,json}",
     "migrate": "node-pg-migrate -m src/migrations",
     "devenv:build": "docker-compose -f docker/docker-compose.dev.postgres.yml -f docker/docker-compose.dev.stacks-blockchain.yml -f docker/docker-compose.dev.bitcoind.yml -f docker/docker-compose.dev.rosetta-cli.yml build --no-cache",
     "devenv:deploy": "docker-compose -f docker/docker-compose.dev.postgres.yml -f docker/docker-compose.dev.stacks-blockchain.yml -f docker/docker-compose.dev.bitcoind.yml up",
@@ -139,9 +139,10 @@
     "socket.io": "4.4.0",
     "source-map-support": "0.5.21",
     "split2": "3.2.2",
+    "stacks-encoding-native-js": "0.1.0-alpha.9",
     "strict-event-emitter-types": "2.0.0",
     "ts-unused-exports": "7.0.3",
-    "typescript": "4.5.4",
+    "typescript": "4.6.2",
     "uuid": "8.3.2",
     "winston": "3.3.3",
     "ws": "7.5.6",
@@ -155,15 +156,15 @@
     "@commitlint/cli": "9.1.2",
     "@commitlint/config-conventional": "10.0.0",
     "@stacks/eslint-config": "1.2.0",
-    "@stacks/prettier-config": "0.0.7",
+    "@stacks/prettier-config": "0.0.10",
     "@types/ajv": "1.0.0",
     "@types/bluebird": "3.5.36",
     "@types/bn.js": "4.11.6",
     "@types/cors": "2.8.12",
     "@types/dotenv-flow": "3.2.0",
     "@types/express": "4.17.13",
     "@types/is-ci": "3.0.0",
-    "@types/jest": "27.0.3",
+    "@types/jest": "27.4.1",
     "@types/node": "16.11.17",
     "@types/node-fetch": "2.5.12",
     "@types/pg": "7.14.11",
@@ -172,17 +173,17 @@
     "@types/split2": "2.1.6",
     "@types/supertest": "2.0.11",
     "@types/uuid": "7.0.5",
-    "@typescript-eslint/eslint-plugin": "4.33.0",
-    "@typescript-eslint/parser": "4.33.0",
+    "@typescript-eslint/eslint-plugin": "5.15.0",
+    "@typescript-eslint/parser": "5.15.0",
     "concurrently": "6.5.1",
     "docker-compose": "0.23.14",
-    "eslint": "7.32.0",
-    "eslint-config-prettier": "8.3.0",
-    "eslint-plugin-prettier": "3.4.1",
+    "eslint": "8.11.0",
+    "eslint-config-prettier": "8.5.0",
+    "eslint-plugin-prettier": "4.0.0",
     "eslint-plugin-tsdoc": "0.2.14",
     "husky": "4.3.8",
     "is-ci": "3.0.1",
-    "jest": "27.4.5",
+    "jest": "27.5.1",
     "nock": "13.2.1",
     "nodemon": "2.0.15",
     "pg-connection-string": "2.5.0",
@@ -191,8 +192,8 @@
     "rpc-websocket-client": "1.1.4",
     "socket.io-client": "4.4.0",
     "supertest": "4.0.2",
-    "ts-jest": "27.1.2",
-    "ts-node": "9.1.1",
+    "ts-jest": "27.1.3",
+    "ts-node": "10.7.0",
     "why-is-node-running": "2.2.0"
   },
   "optionalDependencies": {

package.json

@@ -1,13 +1,15 @@
 import {
   abiFunctionToString,
-  BufferReader,
   ClarityAbi,
   ClarityAbiFunction,
-  cvToString,
-  deserializeCV,
   getTypeString,
-  serializeCV,
 } from '@stacks/transactions';
+import {
+  decodeClarityValueList,
+  decodeClarityValueToRepr,
+  decodeClarityValueToTypeName,
+  decodePostConditions,
+} from 'stacks-encoding-native-js';
 
 import {
   AbstractMempoolTransaction,
@@ -22,6 +24,7 @@ import {
   MempoolTransactionStatus,
   Microblock,
   PoisonMicroblockTransactionMetadata,
+  PostCondition,
   RosettaBlock,
   RosettaParentBlockIdentifier,
   RosettaTransaction,
@@ -72,7 +75,6 @@ import {
   unixEpochToIso,
   EMPTY_HASH_256,
 } from '../../helpers';
-import { readClarityValueArray, readTransactionPostConditions } from '../../p2p/tx';
 import { serializePostCondition, serializePostConditionMode } from '../serializers/post-conditions';
 import { getOperations, parseTransactionMemo, processUnlockingEvents } from '../../rosetta-helpers';
 
@@ -214,16 +216,18 @@ export function parseDbEvent(dbEvent: DbEvent): TransactionEvent {
   switch (dbEvent.event_type) {
     case DbEventTypeId.SmartContractLog: {
       const valueBuffer = dbEvent.value;
-      const valueHex = bufferToHexPrefixString(valueBuffer);
-      const valueRepr = cvToString(deserializeCV(valueBuffer));
+      const parsedClarityValue = decodeClarityValueToRepr(valueBuffer);
       const event: TransactionEventSmartContractLog = {
         event_index: dbEvent.event_index,
         event_type: 'smart_contract_log',
         tx_id: dbEvent.tx_id,
         contract_log: {
           contract_id: dbEvent.contract_identifier,
           topic: dbEvent.topic,
-          value: { hex: valueHex, repr: valueRepr },
+          value: {
+            hex: bufferToHexPrefixString(valueBuffer),
+            repr: parsedClarityValue,
+          },
         },
       };
       return event;
@@ -272,8 +276,7 @@ export function parseDbEvent(dbEvent: DbEvent): TransactionEvent {
     }
     case DbEventTypeId.NonFungibleTokenAsset: {
       const valueBuffer = dbEvent.value;
-      const valueHex = bufferToHexPrefixString(valueBuffer);
-      const valueRepr = cvToString(deserializeCV(valueBuffer));
+      const parsedClarityValue = decodeClarityValueToRepr(valueBuffer);
       const event: TransactionEventNonFungibleAsset = {
         event_index: dbEvent.event_index,
         event_type: 'non_fungible_token_asset',
@@ -284,8 +287,8 @@ export function parseDbEvent(dbEvent: DbEvent): TransactionEvent {
           sender: dbEvent.sender || '',
           recipient: dbEvent.recipient || '',
           value: {
-            hex: valueHex,
-            repr: valueRepr,
+            hex: bufferToHexPrefixString(valueBuffer),
+            repr: parsedClarityValue,
           },
         },
       };
@@ -641,13 +644,10 @@ interface GetTxWithEventsArgs extends GetTxArgs {
 }
 
 function parseDbBaseTx(dbTx: DbTx | DbMempoolTx): BaseTransaction {
-  const postConditions =
-    dbTx.post_conditions.byteLength > 2
-      ? readTransactionPostConditions(
-          BufferReader.fromBuffer(dbTx.post_conditions.slice(1))
-        ).map(pc => serializePostCondition(pc))
-      : [];
-
+  const decodedPostConditions = decodePostConditions(dbTx.post_conditions);
+  const normalizedPostConditions = decodedPostConditions.post_conditions.map(pc =>
+    serializePostCondition(pc)
+  );
   const tx: BaseTransaction = {
     tx_id: dbTx.tx_id,
     nonce: dbTx.nonce,
@@ -656,8 +656,8 @@ function parseDbBaseTx(dbTx: DbTx | DbMempoolTx): BaseTransaction {
     sender_address: dbTx.sender_address,
     sponsored: dbTx.sponsored,
     sponsor_address: dbTx.sponsor_address,
-    post_condition_mode: serializePostConditionMode(dbTx.post_conditions.readUInt8(0)),
-    post_conditions: postConditions,
+    post_condition_mode: serializePostConditionMode(decodedPostConditions.post_condition_mode),
+    post_conditions: normalizedPostConditions,
     anchor_mode: getTxAnchorModeString(dbTx.anchor_mode),
   };
   return tx;
@@ -752,31 +752,30 @@ export function parseContractCallMetadata(tx: BaseTx): ContractCallTransactionMe
       throw new Error(`Could not find function name "${functionName}" in ABI for ${contractId}`);
     }
   }
+
+  const functionArgs = tx.contract_call_function_args
+    ? decodeClarityValueList(tx.contract_call_function_args).map((c, fnArgIndex) => {
+        const functionArgAbi = functionAbi
+          ? functionAbi.args[fnArgIndex++]
+          : { name: '', type: undefined };
+        return {
+          hex: c.hex,
+          repr: c.repr,
+          name: functionArgAbi.name,
+          type: functionArgAbi.type
+            ? getTypeString(functionArgAbi.type)
+            : decodeClarityValueToTypeName(c.hex),
+        };
+      })
+    : undefined;
+
   const metadata: ContractCallTransactionMetadata = {
     tx_type: 'contract_call',
     contract_call: {
       contract_id: contractId,
       function_name: functionName,
       function_signature: functionAbi ? abiFunctionToString(functionAbi) : '',
-      function_args: tx.contract_call_function_args
-        ? readClarityValueArray(tx.contract_call_function_args).map((c, fnArgIndex) => {
-            const functionArgAbi = functionAbi
-              ? functionAbi.args[fnArgIndex++]
-              : { name: '', type: undefined };
-            return {
-              hex: bufferToHexPrefixString(serializeCV(c)),
-              repr: cvToString(c),
-              name: functionArgAbi.name,
-              // TODO: This stacks.js function throws when given an empty `list` clarity value.
-              //    This is only used to provide function signature type information if the contract
-              //    ABI is unavailable, which should only happen during rare re-org situations.
-              //    Typically this will be filled in with more accurate type data in a later step before
-              //    being sent to client.
-              // type: getCVTypeString(c),
-              type: functionArgAbi.type ? getTypeString(functionArgAbi.type) : '',
-            };
-          })
-        : undefined,
+      function_args: functionArgs,
     },
   };
   return metadata;
@@ -799,7 +798,7 @@ function parseDbAbstractTx(dbTx: DbTx, baseTx: BaseTransaction): AbstractTransac
     tx_status: getTxStatusString(dbTx.status) as TransactionStatus,
     tx_result: {
       hex: dbTx.raw_result,
-      repr: cvToString(deserializeCV(hexToBuffer(dbTx.raw_result))),
+      repr: decodeClarityValueToRepr(dbTx.raw_result),
     },
     microblock_hash: dbTx.microblock_hash,
     microblock_sequence: dbTx.microblock_sequence,

src/api/controllers/db-controller.ts

@@ -39,7 +39,8 @@ import {
   AddressTransactionsWithTransfersListResponse,
   AddressNonces,
 } from '@stacks/stacks-blockchain-api-types';
-import { ChainID, cvToString, deserializeCV } from '@stacks/transactions';
+import { ChainID } from '@stacks/transactions';
+import { decodeClarityValueToRepr } from 'stacks-encoding-native-js';
 import { validate } from '../validate';
 import { NextFunction, Request, Response } from 'express';
 import {
@@ -354,13 +355,12 @@ export function createAddressRouter(db: DataStore, chainId: ChainID): express.Ro
             recipient: transfer.recipient,
           })),
           nft_transfers: entry.nft_transfers.map(transfer => {
-            const valueHex = bufferToHexPrefixString(transfer.value);
-            const valueRepr = cvToString(deserializeCV(transfer.value));
+            const parsedClarityValue = decodeClarityValueToRepr(transfer.value);
             const nftTransfer = {
               asset_identifier: transfer.asset_identifier,
               value: {
-                hex: valueHex,
-                repr: valueRepr,
+                hex: bufferToHexPrefixString(transfer.value),
+                repr: parsedClarityValue,
               },
               sender: transfer.sender,
               recipient: transfer.recipient,
@@ -497,17 +497,21 @@ export function createAddressRouter(db: DataStore, chainId: ChainID): express.Ro
         blockHeight,
         includeUnanchored,
       });
-      const nft_events = response.results.map(row => ({
-        sender: row.sender,
-        recipient: row.recipient,
-        asset_identifier: row.asset_identifier,
-        value: {
-          hex: bufferToHexPrefixString(row.value),
-          repr: cvToString(deserializeCV(row.value)),
-        },
-        tx_id: bufferToHexPrefixString(row.tx_id),
-        block_height: row.block_height,
-      }));
+      const nft_events = response.results.map(row => {
+        const parsedClarityValue = decodeClarityValueToRepr(row.value);
+        const r = {
+          sender: row.sender,
+          recipient: row.recipient,
+          asset_identifier: row.asset_identifier,
+          value: {
+            hex: bufferToHexPrefixString(row.value),
+            repr: parsedClarityValue,
+          },
+          tx_id: bufferToHexPrefixString(row.tx_id),
+          block_height: row.block_height,
+        };
+        return r;
+      });
       const nftListResponse: AddressNftListResponse = {
         nft_events: nft_events,
         total: response.total,

src/api/routes/address.ts

@@ -1,7 +1,7 @@
 import * as express from 'express';
 import * as BN from 'bn.js';
 import * as btc from 'bitcoinjs-lib';
-import * as c32check from 'c32check';
+import { stacksToBitcoinAddress } from 'stacks-encoding-native-js';
 import * as bodyParser from 'body-parser';
 import { asyncHandler } from '../async-handler';
 import { htmlEscape } from 'escape-goat';
@@ -581,7 +581,7 @@ export function createDebugRouter(db: DataStore): express.Router {
         );
       }
       const [contractAddress, contractName] = poxInfo.contract_id.split('.');
-      const btcAddr = c32check.c32ToB58(sender.stacksAddress);
+      const btcAddr = stacksToBitcoinAddress(sender.stacksAddress);
       const { hashMode, data } = convertBTCAddress(btcAddr);
       const cycles = 3;
       const txOptions: SignedContractCallOptions = {

src/api/routes/debug.ts

@@ -61,7 +61,6 @@ import {
   RosettaOperationType,
 } from '../../rosetta-constants';
 import {
-  bitcoinAddressToSTXAddress,
   getOperations,
   getOptionsFromOperations,
   getSigners,
@@ -77,6 +76,7 @@ import {
   parseTransactionMemo,
 } from './../../../rosetta-helpers';
 import { makeRosettaError, rosettaValidateRequest, ValidSchema } from './../../rosetta-validate';
+import { bitcoinToStacksAddress } from 'stacks-encoding-native-js';
 
 export function createRosettaConstructionRouter(db: DataStore, chainId: ChainID): express.Router {
   const router = express.Router();
@@ -109,7 +109,7 @@ export function createRosettaConstructionRouter(db: DataStore, chainId: ChainID)
           res.status(400).json(RosettaErrors[RosettaErrorsTypes.invalidPublicKey]);
           return;
         }
-        const stxAddress = bitcoinAddressToSTXAddress(btcAddress);
+        const stxAddress = bitcoinToStacksAddress(btcAddress);
 
         const accountIdentifier: RosettaAccountIdentifier = {
           address: stxAddress,