Merge pull request #1694 from hirosystems/feat/event-replay-parquet-based

event-replay optimizations

Author: csgui

.env

@@ -172,12 +172,15 @@ STACKS_NODE_TYPE=L1
 # STACKS_ADDRESS_CACHE_SIZE=10000
 
 # Specify a URL to redirect from /doc. If this URL is not provided, server renders local documentation
-# of openapi.yaml for test / development NODE_ENV. 
+# of openapi.yaml for test / development NODE_ENV.
 # For production, /doc is not served if this env var is not provided.
-# API_DOCS_URL="https://docs.hiro.so/api" 
+# API_DOCS_URL="https://docs.hiro.so/api"
 
-# For use while syncing. Places the API into an "Initial Block Download(IBD)" mode, 
-# forcing it to stop any redundant processing until the node is fully synced up to its peers. 
-# Some examples of processing that are avoided are: 
+# For use while syncing. Places the API into an "Initial Block Download(IBD)" mode,
+# forcing it to stop any redundant processing until the node is fully synced up to its peers.
+# Some examples of processing that are avoided are:
 # REFRESH MATERIALIZED VIEW SQLs that are extremely CPU intensive on the PG instance, Mempool messages, etc.,
 # IBD_MODE_UNTIL_BLOCK=
+
+# Folder with events to be imported by the event-replay.
+STACKS_EVENTS_DIR=./eventssds

Dockerfile

@@ -8,4 +8,18 @@ RUN echo "GIT_TAG=$(git tag --points-at HEAD)" >> .env
 RUN npm config set unsafe-perm true && npm ci && npm run build && npm run build:docs && npm prune --production
 RUN apk del .build-deps
 
+# As no pre-built binaries of duckdb can be found for Alpine (musl based),
+# a rebuild of duckdb package is need.
+#
+# Library used by the event-replay based on parquet files.
+ARG DUCKDB_VERSION=0.8.1
+WORKDIR /duckdb
+RUN apk add --no-cache --virtual .duckdb-build-deps python3 git g++ make
+RUN git clone https://github.com/duckdb/duckdb.git -b v${DUCKDB_VERSION} --depth 1 \
+  && cd duckdb/tools/nodejs \
+  && ./configure && make all
+WORKDIR /app
+RUN npm uninstall duckdb && npm install /duckdb/duckdb/tools/nodejs
+RUN apk del .duckdb-build-deps
+
 CMD ["node", "./lib/index.js"]

README.md

@@ -111,9 +111,23 @@ The feature can be used via program args. For example, if there are breaking cha
 SQL schema, like adding a new column that requires events to be re-played, the following steps
 could be run:
 
-### Event Replay Instructions
+### Event Replay V2
 
-#### V1 BNS Data
+This version of the replay process relies on parquet files processing instead of TSV files.
+
+There are some improvements on the replay process and this version is is, around, 10x times faster than the previous (V1) one.
+
+__Note: the previous event-replay version is still available and can be used as well, for the same purpose.__
+
+#### Instructions
+
+To run the new event-replay, please follow the instructions at [stacks-event-replay](https://github.com/hirosystems/stacks-event-replay#installation) repository.
+
+### Event Replay V1
+
+#### Instructions
+
+##### V1 BNS Data
 
 **Optional but recommended** - If you want the V1 BNS data, there are going to be a few extra steps:
 
@@ -139,7 +153,7 @@ could be run:
     ```
 1. Set the data's location as the value of `BNS_IMPORT_DIR` in your `.env` file.
 
-#### Export and Import
+##### Export and Import
 
 1. Ensure the API process is not running. When stopping the API, let the process exit gracefully so
    that any in-progress SQL writes can finish.

migrations/1620937804009_microblocks.js

@@ -26,7 +26,7 @@ exports.up = pgm => {
       type: 'integer',
       notNull: true,
     },
-    // For the first microblock (sequence number 0), this points to the parent/anchor block hash, 
+    // For the first microblock (sequence number 0), this points to the parent/anchor block hash,
     // for subsequent microblocks it points to the previous microblock's hash.
     microblock_parent_hash: {
       type: 'bytea',

migrations/1621511823381_nft_metadata.js

@@ -20,23 +20,23 @@ exports.up = pgm => {
     image_uri: {
       type: 'string',
       notNull: true,
-    }, 
+    },
     image_canonical_uri: {
-      type: 'string', 
+      type: 'string',
       notNull: true,
-    }, 
+    },
     contract_id: {
-      type: 'string', 
-      notNull: true, 
+      type: 'string',
+      notNull: true,
       unique: true,
     },
     tx_id: {
       type: 'bytea',
       notNull: true,
     },
     sender_address: {
-      type: 'string', 
-      notNull: true, 
+      type: 'string',
+      notNull: true,
     }
   });
 

migrations/1621511832113_ft_metadata.js

@@ -20,31 +20,31 @@ exports.up = pgm => {
     image_uri: {
       type: 'string',
       notNull: true,
-    }, 
+    },
     image_canonical_uri: {
-      type: 'string', 
+      type: 'string',
       notNull: true,
-    }, 
+    },
     contract_id: {
-      type: 'string', 
-      notNull: true, 
+      type: 'string',
+      notNull: true,
       unique: true,
     },
     symbol: {
-      type: 'string', 
-      notNull: true, 
+      type: 'string',
+      notNull: true,
     },
     decimals: {
-      type: 'integer', 
-      notNull: true, 
+      type: 'integer',
+      notNull: true,
     },
     tx_id: {
       type: 'bytea',
       notNull: true,
     },
     sender_address: {
-      type: 'string', 
-      notNull: true, 
+      type: 'string',
+      notNull: true,
     }
   });
 

migrations/1666703991492_pox_events.js

@@ -132,46 +132,46 @@ exports.up = pgm => {
   });
 
   pgm.addConstraint('pox2_events', 'valid_event_specific_columns', `CHECK (
-    CASE name 
-      WHEN 'handle-unlock' THEN 
-        first_cycle_locked IS NOT NULL AND 
+    CASE name
+      WHEN 'handle-unlock' THEN
+        first_cycle_locked IS NOT NULL AND
         first_unlocked_cycle IS NOT NULL
       WHEN 'stack-stx' THEN
-        lock_period IS NOT NULL AND 
-        lock_amount IS NOT NULL AND 
-        start_burn_height IS NOT NULL AND 
+        lock_period IS NOT NULL AND
+        lock_amount IS NOT NULL AND
+        start_burn_height IS NOT NULL AND
         unlock_burn_height IS NOT NULL
       WHEN 'stack-increase' THEN
-        increase_by IS NOT NULL AND 
+        increase_by IS NOT NULL AND
         total_locked IS NOT NULL
       WHEN 'stack-extend' THEN
-        extend_count IS NOT NULL AND 
+        extend_count IS NOT NULL AND
         unlock_burn_height IS NOT NULL
       WHEN 'delegate-stx' THEN
-        amount_ustx IS NOT NULL AND 
+        amount_ustx IS NOT NULL AND
         delegate_to IS NOT NULL
       WHEN 'delegate-stack-stx' THEN
-        lock_period IS NOT NULL AND 
-        lock_amount IS NOT NULL AND 
-        start_burn_height IS NOT NULL AND 
-        unlock_burn_height IS NOT NULL AND 
+        lock_period IS NOT NULL AND
+        lock_amount IS NOT NULL AND
+        start_burn_height IS NOT NULL AND
+        unlock_burn_height IS NOT NULL AND
         delegator IS NOT NULL
       WHEN 'delegate-stack-increase' THEN
-        increase_by IS NOT NULL AND 
+        increase_by IS NOT NULL AND
         total_locked IS NOT NULL AND
         delegator IS NOT NULL
       WHEN 'delegate-stack-extend' THEN
-        extend_count IS NOT NULL AND 
-        unlock_burn_height IS NOT NULL AND 
+        extend_count IS NOT NULL AND
+        unlock_burn_height IS NOT NULL AND
         delegator IS NOT NULL
       WHEN 'stack-aggregation-commit' THEN
-        reward_cycle IS NOT NULL AND 
+        reward_cycle IS NOT NULL AND
         amount_ustx IS NOT NULL
       WHEN 'stack-aggregation-commit-indexed' THEN
-        reward_cycle IS NOT NULL AND 
+        reward_cycle IS NOT NULL AND
         amount_ustx IS NOT NULL
       WHEN 'stack-aggregation-increase' THEN
-        reward_cycle IS NOT NULL AND 
+        reward_cycle IS NOT NULL AND
         amount_ustx IS NOT NULL
       ELSE false
     END