Merge pull request #1308 from hirosystems/beta

merge beta into master

Author: rafaelcr

.github/workflows/ci.yml

@@ -512,6 +512,7 @@ jobs:
             @semantic-release/changelog
             @semantic-release/git
             @semantic-release/exec
+            conventional-changelog-conventionalcommits
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1

CHANGELOG.md

@@ -1,3 +1,56 @@
+## [5.0.0-beta.7](https://github.com/hirosystems/stacks-blockchain-api/compare/v5.0.0-beta.6...v5.0.0-beta.7) (2022-09-07)
+
+
+### Bug Fixes
+
+* filter BNS processing for successful txs only ([#1309](https://github.com/hirosystems/stacks-blockchain-api/issues/1309)) ([6a12936](https://github.com/hirosystems/stacks-blockchain-api/commit/6a129369c6d9fcdc79b5a7ad288d37784cbe77cc))
+
+## [5.0.0-beta.6](https://github.com/hirosystems/stacks-blockchain-api/compare/v5.0.0-beta.5...v5.0.0-beta.6) (2022-09-01)
+
+
+### Features
+
+* add indexes for index_block_hash on BNS tables ([#1304](https://github.com/hirosystems/stacks-blockchain-api/issues/1304)) ([bbf4b2d](https://github.com/hirosystems/stacks-blockchain-api/commit/bbf4b2d2b8c7f6ed30bfda6eaa430d5c2e84cdf5))
+
+## [5.0.0-beta.5](https://github.com/hirosystems/stacks-blockchain-api/compare/v5.0.0-beta.4...v5.0.0-beta.5) (2022-08-31)
+
+
+### Bug Fixes
+
+* detect name transfers and renewals in special circumstances ([#1303](https://github.com/hirosystems/stacks-blockchain-api/issues/1303)) ([cd381a9](https://github.com/hirosystems/stacks-blockchain-api/commit/cd381a95b4d0d3f4bb08e447500153c3f652eff6))
+
+## [5.0.0-beta.4](https://github.com/hirosystems/stacks-blockchain-api/compare/v5.0.0-beta.3...v5.0.0-beta.4) (2022-08-31)
+
+
+### Bug Fixes
+
+* add postgres connection error checking for ECONNRESET code ([03a1896](https://github.com/hirosystems/stacks-blockchain-api/commit/03a1896cff8937a5f39a8b75e5adf51a6344592c))
+
+## [5.0.0-beta.3](https://github.com/hirosystems/stacks-blockchain-api/compare/v5.0.0-beta.2...v5.0.0-beta.3) (2022-08-31)
+
+
+### Bug Fixes
+
+* import BNS v1 data during event replay ([#1301](https://github.com/hirosystems/stacks-blockchain-api/issues/1301)) ([bc59817](https://github.com/hirosystems/stacks-blockchain-api/commit/bc59817aa98dd3a978a27b73d14738b64eb823f9))
+
+## [5.0.0-beta.2](https://github.com/hirosystems/stacks-blockchain-api/compare/v5.0.0-beta.1...v5.0.0-beta.2) (2022-08-26)
+
+
+### Bug Fixes
+
+* bump version ([3863cce](https://github.com/hirosystems/stacks-blockchain-api/commit/3863cce1a64cf7a4c6cffd4f888c049cfd3ada65))
+
+## [5.0.0-beta.1](https://github.com/hirosystems/stacks-blockchain-api/compare/v4.1.2...v5.0.0-beta.1) (2022-08-26)
+
+
+### ⚠ BREAKING CHANGES
+
+* optimize tables and improve canonical treatment of BNS data (#1287)
+
+### Features
+
+* optimize tables and improve canonical treatment of BNS data ([#1287](https://github.com/hirosystems/stacks-blockchain-api/issues/1287)) ([1f64818](https://github.com/hirosystems/stacks-blockchain-api/commit/1f648187b8c701e802a06bac52b077fd10571ff7))
+
 ## [4.1.2](https://github.com/hirosystems/stacks-blockchain-api/compare/v4.1.1...v4.1.2) (2022-08-18)
 
 

package.json

@@ -56,8 +56,12 @@
   "engineStrict": true,
   "release": {
     "plugins": [
-      "@semantic-release/commit-analyzer",
-      "@semantic-release/release-notes-generator",
+      ["@semantic-release/commit-analyzer", {
+        "preset": "conventionalcommits"
+      }],
+      ["@semantic-release/release-notes-generator", {
+        "preset": "conventionalcommits"
+      }],
       [
         "@semantic-release/exec",
         {

readme.md

@@ -98,19 +98,51 @@ For running offline mode set an environment variable `STACKS_API_MODE=offline`
 
 ## Event Replay
 
-The stacks-node is only able to emit events live as they happen. This poses a problem in the scenario where the stacks-blockchain-api needs to
-be upgraded and its database cannot be migrated to a new schema. One way to handle this upgrade is to wipe the stacks-blockchain-api's database
-and stacks-node working directory, and re-sync from scratch.
+The stacks-node is only able to emit events live as they happen. This poses a problem in the
+scenario where the stacks-blockchain-api needs to be upgraded and its database cannot be migrated to
+a new schema. One way to handle this upgrade is to wipe the stacks-blockchain-api's database and
+stacks-node working directory, and re-sync from scratch.
 
-Alternatively, an event-replay feature is available where the API records the HTTP POST requests from the stacks-node event emitter, then streams
-these events back to itself. Essentially simulating a wipe & full re-sync, but much quicker.
+Alternatively, an event-replay feature is available where the API records the HTTP POST requests
+from the stacks-node event emitter, then streams these events back to itself. Essentially simulating
+a wipe & full re-sync, but much quicker.
 
-The feature can be used via program args. For example, if there are breaking changes in the API's sql schema, like adding a new column which requires
-event's to be re-played, the following steps could be ran:
+The feature can be used via program args. For example, if there are breaking changes in the API's
+sql schema, like adding a new column which requires event's to be re-played, the following steps
+could be ran:
 
 ### Event Replay Instructions
 
-1. Ensure the API process is not running. When stopping the API, let the process exit gracefully so that any in-progress SQL writes can finish.
+#### V1 BNS Data
+
+**Optional but recommended** - If you want the V1 BNS data, there are going to be a few extra steps:
+
+1. Download BNS data:
+   ```shell
+   curl -L https://storage.googleapis.com/blockstack-v1-migration-data/export-data.tar.gz -o /stacks-node/bns/export-data.tar.gz
+   ```
+1. Extract it:
+   ```shell
+   tar -xzvf ./bns/export-data.tar.gz -C /stacks-node/bns/
+   ```
+1. Each file in `./bns` will have a corresponding `sha256` value. To Verify, run a script like the
+   following to check the sha256sum:
+
+    ```bash
+    for file in `ls /stacks-node/bns/* | grep -v sha256 | grep -v .tar.gz`; do
+        if [ $(sha256sum $file | awk {'print $1'}) == $(cat ${file}.sha256 ) ]; then
+            echo "sha256 Matched $file"
+        else
+            echo "sha256 Mismatch $file"
+        fi
+    done
+    ```
+1. Set the data's location as the value of `BNS_IMPORT_DIR` in your `.env` file.
+
+#### Export and Import
+
+1. Ensure the API process is not running. When stopping the API, let the process exit gracefully so
+   that any in-progress SQL writes can finish.
 1. Export event data to disk with the `export-events` command:
 
    ```shell
@@ -119,19 +151,25 @@ event's to be re-played, the following steps could be ran:
 1. Update to the new stacks-blockchain-api version.
 1. Perform the event playback using the `import-events` command:
 
-   **WARNING**: This will **drop _all_ tables** from the configured Postgres database, including any tables not automatically added by the API.
+   **WARNING**: This will **drop _all_ tables** from the configured Postgres database, including any
+   tables not automatically added by the API.
 
    ```shell
    node ./lib/index.js import-events --file /tmp/stacks-node-events.tsv --wipe-db --force
    ```
 
    This command has two modes of operation, specified by the `--mode` option:
-   * `archival` (default): The process will import and ingest *all* blockchain events that have happened since the first block.
-   * `pruned`: The import process will ignore some prunable events (mempool, microblocks) until the import block height has reached `chain tip - 256` blocks. This saves a considerable amount of time during import, but sacrifices some historical data. You can use this mode if you're mostly interested in running an API that prioritizes real time information.
-
-Alternatively, instead of performing the `export-events` command in step 1, an environmental variable can be set which enables events to be streamed to a file
-as they are received, while the application is running normally. To enable this feature, set the `STACKS_EXPORT_EVENTS_FILE` env var to the file path where
-events should be appended. Example:
+   * `archival` (default): The process will import and ingest *all* blockchain events that have
+     happened since the first block.
+   * `pruned`: The import process will ignore some prunable events (mempool, microblocks) until the
+     import block height has reached `chain tip - 256` blocks. This saves a considerable amount of
+     time during import, but sacrifices some historical data. You can use this mode if you're mostly
+     interested in running an API that prioritizes real time information.
+
+Alternatively, instead of performing the `export-events` command in step 1, an environmental
+variable can be set which enables events to be streamed to a file as they are received, while the
+application is running normally. To enable this feature, set the `STACKS_EXPORT_EVENTS_FILE` env var
+to the file path where events should be appended. Example:
 ```
 STACKS_EXPORT_EVENTS_FILE=/tmp/stacks-node-events.tsv
 ```

running_an_api.md

@@ -78,34 +78,14 @@ Since we'll need to create some files/dirs for persistent data we'll first creat
 We'll be using:
 
 ```bash
-$ mkdir -p ./stacks-node/{persistent-data/postgres,persistent-data/stacks-blockchain,bns,config}
+$ mkdir -p ./stacks-node/{persistent-data/postgres,persistent-data/stacks-blockchain,config}
 $ docker pull blockstack/stacks-blockchain-api \
     && docker pull blockstack/stacks-blockchain \
     && docker pull postgres:alpine
 $ docker network create stacks-blockchain > /dev/null 2>&1
 $ cd ./stacks-node
 ```
 
-**Optional but recommended**: If you need the v1 BNS data, there are going to be a few extra steps.
-
-1. Download the BNS data:  
-`curl -L https://storage.googleapis.com/blockstack-v1-migration-data/export-data.tar.gz -o ./bns/export-data.tar.gz`
-2. Extract the data:  
-`tar -xzvf ./bns/export-data.tar.gz -C ./bns/`
-3. Each file in `./bns` will have a corresponding `sha256` value.  
-
-To Verify, run a script like the following to check the sha256sum:
-
-```bash
-for file in `ls ./bns/* | grep -v sha256 | grep -v .tar.gz`; do
-    if [ $(sha256sum $file | awk {'print $1'}) == $(cat ${file}.sha256 ) ]; then
-        echo "sha256 Matched $file"
-    else
-        echo "sha256 Mismatch $file"
-    fi
-done
-```
-
 ## Postgres
 
 The `postgres:alpine` image can be run with default settings, the only requirement is that a password Environment Variable is set for the `postgres` user: `POSTGRES_PASSWORD=postgres`
@@ -161,16 +141,9 @@ STACKS_BLOCKCHAIN_API_PORT=3999
 STACKS_BLOCKCHAIN_API_HOST=0.0.0.0
 STACKS_CORE_RPC_HOST=stacks-blockchain
 STACKS_CORE_RPC_PORT=20443
-BNS_IMPORT_DIR=/bns-data
 API_DOCS_URL=https://docs.hiro.so/api
 ```
 
-**Note** that here we are importing the bns data with the env var `BNS_IMPORT`.  
-
-To Disable this import, simply comment the line: `#BNS_IMPORT_DIR=/bns-data`  
-
-***If you leave this enabled***: please allow several minutes for the one-time import to complete before continuing.
-
 The other Environment Variables to pay attention to:
 
 - `PG_HOST`: Set this to your **postgres** instance. In this guide, we'll be using a container named `postgres`.
@@ -184,7 +157,6 @@ docker run -d --rm \
     --name stacks-blockchain-api \
     --net=stacks-blockchain \
     --env-file $(pwd)/.env \
-    -v $(pwd)/bns:/bns-data \
     -p 3700:3700 \
     -p 3999:3999 \
     blockstack/stacks-blockchain-api

running_api_from_source.md

@@ -35,15 +35,15 @@ Since we'll need to create some files/dirs for persistent data,
 we'll first create a base directory structure and set some permissions:
 
 ```bash
-$ sudo mkdir -p /stacks-node/{persistent-data/stacks-blockchain,bns,config,binaries}
+$ sudo mkdir -p /stacks-node/{persistent-data/stacks-blockchain,config,binaries}
 $ sudo chown -R $(whoami) /stacks-node 
 $ cd /stacks-node
 ```
 
 ## Install Requirements
 
 ```bash
-$ PG_VERSION=12 \
+$ PG_VERSION=14 \
   && NODE_VERSION=16 \
   && sudo apt-get update \
   && sudo apt-get install -y \
@@ -65,26 +65,6 @@ $ PG_VERSION=12 \
     nodejs
 ```
 
-**Optional but recommended** - If you want the V1 BNS data, there are going to be a few extra steps:
-
-1. Download the BNS data:  
-`curl -L https://storage.googleapis.com/blockstack-v1-migration-data/export-data.tar.gz -o /stacks-node/bns/export-data.tar.gz`
-2. Extract the data:  
-`tar -xzvf ./bns/export-data.tar.gz -C /stacks-node/bns/`
-3. Each file in `./bns` will have a corresponding `sha256` value.
-
-To Verify, run a script like the following to check the sha256sum:
-
-```bash
-for file in `ls /stacks-node/bns/* | grep -v sha256 | grep -v .tar.gz`; do
-    if [ $(sha256sum $file | awk {'print $1'}) == $(cat ${file}.sha256 ) ]; then
-        echo "sha256 Matched $file"
-    else
-        echo "sha256 Mismatch $file"
-    fi
-done
-```
-
 ## postgres
 
 ### postgres permissions
@@ -127,8 +107,6 @@ $ git clone https://github.com/hirosystems/stacks-blockchain-api /stacks-node/st
 The stacks blockchain api requires several Environment Variables to be set in order to run properly.  
 To reduce complexity, we're going to create a `.env` file that we'll use for these env vars.  
 
-** Note: ** to enable BNS names, uncomment `BNS_IMPORT_DIR` in the below `.env` file. 
-
 Create a new file: `/stacks-node/stacks-blockchain-api/.env` with the following content:
 
 ```bash
@@ -148,7 +126,6 @@ STACKS_BLOCKCHAIN_API_PORT=3999
 STACKS_BLOCKCHAIN_API_HOST=0.0.0.0
 STACKS_CORE_RPC_HOST=localhost
 STACKS_CORE_RPC_PORT=20443
-#BNS_IMPORT_DIR=/stacks-node/bns
 EOF
 $ cd /stacks-node/stacks-blockchain-api && nohup node ./lib/index.js &
 ```

src/api/routes/bns/addresses.ts

@@ -3,13 +3,20 @@ import { asyncHandler } from '../../async-handler';
 import { DataStore } from '../../../datastore/common';
 import { isUnanchoredRequest } from '../../query-helpers';
 import { ChainID } from '@stacks/transactions';
+import {
+  getETagCacheHandler,
+  setETagCacheHeaders,
+} from '../../../api/controllers/cache-controller';
 
 const SUPPORTED_BLOCKCHAINS = ['stacks'];
 
 export function createBnsAddressesRouter(db: DataStore, chainId: ChainID): express.Router {
   const router = express.Router();
+  const cacheHandler = getETagCacheHandler(db);
+
   router.get(
     '/:blockchain/:address',
+    cacheHandler,
     asyncHandler(async (req, res, next) => {
       // Retrieves a list of names owned by the address provided.
       const { blockchain, address } = req.params;
@@ -23,6 +30,7 @@ export function createBnsAddressesRouter(db: DataStore, chainId: ChainID): expre
         includeUnanchored,
         chainId,
       });
+      setETagCacheHeaders(res);
       if (namesByAddress.found) {
         res.json({ names: namesByAddress.result });
       } else {