Skip to content

Commit 6485162

Browse files
andrewklauandrewklau
committed
feat: port block-proxy from Rust to TypeScript
Port the block-proxy service from Rust/axum to TypeScript following monorepo conventions. Same env vars, same endpoints, same behavior. Block-proxy: - Data plane (port 3000): /v0/block/:height, /v0/last_block/final, /healthz, /readyz - Admin plane (port 3001): /metrics (Prometheus), /stats (JSON) - Fallback chain: cache -> S3/MinIO -> fastnear -> NEAR Lake - Singleflight dedup to prevent upstream stampede - Background cache writes, TTL-based eviction loop - README.md with full API docs and architecture diagram - .env.example with safe defaults (fastnear-only works out of box) Indexer-staking: - Switch from nb-blocks-minio to nb-neardata - Replace 6 S3 env vars with optional NEARDATA_URL - Default URL derived from NETWORK (just works without config)
1 parent 4166d0b commit 6485162

30 files changed

+2673
-62
lines changed

‎.github/workflows/workflow-build.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ jobs:
2727
indexer-contract,
2828
explorer-selector,
2929
aggregates,
30+
block-proxy,
3031
]
3132
fail-fast: false
3233
steps:

‎apps/block-proxy/.env.example‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# block-proxy environment configuration
2+
#
3+
# All variables have safe defaults. Zero config = fastnear-only proxy.
4+
# Enable S3 or NEAR Lake by setting their _ENABLED flags and credentials.
5+
6+
# --- Core ---
7+
# PORT=3000
8+
# ADMIN_PORT=3001
9+
# NETWORK=mainnet
10+
# LOG_LEVEL=info
11+
12+
# --- Cache ---
13+
# CACHE_ENABLED=true
14+
# CACHE_DIR=/tmp/block-proxy-cache
15+
# CACHE_TTL_SECS=3600
16+
# RECENT_BLOCK_WINDOW=1000
17+
18+
# --- fastnear (enabled by default, no config needed) ---
19+
# FASTNEAR_ENABLED=true
20+
# FASTNEAR_URL=https://mainnet.neardata.xyz
21+
22+
# --- S3/MinIO (disabled by default, set these if you have a local block store) ---
23+
# S3_ENABLED=false
24+
# S3_ENDPOINT=http://minio:9000
25+
# S3_BUCKET=blocks
26+
# S3_REGION=us-east-1
27+
# S3_ACCESS_KEY=
28+
# S3_SECRET_KEY=
29+
30+
# --- NEAR Lake (disabled by default, requires AWS credentials in environment) ---
31+
# NEAR_LAKE_ENABLED=false
32+
# NEAR_LAKE_BUCKET=near-lake-data-mainnet
33+
# NEAR_LAKE_REGION=eu-central-1
34+
# UPSTREAM_TIMEOUT_SECS=7

‎apps/block-proxy/.eslintrc.cjs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
module.exports = {
2+
extends: ['custom-node'],
3+
root: true,
4+
};

‎apps/block-proxy/Dockerfile‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
FROM node:22-alpine AS base
2+
3+
FROM --platform=$BUILDPLATFORM base AS builder
4+
WORKDIR /app
5+
RUN yarn global add turbo
6+
COPY . .
7+
RUN turbo prune block-proxy --docker
8+
9+
FROM --platform=$BUILDPLATFORM base AS installer
10+
WORKDIR /app
11+
COPY .gitignore .gitignore
12+
COPY --from=builder /app/out/json/ .
13+
COPY --from=builder /app/out/yarn.lock ./yarn.lock
14+
RUN yarn --immutable
15+
COPY --from=builder /app/out/full/ .
16+
COPY turbo.json turbo.json
17+
RUN yarn turbo run build --filter=block-proxy...
18+
19+
FROM base AS runner
20+
ENV NODE_ENV production
21+
USER node
22+
WORKDIR /app
23+
COPY --chown=node:node --from=installer /app .
24+
EXPOSE 3000 3001
25+
CMD ["node", "apps/block-proxy/dist/index.js"]

‎apps/block-proxy/README.md‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
# block-proxy
2+
3+
Caching reverse proxy for NEAR block data. Sits between indexers and upstream block sources (fastnear/neardata, S3/MinIO, NEAR Lake), providing:
4+
5+
- **Local disk cache** with sharded storage and TTL-based eviction
6+
- **Singleflight dedup** — concurrent requests for the same block height are collapsed into one upstream fetch
7+
- **Fallback chain** — cache → S3/MinIO → fastnear → NEAR Lake, with per-source metrics
8+
- **Prometheus metrics** and JSON stats on a separate admin port
9+
10+
## Quick Start
11+
12+
The proxy is URL-compatible with neardata.xyz — any client that fetches `/v0/block/{height}` or `/v0/last_block/final` can point at the proxy instead.
13+
14+
```bash
15+
# Minimal config — fastnear-only proxy, no S3/NEAR Lake
16+
FASTNEAR_ENABLED=true S3_ENABLED=false NEAR_LAKE_ENABLED=false node dist/index.js
17+
```
18+
19+
All env vars have safe defaults. With zero config, the proxy starts on port 3000 serving mainnet blocks via fastnear.
20+
21+
## API
22+
23+
### Data Plane (default port 3000)
24+
25+
| Method | Path | Description |
26+
| ------ | ---------------------- | -------------------------------------------------------------------- |
27+
| GET | `/v0/block/:height` | Fetch block by height. Returns JSON with `X-Upstream-Source` header. |
28+
| GET | `/v0/last_block/final` | Latest finalized block (proxied from fastnear, never cached). |
29+
| GET | `/healthz` | Health check — always returns 200. |
30+
| GET | `/readyz` | Readiness — returns 200 when ready, 503 during startup. |
31+
32+
### Admin Plane (default port 3001)
33+
34+
| Method | Path | Description |
35+
| ------ | ---------- | ------------------------------------------------------------- |
36+
| GET | `/metrics` | Prometheus metrics (text/plain). |
37+
| GET | `/stats` | JSON stats snapshot with hit rates, latencies, dedup savings. |
38+
39+
## Environment Variables
40+
41+
### Core
42+
43+
| Variable | Default | Description |
44+
| ------------ | --------- | ------------------------------------------------------- |
45+
| `PORT` | `3000` | Data server listen port |
46+
| `ADMIN_PORT` | `3001` | Admin server listen port |
47+
| `NETWORK` | `mainnet` | `mainnet` or `testnet` — controls default upstream URLs |
48+
| `LOG_LEVEL` | `info` | Log level (debug, info, warn, error) |
49+
50+
### Cache
51+
52+
| Variable | Default | Description |
53+
| --------------------- | ------------------------ | ------------------------------------------------- |
54+
| `CACHE_ENABLED` | `true` | Enable local disk cache |
55+
| `CACHE_DIR` | `/tmp/block-proxy-cache` | Cache directory path |
56+
| `CACHE_TTL_SECS` | `3600` | TTL for recent block eviction (seconds) |
57+
| `RECENT_BLOCK_WINDOW` | `1000` | Blocks behind tip considered "recent" (evictable) |
58+
| `CACHE_COMPRESSION` | `false` | Reserved for future zstd compression support |
59+
60+
### Upstream: fastnear (neardata.xyz)
61+
62+
| Variable | Default | Description |
63+
| ------------------ | ------------------------ | -------------------------- |
64+
| `FASTNEAR_ENABLED` | `true` | Enable fastnear upstream |
65+
| `FASTNEAR_URL` | _(derived from NETWORK)_ | Override fastnear base URL |
66+
67+
### Upstream: S3/MinIO
68+
69+
| Variable | Default | Description |
70+
| --------------- | ----------------------- | ------------------------ |
71+
| `S3_ENABLED` | `true` | Enable S3/MinIO upstream |
72+
| `S3_ENDPOINT` | _(required if enabled)_ | S3/MinIO endpoint URL |
73+
| `S3_BUCKET` | _(required if enabled)_ | Bucket name |
74+
| `S3_REGION` | `us-east-1` | S3 region |
75+
| `S3_ACCESS_KEY` | _(required if enabled)_ | Access key |
76+
| `S3_SECRET_KEY` | _(required if enabled)_ | Secret key |
77+
78+
### Upstream: NEAR Lake
79+
80+
| Variable | Default | Description |
81+
| ----------------------- | ------------------------ | ------------------------------------ |
82+
| `NEAR_LAKE_ENABLED` | `true` | Enable NEAR Lake upstream |
83+
| `NEAR_LAKE_BUCKET` | _(derived from NETWORK)_ | Override NEAR Lake S3 bucket |
84+
| `NEAR_LAKE_REGION` | `eu-central-1` | NEAR Lake S3 region |
85+
| `UPSTREAM_TIMEOUT_SECS` | `7` | Per-source request timeout (seconds) |
86+
87+
## Architecture
88+
89+
```
90+
┌──────────────┐
91+
indexers ───────► │ block-proxy │
92+
│ :3000 │
93+
└──────┬───────┘
94+
│
95+
┌────────────┼────────────┐
96+
▼ ▼ ▼
97+
┌────────┐ ┌──────────┐ ┌──────────┐
98+
│ cache │ │ S3/MinIO │ │ fastnear │
99+
│ (disk) │ │ │ │ neardata │
100+
└────────┘ └──────────┘ └──────────┘
101+
│
102+
┌────┘
103+
▼
104+
┌──────────┐
105+
│NEAR Lake │
106+
│ (S3) │
107+
└──────────┘
108+
```
109+
110+
Fallback order: cache → S3 → fastnear → NEAR Lake. On any upstream hit, the block is written to cache in the background.
111+
112+
### Singleflight Dedup
113+
114+
When multiple indexers request the same block simultaneously, only one upstream fetch is made. All other requests wait for the leader's result. This prevents upstream stampede.
115+
116+
### Cache Eviction
117+
118+
Every 60 seconds, the eviction loop scans the cache directory:
119+
120+
- Blocks within `RECENT_BLOCK_WINDOW` of the chain tip that are older than `CACHE_TTL_SECS` are evicted
121+
- Historical blocks (older than the window) are permanent and never evicted
122+
123+
## Docker
124+
125+
```bash
126+
docker build -f apps/block-proxy/Dockerfile -t block-proxy .
127+
docker run -p 3000:3000 -p 3001:3001 block-proxy
128+
```
129+
130+
## Connecting Indexers
131+
132+
Since block-proxy is URL-compatible with neardata.xyz, any indexer using `nb-neardata` can point to it:
133+
134+
```bash
135+
# In your indexer's env
136+
NEARDATA_URL=http://localhost:3000
137+
```
138+
139+
The `nb-neardata` package accepts an optional `url` field that overrides the default neardata.xyz endpoint.

‎apps/block-proxy/package.json‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"name": "block-proxy",
3+
"version": "0.1.0",
4+
"author": "NearBlocks",
5+
"license": "Business Source License 1.1",
6+
"type": "module",
7+
"imports": {
8+
"#*": [
9+
"./dist/*.js"
10+
]
11+
},
12+
"scripts": {
13+
"clean": "rimraf dist",
14+
"build": "yarn clean && tsc",
15+
"start": "node dist/index.js",
16+
"dev": "tsx watch --env-file=.env src/index.ts",
17+
"lint": "tsc --noEmit && eslint ./ --fix",
18+
"lint:check": "tsc --noEmit && eslint ./"
19+
},
20+
"dependencies": {
21+
"@aws-sdk/client-s3": "3.712.0",
22+
"envalid": "8.0.0",
23+
"express": "4.19.2",
24+
"prom-client": "15.1.3",
25+
"uuid": "11.0.3"
26+
},
27+
"devDependencies": {
28+
"@types/express": "~4.17",
29+
"@types/node": "~20.8",
30+
"@types/uuid": "~10.0",
31+
"nb-logger": "*",
32+
"nb-tsconfig": "*",
33+
"rimraf": "~5.0",
34+
"tsx": "~4.20.3",
35+
"typescript": "~5.2"
36+
}
37+
}

‎apps/block-proxy/src/admin.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import express from 'express';
2+
3+
import { register } from '#metrics';
4+
import type { AppState } from '#state';
5+
6+
export function createAdminServer(state: AppState): express.Express {
7+
const app = express();
8+
9+
// GET /metrics — Prometheus text format
10+
app.get('/metrics', async (_req, res) => {
11+
res
12+
.set('content-type', register.contentType)
13+
.send(await register.metrics());
14+
});
15+
16+
// GET /stats — JSON stats snapshot
17+
app.get('/stats', (_req, res) => {
18+
const tipHeight = state.tipHeight;
19+
const uptimeSecs = Math.floor((Date.now() - state.startTime) / 1000);
20+
const snapshot = state.stats.snapshot(tipHeight, uptimeSecs, {
21+
fastnear: state.fastnearEnabled,
22+
nearLake: state.nearLakeEnabled,
23+
s3: state.s3Enabled,
24+
});
25+
res.json(snapshot);
26+
});
27+
28+
return app;
29+
}

0 commit comments

Comments
 (0)