Skip to content

Commit e7b8564

Browse files
andrewklauandrewklau
committed
feat: port block-proxy from Rust to TypeScript
Port the block-proxy service from Rust/axum to TypeScript following monorepo conventions. Same env vars, same endpoints, same behavior. Block-proxy: - Data plane (port 3000): /v0/block/:height, /v0/last_block/final, /healthz, /readyz - Admin plane (port 3001): /metrics (Prometheus), /stats (JSON) - Fallback chain: cache -> S3/MinIO -> fastnear -> NEAR Lake - Singleflight dedup to prevent upstream stampede - Background cache writes, TTL-based eviction loop - README.md with full API docs and architecture diagram - .env.example with safe defaults (fastnear-only works out of box) Indexer-staking: - Switch from nb-blocks-minio to nb-neardata - Replace 6 S3 env vars with optional NEARDATA_URL - Default URL derived from NETWORK (just works without config)
1 parent 4166d0b commit e7b8564

30 files changed

+2642
-62
lines changed

‎.github/workflows/workflow-build.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ jobs:
2727
indexer-contract,
2828
explorer-selector,
2929
aggregates,
30+
block-proxy,
3031
]
3132
fail-fast: false
3233
steps:

‎apps/block-proxy/.env.example‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# block-proxy environment configuration
2+
#
3+
# All variables have safe defaults. Zero config = fastnear-only proxy.
4+
# Enable S3 or NEAR Lake by setting their _ENABLED flags and credentials.
5+
6+
# --- Core ---
7+
# PORT=3000
8+
# ADMIN_PORT=3001
9+
# NETWORK=mainnet
10+
# LOG_LEVEL=info
11+
12+
# --- Cache ---
13+
# CACHE_ENABLED=true
14+
# CACHE_DIR=/app/cache
15+
# CACHE_TTL_SECS=3600
16+
17+
# --- fastnear (enabled by default, no config needed) ---
18+
# FASTNEAR_ENABLED=true
19+
# FASTNEAR_URL=https://mainnet.neardata.xyz
20+
21+
# --- S3/MinIO (disabled by default, set these if you have a local block store) ---
22+
# S3_ENABLED=false
23+
# S3_ENDPOINT=http://minio:9000
24+
# S3_BUCKET=blocks
25+
# S3_REGION=us-east-1
26+
# S3_ACCESS_KEY=
27+
# S3_SECRET_KEY=
28+
29+
# --- NEAR Lake (disabled by default, requires AWS credentials in environment) ---
30+
# NEAR_LAKE_ENABLED=false
31+
# NEAR_LAKE_BUCKET=near-lake-data-mainnet
32+
# NEAR_LAKE_REGION=eu-central-1
33+
# UPSTREAM_TIMEOUT_SECS=7

‎apps/block-proxy/.eslintrc.cjs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module.exports = {
2+
extends: ['custom-node'],
3+
ignorePatterns: ['dist'],
4+
root: true,
5+
};

‎apps/block-proxy/Dockerfile‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
FROM node:22-alpine AS base
2+
3+
FROM --platform=$BUILDPLATFORM base AS builder
4+
WORKDIR /app
5+
RUN yarn global add turbo
6+
COPY . .
7+
RUN turbo prune block-proxy --docker
8+
9+
FROM --platform=$BUILDPLATFORM base AS installer
10+
WORKDIR /app
11+
COPY .gitignore .gitignore
12+
COPY --from=builder /app/out/json/ .
13+
COPY --from=builder /app/out/yarn.lock ./yarn.lock
14+
RUN yarn --immutable
15+
COPY --from=builder /app/out/full/ .
16+
COPY turbo.json turbo.json
17+
RUN yarn turbo run build --filter=block-proxy...
18+
19+
FROM base AS runner
20+
ENV NODE_ENV production
21+
USER node
22+
WORKDIR /app
23+
COPY --chown=node:node --from=installer /app .
24+
EXPOSE 3000 3001
25+
CMD ["node", "apps/block-proxy/dist/index.js"]

‎apps/block-proxy/README.md‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
# block-proxy
2+
3+
Caching reverse proxy for NEAR block data. Sits between indexers and upstream block sources (fastnear/neardata, S3/MinIO, NEAR Lake), providing:
4+
5+
- **Local disk cache** with sharded storage and TTL-based eviction
6+
- **Singleflight dedup** — concurrent requests for the same block height are collapsed into one upstream fetch
7+
- **Fallback chain** — cache → S3/MinIO → fastnear → NEAR Lake, with per-source metrics
8+
- **Prometheus metrics** and JSON stats on a separate admin port
9+
10+
## Quick Start
11+
12+
The proxy is URL-compatible with neardata.xyz — any client that fetches `/v0/block/{height}` or `/v0/last_block/final` can point at the proxy instead.
13+
14+
```bash
15+
# Minimal config — fastnear-only proxy, no S3/NEAR Lake
16+
FASTNEAR_ENABLED=true S3_ENABLED=false NEAR_LAKE_ENABLED=false node dist/index.js
17+
```
18+
19+
All env vars have safe defaults. With zero config, the proxy starts on port 3000 serving mainnet blocks via fastnear.
20+
21+
## API
22+
23+
### Data Plane (default port 3000)
24+
25+
| Method | Path | Description |
26+
| ------ | ---------------------- | -------------------------------------------------------------------- |
27+
| GET | `/v0/block/:height` | Fetch block by height. Returns JSON with `X-Upstream-Source` header. |
28+
| GET | `/v0/last_block/final` | Latest finalized block (proxied from fastnear, never cached). |
29+
| GET | `/healthz` | Health check — always returns 200. |
30+
| GET | `/readyz` | Readiness — returns 200 when ready, 503 during startup. |
31+
32+
### Admin Plane (default port 3001)
33+
34+
| Method | Path | Description |
35+
| ------ | ---------- | ------------------------------------------------------------- |
36+
| GET | `/metrics` | Prometheus metrics (text/plain). |
37+
| GET | `/stats` | JSON stats snapshot with hit rates, latencies, dedup savings. |
38+
39+
## Environment Variables
40+
41+
### Core
42+
43+
| Variable | Default | Description |
44+
| ------------ | --------- | ------------------------------------------------------- |
45+
| `PORT` | `3000` | Data server listen port |
46+
| `ADMIN_PORT` | `3001` | Admin server listen port |
47+
| `NETWORK` | `mainnet` | `mainnet` or `testnet` — controls default upstream URLs |
48+
| `LOG_LEVEL` | `info` | Log level (debug, info, warn, error) |
49+
50+
### Cache
51+
52+
| Variable | Default | Description |
53+
| ------------------- | ------------ | -------------------------------------------- |
54+
| `CACHE_ENABLED` | `true` | Enable local disk cache |
55+
| `CACHE_DIR` | `/app/cache` | Cache directory path |
56+
| `CACHE_TTL_SECS` | `3600` | TTL for cached block eviction (seconds) |
57+
| `CACHE_COMPRESSION` | `false` | Reserved for future zstd compression support |
58+
59+
### Upstream: fastnear (neardata.xyz)
60+
61+
| Variable | Default | Description |
62+
| ------------------ | ------------------------ | -------------------------- |
63+
| `FASTNEAR_ENABLED` | `true` | Enable fastnear upstream |
64+
| `FASTNEAR_URL` | _(derived from NETWORK)_ | Override fastnear base URL |
65+
66+
### Upstream: S3/MinIO
67+
68+
| Variable | Default | Description |
69+
| --------------- | ----------------------- | ------------------------ |
70+
| `S3_ENABLED` | `false` | Enable S3/MinIO upstream |
71+
| `S3_ENDPOINT` | _(required if enabled)_ | S3/MinIO endpoint URL |
72+
| `S3_BUCKET` | _(required if enabled)_ | Bucket name |
73+
| `S3_REGION` | `us-east-1` | S3 region |
74+
| `S3_ACCESS_KEY` | _(required if enabled)_ | Access key |
75+
| `S3_SECRET_KEY` | _(required if enabled)_ | Secret key |
76+
77+
### Upstream: NEAR Lake
78+
79+
| Variable | Default | Description |
80+
| ----------------------- | ------------------------ | ------------------------------------ |
81+
| `NEAR_LAKE_ENABLED` | `false` | Enable NEAR Lake upstream |
82+
| `NEAR_LAKE_BUCKET` | _(derived from NETWORK)_ | Override NEAR Lake S3 bucket |
83+
| `NEAR_LAKE_REGION` | `eu-central-1` | NEAR Lake S3 region |
84+
| `UPSTREAM_TIMEOUT_SECS` | `7` | Per-source request timeout (seconds) |
85+
86+
## Architecture
87+
88+
```
89+
┌──────────────┐
90+
indexers ───────► │ block-proxy │
91+
│ :3000 │
92+
└──────┬───────┘
93+
│
94+
┌────────────┼────────────┐
95+
▼ ▼ ▼
96+
┌────────┐ ┌──────────┐ ┌──────────┐
97+
│ cache │ │ S3/MinIO │ │ fastnear │
98+
│ (disk) │ │ │ │ neardata │
99+
└────────┘ └──────────┘ └──────────┘
100+
│
101+
┌────┘
102+
▼
103+
┌──────────┐
104+
│NEAR Lake │
105+
│ (S3) │
106+
└──────────┘
107+
```
108+
109+
Fallback order: cache → S3 → fastnear → NEAR Lake. On any upstream hit, the block is written to cache in the background.
110+
111+
### Singleflight Dedup
112+
113+
When multiple indexers request the same block simultaneously, only one upstream fetch is made. All other requests wait for the leader's result. This prevents upstream stampede.
114+
115+
### Cache Eviction
116+
117+
Every 60 seconds, the eviction loop scans the cache directory and removes any cached block file older than `CACHE_TTL_SECS`.
118+
119+
## Docker
120+
121+
```bash
122+
docker build -f apps/block-proxy/Dockerfile -t block-proxy .
123+
docker run -p 3000:3000 -p 3001:3001 block-proxy
124+
```
125+
126+
## Connecting Indexers
127+
128+
Since block-proxy is URL-compatible with neardata.xyz, any indexer using `nb-neardata` can point to it:
129+
130+
```bash
131+
# In your indexer's env
132+
NEARDATA_URL=http://localhost:3000
133+
```
134+
135+
The `nb-neardata` package accepts an optional `url` field that overrides the default neardata.xyz endpoint.

‎apps/block-proxy/package.json‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"name": "block-proxy",
3+
"version": "0.1.0",
4+
"author": "NearBlocks",
5+
"license": "Business Source License 1.1",
6+
"type": "module",
7+
"imports": {
8+
"#*": [
9+
"./dist/*.js"
10+
]
11+
},
12+
"scripts": {
13+
"clean": "rimraf dist",
14+
"build": "yarn clean && tsc",
15+
"start": "node dist/index.js",
16+
"dev": "tsx watch --env-file=.env src/index.ts",
17+
"lint": "tsc --noEmit && eslint ./ --fix",
18+
"lint:check": "tsc --noEmit && eslint ./"
19+
},
20+
"dependencies": {
21+
"@aws-sdk/client-s3": "3.712.0",
22+
"envalid": "8.0.0",
23+
"express": "4.19.2",
24+
"prom-client": "15.1.3",
25+
"uuid": "11.0.3"
26+
},
27+
"devDependencies": {
28+
"@types/express": "~4.17",
29+
"@types/node": "~20.8",
30+
"@types/uuid": "~10.0",
31+
"nb-logger": "*",
32+
"nb-tsconfig": "*",
33+
"rimraf": "~5.0",
34+
"tsx": "~4.20.3",
35+
"typescript": "~5.2"
36+
}
37+
}

‎apps/block-proxy/src/admin.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import express from 'express';
2+
3+
import { register } from '#metrics';
4+
import type { AppState } from '#state';
5+
6+
export function createAdminServer(state: AppState): express.Express {
7+
const app = express();
8+
9+
// GET /metrics — Prometheus text format
10+
app.get('/metrics', async (_req, res) => {
11+
res
12+
.set('content-type', register.contentType)
13+
.send(await register.metrics());
14+
});
15+
16+
// GET /stats — JSON stats snapshot
17+
app.get('/stats', (_req, res) => {
18+
const tipHeight = state.tipHeight;
19+
const uptimeSecs = Math.floor((Date.now() - state.startTime) / 1000);
20+
const snapshot = state.stats.snapshot(tipHeight, uptimeSecs, {
21+
fastnear: state.fastnearEnabled,
22+
nearLake: state.nearLakeEnabled,
23+
s3: state.s3Enabled,
24+
});
25+
res.json(snapshot);
26+
});
27+
28+
return app;
29+
}

0 commit comments

Comments
 (0)