Skip to content

Commit 9e1ba61

Browse files
feedthejimclaude
feedthejim
and
claude
committed
chore: add dev boot profiling and benchmark scripts
- benchmark-boot-time.sh: Wall-clock benchmark with two metrics - benchmark-next-dev-boot.js: Multi-iteration benchmark with stats - profile-next-dev-boot.js: CPU profiling infrastructure - analyze-*.js: Profile and bundle analysis tools - trace-cli-startup.js: Module loading trace 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 1b5f052 commit 9e1ba61

File tree

7 files changed

+1424
-0
lines changed

7 files changed

+1424
-0
lines changed

‎.github/workflows/test_dev_boot_time.yml‎

Lines changed: 363 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,363 @@
1+
name: Dev Server Boot Time
2+
3+
on:
4+
pull_request:
5+
workflow_dispatch:
6+
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
9+
cancel-in-progress: true
10+
11+
env:
12+
NEXT_TELEMETRY_DISABLED: 1
13+
NODE_LTS_VERSION: 22
14+
# Enable Turbo remote caching - canary builds are likely already cached from other CI runs
15+
TURBO_TEAM: 'vercel'
16+
TURBO_CACHE: 'remote:rw'
17+
TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
18+
19+
jobs:
20+
benchmark-boot-time:
21+
name: Benchmark Dev Server Boot Time
22+
runs-on: ubuntu-latest
23+
timeout-minutes: 30
24+
25+
steps:
26+
- name: Checkout PR branch
27+
uses: actions/checkout@v4
28+
with:
29+
fetch-depth: 0
30+
31+
- name: Setup Node.js
32+
uses: actions/setup-node@v4
33+
with:
34+
node-version: ${{ env.NODE_LTS_VERSION }}
35+
36+
- name: Setup pnpm
37+
run: |
38+
npm i -g corepack@0.31
39+
corepack enable
40+
corepack prepare pnpm@9.15.1 --activate
41+
42+
- name: Get pnpm store directory
43+
id: pnpm-cache
44+
run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
45+
46+
- name: Cache pnpm
47+
uses: actions/cache@v4
48+
with:
49+
path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
50+
key: pnpm-store-${{ hashFiles('pnpm-lock.yaml') }}
51+
restore-keys: pnpm-store-
52+
53+
- name: Create test app
54+
run: |
55+
mkdir -p /tmp/boot-test-app/app
56+
cat > /tmp/boot-test-app/package.json << 'EOF'
57+
{
58+
"name": "boot-test-app",
59+
"private": true,
60+
"dependencies": {
61+
"react": "19.0.0",
62+
"react-dom": "19.0.0"
63+
}
64+
}
65+
EOF
66+
cat > /tmp/boot-test-app/app/layout.tsx << 'EOF'
67+
export default function RootLayout({ children }: { children: React.ReactNode }) {
68+
return <html><body>{children}</body></html>
69+
}
70+
EOF
71+
cat > /tmp/boot-test-app/app/page.tsx << 'EOF'
72+
export default function Home() { return <h1>Hello</h1> }
73+
EOF
74+
cd /tmp/boot-test-app && npm install
75+
76+
# ===== BENCHMARK FUNCTION =====
77+
# Measures two metrics:
78+
# - listen_time: when port starts accepting TCP connections
79+
# - ready_time: when first HTTP request succeeds
80+
- name: Create benchmark script
81+
run: |
82+
cat > /tmp/benchmark.sh << 'SCRIPT'
83+
#!/bin/bash
84+
set -e
85+
NEXT_BIN=$1
86+
TEST_DIR=$2
87+
RUNS=${3:-5}
88+
PORT=3456
89+
LOG_FILE=/tmp/next-server.log
90+
91+
# Sanity check: verify binary exists
92+
if [ ! -f "$NEXT_BIN" ]; then
93+
echo "ERROR: Next.js binary not found at $NEXT_BIN" >&2
94+
echo "COLD_LISTEN=0"
95+
echo "COLD_READY=0"
96+
echo "WARM_LISTEN=0"
97+
echo "WARM_READY=0"
98+
exit 0
99+
fi
100+
101+
# Kill any existing process on port
102+
pkill -f "next.*$PORT" 2>/dev/null || true
103+
sleep 0.5
104+
105+
benchmark_run() {
106+
local clean=$1
107+
[ "$clean" = "true" ] && rm -rf "$TEST_DIR/.next"
108+
109+
# Kill any lingering process
110+
pkill -f "next.*$PORT" 2>/dev/null || true
111+
sleep 0.3
112+
113+
local start_ms=$(date +%s%3N)
114+
# Capture output to log file for debugging
115+
"$NEXT_BIN" dev --turbopack --port $PORT "$TEST_DIR" > "$LOG_FILE" 2>&1 &
116+
local pid=$!
117+
118+
local listen_ms=""
119+
local ready_ms=""
120+
121+
# Wait for port to listen (max 45s = 900 * 50ms)
122+
for i in $(seq 1 900); do
123+
if nc -z localhost $PORT 2>/dev/null; then
124+
listen_ms=$(date +%s%3N)
125+
break
126+
fi
127+
# Check if process died
128+
if ! kill -0 $pid 2>/dev/null; then
129+
echo "ERROR: Server process died. Last 20 lines of log:" >&2
130+
tail -20 "$LOG_FILE" >&2
131+
break
132+
fi
133+
sleep 0.05
134+
done
135+
136+
# Wait for HTTP response (max 45s)
137+
if [ -n "$listen_ms" ]; then
138+
for i in $(seq 1 900); do
139+
if curl -s -m 2 "http://localhost:$PORT" > /dev/null 2>&1; then
140+
ready_ms=$(date +%s%3N)
141+
break
142+
fi
143+
sleep 0.05
144+
done
145+
fi
146+
147+
kill $pid 2>/dev/null || true
148+
wait $pid 2>/dev/null || true
149+
150+
if [ -n "$listen_ms" ] && [ -n "$ready_ms" ]; then
151+
echo "$((listen_ms - start_ms)),$((ready_ms - start_ms))"
152+
else
153+
echo "TIMEOUT: listen_ms=$listen_ms ready_ms=$ready_ms" >&2
154+
if [ -f "$LOG_FILE" ]; then
155+
echo "Last 30 lines of server log:" >&2
156+
tail -30 "$LOG_FILE" >&2
157+
fi
158+
echo "TIMEOUT,TIMEOUT"
159+
fi
160+
}
161+
162+
run_series() {
163+
local name=$1
164+
local clean=$2
165+
local listen_sum=0
166+
local ready_sum=0
167+
local count=0
168+
169+
for i in $(seq 1 $RUNS); do
170+
result=$(benchmark_run "$clean")
171+
listen=$(echo "$result" | cut -d',' -f1)
172+
ready=$(echo "$result" | cut -d',' -f2)
173+
if [ "$listen" != "TIMEOUT" ]; then
174+
listen_sum=$((listen_sum + listen))
175+
ready_sum=$((ready_sum + ready))
176+
count=$((count + 1))
177+
echo "${name} run $i: listen=${listen}ms ready=${ready}ms" >&2
178+
else
179+
echo "${name} run $i: TIMEOUT" >&2
180+
fi
181+
done
182+
183+
if [ $count -gt 0 ]; then
184+
echo "$((listen_sum / count)),$((ready_sum / count))"
185+
else
186+
echo "0,0"
187+
fi
188+
}
189+
190+
# Run cold benchmark
191+
echo "=== Cold Start Benchmark ===" >&2
192+
cold=$(run_series "cold" "true")
193+
echo "COLD_LISTEN=$(echo $cold | cut -d',' -f1)"
194+
echo "COLD_READY=$(echo $cold | cut -d',' -f2)"
195+
196+
# Warmup for bytecode cache (only if cold benchmarks succeeded)
197+
cold_listen=$(echo $cold | cut -d',' -f1)
198+
if [ "$cold_listen" != "0" ]; then
199+
echo "=== Warming up bytecode cache (12s) ===" >&2
200+
rm -rf "$TEST_DIR/.next"
201+
"$NEXT_BIN" dev --turbopack --port $PORT "$TEST_DIR" > "$LOG_FILE" 2>&1 &
202+
pid=$!
203+
for i in $(seq 1 400); do
204+
curl -s -m 2 "http://localhost:$PORT" > /dev/null 2>&1 && break
205+
sleep 0.05
206+
done
207+
sleep 12
208+
kill $pid 2>/dev/null || true
209+
wait $pid 2>/dev/null || true
210+
211+
# Run warm benchmark
212+
echo "=== Warm Start Benchmark ===" >&2
213+
warm=$(run_series "warm" "false")
214+
echo "WARM_LISTEN=$(echo $warm | cut -d',' -f1)"
215+
echo "WARM_READY=$(echo $warm | cut -d',' -f2)"
216+
else
217+
echo "Skipping warm benchmark - cold benchmark failed" >&2
218+
echo "WARM_LISTEN=0"
219+
echo "WARM_READY=0"
220+
fi
221+
SCRIPT
222+
chmod +x /tmp/benchmark.sh
223+
224+
# ===== BASELINE (canary) =====
225+
- name: Checkout canary
226+
run: git checkout origin/canary
227+
228+
- name: Get canary commit hash
229+
id: canary-hash
230+
run: echo "hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
231+
232+
- name: Cache canary Next.js build
233+
id: cache-canary
234+
uses: actions/cache@v4
235+
with:
236+
path: packages/next/dist
237+
key: next-canary-dist-${{ steps.canary-hash.outputs.hash }}
238+
239+
- name: Install dependencies (canary)
240+
run: pnpm install --frozen-lockfile
241+
242+
- name: Build Next.js (canary)
243+
if: steps.cache-canary.outputs.cache-hit != 'true'
244+
run: pnpm turbo run build --filter=next...
245+
246+
- name: Link Next.js (canary)
247+
run: |
248+
cd /tmp/boot-test-app
249+
npm link ${{ github.workspace }}/packages/next
250+
251+
- name: Verify Next.js binary (canary)
252+
run: |
253+
echo "Checking Next.js binary..."
254+
ls -la ${{ github.workspace }}/packages/next/dist/bin/next || echo "Binary not found!"
255+
echo "Testing Next.js version..."
256+
node ${{ github.workspace }}/packages/next/dist/bin/next --version || echo "Version check failed!"
257+
258+
- name: Benchmark canary
259+
id: canary
260+
run: |
261+
cd /tmp/boot-test-app
262+
/tmp/benchmark.sh "${{ github.workspace }}/packages/next/dist/bin/next" /tmp/boot-test-app 5 >> $GITHUB_OUTPUT
263+
264+
# ===== PR BRANCH =====
265+
- name: Checkout PR
266+
run: git checkout ${{ github.sha }}
267+
268+
- name: Install dependencies (PR)
269+
run: pnpm install --frozen-lockfile
270+
271+
- name: Build Next.js (PR)
272+
run: pnpm turbo run build --filter=next...
273+
274+
- name: Link Next.js (PR)
275+
run: |
276+
cd /tmp/boot-test-app
277+
rm -rf node_modules/next node_modules/.next
278+
npm link ${{ github.workspace }}/packages/next
279+
280+
- name: Verify Next.js binary (PR)
281+
run: |
282+
echo "Checking Next.js binary..."
283+
ls -la ${{ github.workspace }}/packages/next/dist/bin/next || echo "Binary not found!"
284+
echo "Testing Next.js version..."
285+
node ${{ github.workspace }}/packages/next/dist/bin/next --version || echo "Version check failed!"
286+
287+
- name: Benchmark PR
288+
id: pr
289+
run: |
290+
cd /tmp/boot-test-app
291+
/tmp/benchmark.sh "${{ github.workspace }}/packages/next/dist/bin/next" /tmp/boot-test-app 5 >> $GITHUB_OUTPUT
292+
293+
# ===== RESULTS =====
294+
- name: Calculate and report results
295+
run: |
296+
# Canary results
297+
CANARY_LISTEN=${{ steps.canary.outputs.COLD_LISTEN }}
298+
CANARY_READY=${{ steps.canary.outputs.COLD_READY }}
299+
CANARY_WARM_LISTEN=${{ steps.canary.outputs.WARM_LISTEN }}
300+
CANARY_WARM_READY=${{ steps.canary.outputs.WARM_READY }}
301+
302+
# PR results
303+
PR_LISTEN=${{ steps.pr.outputs.COLD_LISTEN }}
304+
PR_READY=${{ steps.pr.outputs.COLD_READY }}
305+
PR_WARM_LISTEN=${{ steps.pr.outputs.WARM_LISTEN }}
306+
PR_WARM_READY=${{ steps.pr.outputs.WARM_READY }}
307+
308+
# Calculate differences
309+
COLD_LISTEN_DIFF=$((PR_LISTEN - CANARY_LISTEN))
310+
COLD_READY_DIFF=$((PR_READY - CANARY_READY))
311+
WARM_LISTEN_DIFF=$((PR_WARM_LISTEN - CANARY_WARM_LISTEN))
312+
WARM_READY_DIFF=$((PR_WARM_READY - CANARY_WARM_READY))
313+
314+
# Status indicators
315+
status() {
316+
if [ $1 -lt -20 ]; then echo "🟢";
317+
elif [ $1 -gt 50 ]; then echo "🔴";
318+
else echo "🟡"; fi
319+
}
320+
321+
COLD_LISTEN_STATUS=$(status $COLD_LISTEN_DIFF)
322+
COLD_READY_STATUS=$(status $COLD_READY_DIFF)
323+
WARM_LISTEN_STATUS=$(status $WARM_LISTEN_DIFF)
324+
WARM_READY_STATUS=$(status $WARM_READY_DIFF)
325+
326+
# Generate summary
327+
cat >> $GITHUB_STEP_SUMMARY << EOF
328+
## Dev Server Boot Time Benchmark
329+
330+
### Cold Start (fresh .next)
331+
| Metric | Canary | PR | Diff |
332+
|--------|--------|-----|------|
333+
| Port listening | ${CANARY_LISTEN}ms | ${PR_LISTEN}ms | ${COLD_LISTEN_STATUS} ${COLD_LISTEN_DIFF}ms |
334+
| First request | ${CANARY_READY}ms | ${PR_READY}ms | ${COLD_READY_STATUS} ${COLD_READY_DIFF}ms |
335+
336+
### Warm Start (with cache)
337+
| Metric | Canary | PR | Diff |
338+
|--------|--------|-----|------|
339+
| Port listening | ${CANARY_WARM_LISTEN}ms | ${PR_WARM_LISTEN}ms | ${WARM_LISTEN_STATUS} ${WARM_LISTEN_DIFF}ms |
340+
| First request | ${CANARY_WARM_READY}ms | ${PR_WARM_READY}ms | ${WARM_READY_STATUS} ${WARM_READY_DIFF}ms |
341+
342+
**Legend:** 🟢 Faster (>20ms improvement) | 🟡 Similar | 🔴 Slower (>50ms regression)
343+
344+
**Metrics:**
345+
- **Port listening**: When server starts accepting TCP connections
346+
- **First request**: When server handles first HTTP request (actual readiness)
347+
EOF
348+
349+
# Log to console
350+
echo "=== RESULTS ==="
351+
echo "Cold: listen ${CANARY_LISTEN}ms→${PR_LISTEN}ms (${COLD_LISTEN_DIFF}ms), ready ${CANARY_READY}ms→${PR_READY}ms (${COLD_READY_DIFF}ms)"
352+
echo "Warm: listen ${CANARY_WARM_LISTEN}ms→${PR_WARM_LISTEN}ms (${WARM_LISTEN_DIFF}ms), ready ${CANARY_WARM_READY}ms→${PR_WARM_READY}ms (${WARM_READY_DIFF}ms)"
353+
354+
- name: Check for regression
355+
run: |
356+
COLD_READY_DIFF=$((${{ steps.pr.outputs.COLD_READY }} - ${{ steps.canary.outputs.COLD_READY }}))
357+
358+
if [ "$COLD_READY_DIFF" -gt 200 ]; then
359+
echo "::error::Cold start regression: ${COLD_READY_DIFF}ms slower (threshold: 200ms)"
360+
exit 1
361+
fi
362+
363+
echo "✅ No significant regression"

0 commit comments

Comments
 (0)