fix: run production start benchmark before dev benchmark

Author: feedthejim

.github/actions/next-stats-action/src/run/collect-stats.js

@@ -1,4 +1,5 @@
 const path = require('path')
+const net = require('net')
 const fs = require('fs/promises')
 const getPort = require('get-port')
 const fetch = require('node-fetch')
@@ -10,6 +11,108 @@ const { parse: urlParse } = require('url')
 const benchmarkUrl = require('./benchmark-url')
 const { statsAppDir, diffingDir, benchTitle } = require('../constants')
 
+// Check if a port is accepting TCP connections
+function checkPort(port, timeout = 100) {
+  return new Promise((resolve) => {
+    const socket = new net.Socket()
+    socket.setTimeout(timeout)
+    socket.once('connect', () => {
+      socket.destroy()
+      resolve(true)
+    })
+    socket.once('timeout', () => {
+      socket.destroy()
+      resolve(false)
+    })
+    socket.once('error', () => {
+      socket.destroy()
+      resolve(false)
+    })
+    socket.connect(port, 'localhost')
+  })
+}
+
+// Wait for port to start accepting TCP connections
+async function waitForPort(port, timeoutMs = 60000) {
+  const start = Date.now()
+  while (Date.now() - start < timeoutMs) {
+    if (await checkPort(port)) {
+      return Date.now() - start
+    }
+    await new Promise((r) => setTimeout(r, 50))
+  }
+  return null
+}
+
+// Wait for HTTP server to respond
+async function waitForHttp(port, timeoutMs = 60000) {
+  const start = Date.now()
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const res = await fetch(`http://localhost:${port}/`, { timeout: 2000 })
+      if (res.ok) {
+        return Date.now() - start
+      }
+    } catch (e) {
+      // Server not ready yet
+    }
+    await new Promise((r) => setTimeout(r, 50))
+  }
+  return null
+}
+
+// Run a single dev server boot benchmark
+async function benchmarkDevBoot(appDevCommand, curDir, port, cleanBuild) {
+  // Clean .next directory for cold start
+  if (cleanBuild) {
+    const nextDir = path.join(curDir, '.next')
+    await fs.rm(nextDir, { recursive: true, force: true })
+  }
+
+  const startTime = Date.now()
+  const devChild = spawn(appDevCommand, {
+    cwd: curDir,
+    env: {
+      PORT: port,
+    },
+    stdio: 'pipe',
+  })
+
+  let exited = false
+  devChild.on('exit', () => {
+    exited = true
+  })
+
+  // Capture output for debugging
+  devChild.stdout.on('data', (data) => process.stdout.write(data))
+  devChild.stderr.on('data', (data) => process.stderr.write(data))
+
+  // Measure time to port listening (TCP level)
+  const listenTime = await waitForPort(port, 60000)
+
+  // Measure time to HTTP ready
+  let readyTime = null
+  if (listenTime !== null && !exited) {
+    readyTime = await waitForHttp(port, 60000)
+  }
+
+  devChild.kill()
+
+  // Wait for process to fully exit to avoid port conflicts on subsequent runs
+  if (!exited) {
+    await new Promise((resolve) => {
+      devChild.on('exit', resolve)
+      // Timeout after 5 seconds in case process doesn't exit cleanly
+      setTimeout(resolve, 5000)
+    })
+  }
+
+  return {
+    listenTime,
+    readyTime,
+  }
+}
+
 async function defaultGetRequiredFiles(nextAppDir, fileName) {
   return [fileName]
 }
@@ -33,6 +136,7 @@ module.exports = async function collectStats(
   const hasPagesToBench =
     Array.isArray(runConfig.pagesToBench) && runConfig.pagesToBench.length > 0
 
+  // Run production start benchmark FIRST (before dev benchmark which cleans .next)
   if (
     !fromDiff &&
     statsConfig.appStartCommand &&
@@ -139,6 +243,81 @@ module.exports = async function collectStats(
     child.kill()
   }
 
+  // Measure dev server boot time if configured (full matrix: cold/warm x listen/ready)
+  // NOTE: This runs AFTER the production start benchmark because it cleans the .next directory
+  if (!fromDiff && statsConfig.appDevCommand && statsConfig.measureDevBoot) {
+    const devPort = await getPort()
+
+    if (!orderedStats['General']) {
+      orderedStats['General'] = {}
+    }
+
+    // 1. Cold start benchmark (clean .next directory)
+    logger('=== Cold Start Benchmark ===')
+    const coldResult = await benchmarkDevBoot(
+      statsConfig.appDevCommand,
+      curDir,
+      devPort,
+      true // clean build
+    )
+
+    if (coldResult.listenTime !== null) {
+      orderedStats['General']['nextDevColdListenDuration (ms)'] =
+        coldResult.listenTime
+    }
+    if (coldResult.readyTime !== null) {
+      orderedStats['General']['nextDevColdReadyDuration (ms)'] =
+        coldResult.readyTime
+    }
+
+    // 2. Warm up bytecode cache by running server for ~10 seconds
+    if (coldResult.readyTime !== null) {
+      logger('=== Warming up bytecode cache (10s) ===')
+      const warmupChild = spawn(statsConfig.appDevCommand, {
+        cwd: curDir,
+        env: {
+          PORT: devPort,
+        },
+        stdio: 'pipe',
+      })
+
+      // Wait for server to be ready
+      await waitForHttp(devPort, 60000)
+
+      // Let it run for 10 seconds to warm bytecode cache
+      await new Promise((r) => setTimeout(r, 10000))
+
+      warmupChild.kill()
+
+      // Wait for warmup server to fully exit to avoid port conflicts
+      await new Promise((resolve) => {
+        warmupChild.on('exit', resolve)
+        // Timeout after 5 seconds in case process doesn't exit cleanly
+        setTimeout(resolve, 5000)
+      })
+
+      // 3. Warm start benchmark (keep .next directory)
+      logger('=== Warm Start Benchmark ===')
+      const warmResult = await benchmarkDevBoot(
+        statsConfig.appDevCommand,
+        curDir,
+        devPort,
+        false // keep build
+      )
+
+      if (warmResult.listenTime !== null) {
+        orderedStats['General']['nextDevWarmListenDuration (ms)'] =
+          warmResult.listenTime
+      }
+      if (warmResult.readyTime !== null) {
+        orderedStats['General']['nextDevWarmReadyDuration (ms)'] =
+          warmResult.readyTime
+      }
+    }
+
+    logger('=== Dev Boot Benchmark Complete ===')
+  }
+
   for (const fileGroup of runConfig.filesToTrack) {
     const {
       getRequiredFiles = defaultGetRequiredFiles,