diff --git a/.github/workflows/scheduled_bench.yml b/.github/workflows/scheduled_bench.yml index b35a5f27c..e2c3b87ed 100644 --- a/.github/workflows/scheduled_bench.yml +++ b/.github/workflows/scheduled_bench.yml @@ -11,6 +11,9 @@ jobs: - uses: actions/checkout@v4 - name: Init env uses: ./.github/actions/env + - name: Set Scheduled env + shell: bash + run: echo "SCHEDULED_JOB=true" >> $GITHUB_ENV - name: Build rspack run: node bin/cli.js build - name: Run benchmark diff --git a/bench.config.js b/bench.config.js index 7fb6c93ee..7fcd434bf 100644 --- a/bench.config.js +++ b/bench.config.js @@ -1,17 +1,71 @@ +const isScheduled = !!process.env.SCHEDULED_JOB; + +// HMR will run 10 times in build plugin, so we should not start multiple instances of Rspack. +// However, we still need to run multiple instances of Rspack when executing scheduled tasks for longer runtimes. +const hmrRuns = isScheduled ? 10 : 1; + export default { jobs: [ - "10000_development-mode", - "10000_development-mode_hmr", - "10000_production-mode", - "arco-pro_development-mode", - "arco-pro_development-mode_intercept-plugin", - "arco-pro_development-mode_hmr", - "arco-pro_development-mode_hmr_intercept-plugin", - "arco-pro_production-mode", - "arco-pro_production-mode_intercept-plugin", - "threejs_development-mode_10x", - "threejs_development-mode_10x_hmr", - "threejs_production-mode_10x" + { + name: "10000_development-mode", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "10000_development-mode_hmr", + runs: hmrRuns, + compareMetrics: ["stats"] + }, + { + name: "10000_production-mode", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "arco-pro_development-mode", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "arco-pro_development-mode_intercept-plugin", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "arco-pro_development-mode_hmr", + runs: hmrRuns, + compareMetrics: ["stats"] + }, + { + name: "arco-pro_development-mode_hmr_intercept-plugin", + runs: hmrRuns, + compareMetrics: ["stats"] + }, + { + name: "arco-pro_production-mode", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "arco-pro_production-mode_intercept-plugin", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "threejs_development-mode_10x", + runs: 10, + compareMetrics: ["exec"] + }, + { + name: "threejs_development-mode_10x_hmr", + runs: hmrRuns, + compareMetrics: ["stats"] + }, + { + name: "threejs_production-mode_10x", + runs: 10, + compareMetrics: ["exec"] + } ], rspackDirectory: process.env.RSPACK_DIR }; diff --git a/bin/cli.js b/bin/cli.js index 78a5effd2..ac75e21fe 100644 --- a/bin/cli.js +++ b/bin/cli.js @@ -68,12 +68,13 @@ const { const cwd = process.cwd(); -const configPath = join(process.cwd(), "bench.config.js"); -const config = (await import(configPath)).default; +const benchConfigPath = join(process.cwd(), "bench.config.js"); +const benchConfig = (await import(benchConfigPath)).default; -const jobs = config.jobs ?? []; -const rspackDirectory = config.rspackDirectory ?? join(cwd, ".rspack"); -const benchmarkDirectory = config.benchmarkDirectory ?? join(cwd, "output"); +const jobs = benchConfig.jobs ?? []; +const rspackDirectory = benchConfig.rspackDirectory ?? join(cwd, ".rspack"); +const benchmarkDirectory = + benchConfig.benchmarkDirectory ?? join(cwd, "output"); if (!command || command === "build") { const fetchUrl = `https://github.com/${repository}`; @@ -125,17 +126,18 @@ if (!command || command === "bench") { console.log( [ `Running jobs for shard ${currentIndex}/${totalShards}:`, - ...shardJobs + ...shardJobs.map(job => job.name) ].join("\n * ") ); for (const job of shardJobs) { const start = Date.now(); - const result = await run(job); + const result = await run(job.name, job.runs); + const message = `${job.name} was run ${job.runs} times, with the following results:`; if (isGitHubActions) { - actionsCore.startGroup(`${job} result is`); + actionsCore.startGroup(message); } else { - console.log(`${job} result is`); + console.log(message); } console.log(formatResultTable(result, { verbose: true })); @@ -143,11 +145,11 @@ if (!command || command === "bench") { if (isGitHubActions) { actionsCore.endGroup(); const cost = Math.ceil((Date.now() - start) / 1000); - console.log(`Cost for \`${job}\`: ${cost} s`); + console.log(`Cost for \`${job.name}\`: ${cost} s`); } await writeFile( - join(benchmarkDirectory, `${job}.json`), + join(benchmarkDirectory, `${job.name}.json`), JSON.stringify(result, null, 2) ); } @@ -157,5 +159,5 @@ if (!command || command === "bench") { } if (!command || command === "compare") { - compare(base, current, benchmarkDirectory); + compare(base, current, benchmarkDirectory, jobs); } diff --git a/docs/index.html b/docs/index.html index 010628b62..9ffb16bd6 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1,4 +1,4 @@ - + rspack/benchmark diff --git a/docs/index.js b/docs/index.js index c84504593..220d6c6ce 100644 --- a/docs/index.js +++ b/docs/index.js @@ -329,8 +329,8 @@ class BenchmarkChart { context.dataset.yAxisID === "size" ? formatSize(value, value) : context.dataset.yAxisID === "ratio" - ? formatRatio(value, value) - : formatTime(value, value); + ? formatRatio(value, value) + : formatTime(value, value); return `${context.dataset.label}: ${text}`; } } diff --git a/lib/addons/hmr.js b/lib/addons/hmr.js index 46ea675d8..037f4dec3 100644 --- a/lib/addons/hmr.js +++ b/lib/addons/hmr.js @@ -3,7 +3,6 @@ import { Addon } from "./common.js"; export default class extends Addon { async afterSetup(ctx) { ctx.rspackArgs.push("--watch"); - ctx.runTimes = 5; ctx.config = ctx.config + ` diff --git a/lib/bench.js b/lib/bench.js index 2b96fb2f9..bfa38f70f 100644 --- a/lib/bench.js +++ b/lib/bench.js @@ -5,7 +5,7 @@ import { useAddons, dirExist } from "./utils.js"; const dirname = path.resolve(fileURLToPath(import.meta.url), ".."); -export async function run(benchmarkName) { +export async function run(benchmarkName, runs) { const [caseName, ...addonNames] = benchmarkName.split("_"); const scenario = getScenario(caseName); const addons = await Promise.all( @@ -20,6 +20,8 @@ export async function run(benchmarkName) { await useAddons(addons, "beforeSetup"); const ctx = await scenario.setup(); + ctx.runTimes = runs; + await useAddons(addons, "afterSetup", ctx); try { diff --git a/lib/compare.js b/lib/compare.js index 37aa6a4e7..16949f000 100644 --- a/lib/compare.js +++ b/lib/compare.js @@ -71,9 +71,7 @@ async function getResults(date, index, benchmarkDirectory) { ); } -export async function compare(base, current, benchmarkDirectory) { - const compareMetric = ["exec"]; - +export async function compare(base, current, benchmarkDirectory, jobs) { const index = await fetchIndex(); if (base === "latest") { base = index[index.length - 1].date; @@ -86,13 +84,20 @@ export async function compare(base, current, benchmarkDirectory) { ]); const baseData = {}; const currentData = {}; - for (const metric of compareMetric) { - for (const { name, result } of baseResults) { + + for (const { name, result } of baseResults) { + const job = jobs.find(job => job.name === name); + const compareMetrics = job ? job.compareMetrics : []; + for (const metric of compareMetrics) { const tag = `${name} + ${metric}`; baseData[tag] = result[metric]; } + } - for (const { name, result } of currentResults) { + for (const { name, result } of currentResults) { + const job = jobs.find(job => job.name === name); + const compareMetrics = job ? job.compareMetrics : []; + for (const metric of compareMetrics) { const tag = `${name} + ${metric}`; currentData[tag] = result[metric]; } diff --git a/lib/display.js b/lib/display.js index 5b050b1d6..e37f74fd5 100644 --- a/lib/display.js +++ b/lib/display.js @@ -135,7 +135,7 @@ export function formatResultTable(result, { verbose, limit, threshold }) { con: l => f(l.name, l.confidence), con: l => f(l.name, l.confidence), n: l => `${l.count}` - } + } : undefined) }; return formatTable(entries, columns); diff --git a/lib/scenarios/build-plugin.cjs b/lib/scenarios/build-plugin.cjs index c2d08c535..226e73f55 100644 --- a/lib/scenarios/build-plugin.cjs +++ b/lib/scenarios/build-plugin.cjs @@ -8,22 +8,16 @@ module.exports = class BuildPlugin { compiler.hooks.watchRun.tap("BuildPlugin", () => { isWatching = true; }); - (compiler.hooks.afterDone || compiler.hooks.done).tap("BuildPlugin", () => { - setTimeout(() => { - if (counter === WARMUP_BUILDS) console.log("#!# start"); - if (isWatching && counter <= TOTAL_BUILDS) console.log("#!# next"); - }, 10); + compiler.hooks.afterDone.tap("BuildPlugin", () => { + if (counter === WARMUP_BUILDS) console.log("#!# start"); + if (isWatching && counter <= TOTAL_BUILDS) console.log("#!# next"); }); compiler.hooks.done.tap("BuildPlugin", stats => { if (isWatching) { counter++; if (counter <= WARMUP_BUILDS) return; if (counter > TOTAL_BUILDS) { - if (compiler.watching) { - compiler.watching.close(); - } else { - process.nextTick(() => process.exit(0)); - } + process.nextTick(() => process.exit(0)); } } const { logging, time } = stats.toJson({ @@ -69,10 +63,7 @@ module.exports = class BuildPlugin { if (type !== "time") return; const ms = args[1] * 1000 + args[2] / 1000000; console.log( - `#!# ${name}.${args[0].replace( - /restore cache content \d.+$/, - "restore cache content" - )} = ${ms}` + `#!# ${name}.${args[0].replace(/restore cache content \d.+$/, "restore cache content")} = ${ms}` ); }; } diff --git a/lib/scenarios/index.js b/lib/scenarios/index.js index e95b88096..db4964c56 100644 --- a/lib/scenarios/index.js +++ b/lib/scenarios/index.js @@ -18,7 +18,8 @@ async function runRspack(ctx) { let counter = 0; let dataSetCounter = -1; let promise = Promise.resolve(); - const data = {}; + let data = {}; + const dataSet = []; const processLine = line => { if (line === "#!# start") { start = Date.now(); @@ -26,7 +27,11 @@ async function runRspack(ctx) { } else if (line === "#!# next") { promise = promise.then(async () => { counter++; - if (dataSetCounter >= 0) dataSetCounter++; + if (dataSetCounter >= 0) { + dataSetCounter++; + data = {}; + dataSet.push(data); + } await new Promise(r => setTimeout(r, Math.max(300, 1000 / counter))); for (const item of ctx.hmrConfig) { const content = item.generateContent( @@ -38,7 +43,7 @@ async function runRspack(ctx) { }); } else if (line.startsWith("#!#")) { const [, name, valueStr] = /^#!# (.+) = ((\d|\.|e|-)+)$/.exec(line); - data[name] = (data[name] || 0) + +valueStr; + data[name] = (data[name] || 0) + Number(valueStr); } }; let remainingLine = ""; @@ -63,7 +68,7 @@ async function runRspack(ctx) { } } data["dist size"] = await getDirSizes("dist"); - return data; + return dataSet; } export function getScenario(caseName) { @@ -137,7 +142,8 @@ module.exports.plugins.push(new (require("../../lib/scenarios/build-plugin.cjs") for (let i = 0; i < ctx.runTimes; i++) { await clearCaches(ctx.caseDir); - ctx.runData.push(await runRspack(ctx)); + const result = await runRspack(ctx); + ctx.runData.push(...result); const runtime = Date.now() - start; if (runtime > ctx.timeout) break; diff --git a/lib/scenarios/utils.js b/lib/scenarios/utils.js index 6fcae0861..94c1b031b 100644 --- a/lib/scenarios/utils.js +++ b/lib/scenarios/utils.js @@ -68,8 +68,14 @@ const tDist95Two = n => { }; export function calcStatistics(data) { const stats = {}; - for (const key of Object.keys(data[0])) { - const values = data.map(r => r[key] || 0); + const keys = Array.from(data.reduce((set, item) => { + for (const key of Object.keys(item)) { + set.add(key); + } + return set; + }, new Set())); + for (const key of keys) { + const values = data.map(r => r[key]).filter(d => typeof d === 'number'); if (typeof values[0] === "object") { stats[key] = calcStatistics(values); } else { diff --git a/lib/utils.js b/lib/utils.js index dec4d9303..12b99cbea 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -21,7 +21,7 @@ export async function runCommand( ? { ...process.env, ...env - } + } : undefined }); if (hasOnData) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ed37658d8..401209341 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,10 @@ importers: zx: specifier: ^8.1.1 version: 8.1.1 + devDependencies: + prettier: + specifier: ^2.6.2 + version: 2.8.8 cases/10000: {}