Revert "feat: only run 1 time when not in scheduled job (#28)" (#29)

This reverts commit f6d9e69.

Author: SyMind

.github/workflows/scheduled_bench.yml

@@ -11,9 +11,6 @@ jobs:
       - uses: actions/checkout@v4
       - name: Init env
         uses: ./.github/actions/env
-      - name: Set Scheduled env
-        shell: bash
-        run: echo "SCHEDULED_JOB=true" >> $GITHUB_ENV
       - name: Build rspack
         run: node bin/cli.js build
       - name: Run benchmark

bench.config.js

@@ -1,71 +1,17 @@
-const isScheduled = !!process.env.SCHEDULED_JOB;
-
-// HMR will run 10 times in build plugin, so we should not start multiple instances of Rspack.
-// However, we still need to run multiple instances of Rspack when executing scheduled tasks for longer runtimes.
-const hmrRuns = isScheduled ? 10 : 1;
-
 export default {
 	jobs: [
-		{
-			name: "10000_development-mode",
-			runs: 10,
-			compareMetrics: ["exec"]
-		},
-		{
-			name: "10000_development-mode_hmr",
-			runs: hmrRuns,
-			compareMetrics: ["stats"]
-		},
-		{
-			name: "10000_production-mode",
-			runs: 10,
-			compareMetrics: ["exec"]
-		},
-		{
-			name: "arco-pro_development-mode",
-			runs: 10,
-			compareMetrics: ["exec"]
-		},
-		{
-			name: "arco-pro_development-mode_intercept-plugin",
-			runs: 10,
-			compareMetrics: ["exec"]
-		},
-		{
-			name: "arco-pro_development-mode_hmr",
-			runs: hmrRuns,
-			compareMetrics: ["stats"]
-		},
-		{
-			name: "arco-pro_development-mode_hmr_intercept-plugin",
-			runs: hmrRuns,
-			compareMetrics: ["stats"]
-		},
-		{
-			name: "arco-pro_production-mode",
-			runs: 10,
-			compareMetrics: ["exec"]
-		},
-		{
-			name: "arco-pro_production-mode_intercept-plugin",
-			runs: 10,
-			compareMetrics: ["exec"]
-		},
-		{
-			name: "threejs_development-mode_10x",
-			runs: 10,
-			compareMetrics: ["exec"]
-		},
-		{
-			name: "threejs_development-mode_10x_hmr",
-			runs: hmrRuns,
-			compareMetrics: ["stats"]
-		},
-		{
-			name: "threejs_production-mode_10x",
-			runs: 10,
-			compareMetrics: ["exec"]
-		}
+		"10000_development-mode",
+		"10000_development-mode_hmr",
+		"10000_production-mode",
+		"arco-pro_development-mode",
+		"arco-pro_development-mode_intercept-plugin",
+		"arco-pro_development-mode_hmr",
+		"arco-pro_development-mode_hmr_intercept-plugin",
+		"arco-pro_production-mode",
+		"arco-pro_production-mode_intercept-plugin",
+		"threejs_development-mode_10x",
+		"threejs_development-mode_10x_hmr",
+		"threejs_production-mode_10x"
 	],
 	rspackDirectory: process.env.RSPACK_DIR
 };

bin/cli.js

@@ -68,13 +68,12 @@ const {
 
 const cwd = process.cwd();
 
-const benchConfigPath = join(process.cwd(), "bench.config.js");
-const benchConfig = (await import(benchConfigPath)).default;
+const configPath = join(process.cwd(), "bench.config.js");
+const config = (await import(configPath)).default;
 
-const jobs = benchConfig.jobs ?? [];
-const rspackDirectory = benchConfig.rspackDirectory ?? join(cwd, ".rspack");
-const benchmarkDirectory =
-	benchConfig.benchmarkDirectory ?? join(cwd, "output");
+const jobs = config.jobs ?? [];
+const rspackDirectory = config.rspackDirectory ?? join(cwd, ".rspack");
+const benchmarkDirectory = config.benchmarkDirectory ?? join(cwd, "output");
 
 if (!command || command === "build") {
 	const fetchUrl = `https://github.com/${repository}`;
@@ -126,30 +125,29 @@ if (!command || command === "bench") {
 		console.log(
 			[
 				`Running jobs for shard ${currentIndex}/${totalShards}:`,
-				...shardJobs.map(job => job.name)
+				...shardJobs
 			].join("\n  * ")
 		);
 
 		for (const job of shardJobs) {
 			const start = Date.now();
-			const result = await run(job.name, job.runs);
-			const message = `${job.name} was run ${job.runs} times, with the following results:`;
+			const result = await run(job);
 			if (isGitHubActions) {
-				actionsCore.startGroup(message);
+				actionsCore.startGroup(`${job} result is`);
 			} else {
-				console.log(message);
+				console.log(`${job} result is`);
 			}
 
 			console.log(formatResultTable(result, { verbose: true }));
 
 			if (isGitHubActions) {
 				actionsCore.endGroup();
 				const cost = Math.ceil((Date.now() - start) / 1000);
-				console.log(`Cost for \`${job.name}\`: ${cost} s`);
+				console.log(`Cost for \`${job}\`: ${cost} s`);
 			}
 
 			await writeFile(
-				join(benchmarkDirectory, `${job.name}.json`),
+				join(benchmarkDirectory, `${job}.json`),
 				JSON.stringify(result, null, 2)
 			);
 		}
@@ -159,5 +157,5 @@ if (!command || command === "bench") {
 }
 
 if (!command || command === "compare") {
-	compare(base, current, benchmarkDirectory, jobs);
+	compare(base, current, benchmarkDirectory);
 }

docs/index.html

@@ -1,4 +1,4 @@
-<!doctype html>
+<!DOCTYPE html>
 <html>
 	<head>
 		<title>rspack/benchmark</title>

docs/index.js

@@ -329,8 +329,8 @@ class BenchmarkChart {
 									context.dataset.yAxisID === "size"
 										? formatSize(value, value)
 										: context.dataset.yAxisID === "ratio"
-											? formatRatio(value, value)
-											: formatTime(value, value);
+										? formatRatio(value, value)
+										: formatTime(value, value);
 								return `${context.dataset.label}: ${text}`;
 							}
 						}

lib/addons/hmr.js

@@ -3,6 +3,7 @@ import { Addon } from "./common.js";
 export default class extends Addon {
 	async afterSetup(ctx) {
 		ctx.rspackArgs.push("--watch");
+		ctx.runTimes = 5;
 		ctx.config =
 			ctx.config +
 			`

lib/bench.js

@@ -5,7 +5,7 @@ import { useAddons, dirExist } from "./utils.js";
 
 const dirname = path.resolve(fileURLToPath(import.meta.url), "..");
 
-export async function run(benchmarkName, runs) {
+export async function run(benchmarkName) {
 	const [caseName, ...addonNames] = benchmarkName.split("_");
 	const scenario = getScenario(caseName);
 	const addons = await Promise.all(
@@ -20,8 +20,6 @@ export async function run(benchmarkName, runs) {
 
 	await useAddons(addons, "beforeSetup");
 	const ctx = await scenario.setup();
-	ctx.runTimes = runs;
-
 	await useAddons(addons, "afterSetup", ctx);
 
 	try {

lib/compare.js

@@ -71,7 +71,9 @@ async function getResults(date, index, benchmarkDirectory) {
 	);
 }
 
-export async function compare(base, current, benchmarkDirectory, jobs) {
+export async function compare(base, current, benchmarkDirectory) {
+	const compareMetric = ["exec"];
+
 	const index = await fetchIndex();
 	if (base === "latest") {
 		base = index[index.length - 1].date;
@@ -84,20 +86,13 @@ export async function compare(base, current, benchmarkDirectory, jobs) {
 	]);
 	const baseData = {};
 	const currentData = {};
-
-	for (const { name, result } of baseResults) {
-		const job = jobs.find(job => job.name === name);
-		const compareMetrics = job ? job.compareMetrics : [];
-		for (const metric of compareMetrics) {
+	for (const metric of compareMetric) {
+		for (const { name, result } of baseResults) {
 			const tag = `${name} + ${metric}`;
 			baseData[tag] = result[metric];
 		}
-	}
 
-	for (const { name, result } of currentResults) {
-		const job = jobs.find(job => job.name === name);
-		const compareMetrics = job ? job.compareMetrics : [];
-		for (const metric of compareMetrics) {
+		for (const { name, result } of currentResults) {
 			const tag = `${name} + ${metric}`;
 			currentData[tag] = result[metric];
 		}

lib/display.js

@@ -135,7 +135,7 @@ export function formatResultTable(result, { verbose, limit, threshold }) {
 					con: l => f(l.name, l.confidence),
 					con: l => f(l.name, l.confidence),
 					n: l => `${l.count}`
-				}
+			  }
 			: undefined)
 	};
 	return formatTable(entries, columns);

lib/utils.js

@@ -21,7 +21,7 @@ export async function runCommand(
 			? {
 					...process.env,
 					...env
-				}
+			  }
 			: undefined
 	});
 	if (hasOnData) {