chore: add bench pr yaml (#52)

Timeless0911 · web-flow · commit c55456426d2f · 2023-11-16T19:13:48.000+08:00
diff --git a/.github/workflows/pr-bench.yaml b/.github/workflows/pr-bench.yaml
@@ -0,0 +1,104 @@
+name: Bench PR
+
+on:
+  workflow_dispatch:
+    inputs:
+      prNumber:
+        description: 'PR number'
+        required: true
+        type: string
+      product:
+        description: 'product name of which to bench'
+        required: true
+        type: string
+      repo:
+        description: 'repo name of which to bench'
+        required: true
+        type: string
+
+jobs:
+  create-comment:
+    runs-on: ubuntu-latest
+    outputs:
+      comment-id: ${{ steps.create-comment.outputs.result }}
+    steps:
+      - id: create-comment
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.RSPACK_BOT_ACCESS_TOKEN }}
+          result-encoding: string
+          script: |
+            const url = `${context.serverUrl}//${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
+            const urlLink = `[Open](${url})`
+
+            const { data: comment } = await github.rest.issues.createComment({
+              issue_number: context.payload.inputs.prNumber,
+              owner: context.repo.owner,
+              repo: context.payload.inputs.repo,
+              body: `⏳ Triggered benchmark: ${urlLink}`
+            })
+            return comment.id
+  run-bench:
+    runs-on: ${{ fromJSON(vars.SELF_LINUX_LABELS || '"ubuntu-latest"') }}
+    timeout-minutes: 30
+    needs: create-comment
+    outputs:
+      diff-result: ${{ steps.print-results.outputs.diff-result }}
+    steps:
+      - name: Print runner context
+        env:
+          RUNNER_CONTEXT: ${{ toJson(runner) }}
+        run: echo "$RUNNER_CONTEXT"
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          cache-dependency-path: pnpm-lock.yaml
+
+      - name: Install Pnpm
+        run: corepack enable && pnpm -v && pnpm store path
+
+      - name: Install Dependencies
+        run: pnpm run install:scripts
+
+      - id: print-results
+        name: 🚀 Run specified benchmark and Print results
+        run: |
+          result=$(cd scripts && PR_NUMBER=${{ inputs.prNumber }} pnpm start ${{ inputs.product }})
+          echo "$result"
+          echo "diff-result=${result//$'\n'/'@@'}" >> $GITHUB_OUTPUT
+  update-comment:
+    runs-on: ubuntu-latest
+    needs: [create-comment, run-bench]
+    if: always()
+    steps:
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.RSPACK_BOT_ACCESS_TOKEN }}
+          script: |
+            const diffResult = `${{ needs.run-bench.outputs.diff-result }}`
+            let result = "task ${{ needs.run-bench.result }}"
+            if (diffResult) {
+              result = diffResult.replace(/@@/g, "\n");
+            }
+
+            const url = `${context.serverUrl}//${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
+            const urlLink = `[Open](${url})`
+            const body = `
+            📝 Benchmark detail: ${urlLink}
+
+            ${result}
+            `
+
+            await github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: context.payload.inputs.repo,
+              comment_id: `${{ needs.create-comment.outputs.comment-id }}`,
+              body
+            })
diff --git a/scripts/src/main.ts b/scripts/src/main.ts
@@ -1,12 +1,20 @@
 import logger from 'consola';
 import { dev } from './runners/dev';
 import { build } from './runners/build';
-import { cloneRepo, getDataPath, mergeMetrics } from './shared';
+import {
+  DefaultBenchCase,
+  cloneRepo,
+  getDataPath,
+  mergeMetrics,
+} from './shared';
 import { remove } from 'fs-extra';
 import { yarnInstall } from './runners/yarn-install';
+import { compare } from './shared/compare';
 
 const productName = process.argv[2] || 'MODERNJS_FRAMEWORK';
-const caseName = process.argv[3] || 'app-minimal';
+const caseName =
+  process.argv[3] ||
+  DefaultBenchCase[productName as keyof typeof DefaultBenchCase];
 
 async function main() {
   const dataPath = getDataPath(productName);
@@ -35,7 +43,11 @@ async function main() {
       console.log('failed to collect install size metrics:', err);
     }
 
-    await mergeMetrics(productName, caseName);
+    const jsonPath = await mergeMetrics(productName, caseName);
+
+    if (process.env.PR_NUMBER) {
+      await compare(jsonPath);
+    }
   } else {
     logger.error(`Case not found: ${productName} ${caseName}`);
   }
diff --git a/scripts/src/shared/compare.ts b/scripts/src/shared/compare.ts
@@ -0,0 +1,73 @@
+import { readJson } from 'fs-extra';
+import { Metrics } from './types';
+
+const formatFileSize = (size: number, target = 'KB') => {
+  if (target === 'KB') {
+    size = size / 1024;
+  } else if (target === 'MB') {
+    size = size / 1024 / 1024;
+  }
+  return Number(size.toFixed(2));
+};
+
+const formatSecond = (ms: number) => Number((ms / 1000).toFixed(2));
+
+function formatValue(value: number, property: string) {
+  if (property.endsWith('Time')) {
+    return String(formatSecond(value)) + 's';
+  } else if (property === 'installSize') {
+    return String(formatFileSize(value, 'MB')) + 'MB';
+  } else if (property.endsWith('Size')) {
+    return String(formatFileSize(value)) + 'KB';
+  } else {
+    return String(value);
+  }
+}
+
+function generateTable(
+  base: Record<string, number>,
+  current: Record<string, number>,
+) {
+  const properties = Object.keys(base);
+
+  const maxPropertyLength = Math.max(...properties.map(p => p.length));
+
+  const table = [
+    `| Metric${' '.repeat(maxPropertyLength - 6)} | Base${' '.repeat(
+      6,
+    )} | Current${' '.repeat(3)} | %${' '.repeat(9)} |`,
+    `|${'-'.repeat(maxPropertyLength + 2)}|${'-'.repeat(12)}|${'-'.repeat(
+      12,
+    )}|${'-'.repeat(12)}|`,
+  ];
+
+  properties.forEach(property => {
+    if (property === 'time') return;
+    const percent =
+      ((current[property] - base[property]) * 100) / base[property];
+    const formattedPercent =
+      percent > 0 ? `+${percent.toFixed(2)}%` : `${percent.toFixed(2)}%`;
+
+    const row = `| ${property.padEnd(maxPropertyLength)} | ${formatValue(
+      base[property],
+      property,
+    ).padEnd(10)} | ${formatValue(current[property], property).padEnd(
+      10,
+    )} | ${formattedPercent.padEnd(10)} |`;
+    table.push(row);
+  });
+
+  return table.join('\n');
+}
+
+export async function compare(jsonPath: string) {
+  const allMetrics: Metrics[] = await readJson(jsonPath);
+  const keys = Object.keys(allMetrics);
+  const currentKey = keys[keys.length - 1];
+  const baseKey = keys[keys.length - 2];
+  const current = allMetrics[currentKey as any];
+  const base = allMetrics[baseKey as any];
+
+  const formatTable = generateTable(base, current);
+  console.log(formatTable);
+}
diff --git a/scripts/src/shared/fs.ts b/scripts/src/shared/fs.ts
@@ -102,7 +102,10 @@ const cleanData = (nums: number[]) => {
   return nums;
 };
 
-export async function mergeMetrics(productName: string, caseName: string) {
+export async function mergeMetrics(
+  productName: string,
+  caseName: string,
+): Promise<string> {
   const { jsonPath, jsonName, remoteURL } = await getMetricsPath(
     productName,
     caseName,
@@ -143,4 +146,6 @@ export async function mergeMetrics(productName: string, caseName: string) {
     await outputJson(jsonPath, allData);
     logger.success(`Successfully merged metrics to ${jsonName}.`);
   }
+
+  return jsonPath;
 }
diff --git a/scripts/src/shared/git.ts b/scripts/src/shared/git.ts
@@ -15,13 +15,14 @@ export async function cloneRepo(productName: string, caseName: string) {
   const repoName = getRepoName(productName);
   const localRepoPath = getRepoPath(repoName);
   if (!(await pathExists(localRepoPath))) {
-    const { GITHUB_ACTOR, GITHUB_TOKEN, COMMIT_ID } = process.env;
+    const { GITHUB_ACTOR, GITHUB_TOKEN, COMMIT_ID, PR_NUMBER } = process.env;
     const repoURL = GITHUB_TOKEN
       ? `https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/web-infra-dev/${repoName}.git`
       : `git@github.com:web-infra-dev/${repoName}.git`;
 
     const options = ['clone', '--single-branch'];
-    if (!COMMIT_ID) {
+
+    if (!COMMIT_ID && !PR_NUMBER) {
       options.push('--depth', '1');
     }
 
@@ -38,6 +39,24 @@ export async function cloneRepo(productName: string, caseName: string) {
         stdout: 'inherit',
       });
     }
+
+    if (PR_NUMBER) {
+      const branchName = String(new Date().valueOf());
+      await execa(
+        'git',
+        ['fetch', 'origin', `pull/${PR_NUMBER}/head:${branchName}`],
+        {
+          cwd: localRepoPath,
+          stderr: 'inherit',
+          stdout: 'inherit',
+        },
+      );
+      await execa('git', ['checkout', `${branchName}`], {
+        cwd: localRepoPath,
+        stderr: 'inherit',
+        stdout: 'inherit',
+      });
+    }
   }
 
   if (process.env.ONLY_INSTALL_SIZE === 'true') {
diff --git a/scripts/src/shared/product.ts b/scripts/src/shared/product.ts
@@ -13,3 +13,10 @@ export enum REPO_NAME {
   RSPRESS = 'rspress',
   RSPACK = 'rspack',
 }
+
+export const DefaultBenchCase = {
+  MODERNJS_FRAMEWORK: 'app-arco-pro-rspack',
+  MODERNJS_MODULE: 'module-library',
+  RSPRESS: 'rspress-website',
+  RSBUILD: 'rsbuild-react',
+};
