diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 760fed9..3185e4c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -25,9 +25,9 @@ on: - retrolab - testing reference_branch: - description: "Reference branch on the JupyterLab repository (default: master)" + description: "Reference branch on the JupyterLab repository (default: main)" required: false - default: "master" + default: "main" reference_project: description: "Playwright project to execute on the reference version" required: false @@ -81,9 +81,9 @@ jobs: # Repository to clone for scheduled benchmark challenger: ${{ github.event.inputs.challenger || 'jupyterlab/jupyterlab' }} # Branch to checkout for scheduled benchmark - challenger_branch: ${{ github.event.inputs.challenger_branch || 'master' }} + challenger_branch: ${{ github.event.inputs.challenger_branch || 'main' }} challenger_project: ${{ github.event.inputs.challenger_project || 'jupyterlab' }} - reference_branch: ${{ github.event.inputs.reference_branch || 'master' }} + reference_branch: ${{ github.event.inputs.reference_branch || 'main' }} reference_project: ${{ github.event.inputs.reference_project || 'jupyterlab' }} # Which browser to use (one of 'chromium', 'firefox', 'webkit') browser: ${{ github.event.inputs.browser || 'chromium' }} diff --git a/.github/workflows/memory-leak.yml b/.github/workflows/memory-leak.yml index 0ffa6bc..bf484c3 100644 --- a/.github/workflows/memory-leak.yml +++ b/.github/workflows/memory-leak.yml @@ -23,5 +23,5 @@ jobs: uses: ./.github/workflows/run-memory-leak.yml with: repository: ${{ github.event.inputs.repository || 'jupyterlab/jupyterlab' }} - branch: ${{ github.event.inputs.branch || 'master' }} + branch: ${{ github.event.inputs.branch || 'main' }} samples: ${{ github.event.inputs.samples || '7' }} diff --git a/.github/workflows/profiler.yml b/.github/workflows/profiler.yml index 09ab673..282234e 100644 --- a/.github/workflows/profiler.yml +++ b/.github/workflows/profiler.yml @@ -10,7 +10,7 @@ on: challenger_branch: description: "Git repository reference to the challenger branch" required: true - default: "master" + default: "main" challenger_project: description: "Playwright project to execute (windowingMode JupyterLab 4; renderCellOnIdle: JupyterLab 2.3 or 3.x)" required: false @@ -27,9 +27,9 @@ on: - retrolab - testing reference_branch: - description: "Reference branch on the JupyterLab repository (default: master)" + description: "Reference branch on the JupyterLab repository (default: main)" required: false - default: "master" + default: "main" reference_project: description: "Playwright project to execute on the reference version" required: false @@ -79,9 +79,9 @@ jobs: uses: ./.github/workflows/run-profiler.yml with: challenger: ${{ github.event.inputs.challenger || 'jupyterlab/jupyterlab' }} - challenger_branch: ${{ github.event.inputs.challenger_branch || 'master' }} + challenger_branch: ${{ github.event.inputs.challenger_branch || 'main' }} challenger_project: ${{ github.event.inputs.challenger_project || 'jupyterlab' }} - reference_branch: ${{ github.event.inputs.reference_branch || 'master' }} + reference_branch: ${{ github.event.inputs.reference_branch || 'main' }} reference_project: ${{ github.event.inputs.reference_project || 'jupyterlab' }} browser: ${{ github.event.inputs.browser || 'chromium' }} samples: ${{ github.event.inputs.samples || '25' }} diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml index 527dc16..adb4cd1 100644 --- a/.github/workflows/run-benchmark.yml +++ b/.github/workflows/run-benchmark.yml @@ -24,9 +24,9 @@ on: default: "jupyterlab" type: "string" reference_branch: - description: "Reference branch on the JupyterLab repository (default: master)" + description: "Reference branch on the JupyterLab repository (default: main)" required: false - default: "master" + default: "main" type: string reference_project: description: "Playwright project to execute on the reference version" diff --git a/.github/workflows/run-profiler.yml b/.github/workflows/run-profiler.yml index d7e25b3..4d94c3f 100644 --- a/.github/workflows/run-profiler.yml +++ b/.github/workflows/run-profiler.yml @@ -20,9 +20,9 @@ on: default: "jupyterlab" type: "string" reference_branch: - description: "Reference branch on the JupyterLab repository (default: master)" + description: "Reference branch on the JupyterLab repository (default: main)" required: false - default: "master" + default: "main" type: string reference_project: description: "Playwright project to execute on the reference version" @@ -44,6 +44,11 @@ on: required: false default: "profiler-assets" type: "string" + results_name: + description: "Uploaded results name" + required: false + default: "profiler-results" + type: "string" grep: description: "Tests to include" required: false @@ -220,6 +225,14 @@ jobs: benchmarks/tests/report-reference benchmarks/tests/report-challenger + - name: Upload UI Profiler assets + if: always() + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.results_name }} + path: | + benchmarks/tests/results/ + - name: Print JupyterLab logs if: always() run: | diff --git a/.github/workflows/scheduled-benchmark.yml b/.github/workflows/scheduled-benchmark.yml index 6865e32..947a057 100644 --- a/.github/workflows/scheduled-benchmark.yml +++ b/.github/workflows/scheduled-benchmark.yml @@ -21,8 +21,8 @@ jobs: # Repository to clone for scheduled benchmark challenger: 'jupyterlab/jupyterlab' # Branch to checkout for scheduled benchmark - challenger_branch: 'master' - reference_branch: 'master' + challenger_branch: 'main' + reference_branch: 'main' # Which browser to use (one of 'chromium', 'firefox', 'webkit') browser: 'chromium' # How many samples to compute the statistical distribution @@ -42,8 +42,8 @@ jobs: # Repository to clone for scheduled benchmark challenger: 'jupyterlab/jupyterlab' # Branch to checkout for scheduled benchmark - challenger_branch: 'master' - reference_branch: 'master' + challenger_branch: 'main' + reference_branch: 'main' # Which browser to use (one of 'chromium', 'firefox', 'webkit') browser: 'chromium' # How many samples to compute the statistical distribution @@ -63,8 +63,8 @@ jobs: # Repository to clone for scheduled benchmark challenger: 'jupyterlab/jupyterlab' # Branch to checkout for scheduled benchmark - challenger_branch: 'master' - reference_branch: 'master' + challenger_branch: 'main' + reference_branch: 'main' # Which browser to use (one of 'chromium', 'firefox', 'webkit') browser: 'chromium' # How many samples to compute the statistical distribution @@ -84,8 +84,8 @@ jobs: # Repository to clone for scheduled benchmark challenger: 'jupyterlab/jupyterlab' # Branch to checkout for scheduled benchmark - challenger_branch: 'master' - reference_branch: 'master' + challenger_branch: 'main' + reference_branch: 'main' # Which browser to use (one of 'chromium', 'firefox', 'webkit') browser: 'chromium' # How many samples to compute the statistical distribution @@ -105,8 +105,8 @@ jobs: # Repository to clone for scheduled benchmark challenger: 'jupyterlab/jupyterlab' # Branch to checkout for scheduled benchmark - challenger_branch: 'master' - reference_branch: 'master' + challenger_branch: 'main' + reference_branch: 'main' # Which browser to use (one of 'chromium', 'firefox', 'webkit') browser: 'chromium' # How many samples to compute the statistical distribution diff --git a/.github/workflows/scheduled-memory-leak.yml b/.github/workflows/scheduled-memory-leak.yml index 493f037..8c50894 100644 --- a/.github/workflows/scheduled-memory-leak.yml +++ b/.github/workflows/scheduled-memory-leak.yml @@ -11,4 +11,4 @@ jobs: uses: ./.github/workflows/run-memory-leak.yml with: repository: jupyterlab/jupyterlab - branch: master + branch: main diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e9f5127..2f6e66d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,7 +57,7 @@ jobs: uses: actions/checkout@v3 with: repository: jupyterlab/jupyterlab - ref: master + ref: main path: reference - name: Install dependencies @@ -153,4 +153,4 @@ jobs: uses: ./.github/workflows/run-memory-leak.yml with: repository: jupyterlab/jupyterlab - branch: master + branch: main diff --git a/tests/jupyterlab/ui-profiler.specs.ts b/tests/jupyterlab/ui-profiler.specs.ts index 2bceb80..3534397 100644 --- a/tests/jupyterlab/ui-profiler.specs.ts +++ b/tests/jupyterlab/ui-profiler.specs.ts @@ -1,4 +1,4 @@ -import { benchmark, galata } from "@jupyterlab/galata"; +import { benchmark, galata, IJupyterLabPageFixture } from "@jupyterlab/galata"; import { JSONObject } from "@lumino/coreutils"; import type { IBenchmarkResult, @@ -157,6 +157,21 @@ test.afterEach(async ({ tmpPath, baseURL }) => { await contents.deleteDirectory(tmpPath); }); +async function acceptKernelDialog(page: IJupyterLabPageFixture) { + const dialogLocator = page.locator(".jp-Dialog"); + try { + // Wait up to three seconds for the kernel selection dialog to appear + await dialogLocator.waitFor({ timeout: 3000 }); + } catch { + // no-op + } + // If the kernel dialog shows up, accept default kernel + if (await dialogLocator.isVisible()) { + await page.click(".jp-Dialog .jp-mod-accept"); + await dialogLocator.waitFor({ state: "detached" }); + } +} + test.describe("Measure execution time", () => { for (const [id, scenario] of Object.entries(scenarios)) { for (const file of fileNames) { @@ -180,6 +195,7 @@ test.describe("Measure execution time", () => { if (openNotebook) { await page.notebook.openByPath(notebookPath); + await acceptKernelDialog(page); } const result = (await profiler.runBenchmark( @@ -206,9 +222,18 @@ test.describe("Measure execution time", () => { time: time, project: testInfo.project.name, profiler: true, + granular: true, }) ); } + testInfo.attach(`${reference}-${id}:execution-time.json`, { + body: JSON.stringify({ + ...result, + reference, + backgroundTab: file, + }), + contentType: "application/json", + }); }); } } @@ -239,6 +264,7 @@ test.describe("Benchmark style sheets @slow", () => { if (openNotebook) { await page.notebook.openByPath(notebookPath); + await acceptKernelDialog(page); } const result = (await profiler.runBenchmark( @@ -257,7 +283,11 @@ test.describe("Benchmark style sheets @slow", () => { >; testInfo.attach(`${reference}-${id}:style-sheet.json`, { - body: JSON.stringify(result), + body: JSON.stringify({ + ...result, + reference, + backgroundTab: file, + }), contentType: "application/json", }); }); @@ -287,6 +317,7 @@ test.describe("Benchmark style rules @slow", () => { if (openNotebook) { await page.notebook.openByPath(notebookPath); + await acceptKernelDialog(page); } const result = (await profiler.runBenchmark( @@ -305,7 +336,11 @@ test.describe("Benchmark style rules @slow", () => { >; testInfo.attach(`${reference}-${id}:style-rule.json`, { - body: JSON.stringify(result), + body: JSON.stringify({ + ...result, + reference, + backgroundTab: file, + }), contentType: "application/json", }); }); diff --git a/tests/package.json b/tests/package.json index 1f76a81..e823fde 100644 --- a/tests/package.json +++ b/tests/package.json @@ -27,5 +27,8 @@ "prettier": "^2.8.8", "rimraf": "^3.0.2", "typescript": "~4.9.0" + }, + "dependencies": { + "@actions/core": "^1.10.0" } } diff --git a/tests/playwright.config.ts b/tests/playwright.config.ts index 9f0de0a..e97f90d 100644 --- a/tests/playwright.config.ts +++ b/tests/playwright.config.ts @@ -133,6 +133,10 @@ export default { "@jupyterlab/galata/lib/benchmarkReporter", { outputFile: "lab-benchmark.json" }, ], + [ + "./reporter", {} + ], + ['json', { outputFile: 'test-results.json' }] ], use: { // Browser options diff --git a/tests/reporter.ts b/tests/reporter.ts new file mode 100644 index 0000000..07f40e0 --- /dev/null +++ b/tests/reporter.ts @@ -0,0 +1,161 @@ +import { + FullConfig, + FullResult, + Reporter, + Suite, + TestCase, + TestResult, +} from "@playwright/test/reporter"; +import * as fs from "fs"; + +import { summary, notice, error } from "@actions/core"; + +import type { IBenchmarkResult } from "@jupyterlab/ui-profiler"; + +// TODO: use `Statistic` from ui-profiler? +namespace Statistic { + export function mean(numbers: number[]): number { + if (numbers.length === 0) { + return NaN; + } + return sum(numbers) / numbers.length; + } + + export function round(n: number, precision = 0): number { + const factor = Math.pow(10, precision); + return Math.round(n * factor) / factor; + } + + export function sum(numbers: number[]): number { + if (numbers.length === 0) { + return 0; + } + return numbers.reduce((a, b) => a + b); + } + + /** + * Implements corrected sample standard deviation. + */ + export function standardDeviation(numbers: number[]): number { + if (numbers.length === 0) { + return NaN; + } + const m = mean(numbers); + return Math.sqrt( + (sum(numbers.map((n) => Math.pow(n - m, 2))) * 1) / (numbers.length - 1) + ); + } +} + +interface IAttachment extends IBenchmarkResult { + reference: string; + granular?: boolean; + backgroundTab: string; + name: string; +} + +class UIProfilerReporter implements Reporter { + constructor(options: {}) {} + private _attachments: IAttachment[] = []; + private _reference: string = "unknown"; + + onBegin(config: FullConfig, suite: Suite) { + console.log(`Starting the run with ${suite.allTests().length} tests`); + } + + onTestBegin(test: TestCase, result: TestResult) { + console.log(`Starting test ${test.title}`); + } + + onTestEnd(test: TestCase, result: TestResult) { + console.log(`Finished test ${test.title}: ${result.status}`); + if (result.status !== "passed") { + return; + } + const attachments = result.attachments + .map((raw) => { + const json = JSON.parse( + raw.body?.toString() ?? "{}" + ) as any as IAttachment; + return { ...json, name: raw.name }; + }) + .filter((a) => !a.granular && a.reference); + + for (const attachment of attachments) { + this._attachments.push(attachment); + notice( + attachment.benchmark + + " of " + + attachment.scenario + + " completed in " + + attachment.outcome.totalTime / 1000 + + "s" + ); + } + } + + async onEnd(result: FullResult) { + console.log(`Finished the run: ${result.status}`); + + const dir = "results"; + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir); + } + + const references = new Set(); + for (const attachment of this._attachments) { + // `uploadArtifact` forbids colons in file names + const path = attachment.name.replace(":", "_"); + fs.writeFileSync(dir + "/" + path, JSON.stringify(attachment)); + references.add(attachment.reference); + } + if (references.size > 1) { + throw new Error("More than one reference versions is not supported"); + } else if (references.size == 0) { + throw new Error("No results to report"); + } + const reference = references.values().next().value; + this._reference = reference; + + // Add summary table showing average execution time per scenario (rows) per notebook (columns) + summary.addHeading(this._reference); + + const timeMeasurements = this._attachments.filter( + (a) => a.benchmark === "execution-time" + ); + const scenarios = [...new Set(timeMeasurements.map((a) => a.scenario))]; + const backgrounds = [ + ...new Set(timeMeasurements.map((a) => a.backgroundTab)), + ]; + + summary.addTable([ + [ + { data: "scenario", header: true }, + ...backgrounds.map((b) => { + return { data: b, header: true }; + }), + ], + ...scenarios.map((s) => { + const row = backgrounds.map((b) => { + const result = this._attachments.find( + (a) => a.backgroundTab === b && a.scenario === s + ); + const times = result.outcome.results[0].times; + return ( + Statistic.round(Statistic.mean(times), 2).toString() + + " ± " + + Statistic.round(Statistic.standardDeviation(times), 2).toString() + ); + }); + return [s, ...row]; + }), + ]); + await summary.write(); + } + + printsToStdio() { + return false; + } +} + +export default UIProfilerReporter; diff --git a/yarn.lock b/yarn.lock index 5424f50..a65fee3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5,6 +5,25 @@ __metadata: version: 6 cacheKey: 8 +"@actions/core@npm:^1.10.0": + version: 1.10.0 + resolution: "@actions/core@npm:1.10.0" + dependencies: + "@actions/http-client": ^2.0.1 + uuid: ^8.3.2 + checksum: 0a75621e007ab20d887434cdd165f0b9036f14c22252a2faed33543d8b9d04ec95d823e69ca636a25245574e4585d73e1e9e47a845339553c664f9f2c9614669 + languageName: node + linkType: hard + +"@actions/http-client@npm:^2.0.1": + version: 2.1.0 + resolution: "@actions/http-client@npm:2.1.0" + dependencies: + tunnel: ^0.0.6 + checksum: 25a72a952cc95fb4b3ab086da73a5754dd0957c206637cace69be2e16f018cc1b3d3c40d3bcf89ffd8a5929d5e8445594b498b50db306a50ad7536023f8e3800 + languageName: node + linkType: hard + "@babel/code-frame@npm:7.12.11": version: 7.12.11 resolution: "@babel/code-frame@npm:7.12.11" @@ -438,6 +457,7 @@ __metadata: version: 0.0.0-use.local resolution: "@jupyterlab/benchmarks-tests@workspace:tests" dependencies: + "@actions/core": ^1.10.0 "@jupyterlab/galata": ^4.3.5 "@jupyterlab/ui-profiler": ^0.2.1 "@playwright/test": ^1.30.0 @@ -12229,6 +12249,13 @@ __metadata: languageName: node linkType: hard +"tunnel@npm:^0.0.6": + version: 0.0.6 + resolution: "tunnel@npm:0.0.6" + checksum: c362948df9ad34b649b5585e54ce2838fa583aa3037091aaed66793c65b423a264e5229f0d7e9a95513a795ac2bd4cb72cda7e89a74313f182c1e9ae0b0994fa + languageName: node + linkType: hard + "tweetnacl@npm:^0.14.3, tweetnacl@npm:~0.14.0": version: 0.14.5 resolution: "tweetnacl@npm:0.14.5" @@ -12616,6 +12643,15 @@ __metadata: languageName: node linkType: hard +"uuid@npm:^8.3.2": + version: 8.3.2 + resolution: "uuid@npm:8.3.2" + bin: + uuid: dist/bin/uuid + checksum: 5575a8a75c13120e2f10e6ddc801b2c7ed7d8f3c8ac22c7ed0c7b2ba6383ec0abda88c905085d630e251719e0777045ae3236f04c812184b7c765f63a70e58df + languageName: node + linkType: hard + "v8-compile-cache@npm:^2.0.3": version: 2.3.0 resolution: "v8-compile-cache@npm:2.3.0"