From 91f88929fc2e8dc30ecfdffcda320c3013649f95 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 11 Jun 2023 18:30:31 +0100
Subject: [PATCH 1/8] Update branch names after change in lab

---
 .github/workflows/profiler.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/profiler.yml b/.github/workflows/profiler.yml
index 09ab673..282234e 100644
--- a/.github/workflows/profiler.yml
+++ b/.github/workflows/profiler.yml
@@ -10,7 +10,7 @@ on:
       challenger_branch:
         description: "Git repository reference to the challenger branch"
         required: true
-        default: "master"
+        default: "main"
       challenger_project:
         description: "Playwright project to execute (windowingMode JupyterLab 4; renderCellOnIdle: JupyterLab 2.3 or 3.x)"
         required: false
@@ -27,9 +27,9 @@ on:
           - retrolab
           - testing
       reference_branch:
-        description: "Reference branch on the JupyterLab repository (default: master)"
+        description: "Reference branch on the JupyterLab repository (default: main)"
         required: false
-        default: "master"
+        default: "main"
       reference_project:
         description: "Playwright project to execute on the reference version"
         required: false
@@ -79,9 +79,9 @@ jobs:
     uses: ./.github/workflows/run-profiler.yml
     with:
       challenger: ${{ github.event.inputs.challenger || 'jupyterlab/jupyterlab' }}
-      challenger_branch: ${{ github.event.inputs.challenger_branch || 'master' }}
+      challenger_branch: ${{ github.event.inputs.challenger_branch || 'main' }}
       challenger_project: ${{ github.event.inputs.challenger_project || 'jupyterlab' }}
-      reference_branch: ${{ github.event.inputs.reference_branch || 'master' }}
+      reference_branch: ${{ github.event.inputs.reference_branch || 'main' }}
       reference_project: ${{ github.event.inputs.reference_project || 'jupyterlab' }}
       browser: ${{ github.event.inputs.browser || 'chromium' }}
       samples: ${{ github.event.inputs.samples || '25' }}

From 35d2f6f8c58f0e19c01765bcfb9465181888ec0d Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 11 Jun 2023 18:31:30 +0100
Subject: [PATCH 2/8] Fix issues with kernel dialog, pass more data

---
 tests/jupyterlab/ui-profiler.specs.ts | 41 +++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 3 deletions(-)

diff --git a/tests/jupyterlab/ui-profiler.specs.ts b/tests/jupyterlab/ui-profiler.specs.ts
index 2bceb80..6807ab9 100644
--- a/tests/jupyterlab/ui-profiler.specs.ts
+++ b/tests/jupyterlab/ui-profiler.specs.ts
@@ -1,4 +1,4 @@
-import { benchmark, galata } from "@jupyterlab/galata";
+import { benchmark, galata, IJupyterLabPageFixture } from "@jupyterlab/galata";
 import { JSONObject } from "@lumino/coreutils";
 import type {
   IBenchmarkResult,
@@ -157,6 +157,21 @@ test.afterEach(async ({ tmpPath, baseURL }) => {
   await contents.deleteDirectory(tmpPath);
 });
 
+async function acceptKernelDialog(page: IJupyterLabPageFixture) {
+  const dialogLocator = page.locator(".jp-Dialog");
+  try {
+    // Wait up to three seconds for the kernel selection dialog to appear
+    await dialogLocator.waitFor({ timeout: 3000 });
+  } catch {
+    // no-op
+  }
+  // If the kernel dialog shows up, accept default kernel
+  if (await dialogLocator.isVisible()) {
+    await page.click(".jp-Dialog .jp-mod-accept");
+    await dialogLocator.waitFor({ state: "detached" });
+  }
+}
+
 test.describe("Measure execution time", () => {
   for (const [id, scenario] of Object.entries(scenarios)) {
     for (const file of fileNames) {
@@ -180,6 +195,7 @@ test.describe("Measure execution time", () => {
 
         if (openNotebook) {
           await page.notebook.openByPath(notebookPath);
+          await acceptKernelDialog(page);
         }
 
         const result = (await profiler.runBenchmark(
@@ -206,9 +222,18 @@ test.describe("Measure execution time", () => {
               time: time,
               project: testInfo.project.name,
               profiler: true,
+              granular: true,
             })
           );
         }
+        testInfo.attach(`${reference}-${id}:execution-time.json`, {
+          body: JSON.stringify({
+            ...result,
+            reference,
+            backgroundTab: notebookPath,
+          }),
+          contentType: "application/json",
+        });
       });
     }
   }
@@ -239,6 +264,7 @@ test.describe("Benchmark style sheets @slow", () => {
 
         if (openNotebook) {
           await page.notebook.openByPath(notebookPath);
+          await acceptKernelDialog(page);
         }
 
         const result = (await profiler.runBenchmark(
@@ -257,7 +283,11 @@ test.describe("Benchmark style sheets @slow", () => {
         >;
 
         testInfo.attach(`${reference}-${id}:style-sheet.json`, {
-          body: JSON.stringify(result),
+          body: JSON.stringify({
+            ...result,
+            reference,
+            backgroundTab: notebookPath,
+          }),
           contentType: "application/json",
         });
       });
@@ -287,6 +317,7 @@ test.describe("Benchmark style rules @slow", () => {
 
         if (openNotebook) {
           await page.notebook.openByPath(notebookPath);
+          await acceptKernelDialog(page);
         }
 
         const result = (await profiler.runBenchmark(
@@ -305,7 +336,11 @@ test.describe("Benchmark style rules @slow", () => {
         >;
 
         testInfo.attach(`${reference}-${id}:style-rule.json`, {
-          body: JSON.stringify(result),
+          body: JSON.stringify({
+            ...result,
+            reference,
+            backgroundTab: notebookPath,
+          }),
           contentType: "application/json",
         });
       });

From 7b5e4633d2d5cc858a97dd877a479666560119ff Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 11 Jun 2023 18:33:05 +0100
Subject: [PATCH 3/8] Implement a reporter uploading JSON artifacts and
 reporting summaries

---
 tests/package.json         |   4 ++
 tests/playwright.config.ts |   4 ++
 tests/reporter.ts          | 136 +++++++++++++++++++++++++++++++++++++
 yarn.lock                  |  69 ++++++++++++++++++-
 4 files changed, 212 insertions(+), 1 deletion(-)
 create mode 100644 tests/reporter.ts

diff --git a/tests/package.json b/tests/package.json
index 1f76a81..35f3b0e 100644
--- a/tests/package.json
+++ b/tests/package.json
@@ -27,5 +27,9 @@
     "prettier": "^2.8.8",
     "rimraf": "^3.0.2",
     "typescript": "~4.9.0"
+  },
+  "dependencies": {
+    "@actions/artifact": "^1.1.1",
+    "@actions/core": "^1.10.0"
   }
 }
diff --git a/tests/playwright.config.ts b/tests/playwright.config.ts
index 9f0de0a..e97f90d 100644
--- a/tests/playwright.config.ts
+++ b/tests/playwright.config.ts
@@ -133,6 +133,10 @@ export default {
       "@jupyterlab/galata/lib/benchmarkReporter",
       { outputFile: "lab-benchmark.json" },
     ],
+    [
+      "./reporter", {}
+    ],
+    ['json', {  outputFile: 'test-results.json' }]
   ],
   use: {
     // Browser options
diff --git a/tests/reporter.ts b/tests/reporter.ts
new file mode 100644
index 0000000..93bd37a
--- /dev/null
+++ b/tests/reporter.ts
@@ -0,0 +1,136 @@
+import {
+  FullConfig,
+  FullResult,
+  Reporter,
+  Suite,
+  TestCase,
+  TestResult,
+} from "@playwright/test/reporter";
+import * as fs from "fs";
+
+import { summary, notice } from "@actions/core";
+import * as artifact from "@actions/artifact";
+
+import type { IBenchmarkResult } from "@jupyterlab/ui-profiler";
+
+// TODO: use `Statistic` from ui-profiler?
+namespace Statistic {
+  export function mean(numbers: number[]): number {
+    if (numbers.length === 0) {
+      return NaN;
+    }
+    return sum(numbers) / numbers.length;
+  }
+
+  export function sum(numbers: number[]): number {
+    if (numbers.length === 0) {
+      return 0;
+    }
+    return numbers.reduce((a, b) => a + b);
+  }
+}
+
+interface IAttachment extends IBenchmarkResult {
+  reference: string;
+  granular?: boolean;
+  backgroundTab: string;
+  name: string;
+}
+
+class UIProfilerReporter implements Reporter {
+  constructor(options: {}) {}
+  private _attachments: IAttachment[] = [];
+  private _reference: string = "unknown";
+
+  onBegin(config: FullConfig, suite: Suite) {
+    console.log(`Starting the run with ${suite.allTests().length} tests`);
+  }
+
+  onTestBegin(test: TestCase, result: TestResult) {
+    console.log(`Starting test ${test.title}`);
+  }
+
+  onTestEnd(test: TestCase, result: TestResult) {
+    console.log(`Finished test ${test.title}: ${result.status}`);
+    if (result.status !== "passed") {
+      return;
+    }
+    const attachments = result.attachments
+      .map((raw) => {
+        const json = JSON.parse(
+          raw.body?.toString() ?? "{}"
+        ) as any as IAttachment;
+        return { ...json, name: raw.name };
+      })
+      .filter((a) => !a.granular && a.reference);
+
+    for (const attachment of attachments) {
+      this._attachments.push(attachment);
+      notice(
+        attachment.benchmark +
+          " of " +
+          attachment.scenario +
+          " completed in " +
+          attachment.outcome.totalTime / 1000 +
+          "s"
+      );
+    }
+  }
+
+  onEnd(result: FullResult) {
+    console.log(`Finished the run: ${result.status}`);
+    const artifactClient = artifact.create();
+    const rootDirectory = ".";
+    const options = {
+      continueOnError: true,
+    };
+    const files: string[] = [];
+
+    const references = new Set<string>();
+    for (const attachment of this._attachments) {
+      fs.writeFileSync(attachment.name, JSON.stringify(attachment));
+      files.push(attachment.name);
+      references.add(attachment.reference);
+    }
+    if (references.size > 1) {
+      throw new Error("More than one reference versions is not supported");
+    } else if (references.size == 0) {
+      throw new Error("No results to report");
+    }
+    const reference = references.values().next().value;
+    this._reference = reference;
+
+    const artifactName = "UI profiler " + reference;
+    artifactClient.uploadArtifact(artifactName, files, rootDirectory, options);
+
+    // Add summary table showing average execution time per scenario (rows) per notebook (columns)
+    summary.addHeading(this._reference);
+
+    const timeMeasurements = this._attachments.filter(
+      (a) => a.benchmark === "execution-time"
+    );
+    const scenarios = timeMeasurements.map((a) => a.scenario);
+    const backgrounds = timeMeasurements.map((a) => a.backgroundTab);
+
+    summary.addTable([
+      backgrounds.map((b) => {
+        return { data: b, header: true };
+      }),
+      ...scenarios.map((s) =>
+        backgrounds.map((b) => {
+          const result = this._attachments.find(
+            (a) => a.backgroundTab === b && a.scenario === s
+          );
+          const times = result.outcome.results[0].times;
+          return Statistic.mean(times).toString();
+        })
+      ),
+    ]);
+  }
+
+  printsToStdio() {
+    return false;
+  }
+}
+
+export default UIProfilerReporter;
diff --git a/yarn.lock b/yarn.lock
index 5424f50..0c6fcaa 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5,6 +5,37 @@ __metadata:
   version: 6
   cacheKey: 8
 
+"@actions/artifact@npm:^1.1.1":
+  version: 1.1.1
+  resolution: "@actions/artifact@npm:1.1.1"
+  dependencies:
+    "@actions/core": ^1.9.1
+    "@actions/http-client": ^2.0.1
+    tmp: ^0.2.1
+    tmp-promise: ^3.0.2
+  checksum: 45e170dc64a1023c29a3cfd74407cb3efcf7b4ac3b764f231ed7f3b18d7684eb4459e80ef610dc5859454d9b57df8565ac4cfe9610dc1e6e86b2d21657685517
+  languageName: node
+  linkType: hard
+
+"@actions/core@npm:^1.10.0, @actions/core@npm:^1.9.1":
+  version: 1.10.0
+  resolution: "@actions/core@npm:1.10.0"
+  dependencies:
+    "@actions/http-client": ^2.0.1
+    uuid: ^8.3.2
+  checksum: 0a75621e007ab20d887434cdd165f0b9036f14c22252a2faed33543d8b9d04ec95d823e69ca636a25245574e4585d73e1e9e47a845339553c664f9f2c9614669
+  languageName: node
+  linkType: hard
+
+"@actions/http-client@npm:^2.0.1":
+  version: 2.1.0
+  resolution: "@actions/http-client@npm:2.1.0"
+  dependencies:
+    tunnel: ^0.0.6
+  checksum: 25a72a952cc95fb4b3ab086da73a5754dd0957c206637cace69be2e16f018cc1b3d3c40d3bcf89ffd8a5929d5e8445594b498b50db306a50ad7536023f8e3800
+  languageName: node
+  linkType: hard
+
 "@babel/code-frame@npm:7.12.11":
   version: 7.12.11
   resolution: "@babel/code-frame@npm:7.12.11"
@@ -438,6 +469,8 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@jupyterlab/benchmarks-tests@workspace:tests"
   dependencies:
+    "@actions/artifact": ^1.1.1
+    "@actions/core": ^1.10.0
     "@jupyterlab/galata": ^4.3.5
     "@jupyterlab/ui-profiler": ^0.2.1
     "@playwright/test": ^1.30.0
@@ -10891,7 +10924,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"rimraf@npm:^3.0.2":
+"rimraf@npm:^3.0.0, rimraf@npm:^3.0.2":
   version: 3.0.2
   resolution: "rimraf@npm:3.0.2"
   dependencies:
@@ -12061,6 +12094,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tmp-promise@npm:^3.0.2":
+  version: 3.0.3
+  resolution: "tmp-promise@npm:3.0.3"
+  dependencies:
+    tmp: ^0.2.0
+  checksum: f854f5307dcee6455927ec3da9398f139897faf715c5c6dcee6d9471ae85136983ea06662eba2edf2533bdcb0fca66d16648e79e14381e30c7fb20be9c1aa62c
+  languageName: node
+  linkType: hard
+
 "tmp@npm:^0.0.33":
   version: 0.0.33
   resolution: "tmp@npm:0.0.33"
@@ -12070,6 +12112,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tmp@npm:^0.2.0, tmp@npm:^0.2.1":
+  version: 0.2.1
+  resolution: "tmp@npm:0.2.1"
+  dependencies:
+    rimraf: ^3.0.0
+  checksum: 8b1214654182575124498c87ca986ac53dc76ff36e8f0e0b67139a8d221eaecfdec108c0e6ec54d76f49f1f72ab9325500b246f562b926f85bcdfca8bf35df9e
+  languageName: node
+  linkType: hard
+
 "to-object-path@npm:^0.3.0":
   version: 0.3.0
   resolution: "to-object-path@npm:0.3.0"
@@ -12229,6 +12280,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tunnel@npm:^0.0.6":
+  version: 0.0.6
+  resolution: "tunnel@npm:0.0.6"
+  checksum: c362948df9ad34b649b5585e54ce2838fa583aa3037091aaed66793c65b423a264e5229f0d7e9a95513a795ac2bd4cb72cda7e89a74313f182c1e9ae0b0994fa
+  languageName: node
+  linkType: hard
+
 "tweetnacl@npm:^0.14.3, tweetnacl@npm:~0.14.0":
   version: 0.14.5
   resolution: "tweetnacl@npm:0.14.5"
@@ -12616,6 +12674,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"uuid@npm:^8.3.2":
+  version: 8.3.2
+  resolution: "uuid@npm:8.3.2"
+  bin:
+    uuid: dist/bin/uuid
+  checksum: 5575a8a75c13120e2f10e6ddc801b2c7ed7d8f3c8ac22c7ed0c7b2ba6383ec0abda88c905085d630e251719e0777045ae3236f04c812184b7c765f63a70e58df
+  languageName: node
+  linkType: hard
+
 "v8-compile-cache@npm:^2.0.3":
   version: 2.3.0
   resolution: "v8-compile-cache@npm:2.3.0"

From 254417a34fde98df94c81f177432bd3e4db26191 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 11 Jun 2023 19:57:24 +0100
Subject: [PATCH 4/8] Add SD, await for upload and flush summary

---
 tests/reporter.ts | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/tests/reporter.ts b/tests/reporter.ts
index 93bd37a..5f39195 100644
--- a/tests/reporter.ts
+++ b/tests/reporter.ts
@@ -8,7 +8,7 @@ import {
 } from "@playwright/test/reporter";
 import * as fs from "fs";
 
-import { summary, notice } from "@actions/core";
+import { summary, notice, error } from "@actions/core";
 import * as artifact from "@actions/artifact";
 
 import type { IBenchmarkResult } from "@jupyterlab/ui-profiler";
@@ -28,6 +28,19 @@ namespace Statistic {
     }
     return numbers.reduce((a, b) => a + b);
   }
+
+  /**
+   * Implements corrected sample standard deviation.
+   */
+  export function standardDeviation(numbers: number[]): number {
+    if (numbers.length === 0) {
+      return NaN;
+    }
+    const m = mean(numbers);
+    return Math.sqrt(
+      (sum(numbers.map((n) => Math.pow(n - m, 2))) * 1) / (numbers.length - 1)
+    );
+  }
 }
 
 interface IAttachment extends IBenchmarkResult {
@@ -77,7 +90,7 @@ class UIProfilerReporter implements Reporter {
     }
   }
 
-  onEnd(result: FullResult) {
+  async onEnd(result: FullResult) {
     console.log(`Finished the run: ${result.status}`);
     const artifactClient = artifact.create();
     const rootDirectory = ".";
@@ -101,7 +114,16 @@ class UIProfilerReporter implements Reporter {
     this._reference = reference;
 
     const artifactName = "UI profiler " + reference;
-    artifactClient.uploadArtifact(artifactName, files, rootDirectory, options);
+    const uploadResult = await artifactClient.uploadArtifact(
+      artifactName,
+      files,
+      rootDirectory,
+      options
+    );
+
+    if (uploadResult.failedItems.length > 0) {
+      error("Upload of some artifacts failed");
+    }
 
     // Add summary table showing average execution time per scenario (rows) per notebook (columns)
     summary.addHeading(this._reference);
@@ -122,10 +144,15 @@ class UIProfilerReporter implements Reporter {
             (a) => a.backgroundTab === b && a.scenario === s
           );
           const times = result.outcome.results[0].times;
-          return Statistic.mean(times).toString();
+          return (
+            Statistic.mean(times).toString() +
+            " ± " +
+            Statistic.standardDeviation(times).toString()
+          );
         })
       ),
     ]);
+    await summary.write();
   }
 
   printsToStdio() {

From 1ead362806626e4991791e5ea076666d9ae24048 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 11 Jun 2023 20:00:23 +0100
Subject: [PATCH 5/8] Update all other JupyterLab `master` references to `main`

---
 .github/workflows/benchmark.yml             |  8 ++++----
 .github/workflows/memory-leak.yml           |  2 +-
 .github/workflows/run-benchmark.yml         |  4 ++--
 .github/workflows/run-profiler.yml          |  4 ++--
 .github/workflows/scheduled-benchmark.yml   | 20 ++++++++++----------
 .github/workflows/scheduled-memory-leak.yml |  2 +-
 .github/workflows/test.yml                  |  4 ++--
 7 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 760fed9..3185e4c 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -25,9 +25,9 @@ on:
           - retrolab
           - testing
       reference_branch:
-        description: "Reference branch on the JupyterLab repository (default: master)"
+        description: "Reference branch on the JupyterLab repository (default: main)"
         required: false
-        default: "master"
+        default: "main"
       reference_project:
         description: "Playwright project to execute on the reference version"
         required: false
@@ -81,9 +81,9 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: ${{ github.event.inputs.challenger || 'jupyterlab/jupyterlab' }}
       # Branch to checkout for scheduled benchmark
-      challenger_branch: ${{ github.event.inputs.challenger_branch || 'master' }}
+      challenger_branch: ${{ github.event.inputs.challenger_branch || 'main' }}
       challenger_project: ${{ github.event.inputs.challenger_project || 'jupyterlab' }}
-      reference_branch: ${{ github.event.inputs.reference_branch || 'master' }}
+      reference_branch: ${{ github.event.inputs.reference_branch || 'main' }}
       reference_project: ${{ github.event.inputs.reference_project || 'jupyterlab' }}
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: ${{ github.event.inputs.browser || 'chromium' }}
diff --git a/.github/workflows/memory-leak.yml b/.github/workflows/memory-leak.yml
index 0ffa6bc..bf484c3 100644
--- a/.github/workflows/memory-leak.yml
+++ b/.github/workflows/memory-leak.yml
@@ -23,5 +23,5 @@ jobs:
     uses: ./.github/workflows/run-memory-leak.yml
     with:
       repository: ${{ github.event.inputs.repository || 'jupyterlab/jupyterlab' }}
-      branch: ${{ github.event.inputs.branch || 'master' }}
+      branch: ${{ github.event.inputs.branch || 'main' }}
       samples: ${{ github.event.inputs.samples || '7' }}
diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml
index 527dc16..adb4cd1 100644
--- a/.github/workflows/run-benchmark.yml
+++ b/.github/workflows/run-benchmark.yml
@@ -24,9 +24,9 @@ on:
         default: "jupyterlab"
         type: "string"
       reference_branch:
-        description: "Reference branch on the JupyterLab repository (default: master)"
+        description: "Reference branch on the JupyterLab repository (default: main)"
         required: false
-        default: "master"
+        default: "main"
         type: string
       reference_project:
         description: "Playwright project to execute on the reference version"
diff --git a/.github/workflows/run-profiler.yml b/.github/workflows/run-profiler.yml
index d7e25b3..718ab61 100644
--- a/.github/workflows/run-profiler.yml
+++ b/.github/workflows/run-profiler.yml
@@ -20,9 +20,9 @@ on:
         default: "jupyterlab"
         type: "string"
       reference_branch:
-        description: "Reference branch on the JupyterLab repository (default: master)"
+        description: "Reference branch on the JupyterLab repository (default: main)"
         required: false
-        default: "master"
+        default: "main"
         type: string
       reference_project:
         description: "Playwright project to execute on the reference version"
diff --git a/.github/workflows/scheduled-benchmark.yml b/.github/workflows/scheduled-benchmark.yml
index 6865e32..947a057 100644
--- a/.github/workflows/scheduled-benchmark.yml
+++ b/.github/workflows/scheduled-benchmark.yml
@@ -21,8 +21,8 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: 'jupyterlab/jupyterlab'
       # Branch to checkout for scheduled benchmark
-      challenger_branch: 'master'
-      reference_branch: 'master'
+      challenger_branch: 'main'
+      reference_branch: 'main'
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: 'chromium'
       # How many samples to compute the statistical distribution
@@ -42,8 +42,8 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: 'jupyterlab/jupyterlab'
       # Branch to checkout for scheduled benchmark
-      challenger_branch: 'master'
-      reference_branch: 'master'
+      challenger_branch: 'main'
+      reference_branch: 'main'
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: 'chromium'
       # How many samples to compute the statistical distribution
@@ -63,8 +63,8 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: 'jupyterlab/jupyterlab'
       # Branch to checkout for scheduled benchmark
-      challenger_branch: 'master'
-      reference_branch: 'master'
+      challenger_branch: 'main'
+      reference_branch: 'main'
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: 'chromium'
       # How many samples to compute the statistical distribution
@@ -84,8 +84,8 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: 'jupyterlab/jupyterlab'
       # Branch to checkout for scheduled benchmark
-      challenger_branch: 'master'
-      reference_branch: 'master'
+      challenger_branch: 'main'
+      reference_branch: 'main'
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: 'chromium'
       # How many samples to compute the statistical distribution
@@ -105,8 +105,8 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: 'jupyterlab/jupyterlab'
       # Branch to checkout for scheduled benchmark
-      challenger_branch: 'master'
-      reference_branch: 'master'
+      challenger_branch: 'main'
+      reference_branch: 'main'
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: 'chromium'
       # How many samples to compute the statistical distribution
diff --git a/.github/workflows/scheduled-memory-leak.yml b/.github/workflows/scheduled-memory-leak.yml
index 493f037..8c50894 100644
--- a/.github/workflows/scheduled-memory-leak.yml
+++ b/.github/workflows/scheduled-memory-leak.yml
@@ -11,4 +11,4 @@ jobs:
     uses: ./.github/workflows/run-memory-leak.yml
     with:
       repository: jupyterlab/jupyterlab
-      branch: master
+      branch: main
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e9f5127..2f6e66d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -57,7 +57,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: jupyterlab/jupyterlab
-          ref: master
+          ref: main
           path: reference
 
       - name: Install dependencies
@@ -153,4 +153,4 @@ jobs:
     uses: ./.github/workflows/run-memory-leak.yml
     with:
       repository: jupyterlab/jupyterlab
-      branch: master
+      branch: main

From 388a45308113433190f7278b6719ef596f11e138 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 11 Jun 2023 20:23:11 +0100
Subject: [PATCH 6/8] Replace colons with underscores

---
 tests/reporter.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/reporter.ts b/tests/reporter.ts
index 5f39195..29639bf 100644
--- a/tests/reporter.ts
+++ b/tests/reporter.ts
@@ -101,8 +101,10 @@ class UIProfilerReporter implements Reporter {
 
     const references = new Set<string>();
     for (const attachment of this._attachments) {
-      fs.writeFileSync(attachment.name, JSON.stringify(attachment));
-      files.push(attachment.name);
+      // `uploadArtifact` forbids colons in file names
+      const path = attachment.name.replace(":", "_");
+      fs.writeFileSync(path, JSON.stringify(attachment));
+      files.push(path);
       references.add(attachment.reference);
     }
     if (references.size > 1) {

From 89c198f0dd45c551c2da31d3c07fca5f27b9c1f3 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 11 Jun 2023 21:38:09 +0100
Subject: [PATCH 7/8] Do not use `@actions/artifact` as it can only be used in
 an action

---
 .github/workflows/run-profiler.yml | 13 +++++++++++
 tests/package.json                 |  1 -
 tests/reporter.ts                  | 27 +++++------------------
 yarn.lock                          | 35 ++----------------------------
 4 files changed, 21 insertions(+), 55 deletions(-)

diff --git a/.github/workflows/run-profiler.yml b/.github/workflows/run-profiler.yml
index 718ab61..4d94c3f 100644
--- a/.github/workflows/run-profiler.yml
+++ b/.github/workflows/run-profiler.yml
@@ -44,6 +44,11 @@ on:
         required: false
         default: "profiler-assets"
         type: "string"
+      results_name:
+        description: "Uploaded results name"
+        required: false
+        default: "profiler-results"
+        type: "string"
       grep:
         description: "Tests to include"
         required: false
@@ -220,6 +225,14 @@ jobs:
             benchmarks/tests/report-reference
             benchmarks/tests/report-challenger
 
+      - name: Upload UI Profiler assets
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.results_name }}
+          path: |
+            benchmarks/tests/results/
+
       - name: Print JupyterLab logs
         if: always()
         run: |
diff --git a/tests/package.json b/tests/package.json
index 35f3b0e..e823fde 100644
--- a/tests/package.json
+++ b/tests/package.json
@@ -29,7 +29,6 @@
     "typescript": "~4.9.0"
   },
   "dependencies": {
-    "@actions/artifact": "^1.1.1",
     "@actions/core": "^1.10.0"
   }
 }
diff --git a/tests/reporter.ts b/tests/reporter.ts
index 29639bf..5022289 100644
--- a/tests/reporter.ts
+++ b/tests/reporter.ts
@@ -9,7 +9,6 @@ import {
 import * as fs from "fs";
 
 import { summary, notice, error } from "@actions/core";
-import * as artifact from "@actions/artifact";
 
 import type { IBenchmarkResult } from "@jupyterlab/ui-profiler";
 
@@ -92,19 +91,17 @@ class UIProfilerReporter implements Reporter {
 
   async onEnd(result: FullResult) {
     console.log(`Finished the run: ${result.status}`);
-    const artifactClient = artifact.create();
-    const rootDirectory = ".";
-    const options = {
-      continueOnError: true,
-    };
-    const files: string[] = [];
+
+    const dir = 'results'
+    if (!fs.existsSync(dir)){
+      fs.mkdirSync(dir);
+    }
 
     const references = new Set<string>();
     for (const attachment of this._attachments) {
       // `uploadArtifact` forbids colons in file names
       const path = attachment.name.replace(":", "_");
-      fs.writeFileSync(path, JSON.stringify(attachment));
-      files.push(path);
+      fs.writeFileSync(dir + '/' + path, JSON.stringify(attachment));
       references.add(attachment.reference);
     }
     if (references.size > 1) {
@@ -115,18 +112,6 @@ class UIProfilerReporter implements Reporter {
     const reference = references.values().next().value;
     this._reference = reference;
 
-    const artifactName = "UI profiler " + reference;
-    const uploadResult = await artifactClient.uploadArtifact(
-      artifactName,
-      files,
-      rootDirectory,
-      options
-    );
-
-    if (uploadResult.failedItems.length > 0) {
-      error("Upload of some artifacts failed");
-    }
-
     // Add summary table showing average execution time per scenario (rows) per notebook (columns)
     summary.addHeading(this._reference);
 
diff --git a/yarn.lock b/yarn.lock
index 0c6fcaa..a65fee3 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5,19 +5,7 @@ __metadata:
   version: 6
   cacheKey: 8
 
-"@actions/artifact@npm:^1.1.1":
-  version: 1.1.1
-  resolution: "@actions/artifact@npm:1.1.1"
-  dependencies:
-    "@actions/core": ^1.9.1
-    "@actions/http-client": ^2.0.1
-    tmp: ^0.2.1
-    tmp-promise: ^3.0.2
-  checksum: 45e170dc64a1023c29a3cfd74407cb3efcf7b4ac3b764f231ed7f3b18d7684eb4459e80ef610dc5859454d9b57df8565ac4cfe9610dc1e6e86b2d21657685517
-  languageName: node
-  linkType: hard
-
-"@actions/core@npm:^1.10.0, @actions/core@npm:^1.9.1":
+"@actions/core@npm:^1.10.0":
   version: 1.10.0
   resolution: "@actions/core@npm:1.10.0"
   dependencies:
@@ -469,7 +457,6 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@jupyterlab/benchmarks-tests@workspace:tests"
   dependencies:
-    "@actions/artifact": ^1.1.1
     "@actions/core": ^1.10.0
     "@jupyterlab/galata": ^4.3.5
     "@jupyterlab/ui-profiler": ^0.2.1
@@ -10924,7 +10911,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"rimraf@npm:^3.0.0, rimraf@npm:^3.0.2":
+"rimraf@npm:^3.0.2":
   version: 3.0.2
   resolution: "rimraf@npm:3.0.2"
   dependencies:
@@ -12094,15 +12081,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"tmp-promise@npm:^3.0.2":
-  version: 3.0.3
-  resolution: "tmp-promise@npm:3.0.3"
-  dependencies:
-    tmp: ^0.2.0
-  checksum: f854f5307dcee6455927ec3da9398f139897faf715c5c6dcee6d9471ae85136983ea06662eba2edf2533bdcb0fca66d16648e79e14381e30c7fb20be9c1aa62c
-  languageName: node
-  linkType: hard
-
 "tmp@npm:^0.0.33":
   version: 0.0.33
   resolution: "tmp@npm:0.0.33"
@@ -12112,15 +12090,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"tmp@npm:^0.2.0, tmp@npm:^0.2.1":
-  version: 0.2.1
-  resolution: "tmp@npm:0.2.1"
-  dependencies:
-    rimraf: ^3.0.0
-  checksum: 8b1214654182575124498c87ca986ac53dc76ff36e8f0e0b67139a8d221eaecfdec108c0e6ec54d76f49f1f72ab9325500b246f562b926f85bcdfca8bf35df9e
-  languageName: node
-  linkType: hard
-
 "to-object-path@npm:^0.3.0":
   version: 0.3.0
   resolution: "to-object-path@npm:0.3.0"

From 76d8e95cefce26508d5249b63a8f720f9c3ad037 Mon Sep 17 00:00:00 2001
From: krassowski <5832902+krassowski@users.noreply.github.com>
Date: Sun, 11 Jun 2023 22:11:01 +0100
Subject: [PATCH 8/8] Fix up the summary table

---
 tests/jupyterlab/ui-profiler.specs.ts |  6 ++---
 tests/reporter.ts                     | 39 +++++++++++++++++----------
 2 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/tests/jupyterlab/ui-profiler.specs.ts b/tests/jupyterlab/ui-profiler.specs.ts
index 6807ab9..3534397 100644
--- a/tests/jupyterlab/ui-profiler.specs.ts
+++ b/tests/jupyterlab/ui-profiler.specs.ts
@@ -230,7 +230,7 @@ test.describe("Measure execution time", () => {
           body: JSON.stringify({
             ...result,
             reference,
-            backgroundTab: notebookPath,
+            backgroundTab: file,
           }),
           contentType: "application/json",
         });
@@ -286,7 +286,7 @@ test.describe("Benchmark style sheets @slow", () => {
           body: JSON.stringify({
             ...result,
             reference,
-            backgroundTab: notebookPath,
+            backgroundTab: file,
           }),
           contentType: "application/json",
         });
@@ -339,7 +339,7 @@ test.describe("Benchmark style rules @slow", () => {
           body: JSON.stringify({
             ...result,
             reference,
-            backgroundTab: notebookPath,
+            backgroundTab: file,
           }),
           contentType: "application/json",
         });
diff --git a/tests/reporter.ts b/tests/reporter.ts
index 5022289..07f40e0 100644
--- a/tests/reporter.ts
+++ b/tests/reporter.ts
@@ -21,6 +21,11 @@ namespace Statistic {
     return sum(numbers) / numbers.length;
   }
 
+  export function round(n: number, precision = 0): number {
+    const factor = Math.pow(10, precision);
+    return Math.round(n * factor) / factor;
+  }
+
   export function sum(numbers: number[]): number {
     if (numbers.length === 0) {
       return 0;
@@ -92,8 +97,8 @@ class UIProfilerReporter implements Reporter {
   async onEnd(result: FullResult) {
     console.log(`Finished the run: ${result.status}`);
 
-    const dir = 'results'
-    if (!fs.existsSync(dir)){
+    const dir = "results";
+    if (!fs.existsSync(dir)) {
       fs.mkdirSync(dir);
     }
 
@@ -101,7 +106,7 @@ class UIProfilerReporter implements Reporter {
     for (const attachment of this._attachments) {
       // `uploadArtifact` forbids colons in file names
       const path = attachment.name.replace(":", "_");
-      fs.writeFileSync(dir + '/' + path, JSON.stringify(attachment));
+      fs.writeFileSync(dir + "/" + path, JSON.stringify(attachment));
       references.add(attachment.reference);
     }
     if (references.size > 1) {
@@ -118,26 +123,32 @@ class UIProfilerReporter implements Reporter {
     const timeMeasurements = this._attachments.filter(
       (a) => a.benchmark === "execution-time"
     );
-    const scenarios = timeMeasurements.map((a) => a.scenario);
-    const backgrounds = timeMeasurements.map((a) => a.backgroundTab);
+    const scenarios = [...new Set(timeMeasurements.map((a) => a.scenario))];
+    const backgrounds = [
+      ...new Set(timeMeasurements.map((a) => a.backgroundTab)),
+    ];
 
     summary.addTable([
-      backgrounds.map((b) => {
-        return { data: b, header: true };
-      }),
-      ...scenarios.map((s) =>
-        backgrounds.map((b) => {
+      [
+        { data: "scenario", header: true },
+        ...backgrounds.map((b) => {
+          return { data: b, header: true };
+        }),
+      ],
+      ...scenarios.map((s) => {
+        const row = backgrounds.map((b) => {
           const result = this._attachments.find(
             (a) => a.backgroundTab === b && a.scenario === s
           );
           const times = result.outcome.results[0].times;
           return (
-            Statistic.mean(times).toString() +
+            Statistic.round(Statistic.mean(times), 2).toString() +
             " ± " +
-            Statistic.standardDeviation(times).toString()
+            Statistic.round(Statistic.standardDeviation(times), 2).toString()
           );
-        })
-      ),
+        });
+        return [s, ...row];
+      }),
     ]);
     await summary.write();
   }