Skip to content

Support async and remote workloads #526

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 8 commits into
base: main
Choose a base branch
from
Draft

Support async and remote workloads #526

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4f4789a
fix
camillobruni Sep 25, 2025
61907bf
fix
camillobruni Oct 3, 2025
793493f
fix:
camillobruni Oct 6, 2025
56d6324
regenerate news-site next
camillobruni Oct 6, 2025
901b3bd
update
camillobruni Oct 6, 2025
945089c
add more enums
camillobruni Oct 6, 2025
8a9b320
fix typo
camillobruni Oct 6, 2025
9da34ea
formatting
camillobruni Oct 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion resources/newssite/news-next/dist/404.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="./_next/static/css/a0dca1379a01e5cf.css" as="style"/><link rel="stylesheet" href="./_next/static/css/a0dca1379a01e5cf.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="./_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="./_next/static/chunks/webpack-e50e9853db18b759.js" defer=""></script><script src="./_next/static/chunks/framework-2c79e2a64abdb08b.js" defer=""></script><script src="./_next/static/chunks/main-2ba37e62325cc71b.js" defer=""></script><script src="./_next/static/chunks/pages/_app-77983e68be50f72a.js" defer=""></script><script src="./_next/static/chunks/pages/_error-54de1933a164a1ff.js" defer=""></script><script src="./_next/static/tuwdCnX7HYK_fwpI0QvDm/_buildManifest.js" defer=""></script><script src="./_next/static/tuwdCnX7HYK_fwpI0QvDm/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><div id="settings-container"></div><div id="notifications-container"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"tuwdCnX7HYK_fwpI0QvDm","assetPrefix":".","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="./_next/static/css/a0dca1379a01e5cf.css" as="style"/><link rel="stylesheet" href="./_next/static/css/a0dca1379a01e5cf.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="./_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="./_next/static/chunks/webpack-e50e9853db18b759.js" defer=""></script><script src="./_next/static/chunks/framework-2c79e2a64abdb08b.js" defer=""></script><script src="./_next/static/chunks/main-2ba37e62325cc71b.js" defer=""></script><script src="./_next/static/chunks/pages/_app-77983e68be50f72a.js" defer=""></script><script src="./_next/static/chunks/pages/_error-54de1933a164a1ff.js" defer=""></script><script src="./_next/static/TTK2ht8frjy0sbNzW8tq6/_buildManifest.js" defer=""></script><script src="./_next/static/TTK2ht8frjy0sbNzW8tq6/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><div id="settings-container"></div><div id="notifications-container"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"TTK2ht8frjy0sbNzW8tq6","assetPrefix":".","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
2 changes: 1 addition & 1 deletion ...c/tuwdCnX7HYK_fwpI0QvDm/_buildManifest.js → ...c/TTK2ht8frjy0sbNzW8tq6/_buildManifest.js
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 ...tic/tuwdCnX7HYK_fwpI0QvDm/_ssgManifest.js → ...tic/TTK2ht8frjy0sbNzW8tq6/_ssgManifest.js
View file
Open in desktop
File renamed without changes.
2 changes: 1 addition & 1 deletion ...ic/chunks/pages/index-99c54b3ee91525b8.js → ...ic/chunks/pages/index-84cd729ada3bc18c.js
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion resources/newssite/news-next/dist/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="./_next/static/css/a0dca1379a01e5cf.css" as="style"/><link rel="stylesheet" href="./_next/static/css/a0dca1379a01e5cf.css" data-n-g=""/><link rel="preload" href="./_next/static/css/2cf5163b53bb0adb.css" as="style"/><link rel="stylesheet" href="./_next/static/css/2cf5163b53bb0adb.css" data-n-p=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="./_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="./_next/static/chunks/webpack-e50e9853db18b759.js" defer=""></script><script src="./_next/static/chunks/framework-2c79e2a64abdb08b.js" defer=""></script><script src="./_next/static/chunks/main-2ba37e62325cc71b.js" defer=""></script><script src="./_next/static/chunks/pages/_app-77983e68be50f72a.js" defer=""></script><script src="./_next/static/chunks/743-fd706aeabb7828e3.js" defer=""></script><script src="./_next/static/chunks/pages/index-99c54b3ee91525b8.js" defer=""></script><script src="./_next/static/tuwdCnX7HYK_fwpI0QvDm/_buildManifest.js" defer=""></script><script src="./_next/static/tuwdCnX7HYK_fwpI0QvDm/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><div id="settings-container"></div><div id="notifications-container"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/","query":{},"buildId":"tuwdCnX7HYK_fwpI0QvDm","assetPrefix":".","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="./_next/static/css/a0dca1379a01e5cf.css" as="style"/><link rel="stylesheet" href="./_next/static/css/a0dca1379a01e5cf.css" data-n-g=""/><link rel="preload" href="./_next/static/css/2cf5163b53bb0adb.css" as="style"/><link rel="stylesheet" href="./_next/static/css/2cf5163b53bb0adb.css" data-n-p=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="./_next/static/chunks/polyfills-c67a75d1b6f99dc8.js"></script><script src="./_next/static/chunks/webpack-e50e9853db18b759.js" defer=""></script><script src="./_next/static/chunks/framework-2c79e2a64abdb08b.js" defer=""></script><script src="./_next/static/chunks/main-2ba37e62325cc71b.js" defer=""></script><script src="./_next/static/chunks/pages/_app-77983e68be50f72a.js" defer=""></script><script src="./_next/static/chunks/743-fd706aeabb7828e3.js" defer=""></script><script src="./_next/static/chunks/pages/index-84cd729ada3bc18c.js" defer=""></script><script src="./_next/static/TTK2ht8frjy0sbNzW8tq6/_buildManifest.js" defer=""></script><script src="./_next/static/TTK2ht8frjy0sbNzW8tq6/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><div id="settings-container"></div><div id="notifications-container"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/","query":{},"buildId":"TTK2ht8frjy0sbNzW8tq6","assetPrefix":".","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
2 changes: 1 addition & 1 deletion resources/newssite/news-next/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion resources/newssite/news-next/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
"react-dom": "18.2.0",
"react-router-dom": "^6.11.1",
"react-router-hash-link": "^2.4.3",
"speedometer-utils": "../../shared",
"speedometer-utils": "file:../../shared",
"uuid": "^9.0.0"
}
}
10 changes: 6 additions & 4 deletions resources/newssite/news-next/src/workload-test.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,21 @@
import { BenchmarkStep, BenchmarkSuite } from "speedometer-utils/benchmark.mjs";
import { BenchmarkStep, AsyncBenchmarkStep, AsyncBenchmarkSuite } from "speedometer-utils/benchmark.mjs";
import { forceLayout, getElement } from "speedometer-utils/helpers.mjs";

const suites = {
default: new BenchmarkSuite("default", [
new BenchmarkStep("Navigate-to-US-page", () => {
default: new AsyncBenchmarkSuite("default", [
new AsyncBenchmarkStep("Navigate-to-US-page", async () => {
for (let i = 0; i < 25; i++) {
getElement("#navbar-dropdown-toggle").click();
forceLayout();
getElement("#navbar-dropdown-toggle").click();
forceLayout();
}
await new Promise(r => setTimeout(r, 1000));

getElement("#navbar-navlist-us-link").click();
forceLayout();
}),

new BenchmarkStep("Navigate-to-World-page", () => {
for (let i = 0; i < 25; i++) {
getElement("#navbar-dropdown-toggle").click();
Expand All @@ -36,7 +38,7 @@ const suites = {
getElement("#navbar-navlist-politics-link").click();
forceLayout();
}),
]),
], true),
};

export default suites;
72 changes: 60 additions & 12 deletions resources/shared/benchmark.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* eslint-disable no-case-declarations */
import { TestRunner } from "./test-runner.mjs";
import { TestRunner, AsyncTestRunner } from "./test-runner.mjs";
import { Params } from "./params.mjs";

/**
Expand All @@ -20,15 +20,31 @@ export class BenchmarkStep {
}
}

export class AsyncBenchmarkStep extends BenchmarkStep {
async runAndRecord(params, suite, test, callback) {
const testRunner = new AsyncTestRunner(null, null, params, suite, test, callback);
const result = await testRunner.runTest();
return result;
}
}

export const BenchmarkSuiteType = Object.freeze({
__proto__: null,
sync: "sync",
async: "async",
});

/**
* BenchmarkSuite
*
* A single test suite that contains one or more test steps.
*/
export class BenchmarkSuite {
constructor(name, tests) {
constructor(name, tests, type = BenchmarkSuiteType.sync) {
this.name = name;
this.tests = tests;
this.type = type;
console.assert(this.type in BenchmarkSuiteType);
}

record(_test, syncTime, asyncTime) {
Expand All @@ -54,6 +70,7 @@ export class BenchmarkSuite {

for (const test of this.tests) {
const result = await test.runAndRecord(params, this, test, this.record);
console.assert(result, "Missing test return value", test);
measuredValues.tests[test.name] = result;
measuredValues.total += result.total;
onProgress?.(test.name);
Expand All @@ -71,6 +88,28 @@ export class BenchmarkSuite {
}
}

export class AsyncBenchmarkSuite extends BenchmarkSuite {
constructor(name, tests) {
super(name, tests, BenchmarkSuiteType.async);
}
}


export const MESSAGE_TYPE = Object.freeze({
__proto__: null,
appReady: "app-ready",
suiteStart: "suite-start",
stepComplete: "step-complete",
suiteComplete: "suite-complete",
});

export const MESSAGE_STATUS = Object.freeze({
__proto__: null,
success: "success",
error: "error",
});


/** **********************************************************************
* BenchmarkConnector
*
Expand All @@ -95,31 +134,40 @@ export class BenchmarkConnector {
}

async onMessage(event) {
if (event.data.id !== this.appId || event.data.key !== "benchmark-connector")
const message = event.data;
if (message.appId !== this.appId || message.key !== "benchmark-connector")
return;

switch (event.data.type) {
case "benchmark-suite":
switch (message.type) {
case MESSAGE_TYPE.suiteStart:
const params = new Params(new URLSearchParams(window.location.search));
const suite = this.suites[event.data.name];
const { name } = message.payload;
const suite = this.suites[name];
if (!suite)
console.error(`Suite with the name of "${event.data.name}" not found!`);
const { result } = await suite.runAndRecord(params, (test) => this.sendMessage({ type: "step-complete", status: "success", appId: this.appId, name: this.name, test }));
this.sendMessage({ type: "suite-complete", status: "success", appId: this.appId, result });
console.error(`Suite with the name of "${name}" not found!`);
const onProgress = (test) => this._sendMessage(MESSAGE_TYPE.stepComplete, { name: this.name, test });
const { result } = await suite.runAndRecord(params, onProgress);
this._sendMessage(MESSAGE_TYPE.suiteComplete, { result });
this.disconnect();
break;
default:
console.error(`Message data type not supported: ${event.data.type}`);
console.error(`Message data type not supported: ${message.type}`);
}
}

sendMessage(message) {
_sendMessage(type, payload, status = MESSAGE_STATUS.success) {
const message = {
appId: this.appId,
type: type,
payload: payload,
status: status,
};
window.top.postMessage(message, "*");
}

connect() {
window.addEventListener("message", this.onMessage);
this.sendMessage({ type: "app-ready", status: "success", appId: this.appId });
this._sendMessage({ type: MESSAGE_TYPE.appReady });
}

disconnect() {
Expand Down
8 changes: 4 additions & 4 deletions resources/shared/test-invoker.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ class AsyncRAFTestInvoker extends BaseRAFTestInvoker {

this._asyncCallback();
setTimeout(async () => {
await this._reportCallback();
resolve();
const result = await this._reportCallback();
resolve(result);
}, 0);
};

Expand Down Expand Up @@ -79,8 +79,8 @@ class AsyncRAFTestInvoker extends BaseRAFTestInvoker {
}
}

export const TEST_INVOKER_LOOKUP = {
export const TEST_INVOKER_LOOKUP = Object.freeze({
__proto__: null,
raf: RAFTestInvoker,
async: AsyncRAFTestInvoker,
};
});
13 changes: 7 additions & 6 deletions resources/shared/test-runner.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,15 @@ export class TestRunner {
#callback;
#type;

constructor(frame, page, params, suite, test, callback, type) {
constructor(frame, page, params, suite, test, callback, type = "default") {
this.#suite = suite;
this.#test = test;
this.#params = params;
this.#callback = callback;
this.#page = page;
this.#frame = frame;
this.#type = type;
console.assert(type in TEST_RUNNER_LOOKUP);
}

get page() {
Expand Down Expand Up @@ -69,7 +70,7 @@ export class TestRunner {
const measureAsync = () => {
// Some browsers don't immediately update the layout for paint.
// Force the layout here to ensure we're measuring the layout time.
this.page.layout();
this.page?.layout();

const asyncEndTime = performance.now();
performance.mark(asyncEndLabel);
Expand All @@ -92,18 +93,18 @@ export class TestRunner {
}

export class AsyncTestRunner extends TestRunner {
constructor(frame, page, params, suite, test, callback, type) {
super(frame, page, params, suite, test, callback, type);
constructor(frame, page, params, suite, test, callback) {
super(frame, page, params, suite, test, callback, "async");
}

async _runSyncStep(test, page) {
await test.run(page);
}
}

export const TEST_RUNNER_LOOKUP = {
export const TEST_RUNNER_LOOKUP = Object.freeze({
__proto__: null,
default: TestRunner,
async: AsyncTestRunner,
remote: TestRunner,
};
});
26 changes: 20 additions & 6 deletions resources/suite-runner.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { TEST_RUNNER_LOOKUP } from "./shared/test-runner.mjs";
import { MESSAGE_TYPE } from "./shared/benchmark.mjs";
import { WarmupSuite } from "./benchmark-runner.mjs";

export class SuiteRunner {
Expand Down Expand Up @@ -163,7 +164,7 @@ export class RemoteSuiteRunner extends SuiteRunner {
performance.mark(suitePrepareStartLabel);

// Wait for the app-ready message from the workload.
const appReadyPromise = this._subscribeOnce("app-ready");
const appReadyPromise = this._subscribeOnce(MESSAGE_TYPE.appReady);
await this._loadFrame(this.suite);
const response = await appReadyPromise;
await this.suite.prepare?.(this.page);
Expand All @@ -178,22 +179,32 @@ export class RemoteSuiteRunner extends SuiteRunner {

async _runSuite() {
// Ask workload to run its own tests.
this.frame.contentWindow.postMessage({ id: this.appId, key: "benchmark-connector", type: "benchmark-suite", name: this.suite.config?.name || "default" }, "*");
this._sendMessage(MESSAGE_TYPE.suiteStart, { name: this.suite.config?.name || "default" });
// Capture metrics from the completed tests.
const response = await this._subscribeOnce("suite-complete");
const { result } = await this._subscribeOnce(MESSAGE_TYPE.suiteComplete);

this.suiteResults.tests = {
...this.suiteResults.tests,
...response.result.tests,
...result.tests,
};

this.suiteResults.prepare = this.#prepareTime;
this.suiteResults.total = response.result.total;
this.suiteResults.total = result.total;

this._validateSuiteResults();
await this._updateClient();
}

_sendMessage(type, payload) {
const message = {
appId: this.appId,
key: "benchmark-connector",
type: type,
payload: payload,
};
this.frame.contentWindow.postMessage(message, "*");
}

_handlePostMessage(event) {
const callback = this.postMessageCallbacks.get(event.data.type);
if (callback)
Expand All @@ -217,8 +228,11 @@ export class RemoteSuiteRunner extends SuiteRunner {
_subscribeOnce(type) {
return new Promise((resolve) => {
this._startSubscription(type, (e) => {
const message = e.data;
if (message.appId !== this.appId)
throw new Error(`Got message for invalid app: ${message.appId} instead of {this.appId}`);
this._stopSubscription(type);
resolve(e.data);
resolve(message.payload);
});
});
}
Expand Down
Loading
Loading