Skip to content

Commit 3d26177

Browse files
krausestkrausest
committed
update to lighthouse 3
1 parent cd8720e commit 3d26177

File tree

4 files changed

+176
-84
lines changed

4 files changed

+176
-84
lines changed

‎webdriver-ts-results/src/Common.tsx

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ export interface Result {
2222
min: number;
2323
max: number;
2424
mean: number;
25-
geometricMean: number;
26-
standardDeviation: number;
25+
geometricMean: number|null;
26+
standardDeviation: number|null;
2727
median: number;
2828
values: number[];
2929
count?: number;
@@ -246,8 +246,8 @@ export class ResultTableData {
246246
// https://de.wikipedia.org/wiki/Zweistichproben-t-Test
247247
if (compareWithResults) {
248248
let compareWithMean = compareWithResults.mean;
249-
let stdDev = result.standardDeviation;
250-
let compareWithResultsStdDev = compareWithResults.standardDeviation;
249+
let stdDev = result.standardDeviation || 0;
250+
let compareWithResultsStdDev = compareWithResults.standardDeviation || 0;
251251

252252
let x1 = mean;
253253
let x2 = compareWithMean;
@@ -262,7 +262,7 @@ export class ResultTableData {
262262
statisticalCol = statisticComputeColor(t, p);
263263
statisticalResult = (p*100).toFixed(3)+"%";
264264
}
265-
return new TableResultValueEntry(f.name, mean, standardDeviation, factor, statisticalResult, statisticalCol);
265+
return new TableResultValueEntry(f.name, mean, standardDeviation || 0, factor, statisticalResult, statisticalCol);
266266
}
267267
});
268268
}

‎webdriver-ts/package.json

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,24 @@
2424
"devDependencies": {
2525
"@types/lodash": "4.14.108",
2626
"@types/node": "10.0.8",
27-
"@types/ramda": "^0.25.28",
27+
"@types/ramda": "0.25.28",
2828
"@types/selenium-webdriver": "3.0.8",
29-
"@types/semver": "^5.5.0",
29+
"@types/semver": "5.5.0",
3030
"@types/yargs": "11.0.0",
31+
"ts-node": "7.0.1",
3132
"typescript": "2.8.3"
3233
},
3334
"dependencies": {
3435
"cross-env": "5.1.1",
3536
"chromedriver": "2.41.0",
3637
"dot": "1.1.2",
3738
"jstat": "1.7.1",
38-
"lighthouse": "2.9.4",
39+
"lighthouse": "3.1.1",
3940
"lodash": "4.17.10",
40-
"ramda": "^0.25.0",
41+
"ramda": "0.25.0",
4142
"selenium-webdriver": "4.0.0-alpha.1",
4243
"yargs": "11.0.0",
43-
"npm-check-updates": "^2.14.2",
44-
"semver": "^5.5.0"
44+
"npm-check-updates": "2.14.2",
45+
"semver": "5.5.0"
4546
}
4647
}

‎webdriver-ts/src/common.ts

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,14 @@ export interface ErrorsAndWarning {
1717
warnings: String[];
1818
}
1919

20-
export interface BenchmarkOptions {
21-
outputDirectory: string;
22-
port: string;
20+
export interface BenchmarkDriverOptions {
2321
headless?: boolean;
2422
chromeBinaryPath?: string;
23+
}
24+
25+
export interface BenchmarkOptions extends BenchmarkDriverOptions {
26+
outputDirectory: string;
27+
port: string;
2528
numIterationsForAllBenchmarks: number;
2629
}
2730

‎webdriver-ts/src/forkedBenchmarkRunner.ts

Lines changed: 158 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ import {BenchmarkType, Benchmark, benchmarks, fileName, LighthouseData} from './
44
import {setUseShadowRoot} from './webdriverAccess'
55

66
const lighthouse = require('lighthouse');
7+
const chromeLauncher = require('chrome-launcher');
78

89
import {lhConfig} from './lighthouseConfig';
910
import * as fs from 'fs';
10-
import {JSONResult, config, FrameworkData, BenchmarkError, ErrorsAndWarning, BenchmarkOptions} from './common'
11+
import * as path from 'path';
12+
import {JSONResult, config, FrameworkData, BenchmarkError, ErrorsAndWarning, BenchmarkOptions, BenchmarkDriverOptions} from './common'
1113
import * as R from 'ramda';
1214

1315
var chromedriver:any = require('chromedriver');
@@ -94,32 +96,118 @@ function asString(res: Timingresult[]): string {
9496
return res.reduce((old, cur) => old + "\n" + JSON.stringify(cur), "");
9597
}
9698

97-
async function runLighthouse(protocolResults: any[]): Promise<LighthouseData> {
98-
const traceEvents = protocolResults.filter(e => e.method === 'Tracing.dataCollected').map(e => e.params);
99-
const devtoolsLogs = protocolResults.filter(e => e.method.startsWith('Page') || e.method.startsWith('Network'));
100-
101-
const filenamePrefix = `${process.cwd()}/${Date.now()}`;
102-
const traceFilename = `${filenamePrefix}.trace.json`;
103-
const devtoolslogsFilename = `${filenamePrefix}.devtoolslogs.json`;
104-
105-
fs.writeFileSync(traceFilename, JSON.stringify({traceEvents}));
106-
fs.writeFileSync(devtoolslogsFilename, JSON.stringify(devtoolsLogs));
107-
lhConfig.artifacts.traces = {defaultPass: traceFilename};
108-
lhConfig.artifacts.devtoolsLogs = {defaultPass: devtoolslogsFilename};
109-
110-
const lhResults = await lighthouse('http://example.com/thispage', {}, lhConfig);
111-
fs.unlinkSync(traceFilename);
112-
fs.unlinkSync(devtoolslogsFilename);
113-
114-
const audits = lhResults.audits;
115-
let LighthouseData: LighthouseData = {
116-
TimeToConsistentlyInteractive: 0, // temporarily disabled due to #458 audits['consistently-interactive'].extendedInfo.value.timeInMs,
117-
ScriptBootUpTtime: audits['bootup-time'].rawValue,
118-
MainThreadWorkCost: audits['mainthread-work-breakdown'].rawValue,
119-
TotalByteWeight: audits['total-byte-weight'].rawValue,
99+
// async function runLighthouse(protocolResults: any[]): Promise<LighthouseData> {
100+
// const traceEvents = protocolResults.filter(e => e.method === 'Tracing.dataCollected').map(e => e.params);
101+
// const devtoolsLogs = protocolResults.filter(e => e.method.startsWith('Page') || e.method.startsWith('Network'));
102+
103+
// const filenamePrefix = `${process.cwd()}/${Date.now()}`;
104+
// const traceFilename = `${filenamePrefix}.trace.json`;
105+
// const devtoolslogsFilename = `${filenamePrefix}.devtoolslogs.json`;
106+
107+
// fs.writeFileSync(traceFilename, JSON.stringify({traceEvents}));
108+
// fs.writeFileSync(devtoolslogsFilename, JSON.stringify(devtoolsLogs));
109+
// lhConfig.artifacts.traces = {defaultPass: traceFilename};
110+
// lhConfig.artifacts.devtoolsLogs = {defaultPass: devtoolslogsFilename};
111+
112+
// const lhResults = await lighthouse('http://example.com/thispage', {}, lhConfig);
113+
// fs.unlinkSync(traceFilename);
114+
// fs.unlinkSync(devtoolslogsFilename);
115+
116+
// const audits = lhResults.audits;
117+
// let LighthouseData: LighthouseData = {
118+
// TimeToConsistentlyInteractive: 0, // temporarily disabled due to #458 audits['consistently-interactive'].extendedInfo.value.timeInMs,
119+
// ScriptBootUpTtime: audits['bootup-time'].rawValue,
120+
// MainThreadWorkCost: audits['mainthread-work-breakdown'].rawValue,
121+
// TotalByteWeight: audits['total-byte-weight'].rawValue,
122+
// };
123+
124+
// return LighthouseData;
125+
// }
126+
127+
// async function computeResultsStartup(driver: WebDriver, framework: FrameworkData, benchmark: Benchmark, warnings: String[]): Promise<LighthouseData> {
128+
// let durationJSArr : number[] = await driver.executeScript("return [window.performance.timing.loadEventEnd, window.performance.timing.navigationStart]") as number[];
129+
// let durationJS = (durationJSArr[0] as number) - (durationJSArr[1] as number);
130+
// let reportedDuration = durationJS;
131+
132+
// let entriesBrowser = await driver.manage().logs().get(logging.Type.BROWSER);
133+
// if (config.LOG_DEBUG) console.log("browser entries", entriesBrowser);
134+
135+
// const perfLogEvents = (await fetchEventsFromPerformanceLog(driver));
136+
// let filteredEvents = perfLogEvents.timingResults;
137+
// let protocolResults = perfLogEvents.protocolResults;
138+
// return runLighthouse(protocolResults);
139+
// }
140+
141+
142+
function extractRawValue(results: any, id: string) {
143+
let audits = results.audits;
144+
if (!audits) return null;
145+
let audit_with_id = audits[id];
146+
if (typeof audit_with_id === 'undefined') return null;
147+
if (typeof audit_with_id.rawValue === 'undefined') return null;
148+
return audit_with_id.rawValue;
149+
}
150+
151+
function rmDir(dirPath: string) {
152+
try { var files = fs.readdirSync(dirPath); }
153+
catch(e) { return; }
154+
if (files.length > 0)
155+
for (var i = 0; i < files.length; i++) {
156+
var filePath = path.join(dirPath, files[i]);
157+
if (fs.statSync(filePath).isFile())
158+
fs.unlinkSync(filePath);
159+
else
160+
rmDir(filePath);
161+
}
162+
fs.rmdirSync(dirPath);
163+
};
164+
165+
async function runLighthouse(framework: FrameworkData, benchmarkOptions: BenchmarkOptions): Promise<LighthouseData> {
166+
const opts = {
167+
chromeFlags:
168+
[
169+
"--headless",
170+
"--no-sandbox",
171+
"--no-first-run",
172+
"--enable-automation",
173+
"--disable-infobars",
174+
"--disable-background-networking",
175+
"--disable-background-timer-throttling",
176+
"--disable-cache",
177+
"--disable-translate",
178+
"--disable-sync",
179+
"--disable-extensions",
180+
"--disable-default-apps",
181+
"--window-size=1200,800"
182+
],
183+
onlyCategories: ['performance'],
184+
port: ''
120185
};
121186

122-
return LighthouseData;
187+
try {
188+
if (fs.existsSync('prefs')) rmDir('prefs');
189+
fs.mkdirSync('prefs');
190+
fs.mkdirSync('prefs/Default');
191+
fs.copyFileSync('chromePreferences.json', 'prefs/Default/Preferences');
192+
193+
let options : any = {chromeFlags: opts.chromeFlags, logLevel: "verbose", userDataDir: "prefs"};
194+
if (benchmarkOptions.chromeBinaryPath) options.chromePath = benchmarkOptions.chromeBinaryPath;
195+
let chrome = await chromeLauncher.launch(options);
196+
opts.port = chrome.port;
197+
let results = await lighthouse(`http://localhost:${benchmarkOptions.port}/${framework.uri}/`, opts, null);
198+
await chrome.kill();
199+
200+
let LighthouseData: LighthouseData = {
201+
TimeToConsistentlyInteractive: extractRawValue(results.lhr, 'interactive'),
202+
ScriptBootUpTtime: extractRawValue(results.lhr, 'bootup-time'),
203+
MainThreadWorkCost: extractRawValue(results.lhr, 'mainthread-work-breakdown'),
204+
TotalByteWeight: extractRawValue(results.lhr, 'total-byte-weight')
205+
};
206+
return LighthouseData;
207+
} catch (error) {
208+
console.log("error running lighthouse", error);
209+
throw error;
210+
}
123211
}
124212

125213
async function computeResultsCPU(driver: WebDriver, benchmarkOptions: BenchmarkOptions, framework: FrameworkData, benchmark: Benchmark, warnings: String[]): Promise<number[]> {
@@ -222,21 +310,7 @@ async function computeResultsMEM(driver: WebDriver, benchmarkOptions: BenchmarkO
222310
return results;
223311
}
224312

225-
async function computeResultsStartup(driver: WebDriver, framework: FrameworkData, benchmark: Benchmark, warnings: String[]): Promise<LighthouseData> {
226-
let durationJSArr : number[] = await driver.executeScript("return [window.performance.timing.loadEventEnd, window.performance.timing.navigationStart]") as number[];
227-
let durationJS = (durationJSArr[0] as number) - (durationJSArr[1] as number);
228-
let reportedDuration = durationJS;
229-
230-
let entriesBrowser = await driver.manage().logs().get(logging.Type.BROWSER);
231-
if (config.LOG_DEBUG) console.log("browser entries", entriesBrowser);
232-
233-
const perfLogEvents = (await fetchEventsFromPerformanceLog(driver));
234-
let filteredEvents = perfLogEvents.timingResults;
235-
let protocolResults = perfLogEvents.protocolResults;
236-
return runLighthouse(protocolResults);
237-
}
238-
239-
function buildDriver(benchmarkOptions: BenchmarkOptions) {
313+
function buildDriver(benchmarkOptions: BenchmarkDriverOptions) {
240314
let logPref = new logging.Preferences();
241315
logPref.setLevel(logging.Type.PERFORMANCE, logging.Level.ALL);
242316
logPref.setLevel(logging.Type.BROWSER, logging.Level.ALL);
@@ -418,39 +492,53 @@ async function runMemOrCPUBenchmark(framework: FrameworkData, benchmark: Benchma
418492
async function runStartupBenchmark(framework: FrameworkData, benchmark: Benchmark, benchmarkOptions: BenchmarkOptions ): Promise<ErrorsAndWarning>
419493
{
420494
console.log("benchmarking startup", framework, benchmark.id);
421-
let results : LighthouseData[] = [];
422-
let errors: BenchmarkError[] = [];
423-
let warnings: String[] = [];
424495

425-
let chromeDuration = 0;
426-
try {
427-
for (let i = 0; i<benchmarkOptions.numIterationsForAllBenchmarks; i++) {
428-
let driver = buildDriver(benchmarkOptions);
429-
try {
430-
setUseShadowRoot(framework.useShadowRoot);
431-
await driver.executeScript("console.timeStamp('initBenchmark')");
432-
await initBenchmark(driver, benchmark, framework);
433-
await driver.executeScript("console.timeStamp('runBenchmark')");
434-
await runBenchmark(driver, benchmark, framework);
435-
await driver.executeScript("console.timeStamp('finishedBenchmark')");
436-
await afterBenchmark(driver, benchmark, framework);
437-
await driver.executeScript("console.timeStamp('afterBenchmark')");
438-
await wait(5000);
439-
results.push(await computeResultsStartup(driver, framework, benchmark, warnings));
440-
} catch (e) {
441-
errors.push(await registerError(driver, framework, benchmark, e));
442-
throw e;
443-
} finally {
444-
await driver.close();
445-
await driver.quit();
446-
}
496+
let errors: BenchmarkError[] = [];
497+
let results: LighthouseData[] = [];
498+
for (let i = 0; i <benchmarkOptions.numIterationsForAllBenchmarks; i++) {
499+
try {
500+
results.push(await runLighthouse(framework, benchmarkOptions));
501+
} catch (error) {
502+
errors.push({imageFile: null, exception: error});
503+
throw error;
447504
}
448-
await writeResult({framework: framework, results: results, benchmark: benchmark}, benchmarkOptions.outputDirectory);
449-
} catch (e) {
450-
console.log("ERROR:", e);
451-
if (config.EXIT_ON_ERROR) { throw "Benchmarking failed"}
452505
}
453-
return {errors, warnings};
506+
await writeResult({framework: framework, results: results, benchmark: benchmark}, benchmarkOptions.outputDirectory);
507+
return {errors, warnings: []};
508+
509+
// let results : LighthouseData[] = [];
510+
// let errors: BenchmarkError[] = [];
511+
// let warnings: String[] = [];
512+
513+
// let chromeDuration = 0;
514+
// try {
515+
// for (let i = 0; i<benchmarkOptions.numIterationsForAllBenchmarks; i++) {
516+
// let driver = buildDriver(benchmarkOptions);
517+
// try {
518+
// setUseShadowRoot(framework.useShadowRoot);
519+
// await driver.executeScript("console.timeStamp('initBenchmark')");
520+
// await initBenchmark(driver, benchmark, framework);
521+
// await driver.executeScript("console.timeStamp('runBenchmark')");
522+
// await runBenchmark(driver, benchmark, framework);
523+
// await driver.executeScript("console.timeStamp('finishedBenchmark')");
524+
// await afterBenchmark(driver, benchmark, framework);
525+
// await driver.executeScript("console.timeStamp('afterBenchmark')");
526+
// await wait(5000);
527+
// results.push(await computeResultsStartup(driver, framework, benchmark, warnings));
528+
// } catch (e) {
529+
// errors.push(await registerError(driver, framework, benchmark, e));
530+
// throw e;
531+
// } finally {
532+
// await driver.close();
533+
// await driver.quit();
534+
// }
535+
// }
536+
// await writeResult({framework: framework, results: results, benchmark: benchmark}, benchmarkOptions.outputDirectory);
537+
// } catch (e) {
538+
// console.log("ERROR:", e);
539+
// if (config.EXIT_ON_ERROR) { throw "Benchmarking failed"}
540+
// }
541+
// return {errors, warnings};
454542
}
455543

456544
export async function executeBenchmark(frameworks: FrameworkData[], keyed: boolean, frameworkName: string, benchmarkName: string, benchmarkOptions: BenchmarkOptions): Promise<ErrorsAndWarning> {

0 commit comments

Comments
 (0)