diff --git a/.gitignore b/.gitignore index 90157b1b..c8b09693 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /node_modules -.DS_Store \ No newline at end of file +.DS_Store +/runs-2020-zstd +/runs-zstd \ No newline at end of file diff --git a/index.js b/index.js index f16c2170..cb0d4d7e 100644 --- a/index.js +++ b/index.js @@ -51,9 +51,8 @@ async function process_chunks (path) { const scored_chunk = process_raw_results(chunk_run) if (!result.run_info) { const raw_run_info = scored_chunk.run_info - const matches = raw_run_info - .browser_version.match(/^Servo ([0-9.]+-[a-f0-9]+)?(-dirty)?$/) - const browser_version = matches.length === 3 ? matches[1] : 'Unknown' + const matches = raw_run_info?.browser_version?.match(/^Servo ([0-9.]+-[a-f0-9]+)?(-dirty)?$/) + const browser_version = matches?.length === 3 ? matches[1] : 'Unknown' result.run_info = Object.assign(raw_run_info, { browser_version }) } delete scored_chunk.run_info @@ -67,6 +66,23 @@ async function add_run (runs_dir, chunks_dir, date) { await write_compressed(`./${runs_dir}/${date}.xz`, new_run) } +async function score_single_run (chunks_dir, print_filter) { + const run = await process_chunks(chunks_dir) + const test_to_areas = focus_areas_map(run) + const { area_keys } = get_focus_areas() + const score = score_run(run, run, test_to_areas, print_filter) + const row = [ + ['revision', run.run_info.revision.substring(0, 9)], + ['browser version', run.run_info.browser_version] + ] + + for (const area of area_keys) { + row.push([area, score[area]]) + } + + return row +} + async function recalc_scores (runs_dir) { console.log(`Calculating scores for ${runs_dir} directory...`) @@ -81,10 +97,12 @@ async function recalc_scores (runs_dir) { const { area_keys } = get_focus_areas() for (const [i, r] of all_runs.entries()) { const [date] = r.split('.') + const start = Date.now() console.log(`Reading run ${runs_dir}/${r} (${i}/${run_count})`) const run = await read_compressed(`./${runs_dir}/${r}`) + const start_score = Date.now() console.log(`Calculating score for run ${runs_dir}/${r} (${i}/${run_count})`) - const score = score_run(run, new_run, test_to_areas) + const score = score_run(run, new_run, test_to_areas, () => false) const row = [ date, run.run_info.revision.substring(0, 9), @@ -95,6 +113,13 @@ async function recalc_scores (runs_dir) { row.push(score[area]) } scores.push(row) + + const end = Date.now() + const read_time = start_score - start + const score_time = end - start_score + const total_time = end - start + + console.log(`Done in ${total_time}ms (read in ${read_time}ms; scored in ${score_time}ms).`) } return scores @@ -102,10 +127,22 @@ async function recalc_scores (runs_dir) { async function main () { const mode = process.argv[2] - if (!['--add', '--recalc'].includes(mode)) { + if (!['--add', '--recalc', '--score'].includes(mode)) { throw new Error(`invalid mode specified: ${mode}`) } + if (mode === '--score') { + const input_dir = process.argv[3] + + let filterStr = undefined + if (process.argv[4] === '--filter') { + filterStr = process.argv[5] + } + const result = await score_single_run(input_dir, filterStr ? name => name.includes(filterStr) : () => true) + console.log(result) + return + } + if (mode === '--add') { const chunks = process.argv[3] const date = process.argv[4] diff --git a/package-lock.json b/package-lock.json index 6c802952..c943e54e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "MPL-2.0", "dependencies": { + "chalk": "^5.3.0", "lzma-native": "^8.0.6" }, "devDependencies": { @@ -446,33 +447,17 @@ } }, "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", + "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", + "license": "MIT", "engines": { - "node": ">=10" + "node": "^12.17.0 || ^14.13 || >=16.0.0" }, "funding": { "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/chalk/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -1126,6 +1111,23 @@ "concat-map": "0.0.1" } }, + "node_modules/eslint/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, "node_modules/eslint/node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -1150,6 +1152,19 @@ "node": "*" } }, + "node_modules/eslint/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/espree": { "version": "9.4.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz", @@ -1676,6 +1691,36 @@ "node": ">=12" } }, + "node_modules/http-server/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/http-server/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", @@ -2137,6 +2182,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/log-symbols/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/log-symbols/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -3521,25 +3596,9 @@ "dev": true }, "chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "dependencies": { - "supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", + "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==" }, "chokidar": { "version": "3.5.3", @@ -3807,6 +3866,16 @@ "concat-map": "0.0.1" } }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, "glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -3824,6 +3893,15 @@ "requires": { "brace-expansion": "^1.1.7" } + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } } } }, @@ -4427,6 +4505,27 @@ "secure-compare": "3.0.1", "union": "~0.5.0", "url-join": "^4.0.1" + }, + "dependencies": { + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } } }, "iconv-lite": { @@ -4749,6 +4848,27 @@ "requires": { "chalk": "^4.1.0", "is-unicode-supported": "^0.1.0" + }, + "dependencies": { + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } } }, "lru-cache": { diff --git a/package.json b/package.json index 9bfca181..91a8e02e 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "mocha": "^10.2.0" }, "dependencies": { + "chalk": "^5.3.0", "lzma-native": "^8.0.6" } } diff --git a/process-wpt-results.js b/process-wpt-results.js index 9ee69749..3bbec5ff 100644 --- a/process-wpt-results.js +++ b/process-wpt-results.js @@ -1,4 +1,5 @@ import assert from 'node:assert/strict' +import chalk from 'chalk'; function is_object (val) { return ( @@ -189,7 +190,7 @@ export function get_focus_areas () { return { area_keys, area_names } } -export function score_run (run, against_run, focus_areas_map) { +export function score_run (run, against_run, focus_areas_map, print_filter) { const scores = {} for (const area of Object.keys(FOCUS_AREAS)) { scores[area] = { @@ -198,7 +199,10 @@ export function score_run (run, against_run, focus_areas_map) { } } - for (const [test, { subtests }] of Object.entries(against_run.test_scores)) { + let testNames = Object.keys(against_run.test_scores); + testNames.sort(); + for (const test of testNames) { + const { subtests } = against_run.test_scores[test]; const areas = focus_areas_map[test] for (const area of areas) { @@ -215,17 +219,33 @@ export function score_run (run, against_run, focus_areas_map) { for (const area of areas) { scores[area].total_score += run_test.score } + if (print_filter(test)) { + const passes = run_test.score == 1; + if (passes) { + console.log(chalk.green(`PASS ${test}`)) + } else { + console.log(chalk.red(`FAIL ${test}`)) + } + } } else { - let test_score = 0 + let passed_test_count = 0 for (const subtest of subtest_names) { if (Object.hasOwn(run_test.subtests, subtest)) { - test_score += run_test.subtests[subtest].score + passed_test_count += run_test.subtests[subtest].score } } - test_score /= subtest_names.length + const test_score = passed_test_count / subtest_names.length for (const area of areas) { scores[area].total_score += test_score } + if (print_filter(test)) { + const passes = test_score == 1; + if (passes) { + console.log(chalk.green(`PASS ${test} (${passed_test_count}/${subtest_names.length})`)) + } else { + console.log(chalk.red(`FAIL ${test} (${passed_test_count}/${subtest_names.length})`)) + } + } } }