Add information about number of tests and subtests

This modifies the scores.json file so it now provides more data:
* total_tests: The total number of tests
* total_score: The total scores for the tests
* per_mille: Per mille value of dividing total_score by total_tests
* total_subtests: The total number of subtests
* total_subtests_passed: The number of subtests that pass
* per_mille_subtests: Per mille value of dividing total_subtests by total_subtests_passed

This is based on previous work by Delan Azabani at #35.

Co-authored-by: Delan Azabani <[email protected]>
Signed-off-by: Manuel Rego Casasnovas <[email protected]>

Author: mrego

index.js

@@ -113,13 +113,17 @@ async function main () {
     }
 
     const scores = await recalc_scores('runs-2020')
+    const scores_last_run = scores[scores.length - 1]
 
     const { area_keys, area_names: focus_areas } = get_focus_areas()
 
     console.log('Writing site/scores.json')
     await mkdir('./site', { recursive: true })
     write_json_file(
         './site/scores.json', { area_keys, focus_areas, scores })
+    console.log('Writing site/scores-last-run.json')
+    write_json_file(
+        './site/scores-last-run.json', { area_keys, focus_areas, scores_last_run })
 
     console.log('Done')
 }

process-wpt-results.js

@@ -199,48 +199,44 @@ export function score_run (run, against_run, focus_areas_map) {
     for (const area of Object.keys(FOCUS_AREAS)) {
         scores[area] = {
             total_tests: 0,
-            total_score: 0
+            total_score: 0,
+            total_subtests: 0,
+            total_subtests_passed: 0
         }
     }
 
     for (const [test, { subtests }] of Object.entries(against_run.test_scores)) {
         const areas = focus_areas_map[test]
+        const subtest_names = Object.keys(subtests)
 
         for (const area of areas) {
             scores[area].total_tests += 1
+            scores[area].total_subtests += !subtest_names.length ? 1 : subtest_names.length
         }
 
         const run_test = run.test_scores[test]
 
         // score new tests not present in older runs
         if (!run_test) continue
 
-        const subtest_names = Object.keys(subtests)
         if (!subtest_names.length) {
             for (const area of areas) {
                 scores[area].total_score += run_test.score
+                scores[area].total_subtests_passed += run_test.score
             }
         } else {
-            let test_score = 0
+            let subtests_passed = 0
             for (const subtest of subtest_names) {
                 if (Object.hasOwn(run_test.subtests, subtest)) {
-                    test_score += run_test.subtests[subtest].score
+                    subtests_passed += run_test.subtests[subtest].score
                 }
             }
-            test_score /= subtest_names.length
             for (const area of areas) {
-                scores[area].total_score += test_score
+                scores[area].total_score += subtests_passed / subtest_names.length
+                scores[area].total_subtests_passed += subtests_passed
             }
         }
     }
 
-    return Object.entries(scores).reduce((scores, [area, totals]) => {
-        scores[area] = 0
-        if (totals.total_tests !== 0) {
-            scores[area] = Math.floor(
-                1000 * totals.total_score / totals.total_tests
-            )
-        }
-        return scores
-    }, {})
+    return scores
 }

test/process-wpt-results.test.js

@@ -146,6 +146,12 @@ describe('Process wpt result', () => {
     })
 })
 
+function checkScore (actual, expected) {
+    actual.per_mille = Math.floor(1000 * actual.total_score / actual.total_tests)
+    actual.per_mille_subtests = Math.floor(1000 * actual.total_subtests_passed / actual.total_subtests)
+    assert.deepEqual(actual, expected)
+}
+
 describe('Scoring', () => {
     it('calculates scores for individual tests', () => {
         const run = {
@@ -166,11 +172,109 @@ describe('Scoring', () => {
             test2: ['all']
         }
         let score = score_run(run, run, focus_area_map)
-        assert.deepEqual(score.all, 1000)
+        checkScore(score.all,
+            {
+                total_tests: 2,
+                total_score: 2,
+                per_mille: 1000,
+                total_subtests: 2,
+                total_subtests_passed: 2,
+                per_mille_subtests: 1000
+            })
 
         run.test_scores.test2.score = 0
         score = score_run(run, run, focus_area_map)
-        assert.deepEqual(score.all, 500)
+        checkScore(score.all,
+            {
+                total_tests: 2,
+                total_score: 1,
+                per_mille: 500,
+                total_subtests: 2,
+                total_subtests_passed: 1,
+                per_mille_subtests: 500
+            })
+    })
+    it('calculates subtests count', () => {
+        const run = {
+            test_scores: {
+                test1: {
+                    score: 1,
+                    subtests: {
+                        subtest1: { score: 1 },
+                        subtest2: { score: 1 },
+                        subtest3: { score: 1 }
+                    }
+                },
+                test2: {
+                    score: 0,
+                    subtests: {
+                        subtest1: { score: 1 },
+                        subtest2: { score: 0 }
+                    }
+                }
+            }
+        }
+
+        const focus_area_map = {
+            test1: ['all'],
+            test2: ['all']
+        }
+        const score = score_run(run, run, focus_area_map)
+        checkScore(score.all,
+            {
+                total_tests: 2,
+                total_score: 1.5,
+                per_mille: 750,
+                total_subtests: 5,
+                total_subtests_passed: 4,
+                per_mille_subtests: 800
+            })
+    })
+    it('calculates subtests counts with simple tests', () => {
+        const run = {
+            test_scores: {
+                test1: {
+                    score: 1,
+                    subtests: {
+                        subtest1: { score: 1 },
+                        subtest2: { score: 1 },
+                        subtest3: { score: 1 }
+                    }
+                },
+                test2: {
+                    score: 0,
+                    subtests: {
+                        subtest1: { score: 1 },
+                        subtest2: { score: 0 }
+                    }
+                },
+                test3: {
+                    score: 1,
+                    subtests: {}
+                },
+                test4: {
+                    score: 0,
+                    subtests: {}
+                }
+            }
+        }
+
+        const focus_area_map = {
+            test1: ['all'],
+            test2: ['all'],
+            test3: ['all'],
+            test4: ['all']
+        }
+        const score = score_run(run, run, focus_area_map)
+        checkScore(score.all,
+            {
+                total_tests: 4,
+                total_score: 2.5,
+                per_mille: 625,
+                total_subtests: 7,
+                total_subtests_passed: 5,
+                per_mille_subtests: 714
+            })
     })
     it('calculates scores for subtests', () => {
         const run = {
@@ -187,7 +291,15 @@ describe('Scoring', () => {
         }
 
         const score = score_run(run, run, { test1: ['all'] })
-        assert.equal(score.all, 1000)
+        checkScore(score.all,
+            {
+                total_tests: 1,
+                total_score: 1,
+                per_mille: 1000,
+                total_subtests: 3,
+                total_subtests_passed: 3,
+                per_mille_subtests: 1000
+            })
     })
     it('calculates scores for subtests by averaging', () => {
         const run = {
@@ -204,7 +316,15 @@ describe('Scoring', () => {
         }
 
         const score = score_run(run, run, { test1: ['all'] })
-        assert.equal(score.all, 333)
+        checkScore(score.all,
+            {
+                total_tests: 1,
+                total_score: 1 / 3,
+                per_mille: 333,
+                total_subtests: 3,
+                total_subtests_passed: 1,
+                per_mille_subtests: 333
+            })
     })
     it('calculates scores correctly even subtest name collides with JS builtins', () => {
         const run = {
@@ -237,7 +357,15 @@ describe('Scoring', () => {
         }
 
         const score = score_run(run, against_run, { test1: ['all'], test2: ['all'] })
-        assert.equal(score.all, 500)
+        checkScore(score.all,
+            {
+                total_tests: 2,
+                total_score: 1,
+                per_mille: 500,
+                total_subtests: 2,
+                total_subtests_passed: 1,
+                per_mille_subtests: 500
+            })
     })
     it('calculates scores based only on tests in new runs', () => {
         const old_run = {
@@ -258,11 +386,93 @@ describe('Scoring', () => {
             test1: all, test2: all, test3: all
         }
         let score = score_run(old_run, new_run, focus_map)
-        assert.equal(score.all, 0)
+        checkScore(score.all,
+            {
+                total_tests: 2,
+                total_score: 0,
+                per_mille: 0,
+                total_subtests: 2,
+                total_subtests_passed: 0,
+                per_mille_subtests: 0
+            })
+        old_run.test_scores.test3.score = 1
+        score = score_run(old_run, new_run, focus_map)
+        checkScore(score.all,
+            {
+                total_tests: 2,
+                total_score: 1,
+                per_mille: 500,
+                total_subtests: 2,
+                total_subtests_passed: 1,
+                per_mille_subtests: 500
+            })
+    })
+    it('calculates scores based only on subtests in new runs', () => {
+        const old_run = {
+            test_scores: {
+                test1: {
+                    score: 0,
+                    subtests: {
+                        subtest1: { score: 1 },
+                        subtest2: { score: 0 }
+                    }
+                },
+                test3: {
+                    score: 0,
+                    subtests: {
+                        subtest1: { score: 0 },
+                        subtest2: { score: 1 }
+                    }
+                }
+            }
+        }
+        const new_run = {
+            test_scores: {
+                test2: {
+                    score: 1,
+                    subtests: {
+                        subtest1: { score: 1 },
+                        subtest2: { score: 1 },
+                        subtest3: { score: 1 },
+                        subtest4: { score: 1 }
+                    }
+                },
+                test3: {
+                    score: 1,
+                    subtests: {
+                        subtest1: { score: 1 },
+                        subtest2: { score: 1 }
+                    }
+                }
+            }
+        }
 
+        const all = ['all']
+        const focus_map = {
+            test1: all, test2: all, test3: all
+        }
+        let score = score_run(old_run, new_run, focus_map)
+        checkScore(score.all,
+            {
+                total_tests: 2,
+                total_score: 0.5,
+                per_mille: 250,
+                total_subtests: 6,
+                total_subtests_passed: 1,
+                per_mille_subtests: 166
+            })
         old_run.test_scores.test3.score = 1
+        old_run.test_scores.test3.subtests.subtest1.score = 1
         score = score_run(old_run, new_run, focus_map)
-        assert.equal(score.all, 500)
+        checkScore(score.all,
+            {
+                total_tests: 2,
+                total_score: 1,
+                per_mille: 500,
+                total_subtests: 6,
+                total_subtests_passed: 2,
+                per_mille_subtests: 333
+            })
     })
 })