Add UTF16 sources test (WebKit#162)

- Add explicit plan.allowUtf16 setting
- Add unittest that checks that no js/mjs files or preloads accidentally contain two-byte characters
- Fix workloads where with two-byte comments:
  - Use dash instead of em-dash for segmentation / async-tasks URL comment
  - Fix copied bigint-noble license header and use simple double quotes
  - Replace "•" with " dot " in sunspider comments

Author: camillobruni

JetStreamDriver.js

@@ -711,6 +711,7 @@ class Benchmark {
         this.tags = this.processTags(plan.tags)
         this.iterations = getIterationCount(plan);
         this.isAsync = !!plan.isAsync;
+        this.allowUtf16 = !!plan.allowUtf16;
         this.scripts = null;
         this.preloads = null;
         this.results = [];
@@ -1663,6 +1664,7 @@ let BENCHMARKS = [
             babylonBlob: "./ARES-6/Babylon/babylon-blob.js",
         },
         tags: ["Default", "ARES"],
+        allowUtf16: true,
     }),
     // CDJS
     new DefaultBenchmark({
@@ -2229,6 +2231,7 @@ let BENCHMARKS = [
         async: true,
         deterministicRandom: true,
         exposeBrowserTest: true,
+        allowUtf16: true,
         tags: ["Wasm"],
     }),
     new WasmLegacyBenchmark({
@@ -2250,6 +2253,7 @@ let BENCHMARKS = [
         async: true,
         deterministicRandom: true,
         exposeBrowserTest: true,
+        allowUtf16: true,
         tags: ["Wasm"],
     }),
     new WasmEMCCBenchmark({
@@ -2264,6 +2268,7 @@ let BENCHMARKS = [
         iterations: 30,
         worstCaseCount: 3,
         deterministicRandom: true,
+        allowUtf16: true,
         tags: ["Default", "Wasm"],
     }),
     // WorkerTests
@@ -2577,6 +2582,7 @@ for (const name of WTB_TESTS) {
         ],
         iterations: 5,
         worstCaseCount: 1,
+        allowUtf16: true,
         tags: ["Default", "WTB"],
     }));
 }

SunSpider/crypto-aes.js

@@ -57,12 +57,12 @@ function ShiftRows(s, Nb) {    // shift row r of state S left by r bytes [§5.1.
 function MixColumns(s, Nb) {   // combine bytes of each col of state S [§5.1.3]
   for (var c=0; c<4; c++) {
     var a = new Array(4);  // 'a' is a copy of the current column from 's'
-    var b = new Array(4);  // 'b' is a•{02} in GF(2^8)
+    var b = new Array(4);  // 'b' is a dot {02} in GF(2^8)
     for (var i=0; i<4; i++) {
       a[i] = s[i][c];
       b[i] = s[i][c]&0x80 ? s[i][c]<<1 ^ 0x011b : s[i][c]<<1;
     }
-    // a[n] ^ b[n] is a•{03} in GF(2^8)
+    // a[n] ^ b[n] is a dot {03} in GF(2^8)
     s[0][c] = b[0] ^ a[1] ^ b[1] ^ a[2] ^ a[3]; // 2*a0 + 3*a1 + a2 + a3
     s[1][c] = a[0] ^ b[1] ^ a[2] ^ b[2] ^ a[3]; // a0 * 2*a1 + 3*a2 + a3
     s[2][c] = a[0] ^ a[1] ^ b[2] ^ a[3] ^ b[3]; // a0 + a1 + 2*a2 + 3*a3

bigint/noble-bls12-381-bundle.js

@@ -3,7 +3,7 @@
 // Copyright (c) 2019 Paul Miller (https://paulmillr.com)
 
 // Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the “Software”), to deal
+// of this software and associated documentation files (the "Software"), to deal
 // in the Software without restriction, including without limitation the rights
 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 // copies of the Software, and to permit persons to whom the Software is
@@ -12,7 +12,7 @@
 // The above copyright notice and this permission notice shall be included in
 // all copies or substantial portions of the Software.
 
-// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

bigint/noble-ed25519-bundle.js

@@ -3,7 +3,7 @@
 // Copyright (c) 2019 Paul Miller (https://paulmillr.com)
 
 // Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the “Software”), to deal
+// of this software and associated documentation files (the "Software"), to deal
 // in the Software without restriction, including without limitation the rights
 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 // copies of the Software, and to permit persons to whom the Software is
@@ -12,7 +12,7 @@
 // The above copyright notice and this permission notice shall be included in
 // all copies or substantial portions of the Software.
 
-// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

bigint/noble-secp256k1-bundle.js

@@ -3,7 +3,7 @@
 // Copyright (c) 2019 Paul Miller (https://paulmillr.com)
 
 // Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the “Software”), to deal
+// of this software and associated documentation files (the "Software"), to deal
 // in the Software without restriction, including without limitation the rights
 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 // copies of the Software, and to permit persons to whom the Software is
@@ -12,7 +12,7 @@
 // The above copyright notice and this permission notice shall be included in
 // all copies or substantial portions of the Software.
 
-// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

tests/unit-tests.js

@@ -114,6 +114,45 @@ function assertThrows(message, func) {
   }
 })();
 
+(function checkUtf16Sources() {
+  // Test that only explicitly UTF16-enabled benchmarks can have sources
+  // with non-8-byte characters.
+  const twoByteCharsRegex = /[^\x00-\xFF]/g;
+  const jsFileRegex = /\.(js|mjs)$/;
+
+  function checkFile(benchmarkName, file, type) {
+      if (!jsFileRegex.test(file))
+          return;
+      const content = read(file);
+      const match = content.match(twoByteCharsRegex);
+      if (!match)
+          return;
+      const uniqueMatches = Array.from(new Set(match));
+      const offendingChars = uniqueMatches.map(char => {
+          const hex = char.charCodeAt(0).toString(16).padStart(4, "0");
+          return `\n        - \\u${hex}: '${char}'`;
+      }).join("");
+      throw new Error(
+        `Benchmark '${benchmarkName}' has two-byte characters in ${type} '${file}':\n` +
+        `    Offending characters: ${offendingChars}`);
+  }
+
+  for (const benchmark of BENCHMARKS) {
+    if (benchmark.allowUtf16)
+        continue;
+
+    for (const file of benchmark.files) {
+        checkFile(benchmark.name, file, "file");
+    }
+
+    if (benchmark.plan.preload) {
+        for (const [name, file] of Object.entries(benchmark.plan.preload)) {
+            checkFile(benchmark.name, file, `preload.${name}`);
+        }
+    }
+  }
+})();
+
 function validateIterationSources(sources) {
   for (const source of sources) {
     assertTrue(typeof source == "string");

worker/async-task.js

@@ -123,7 +123,7 @@ var Statistics = new (function () {
         var sumOfSampleVarianceOverSampleSize = stat1.variance / stat1.size + stat2.variance / stat2.size;
         var t = Math.abs((stat1.mean - stat2.mean) / Math.sqrt(sumOfSampleVarianceOverSampleSize));
 
-        // http://en.wikipedia.org/wiki/Welch–Satterthwaite_equation
+        // http://en.wikipedia.org/wiki/Welch-Satterthwaite_equation
         var degreesOfFreedom = sumOfSampleVarianceOverSampleSize * sumOfSampleVarianceOverSampleSize
             / (stat1.variance * stat1.variance / stat1.size / stat1.size / stat1.degreesOfFreedom
                 + stat2.variance * stat2.variance / stat2.size / stat2.size / stat2.degreesOfFreedom);

worker/bomb-subtests/crypto-aes.js

@@ -57,12 +57,12 @@ function ShiftRows(s, Nb) {    // shift row r of state S left by r bytes [§5.1.
 function MixColumns(s, Nb) {   // combine bytes of each col of state S [§5.1.3]
   for (var c=0; c<4; c++) {
     var a = new Array(4);  // 'a' is a copy of the current column from 's'
-    var b = new Array(4);  // 'b' is a•{02} in GF(2^8)
+    var b = new Array(4);  // 'b' is a dot {02} in GF(2^8)
     for (var i=0; i<4; i++) {
       a[i] = s[i][c];
       b[i] = s[i][c]&0x80 ? s[i][c]<<1 ^ 0x011b : s[i][c]<<1;
     }
-    // a[n] ^ b[n] is a•{03} in GF(2^8)
+    // a[n] ^ b[n] is a dot {03} in GF(2^8)
     s[0][c] = b[0] ^ a[1] ^ b[1] ^ a[2] ^ a[3]; // 2*a0 + 3*a1 + a2 + a3
     s[1][c] = a[0] ^ b[1] ^ a[2] ^ b[2] ^ a[3]; // a0 * 2*a1 + 3*a2 + a3
     s[2][c] = a[0] ^ a[1] ^ b[2] ^ a[3] ^ b[3]; // a0 + a1 + 2*a2 + 3*a3