Run V8 spec conformance test suite (#243)

The shell script runs the tests and diffs stdout against v8.txt.
Lines added/removed means tests were broken/fixed.

Future work is to a) fix failing tests, and b) enable more tests.

A number are disabled for various reasons and mjsunit subdirectories are
currently skipped. Need to decide on a case-by-case basis what is and
isn't relevant to us.

At the moment about 430 tests are run of which approx. 80% pass.

Author: bnoordhuis

.github/workflows/ci.yml

@@ -38,6 +38,10 @@ jobs:
         if: ${{ matrix.buildType == 'Release' }}
         run: |
           time make test262
+      - name: test v8 mjsunit
+        if: ${{ matrix.buildType == 'Release' }}
+        run: |
+          ./v8.sh
   linux-32bits:
     runs-on: ubuntu-latest
     defaults:

qjs.c

@@ -41,6 +41,8 @@
 extern const uint8_t qjsc_repl[];
 extern const uint32_t qjsc_repl_size;
 
+static JSCFunctionListEntry argv0;
+
 static int eval_buf(JSContext *ctx, const void *buf, int buf_len,
                     const char *filename, int eval_flags)
 {
@@ -94,6 +96,22 @@ static int eval_file(JSContext *ctx, const char *filename, int module)
     return ret;
 }
 
+static JSValue js_gc(JSContext *ctx, JSValue this_val,
+                     int argc, JSValue *argv)
+{
+    JS_RunGC(JS_GetRuntime(ctx));
+    return JS_UNDEFINED;
+}
+
+static const JSCFunctionListEntry navigator_obj[] = {
+    JS_PROP_STRING_DEF("userAgent", "quickjs-ng", JS_PROP_ENUMERABLE),
+};
+
+static const JSCFunctionListEntry global_obj[] = {
+    JS_CFUNC_DEF("gc", 0, js_gc),
+    JS_OBJECT_DEF("navigator", navigator_obj, countof(navigator_obj), JS_PROP_C_W_E),
+};
+
 /* also used to initialize the worker context */
 static JSContext *JS_NewCustomContext(JSRuntime *rt)
 {
@@ -105,11 +123,9 @@ static JSContext *JS_NewCustomContext(JSRuntime *rt)
     js_init_module_std(ctx, "std");
     js_init_module_os(ctx, "os");
 
-    /* navigator.userAgent */
     JSValue global = JS_GetGlobalObject(ctx);
-    JSValue navigator = JS_NewObject(ctx);
-    JS_DefinePropertyValueStr(ctx, navigator, "userAgent", JS_NewString(ctx, "quickjs-ng"), JS_PROP_ENUMERABLE);
-    JS_DefinePropertyValueStr(ctx, global, "navigator", navigator, JS_PROP_ENUMERABLE);
+    JS_SetPropertyFunctionList(ctx, global, global_obj, countof(global_obj));
+    JS_SetPropertyFunctionList(ctx, global, &argv0, 1);
     JS_FreeValue(ctx, global);
 
     return ctx;
@@ -283,6 +299,9 @@ int main(int argc, char **argv)
     int i, include_count = 0;
     size_t stack_size = 0;
 
+    argv0 = (JSCFunctionListEntry)JS_PROP_STRING_DEF("argv0", argv[0],
+                                                     JS_PROP_C_W_E);
+
     /* cannot use getopt because we want to pass the command line to
        the script */
     optind = 1;

quickjs-libc.c

@@ -3739,7 +3739,7 @@ JSModuleDef *js_init_module_os(JSContext *ctx, const char *module_name)
 /**********************************************************/
 
 static JSValue js_print(JSContext *ctx, JSValue this_val,
-                              int argc, JSValue *argv)
+                        int argc, JSValue *argv)
 {
     int i;
     const char *str;
@@ -3755,6 +3755,7 @@ static JSValue js_print(JSContext *ctx, JSValue this_val,
         JS_FreeCString(ctx, str);
     }
     putchar('\n');
+    fflush(stdout);
     return JS_UNDEFINED;
 }
 

v8-tweak.js

@@ -0,0 +1,11 @@
+;(function() {
+    "use strict"
+    const print = globalThis.print
+    globalThis.print = function() {}    // print nothing, v8 tests are chatty
+    let count = 0                       // rate limit to avoid excessive logs
+    globalThis.failWithMessage = function(message) {
+        if (count > 99) return
+        if (++count > 99) return print("<output elided>")
+        print(String(message).slice(0, 128))
+    }
+})()

v8.js

@@ -0,0 +1,52 @@
+import * as std from "std"
+import * as os from "os"
+
+argv0 = realpath(argv0)
+const tweak = realpath("v8-tweak.js")
+const dir = "test262/implementation-contributed/v8/mjsunit"
+
+const exclude = [
+    "array-concat.js",              // slow
+    "array-isarray.js",             // unstable output due to stack overflow
+    "ascii-regexp-subject.js",      // slow
+    "cyclic-array-to-string.js",    // unstable output due to stack overflow
+    "error-tostring.js",            // unstable output due to stack overflow
+    "regexp.js",                    // invalid, legitimate early SyntaxError
+    "regexp-capture-3.js",          // slow
+    "string-replace.js",            // unstable output
+
+    "mjsunit-assertion-error.js",
+    "mjsunit.js",
+    "mjsunit_suppressions.js",
+
+    "verify-assert-false.js",       // self check
+    "verify-check-false.js",        // self check
+]
+
+let files = scriptArgs.slice(1) // run only these tests
+if (files.length === 0) files = os.readdir(dir)[0].sort()
+
+for (const file of files) {
+    if (!file.endsWith(".js")) continue
+    if (exclude.includes(file)) continue
+    const source = std.loadFile(dir + "/" + file)
+    if (/^(im|ex)port/m.test(source)) continue // TODO support modules
+    if (source.includes('// Files:')) continue // TODO support includes
+    // exclude tests that use V8 intrinsics like %OptimizeFunctionOnNextCall
+    if (source.includes ("--allow-natives-syntax")) continue
+    // exclude tests that use V8 extensions
+    if (source.includes ("--expose-externalize-string")) continue
+    const env =
+        source.match(/environment variables:.*TZ=(?<TZ>[\S]+)/i)?.groups
+    print(`=== ${file}`)
+    // the fixed --stack-size is necessary to keep output of stack overflowing
+    // tests stable; their stack traces are somewhat arbitrary otherwise
+    const args = [argv0, "--stack-size", `${2048 * 1024}`,
+                  "-I", "mjsunit.js", "-I", tweak, file]
+    const opts = {block:true, cwd:dir, env:env, usePath:false}
+    os.exec(args, opts)
+}
+
+function realpath(path) {
+    return os.realpath(path)[0]
+}

v8.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+: ${QJS:=build/qjs}
+"$QJS" v8.js $* 2>&1 | tee v8.txt$$
+diff -uw v8.txt v8.txt$$ || exit 1
+rm v8.txt$$