Merge pull request #2 from danleh/sqlite-wasm

Sqlite Wasm build: build script, README, shell polyfill

Author: kmiller68

JetStreamDriver.js

@@ -1109,16 +1109,17 @@ class WasmBenchmark extends Benchmark {
             };
 
             oldPrint = globalObject.print;
+            oldConsoleLog = globalObject.console.log;
             globalObject.print = globalObject.printErr = (...args) => {
                 if (verbose)
-                    console.log('Intercepted print: ', ...args);
+                    oldConsoleLog('Intercepted print: ', ...args);
             };
 
             let Module = {
                 preRun: [],
                 postRun: [],
-                print: function() { },
-                printErr: function() { },
+                print: globalObject.print,
+                printErr: globalObject.print,
                 setStatus: function(text) {
                 },
                 totalDependencies: 0,
@@ -1822,6 +1823,19 @@ const testPlans = [
         benchmarkClass: WasmBenchmark,
         testGroup: WasmGroup
     },
+    {
+        name: "sqlite3-wasm",
+        files: [
+            "./sqlite3/polyfills.js",
+            "./sqlite3/build/jswasm/speedtest1.js",
+            "./sqlite3/benchmark.js",
+        ],
+        preload: {
+            wasmBinary: "./sqlite3/build/jswasm/speedtest1.wasm"
+        },
+        benchmarkClass: WasmBenchmark,
+        testGroup: WasmGroup
+    },
     {
         name: "tfjs-wasm",
         files: [

cli.js

@@ -35,6 +35,8 @@ const isSpiderMonkey = typeof newGlobal !== "undefined";
 if (isSpiderMonkey)
     globalThis.readFile = readRelativeToScript;
 
+if (typeof arguments !== "undefined" && arguments.length > 0)
+    testList = arguments.slice();
 if (typeof testList === "undefined")
     testList = undefined;
 

sqlite3/.gitignore

@@ -0,0 +1,2 @@
+/sqlite-src-*/
+/sqlite-src-*.zip

sqlite3/README.md

@@ -0,0 +1,51 @@
+# Official sqlite3 benchmark, compiled to WebAssembly
+
+This is a Wasm (shell-only) build of SQLite's official `speedtest1.c` benchmark program.
+Quoting from https://sqlite.org/cpu.html:
+> This program strives to exercise the SQLite library in a way that is typical of real-world applications. Of course, every application is different, and so no test program can exactly mirror the behavior of all applications. The speedtest1.c program is updated from time to time as the SQLite developers' understanding of what constitutes "typical" usage evolves.
+
+## Build Instructions
+
+See `build.sh` or just run it.
+See `build.log` for the last build time, used sources, and toolchain versions.
+
+Prerequisites:
+- Emscripten: https://emscripten.org. Make sure `emcc` is on your `PATH`, e.g., by running `source /path/to/emsdk_env.sh`.
+- `wasm-strip`: Install WABT, e.g., via `sudo apt install wabt` or by compiling yourself from the source at https://github.com/WebAssembly/wabt.
+
+Since this benchmark is meant for the developers of SQLite, it is not part of the official Wasm build of SQLite at https://sqlite.org/wasm/doc/trunk/index.md and also not contained in the "amalgamation" single-file source code of SQLite (see https://sqlite.org/amalgamation.html).
+So we need to build it ourselves from the full sources.
+You can download the full SQLite source tree from https://sqlite.org/download.html.
+See under "Alternative Source Code Formats" or search for "Snapshot of the complete (raw) source tree for SQLite".
+
+## Running in JS shells
+
+The SQLite developers only maintain compatibility of the Wasm build for running inside browsers, so `benchmark.js` contains some polyfills to run it JavaScript shells (such as `d8` (V8), `js` (SpiderMonkey), and `jsc` (JavaScriptCore)).
+Ideally, it should run just via `$shell benchmark.js` from the current directory.
+
+To keep the shell runner and browser results consistent, the benchmark is configured to never use OPFS (Origin Private File System) as the underlying storage layer ("VFS" in SQLite), since that is not available in shells.
+It might thus show slightly different performance characteristics compared to the upstream `speedtest1.html` running with OPFS.
+
+## Running the upstream version of the benchmark in browsers
+
+Start a webserver in `build/` that serves with the correct headers (CORS/COOP/COEP) set for Wasm execution.
+E.g., this simple Python server will do:
+```
+#!/usr/bin/env python3
+from http.server import HTTPServer, SimpleHTTPRequestHandler, test
+import sys
+
+class CORSRequestHandler (SimpleHTTPRequestHandler):
+    def end_headers (self):
+        self.send_header('Cross-Origin-Embedder-Policy', 'require-corp')
+        self.send_header('Cross-Origin-Opener-Policy', 'same-origin')
+
+        SimpleHTTPRequestHandler.end_headers(self)
+
+if __name__ == '__main__':
+    test(CORSRequestHandler, HTTPServer, port=int(sys.argv[1]) if len(sys.argv) > 1 else 8083)
+```
+
+Then browse to http://localhost:8083/speedtest1.html.
+(Note that, e.g., Chrome requires `localhost`, not an IP!)
+Because the computation happens on the main thread, it may hang for a little while but should show the console output afterwards.

sqlite3/benchmark.js

@@ -0,0 +1,73 @@
+// Copyright 2024 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+const inJetStreamRunner = typeof globalThis.benchmarkTime !== "undefined";
+if (!inJetStreamRunner) {
+  load("polyfills.js");
+
+  // Exports `sqlite3InitModule()` and contains the main code.
+  load("build/jswasm/speedtest1.js");
+
+  // Load Wasm binary from disk.
+  globalThis.Module = {
+    wasmBinary: read("build/jswasm/speedtest1.wasm", "binary"),
+  };
+}
+
+// Make sure we never initialize OPFS by removing one of it's APIs (see
+// `installOpfsVfs` in the generated JavaScript code of sqlite).
+// We never want to use it anyway (see VFS config below) and this way we don't
+// waste cycles on the browser runner to initialize it.
+delete globalThis.FileSystemHandle;
+
+// Simplified from inline JavaScript in `speedtest1.html`.
+function runTests(sqlite3Module) {
+  // Configure the VFS to use.
+  // Don't use OPFS, WASMFS (which is on top of OPFS), or kvvfs, since they all
+  // use persistent browser storage (localStorage or OPFS), which is not
+  // available in JavaScript shells.
+  // Also don't use memfs, since that crashes with a NULL function pointer.
+  // Instead, make the default VFS explicit.
+  const capi = sqlite3Module.capi
+  console.log("Available SQLite VFS:", capi.sqlite3_js_vfs_list());
+  const vfs = "unix";
+  console.log("Using VFS:", vfs);
+  const pVfs = capi.sqlite3_vfs_find(vfs);
+  if (!pVfs) {
+    console.error("Error: Unknown VFS:", vfs);
+    return;
+  }
+
+  // These arguments should match the upstream browser runner `speedtest1.html`.
+  let argv = [
+    "speedtest1",
+    "--singlethread",
+    //"--nomutex",
+    //"--nosync",
+    //"--memdb", // note that memdb trumps the filename arg
+    "--nomemstat",
+    "--big-transactions" /*important for tests 410 and 510!*/,
+    "--size", "20", // To speedup, default is 100 (and takes about 4s).
+    "--vfs", vfs, // See VFS comment above.
+  ];
+
+  console.log("Calling main with argv:", argv);
+  const wasm = sqlite3Module.wasm;
+  wasm.scopedAllocPush();  // Required for `scopedAllocMainArgv()`.
+  wasm.xCall("wasm_main", argv.length, wasm.scopedAllocMainArgv(argv));
+  wasm.scopedAllocPop();
+}
+
+async function doRun() {
+  let start = benchmarkTime();
+  const sqliteModule = await sqlite3InitModule(Module);
+  reportCompileTime(benchmarkTime() - start);
+
+  start = benchmarkTime();
+  runTests(sqliteModule);
+  reportRunTime(benchmarkTime() - start);
+}
+if (!inJetStreamRunner) {
+  sqlite3InitModule(Module).then(runTests);
+}

sqlite3/build.log

@@ -0,0 +1,11 @@
+Built on 2024-12-05 14:24:19+01:00
+
+Toolchain versions
+emcc (Emscripten gcc/clang-like replacement + linker emulating GNU ld) 3.1.71 (4171ae200b77a6c266b0e1ebb507d61d1ade3501)
+wasm-strip 1.0.34
+
+Getting sources from https://sqlite.org/2024/sqlite-src-3470100.zip
+
+Building...
+Copying files from sqlite-src-*/ext/wasm/ into build/
+Build success

sqlite3/build.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+# Cleanup old files.
+rm sqlite-src-*.zip
+rm -rf sqlite-src-*/
+rm -rf build/
+
+BUILD_LOG="$(realpath build.log)"
+echo -e "Built on $(date --rfc-3339=seconds)\n" | tee "$BUILD_LOG"
+
+echo "Toolchain versions" | tee -a "$BUILD_LOG"
+emcc --version | head -n1 | tee -a "$BUILD_LOG"
+echo -e "wasm-strip $(wasm-strip --version)\n" | tee -a "$BUILD_LOG"
+
+# Check https://sqlite.org/download.html and update the source link, if needed.
+SQLITE_SRC_URL="https://sqlite.org/2024/sqlite-src-3470100.zip"
+echo -e "Getting sources from $SQLITE_SRC_URL\n" | tee -a "$BUILD_LOG"
+SQLITE_SRC_FILE="$(basename $SQLITE_SRC_URL)"
+curl -o "$SQLITE_SRC_FILE" $SQLITE_SRC_URL
+unzip "$SQLITE_SRC_FILE"
+
+# Paths and information in make output could be sensitive, so don't save in log.
+echo "Building..." | tee -a "$BUILD_LOG"
+pushd sqlite-src-*/
+./configure
+cd ext/wasm
+make dist
+popd
+
+echo "Copying files from sqlite-src-*/ext/wasm/ into build/" | tee -a "$BUILD_LOG"
+mkdir -p build/{common,jswasm} | tee -a "$BUILD_LOG"
+cp sqlite-src-*/ext/wasm/jswasm/speedtest1.{js,wasm} build/jswasm/ | tee -a "$BUILD_LOG"
+# The next files are only needed for the upstream browser build, not the
+# JetStream version, hence don't copy them by default.
+# cp sqlite-src-*/ext/wasm/speedtest1.html build/ | tee -a "$BUILD_LOG"
+# cp sqlite-src-*/ext/wasm/common/{emscripten.css,SqliteTestUtil.js,testing.css} build/common/ | tee -a "$BUILD_LOG"
+
+echo "Build success" | tee -a "$BUILD_LOG"