Pull upstream changes

.changeset/proud-moose-report.md

@@ -0,0 +1,5 @@
+---
+'@journeyapps/wa-sqlite': patch
+---
+
+Fix issue where OPFS VFS could freeze in infinite loop

Makefile

@@ -84,6 +84,7 @@ EMFLAGS_COMMON = \
 	-s INVOKE_RUN \
 	-s ENVIRONMENT="web,worker" \
 	-s STACK_SIZE=512KB \
+	-s WASM_BIGINT=0 \
 	$(EMFLAGS_EXTRA)
 
 EMFLAGS_DEBUG = \

demo/demo.js

@@ -77,6 +77,7 @@ window.addEventListener('DOMContentLoaded', async function() {
     let time = performance.now();
     worker.postMessage(queries);
     worker.addEventListener('message', async function(event) {
+      timestamp.textContent += ` ${(performance.now() - time).toFixed(1)} milliseconds`;
       if (event.data.results) {
         // Format the results as tables.
         event.data.results
@@ -85,7 +86,6 @@ window.addEventListener('DOMContentLoaded', async function() {
       } else {
         output.innerHTML = `<pre>${event.data.error.message}</pre>`;
       }
-      timestamp.textContent += ` ${Math.trunc(performance.now() - time) / 1000} seconds`;
       button.disabled = false;
     }, { once: true });
   });

demo/file/index.js

@@ -1,5 +1,6 @@
 import * as VFS from "../../src/VFS.js";
-import { IDBBatchAtomicVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
+import { IDBBatchAtomicVFS as MyVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
+// import { IDBMirrorVFS as MyVFS } from "../../src/examples/IDBMirrorVFS.js";
 
 const SEARCH_PARAMS = new URLSearchParams(location.search);
 const IDB_NAME = SEARCH_PARAMS.get('idb') ?? 'sqlite-vfs';
@@ -43,7 +44,7 @@ document.getElementById('file-fetch').addEventListener('click', async () => {
   let vfs;
   try {
     log(`Importing to IndexedDB ${IDB_NAME}, path ${DB_NAME}`);
-    vfs = await IDBBatchAtomicVFS.create(IDB_NAME, null);
+    vfs = await MyVFS.create(IDB_NAME, null);
 
     // @ts-ignore
     const importURL = document.getElementById('file-url').value;
@@ -69,7 +70,7 @@ document.getElementById('file-import').addEventListener('change', async event =>
   let vfs;
   try {
     log(`Importing to IndexedDB ${IDB_NAME}, path ${DB_NAME}`);
-    vfs = await IDBBatchAtomicVFS.create(IDB_NAME, null);
+    vfs = await MyVFS.create(IDB_NAME, null);
     // @ts-ignore
     await importDatabase(vfs, DB_NAME, event.target.files[0].stream());
     log('Import complete');
@@ -87,7 +88,7 @@ document.getElementById('file-import').addEventListener('change', async event =>
 });
 
 /**
- * @param {IDBBatchAtomicVFS} vfs 
+ * @param {MyVFS} vfs 
  * @param {string} path 
  * @param {ReadableStream} stream 
  */

demo/file/service-worker.js

@@ -1,5 +1,6 @@
 import * as VFS from "../../src/VFS.js";
-import { IDBBatchAtomicVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
+import { IDBBatchAtomicVFS as MyVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
+// import { IDBMirrorVFS as MyVFS } from "../../src/examples/IDBMirrorVFS.js";
 
 // Install the service worker as soon as possible.
 globalThis.addEventListener('install', (/** @type {ExtendableEvent} */ event) => {
@@ -26,7 +27,7 @@ globalThis.addEventListener('fetch', async (/** @type {FetchEvent} */ event) =>
 
   return event.respondWith((async () => {
     // Create the VFS and streaming source using the request parameters.
-    const vfs = await IDBBatchAtomicVFS.create(url.searchParams.get('idb'), null);
+    const vfs = await MyVFS.create(url.searchParams.get('idb'), null);
     const path = url.searchParams.get('db');
     const source = new DatabaseSource(vfs, path);
 

demo/file/verifier.js

@@ -1,6 +1,7 @@
 import SQLiteESMFactory from '../../dist/wa-sqlite-async.mjs';
 import * as SQLite from '../../src/sqlite-api.js';
-import { IDBBatchAtomicVFS } from '../../src/examples/IDBBatchAtomicVFS.js';
+import { IDBBatchAtomicVFS as MyVFS } from "../../src/examples/IDBBatchAtomicVFS.js";
+// import { IDBMirrorVFS as MyVFS } from "../../src/examples/IDBMirrorVFS.js";
 
 const SEARCH_PARAMS = new URLSearchParams(location.search);
 const IDB_NAME = SEARCH_PARAMS.get('idb') ?? 'sqlite-vfs';
@@ -10,7 +11,7 @@ const DB_NAME = SEARCH_PARAMS.get('db') ?? 'sqlite.db';
   const module = await SQLiteESMFactory();
   const sqlite3 = SQLite.Factory(module);
 
-  const vfs = await IDBBatchAtomicVFS.create(IDB_NAME, module);
+  const vfs = await MyVFS.create(IDB_NAME, module);
   // @ts-ignore
   sqlite3.vfs_register(vfs, true);
 

package.json

@@ -38,8 +38,8 @@
     "@changesets/cli": "^2.26.2",
     "@types/jasmine": "^5.1.4",
     "@web/dev-server": "^0.4.6",
-    "@web/test-runner": "^0.18.2",
-    "@web/test-runner-core": "^0.13.3",
+    "@web/test-runner": "^0.20.0",
+    "@web/test-runner-core": "^0.13.4",
     "comlink": "^4.4.1",
     "jasmine-core": "^4.5.0",
     "monaco-editor": "^0.34.1",

src/WebLocksMixin.js

@@ -82,8 +82,10 @@ export const WebLocksMixin = superclass => class extends superclass {
    */
   async jUnlock(fileId, lockType) {
     try {
+      // SQLite can call xUnlock() without ever calling xLock() so
+      // the state may not exist.
       const lockState = this.#mapIdToState.get(fileId);
-      if (lockType >= lockState.type) return VFS.SQLITE_OK;
+      if (!(lockType < lockState?.type)) return VFS.SQLITE_OK;
 
       switch (this.#options.lockPolicy) {
         case 'exclusive':
@@ -122,17 +124,17 @@ export const WebLocksMixin = superclass => class extends superclass {
   }
 
   /**
-   * @param {number} pFile
+   * @param {number} fileId
    * @param {number} op
    * @param {DataView} pArg
    * @returns {number|Promise<number>}
    */
-  jFileControl(pFile, op, pArg) {
-    const lockState = this.#mapIdToState.get(pFile) ??
+  jFileControl(fileId, op, pArg) {
+    const lockState = this.#mapIdToState.get(fileId) ??
       (() => {
         // Call jLock() to create the lock state.
-        this.jLock(pFile, VFS.SQLITE_LOCK_NONE);
-        return this.#mapIdToState.get(pFile);
+        this.jLock(fileId, VFS.SQLITE_LOCK_NONE);
+        return this.#mapIdToState.get(fileId);
       })();
     if (op === WebLocksMixin.WRITE_HINT_OP_CODE &&
         this.#options.lockPolicy === 'shared+hint'){

src/examples/IDBBatchAtomicVFS.js

@@ -3,6 +3,11 @@ import { FacadeVFS } from '../FacadeVFS.js';
 import * as VFS from '../VFS.js';
 import { WebLocksMixin } from '../WebLocksMixin.js';
 
+const RETRYABLE_ERRORS = new Set([
+  'TransactionInactiveError',
+  'InvalidStateError'
+]);
+
 /**
  * @typedef Metadata
  * @property {string} name
@@ -482,25 +487,27 @@ export class IDBBatchAtomicVFS extends WebLocksMixin(FacadeVFS) {
           break;
         case VFS.SQLITE_FCNTL_SYNC:
           this.log?.('xFileControl', file.path, 'SYNC');
-          const commitMetadata = Object.assign({}, file.metadata);
-          const prevFileSize = file.rollback.fileSize
-          this.#idb.q(({ metadata, blocks }) => {
-            metadata.put(commitMetadata);
-
-            // Remove old page versions.
-            for (const offset of file.changedPages) {
-              if (offset < prevFileSize) {
-                const range = IDBKeyRange.bound(
-                  [file.path, -offset, commitMetadata.version],
-                  [file.path, -offset, Infinity],
-                  true);
-                blocks.delete(range);
+          if (file.rollback) {
+            const commitMetadata = Object.assign({}, file.metadata);
+            const prevFileSize = file.rollback.fileSize
+            this.#idb.q(({ metadata, blocks }) => {
+              metadata.put(commitMetadata);
+
+              // Remove old page versions.
+              for (const offset of file.changedPages) {
+                if (offset < prevFileSize) {
+                  const range = IDBKeyRange.bound(
+                    [file.path, -offset, commitMetadata.version],
+                    [file.path, -offset, Infinity],
+                    true);
+                  blocks.delete(range);
+                }
               }
-            }
-            file.changedPages.clear();
-          }, 'rw', file.txOptions);
-          file.needsMetadataSync = false;
-          file.rollback = null;
+              file.changedPages.clear();
+            }, 'rw', file.txOptions);
+            file.needsMetadataSync = false;
+            file.rollback = null;
+          }
           break;
         case VFS.SQLITE_FCNTL_BEGIN_ATOMIC_WRITE:
           // Every write transaction is atomic, so this is a no-op.
@@ -717,21 +724,21 @@ export class IDBContext {
         });
       }
 
-      // @ts-ignore
-      // Create object store proxies.
-      const objectStores = [...tx.objectStoreNames].map(name => {
-        return [name, this.proxyStoreOrIndex(tx.objectStore(name))];
-      });
-
       try {
+        // @ts-ignore
+        // Create object store proxies.
+        const objectStores = [...tx.objectStoreNames].map(name => {
+          return [name, this.proxyStoreOrIndex(tx.objectStore(name))];
+        });
+
         // Execute the function.
         return await f(Object.fromEntries(objectStores));
       } catch (e) {
         // Use a new transaction if this one was inactive. This will
         // happen if the last request in the transaction completed
         // in a previous task but the transaction has not yet committed.
-        if (!i && e.name === 'TransactionInactiveError') {
-          this.log?.('TransactionInactiveError, retrying');
+        if (!i && RETRYABLE_ERRORS.has(e.name)) {
+          this.log?.(`${e.name}, retrying`);
           tx = null;
           continue;
         }

src/examples/IDBMirrorVFS.js

@@ -332,13 +332,12 @@ export class IDBMirrorVFS extends FacadeVFS {
       const file = this.#mapIdToFile.get(fileId);
 
       if (file.flags & VFS.SQLITE_OPEN_MAIN_DB) {
-        if (!file.txActive) {
-          file.txActive = {
-            path: file.path,
-            txId: file.viewTx.txId + 1,
-            blocks: new Map(),
-            fileSize: file.blockSize * file.blocks.size,
-          };
+        this.#requireTxActive(file);
+        // SQLite is not necessarily written sequentially, so fill in the
+        // unwritten blocks here.
+        for (let fillOffset = file.txActive.fileSize;
+             fillOffset < iOffset; fillOffset += pData.byteLength) {
+          file.txActive.blocks.set(fillOffset, new Uint8Array(pData.byteLength));
         }
         file.txActive.blocks.set(iOffset, pData.slice());
         file.txActive.fileSize = Math.max(file.txActive.fileSize, iOffset + pData.byteLength);
@@ -375,6 +374,7 @@ export class IDBMirrorVFS extends FacadeVFS {
       const file = this.#mapIdToFile.get(fileId);
 
       if (file.flags & VFS.SQLITE_OPEN_MAIN_DB) {
+        this.#requireTxActive(file);
         file.txActive.fileSize = iSize;
       } else {
         // All files that are not main databases are stored in a single
@@ -717,6 +717,20 @@ export class IDBMirrorVFS extends FacadeVFS {
     file.txWriteHint = false;
   }
 
+  /**
+   * @param {File} file 
+   */
+  #requireTxActive(file) {
+    if (!file.txActive) {
+      file.txActive = {
+        path: file.path,
+        txId: file.viewTx.txId + 1,
+        blocks: new Map(),
+        fileSize: file.blockSize * file.blocks.size,
+      };
+    }
+  }
+
   /**
    * @param {string} path 
    * @returns {Promise}

src/examples/OPFSAdaptiveVFS.js

@@ -413,7 +413,7 @@ export class OPFSAdaptiveVFS extends WebLocksMixin(FacadeVFS) {
       this.lastError = e;
       return VFS.SQLITE_IOERR;
     }
-    return VFS.SQLITE_NOTFOUND;
+    return super.jFileControl(fileId, op, pArg);
   }
 
   jGetLastError(zBuf) {

src/examples/OPFSCoopSyncVFS.js

@@ -437,7 +437,7 @@ export class OPFSCoopSyncVFS extends FacadeVFS {
         if (file.persistentFile.isHandleRequested) {
             // Another connection wants the access handle.
           this.#releaseAccessHandle(file);
-          this.isHandleRequested = false;
+          file.persistentFile.isHandleRequested = false;
         }
         file.persistentFile.isFileLocked = false;
       }

src/examples/OPFSPermutedVFS.js

@@ -463,6 +463,9 @@ export class OPFSPermutedVFS extends FacadeVFS {
       const file = this.#mapIdToFile.get(fileId);
       if ((file.flags & VFS.SQLITE_OPEN_MAIN_DB) && !file.txIsOverwrite) {
         file.abortController.signal.throwIfAborted();
+        if (!file.txActive) {
+          this.#beginTx(file);
+        }
         file.txActive.fileSize = iSize;
 
         // Remove now obsolete pages from file.txActive.pages

src/examples/README.md

@@ -19,7 +19,7 @@ Changing the page size after the database is created is not supported (this is a
 ### IDBMirrorVFS
 This VFS keeps all files in memory, persisting database files to IndexedDB. It works on all contexts.
 
-IDBBatchAtomicVFS can trade durability for performance by setting `PRAGMA synchronous=normal`.
+IDBMirrorVFS can trade durability for performance by setting `PRAGMA synchronous=normal`.
 
 Changing the page size after the database is created is not supported.
 

web-test-runner.config.mjs

@@ -8,9 +8,12 @@ export default /** @type {import("@web/test-runner").TestRunnerConfig} */ ({
       defaultTimeoutInterval: 5 * 60 * 1000
     },
   },
+  browserLogs: true,
+  browserStartTimeout: 60_000,
   nodeResolve: true,
   files: ['./test/*.test.js'],
   concurrency: 1,
+  concurrentBrowsers: 1,
   browsers: [
     chromeLauncher({
       launchOptions: {