New OPFS mode, sqlite3_web updates

Author: simolus3

sqlite3/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.9.4
+
+- `SimpleOpfsFileSystem`: Allow opening with `readwrite-unsafe`, which can be used to implement
+  multi-tab OPFS databases on Chrome with an outer locking scheme.
+
 ## 2.9.3
 
 - Allow iterating over statements after `SQLITE_BUSY` errors.

sqlite3/lib/src/wasm/js_interop/new_file_system_access.dart

@@ -52,21 +52,27 @@ extension FileSystemDirectoryHandleApi on FileSystemDirectoryHandle {
     return getFileHandle(name, FileSystemGetFileOptions(create: create)).toDart;
   }
 
-  Future<FileSystemDirectoryHandle> getDirectory(String name,
-      {bool create = false}) {
+  Future<FileSystemDirectoryHandle> getDirectory(
+    String name, {
+    bool create = false,
+  }) {
     return getDirectoryHandle(
-            name, FileSystemGetDirectoryOptions(create: create))
-        .toDart;
+      name,
+      FileSystemGetDirectoryOptions(create: create),
+    ).toDart;
   }
 
   Future<void> remove(String name, {bool recursive = false}) {
-    return removeEntry(name, FileSystemRemoveOptions(recursive: recursive))
-        .toDart;
+    return removeEntry(
+      name,
+      FileSystemRemoveOptions(recursive: recursive),
+    ).toDart;
   }
 
   Stream<FileSystemHandle> list() {
-    return AsyncJavaScriptIteratable<JSArray>(this)
-        .map((data) => data.toDart[1] as FileSystemHandle);
+    return AsyncJavaScriptIteratable<JSArray>(
+      this,
+    ).map((data) => data.toDart[1] as FileSystemHandle);
   }
 
   Stream<FileSystemHandle> getFilesRecursively() async* {
@@ -79,3 +85,22 @@ extension FileSystemDirectoryHandleApi on FileSystemDirectoryHandle {
     }
   }
 }
+
+// https://github.com/whatwg/fs/blob/main/proposals/MultipleReadersWriters.md
+extension ProposedLockingSchemeApi on FileSystemFileHandle {
+  external JSPromise<FileSystemSyncAccessHandle> createSyncAccessHandle(
+    FileSystemCreateSyncAccessHandleOptions options,
+  );
+}
+
+// https://github.com/whatwg/fs/blob/main/proposals/MultipleReadersWriters.md#modes-of-creating-a-filesystemsyncaccesshandle
+@anonymous
+extension type FileSystemCreateSyncAccessHandleOptions._(JSObject _)
+    implements JSObject {
+  external factory FileSystemCreateSyncAccessHandleOptions({JSString? mode});
+
+  static FileSystemCreateSyncAccessHandleOptions unsafeReadWrite() {
+    return FileSystemCreateSyncAccessHandleOptions(
+        mode: 'readwrite-unsafe'.toJS);
+  }
+}

sqlite3/lib/src/wasm/vfs/simple_opfs.dart

@@ -25,9 +25,7 @@ enum FileType {
 
   const FileType(this.filePath);
 
-  static final byName = {
-    for (final entry in values) entry.filePath: entry,
-  };
+  static final byName = {for (final entry in values) entry.filePath: entry};
 }
 
 /// A [VirtualFileSystem] for the `sqlite3` wasm library based on the [file system access API].
@@ -96,15 +94,24 @@ final class SimpleOpfsFileSystem extends BaseVirtualFileSystem {
   /// Throws a [VfsException] if OPFS is not available - please note that
   /// this file system implementation requires a recent browser and only works
   /// in dedicated web workers.
-  static Future<SimpleOpfsFileSystem> loadFromStorage(String path,
-      {String vfsName = 'simple-opfs'}) async {
+  ///
+  /// When [readWriteUnsafe] is passed, the synchronous file handles are opened
+  /// using the [proposed lock mode](https://github.com/whatwg/fs/blob/main/proposals/MultipleReadersWriters.md).
+  /// This mode is currently not supported across browsers, but can be used on
+  /// Chrome for faster database access across tabs.
+  static Future<SimpleOpfsFileSystem> loadFromStorage(
+    String path, {
+    String vfsName = 'simple-opfs',
+    bool readWriteUnsafe = false,
+  }) async {
     final storage = storageManager;
     if (storage == null) {
       throw VfsException(SqlError.SQLITE_ERROR);
     }
 
     final (_, directory) = await _resolveDir(path);
-    return inDirectory(directory, vfsName: vfsName);
+    return inDirectory(directory,
+        vfsName: vfsName, readWriteUnsafe: readWriteUnsafe);
   }
 
   /// Deletes the file system directory handle that would store sqlite3
@@ -134,14 +141,27 @@ final class SimpleOpfsFileSystem extends BaseVirtualFileSystem {
   /// Loads an [SimpleOpfsFileSystem] in the desired [root] directory, which must be
   /// a Dart wrapper around a [FileSystemDirectoryHandle].
   ///
+  /// When [readWriteUnsafe] is passed, the synchronous file handles are opened
+  /// using the [proposed lock mode](https://github.com/whatwg/fs/blob/main/proposals/MultipleReadersWriters.md).
+  /// This mode is currently not supported across browsers, but can be used on
+  /// Chrome for faster database access across tabs.
+  ///
   /// [FileSystemDirectoryHandle]: https://developer.mozilla.org/en-US/docs/Web/API/FileSystemDirectoryHandle
   static Future<SimpleOpfsFileSystem> inDirectory(
     FileSystemDirectoryHandle root, {
     String vfsName = 'simple-opfs',
+    bool readWriteUnsafe = false,
   }) async {
     Future<FileSystemSyncAccessHandle> open(String name) async {
       final handle = await root.openFile(name, create: true);
-      return await handle.createSyncAccessHandle().toDart;
+
+      final syncHandlePromise = readWriteUnsafe
+          ? ProposedLockingSchemeApi(handle).createSyncAccessHandle(
+              FileSystemCreateSyncAccessHandleOptions.unsafeReadWrite(),
+            )
+          : handle.createSyncAccessHandle();
+
+      return await syncHandlePromise.toDart;
     }
 
     final meta = await open('meta');

sqlite3/pubspec.yaml

@@ -1,6 +1,6 @@
 name: sqlite3
 description: Provides lightweight yet convenient bindings to SQLite by using dart:ffi
-version: 2.9.3
+version: 2.9.4
 homepage: https://github.com/simolus3/sqlite3.dart/tree/main/sqlite3
 issue_tracker: https://github.com/simolus3/sqlite3.dart/issues
 resolution: workspace

sqlite3_web/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.4.0
+
+- Remove `userVersion` and `setUserVersion`. Users should run the pragma statements manually.
+- Add `requestLock`, which can be used to make multiple database calls in a locked context.
+- Allow aborting requests.
+- Add support for a new Web FS (OPFS) access mode based on `readwrite-unsafe`.
+
 ## 0.3.2
 
 - Allow workers to send requests to clients. Previously, only the other

sqlite3_web/README.md

@@ -28,7 +28,7 @@ And that is what this package can do for you:
 ## Getting started
 
 Note: While this package can be used by end applications, it is meant as a
-building block for database packages like `sqlite3_async` or `drift`. Using
+building block for database packages like `sqlite_async` or `drift`. Using
 these packages helps avoid some setup work.
 
 Workers are responsible for opening databases and exposing them through message
@@ -75,6 +75,36 @@ Future<void> connectToDatabase() async {
 }
 ```
 
+## Lock and transaction management
+
+This package provides the `select` and `execute` method returning:
+
+1. The `autocommit` state of the database after running the statement.
+2. The last insert rowid value.
+3. For `select`, results.
+
+Because each database typically only uses a single connection (likely hosted in a shared worker),
+implementing transactions requires multiple statements to run without interference from others.
+
+The `requestLock` API is a helpful building block for this. In its callback, no other tab will
+have access to the database:
+
+```dart
+await db.requestLock((lock) async {
+  await db.execute('BEGIN', token: lock);
+  await db.execute(
+    '... other statements',
+    token: lock,
+    checkInTransaction: true,
+  );
+  await db.execute('COMMIT', token: lock, checkInTransaction: true);
+});
+```
+
+Inside a lock context, the `LockToken` must be passed to `select` and `execute`.
+Additionally, the `checkInTransaction` flag can be used to verify that the database
+is not in autocommit mode before running a statement.
+
 ## Custom requests
 
 In some cases, it may be useful to re-use the worker communication scheme
@@ -86,3 +116,10 @@ open transactions can be implemented in the worker by overriding
 `handleCustomRequest` in `WorkerDatabase`. You can encode requests and
 responses as arbitrary values exchanged as `JSAny?`.
 On the client side, requests can be issued with `Database.customRequest`.
+
+## Testing this package
+
+This package uses both regular `dart test` tests and integration tests.
+All tests need a compatible `sqlite3.wasm` file in `web/`.
+
+Integration tests also require `geckodriver` and `chromedriver` to be installed.

sqlite3_web/example/index.html

@@ -7,7 +7,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta name="scaffolded-by" content="https://github.com/dart-lang/sdk">
     <title>sqlite on the web</title>
-    <script defer src="main.dart.js"></script>
+    <script defer src="main.js"></script>
 </head>
 
 <body>
@@ -17,28 +17,26 @@ <h1>sqlite3 web demo</h1>
 This demo can be used to open databases in different storage implementations and
 access modes (e.g. through shared or dedicated workers).
 
-To open a database, open the consule and run <code>await open('name', 'storage', 'access')</code>.
-Supported values for <code>storage</code> are:
+To open a database, open the consule and run <code>await open('name', 'implementation')</code>.
+Supported values for <code>implementation</code> are:
 <ul>
-    <li><code>opfs</code>: Uses the <a href="https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system">Origin private file system</a></li>
-    <li><code>indexedDb</code>: Uses a virtualized file system implemented by storing chunks in IndexedDB</li>
-    <li><code>inMemory</code>: Only stores files in memory without persistence.</li>
-</ul>
-
-Supported values for <code>access</code> are:
-
-<ul>
-    <li><code>throughSharedWorker</code>: Open the database in a shared worker. Different tabs will use the same sqlite3 connection</li>
-    <li><code>throughDedicatedWorker</code>: Open the database in a dedicated worker. Requires shared array buffers when used with OPFS, and there is no synchronization for IndexedDB in dedicated workers.</li>
-    <li><code>inCurrentContext</code>: Open the database in the current context, without a worker</li>
+    <li><code>inMemoryLocal</code>: Only stores files in memory without persistence.</li>
+    <li><code>inMemoryShared</code>: In-memory database, but shared across tabs with a shared worker.</li>
+    <li><code>indexedDbUnsafeLocal</code>: Store data in IndexedDB, using each tab unisolated access to the database. Because data is just read once on startup, this is not a good file system implementation and prone to data corruption when used across multiple tabs.</li>
+    <li><code>indexedDbUnsafeWorker</code>: Like local, but in a shared worker.</li>
+    <li><code>indexedDbShared</code>: Safely store data in IndexedDB with a shared worker. Limited durability guarantees compared to OPFS.</li>
+
+    <li><code>opfsWithExternalLocks</code>: Uses the <a href="https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system">Origin private file system</a>. This uses the experimental <code>readwrite-unsafe</code> mode only available on Chrome.</li>
+    <li><code>opfsAtomics</code>: OPFS storage, but uses a pair of dedicated workers using shared memory and atomics to syncify requests. This requires COOP + COEP headers.</li>
+    <li><code>opfsShared</code>: OPFS storage, using a dedicated worker in a shared worker. This is only supported on Firefox.</li>
 </ul>
 
 After opening a database, you can use <code>execute</code> in the console to run
 SQL on it, e.g:
 
 <code>
 <pre>
-let db = await open('test', 'inMemory', 'throughSharedWorker');
+let db = await open('test', 'opfsWithExternalLocks');
 await execute(db, 'create table foo (bar);');
 </pre>
 </code>

sqlite3_web/example/main.dart

@@ -12,13 +12,12 @@ void main() async {
   final features = await sqlite.runFeatureDetection();
   print('got features: $features');
 
-  globalContext['open'] =
-      (JSString name, JSString storage, JSString accessMode) {
+  globalContext['open'] = (JSString name, JSString implementation) {
     return Future(() async {
       final database = await sqlite.connect(
-          name.toDart,
-          StorageMode.values.byName(storage.toDart),
-          AccessMode.values.byName(accessMode.toDart));
+        name.toDart,
+        DatabaseImplementation.values.byName(implementation.toDart),
+      );
 
       database.updates.listen((update) {
         print('Update on $name: $update');

sqlite3_web/example/main.js

@@ -0,0 +1,23 @@
+(async () => {
+    const thisScript = document.currentScript;
+    const params = new URL(document.location.toString()).searchParams;
+    const wasmOption = params.get("wasm");
+
+    function relativeURL(ref) {
+      const base = thisScript?.src ?? document.baseURI;
+      return new URL(ref, base).toString();
+    }
+
+    if (wasmOption == "1") {
+        let { compileStreaming } = await import("./main.mjs");
+
+        let app = await compileStreaming(fetch(relativeURL("main.wasm")));
+        let module = await app.instantiate({});
+        module.invokeMain();
+    } else {
+        const scriptTag = document.createElement("script");
+        scriptTag.type = "application/javascript";
+        scriptTag.src = relativeURL("./main.dart2js.js");
+        document.head.append(scriptTag);
+    }
+})();

sqlite3_web/lib/protocol_utils.dart

@@ -28,11 +28,11 @@ List<Object?> deserializeParameters(JSArray values, JSArrayBuffer? types) {
 /// Serializes a [ResultSet] into a serializable [JSObject].
 JSObject serializeResultSet(ResultSet resultSet) {
   final msg = JSObject();
-  RowsResponse(resultSet: resultSet, requestId: 0).serialize(msg, []);
+  RowsResponse.serializeResultSet(msg, [], resultSet);
   return msg;
 }
 
 /// Deserializes a result set from the format in [serializeResultSet].
 ResultSet deserializeResultSet(JSObject object) {
-  return RowsResponse.deserialize(object).resultSet;
+  return RowsResponse.deserializeResultSet(object);
 }