Merge branch 'dynamicbuffer'

Author: simolus3

sqlite3/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.4.7
+
+- Web: Improve performance of in-memory and IndexedDB file system implementations.
+
 ## 2.4.6
 
 - WebAssembly: Call `_initialize` function of sqlite3 module if one is present.

sqlite3/lib/src/wasm/vfs/indexed_db.dart

@@ -8,6 +8,7 @@ import 'dart:math';
 import 'dart:typed_data';
 
 import 'package:meta/meta.dart';
+import 'package:typed_data/typed_buffers.dart';
 import 'package:web/web.dart' as web;
 
 import '../../constants.dart';
@@ -533,7 +534,12 @@ final class IndexedDbFileSystem extends BaseVirtualFileSystem {
       final name = entry.key;
       final fileId = entry.value;
 
-      _memory.fileData[name] = await _asynchronous.readFully(fileId);
+      final buffer = Uint8Buffer();
+      final data = await _asynchronous.readFully(fileId);
+      buffer.length = data.length;
+      buffer.setRange(0, data.length, data);
+
+      _memory.fileData[name] = buffer;
     }
   }
 
@@ -642,18 +648,24 @@ class _IndexedDbFile implements VirtualFileSystemFile {
   void xWrite(Uint8List buffer, int fileOffset) {
     vfs._checkClosed();
 
-    final previousContent = vfs._memory.fileData[path] ?? Uint8List(0);
+    if (vfs._inMemoryOnlyFiles.contains(path)) {
+      // There's nothing to persist, so we just forward the write to the in-
+      // memory buffer.
+      memoryFile.xWrite(buffer, fileOffset);
+      return;
+    }
+
+    final previousContent = vfs._memory.fileData[path] ?? Uint8Buffer();
+    final previousList =
+        previousContent.buffer.asUint8List(0, previousContent.length);
     memoryFile.xWrite(buffer, fileOffset);
 
-    if (!vfs._inMemoryOnlyFiles.contains(path)) {
-      // We need to copy the buffer for the write because it will become invalid
-      // after this synchronous method returns.
-      final copy = Uint8List(buffer.length);
-      copy.setAll(0, buffer);
+    // We need to copy the buffer for the write because it will become invalid
+    // after this synchronous method returns.
+    final copy = Uint8List(buffer.length)..setAll(0, buffer);
 
-      vfs._submitWork(_WriteFileWorkItem(vfs, path, previousContent)
-        ..writes.add(_OffsetAndBuffer(fileOffset, copy)));
-    }
+    vfs._submitWork(_WriteFileWorkItem(vfs, path, previousList)
+      ..writes.add(_OffsetAndBuffer(fileOffset, copy)));
   }
 }
 

sqlite3/lib/src/wasm/vfs/memory.dart

@@ -2,13 +2,14 @@ import 'dart:math';
 import 'dart:typed_data';
 
 import 'package:path/path.dart' as p;
+import 'package:typed_data/typed_buffers.dart';
 
 import '../../constants.dart';
 import '../../vfs.dart';
 import 'utils.dart';
 
 final class InMemoryFileSystem extends BaseVirtualFileSystem {
-  final Map<String, Uint8List?> fileData = {};
+  final Map<String, Uint8Buffer?> fileData = {};
 
   InMemoryFileSystem({super.name = 'dart-memory', super.random});
 
@@ -34,7 +35,7 @@ final class InMemoryFileSystem extends BaseVirtualFileSystem {
       final create = flags & SqlFlag.SQLITE_OPEN_CREATE;
 
       if (create != 0) {
-        fileData[pathStr] = Uint8List(0);
+        fileData[pathStr] = Uint8Buffer();
       } else {
         throw VfsException(SqlError.SQLITE_CANTOPEN);
       }
@@ -69,7 +70,8 @@ class _InMemoryFile extends BaseVfsFile {
     if (file == null || file.length <= offset) return 0;
 
     final available = min(buffer.length, file.length - offset);
-    buffer.setRange(0, available, file, offset);
+    final list = file.buffer.asUint8List(0, file.length);
+    buffer.setRange(0, available, list, offset);
     return available;
   }
 
@@ -102,12 +104,12 @@ class _InMemoryFile extends BaseVfsFile {
   void xTruncate(int size) {
     final file = vfs.fileData[path];
 
-    final result = Uint8List(size);
-    if (file != null) {
-      result.setRange(0, min(size, file.length), file);
+    if (file == null) {
+      vfs.fileData[path] = Uint8Buffer();
+      vfs.fileData[path]!.length = size;
+    } else {
+      file.length = size;
     }
-
-    vfs.fileData[path] = result;
   }
 
   @override
@@ -117,19 +119,17 @@ class _InMemoryFile extends BaseVfsFile {
 
   @override
   void xWrite(Uint8List buffer, int fileOffset) {
-    final file = vfs.fileData[path] ?? Uint8List(0);
-    final increasedSize = fileOffset + buffer.length - file.length;
+    var file = vfs.fileData[path];
 
-    if (increasedSize <= 0) {
-      // Can write directy
-      file.setRange(fileOffset, fileOffset + buffer.length, buffer);
-    } else {
-      // We need to grow the file first
-      final newFile = Uint8List(file.length + increasedSize)
-        ..setAll(0, file)
-        ..setAll(fileOffset, buffer);
+    if (file == null) {
+      file = Uint8Buffer();
+      vfs.fileData[path] = file;
+    }
 
-      vfs.fileData[path] = newFile;
+    var endIndex = fileOffset + buffer.length;
+    if (endIndex > file.length) {
+      file.length = endIndex;
     }
+    file.setRange(fileOffset, endIndex, buffer);
   }
 }

sqlite3/pubspec.yaml

@@ -1,6 +1,6 @@
 name: sqlite3
 description: Provides lightweight yet convenient bindings to SQLite by using dart:ffi
-version: 2.4.6
+version: 2.4.7
 homepage: https://github.com/simolus3/sqlite3.dart/tree/main/sqlite3
 issue_tracker: https://github.com/simolus3/sqlite3.dart/issues
 
@@ -27,6 +27,7 @@ dependencies:
   meta: ^1.3.0
   path: ^1.8.0
   web: ^1.0.0
+  typed_data: ^1.3.2
 
 dev_dependencies:
   analyzer: ^6.4.1