Add a GrowingByteBuffer for VFS implementations.

Author: rkistner

sqlite3/lib/src/wasm/vfs/growing_buffer.dart

@@ -0,0 +1,81 @@
+import 'dart:typed_data';
+
+/// A utility class that manages a growing byte buffer.
+/// It dynamically increases its capacity in factors of 2 when needed.
+class GrowingByteBuffer {
+  Uint8List _buffer;
+  int _capacity;
+  int _length = 0;
+
+  /// Creates a [GrowingByteBuffer] with an optional initial capacity.
+  /// The default initial capacity is 1024 bytes.
+  GrowingByteBuffer([int initialCapacity = 1024])
+      : _capacity = initialCapacity,
+        _buffer = Uint8List(initialCapacity);
+
+  /// Adds [data] to the buffer, expanding its capacity if necessary.
+  void add(Uint8List data) {
+    _ensureCapacity(_length + data.length);
+    _buffer.setRange(_length, _length + data.length, data);
+    _length += data.length;
+  }
+
+  /// Writes [data] into the buffer starting at [offset], expanding capacity if necessary.
+  /// If the write operation extends beyond the current length, the length is updated accordingly.
+  void write(Uint8List data, int offset) {
+    if (offset < 0) {
+      throw ArgumentError("Offset must be non-negative");
+    }
+    int endPosition = offset + data.length;
+    _ensureCapacity(endPosition);
+    _buffer.setRange(offset, endPosition, data);
+    if (endPosition > _length) {
+      _length = endPosition;
+    }
+  }
+
+  /// Truncates the buffer to a specific [newSize].
+  /// If [newSize] is less than the current length, the buffer is truncated.
+  /// If [newSize] is greater than the current length, the buffer length is extended with zeros.
+  void truncate(int newSize) {
+    if (newSize < 0) {
+      throw ArgumentError("newSize must be non-negative");
+    }
+    _ensureCapacity(newSize);
+    if (newSize > _length) {
+      // Zero out the new extended area
+      _buffer.fillRange(_length, newSize, 0);
+    }
+    _length = newSize;
+  }
+
+  /// Returns a [Uint8List] containing the data up to the current length.
+  Uint8List toUint8List() {
+    return Uint8List.view(_buffer.buffer, 0, _length);
+  }
+
+  /// Ensures that the buffer has enough capacity to hold [requiredCapacity] bytes.
+  void _ensureCapacity(int requiredCapacity) {
+    if (requiredCapacity > _capacity) {
+      // Double the capacity until it is sufficient.
+      int newCapacity = _capacity;
+      while (newCapacity < requiredCapacity) {
+        newCapacity *= 2;
+      }
+      // Allocate a new buffer and copy existing data.
+      print('resizing to $newCapacity');
+      Uint8List newBuffer = Uint8List(newCapacity);
+      newBuffer.setRange(0, _length, _buffer);
+      _buffer = newBuffer;
+      _capacity = newCapacity;
+    }
+  }
+
+  /// Returns the current length of the data in the buffer.
+  int get length => _length;
+
+  /// Resets the buffer length to zero without altering its capacity.
+  void reset() {
+    _length = 0;
+  }
+}

sqlite3/lib/src/wasm/vfs/indexed_db.dart

@@ -8,6 +8,7 @@ import 'dart:math';
 import 'dart:typed_data';
 
 import 'package:meta/meta.dart';
+import 'package:sqlite3/src/wasm/vfs/growing_buffer.dart';
 import 'package:web/web.dart' as web;
 
 import '../../constants.dart';
@@ -533,7 +534,11 @@ final class IndexedDbFileSystem extends BaseVirtualFileSystem {
       final name = entry.key;
       final fileId = entry.value;
 
-      _memory.fileData[name] = await _asynchronous.readFully(fileId);
+      final buffer = GrowingByteBuffer();
+      final data = await _asynchronous.readFully(fileId);
+      buffer.add(data);
+
+      _memory.fileData[name] = buffer;
     }
   }
 
@@ -642,7 +647,7 @@ class _IndexedDbFile implements VirtualFileSystemFile {
   void xWrite(Uint8List buffer, int fileOffset) {
     vfs._checkClosed();
 
-    final previousContent = vfs._memory.fileData[path] ?? Uint8List(0);
+    final previousContent = vfs._memory.fileData[path] ?? GrowingByteBuffer();
     memoryFile.xWrite(buffer, fileOffset);
 
     if (!vfs._inMemoryOnlyFiles.contains(path)) {
@@ -651,8 +656,9 @@ class _IndexedDbFile implements VirtualFileSystemFile {
       final copy = Uint8List(buffer.length);
       copy.setAll(0, buffer);
 
-      vfs._submitWork(_WriteFileWorkItem(vfs, path, previousContent)
-        ..writes.add(_OffsetAndBuffer(fileOffset, copy)));
+      vfs._submitWork(
+          _WriteFileWorkItem(vfs, path, previousContent.toUint8List())
+            ..writes.add(_OffsetAndBuffer(fileOffset, copy)));
     }
   }
 }

sqlite3/lib/src/wasm/vfs/memory.dart

@@ -2,13 +2,14 @@ import 'dart:math';
 import 'dart:typed_data';
 
 import 'package:path/path.dart' as p;
+import 'growing_buffer.dart';
 
 import '../../constants.dart';
 import '../../vfs.dart';
 import 'utils.dart';
 
 final class InMemoryFileSystem extends BaseVirtualFileSystem {
-  final Map<String, Uint8List?> fileData = {};
+  final Map<String, GrowingByteBuffer?> fileData = {};
 
   InMemoryFileSystem({super.name = 'dart-memory', super.random});
 
@@ -34,7 +35,7 @@ final class InMemoryFileSystem extends BaseVirtualFileSystem {
       final create = flags & SqlFlag.SQLITE_OPEN_CREATE;
 
       if (create != 0) {
-        fileData[pathStr] = Uint8List(0);
+        fileData[pathStr] = GrowingByteBuffer();
       } else {
         throw VfsException(SqlError.SQLITE_CANTOPEN);
       }
@@ -69,7 +70,7 @@ class _InMemoryFile extends BaseVfsFile {
     if (file == null || file.length <= offset) return 0;
 
     final available = min(buffer.length, file.length - offset);
-    buffer.setRange(0, available, file, offset);
+    buffer.setRange(0, available, file.toUint8List(), offset);
     return available;
   }
 
@@ -102,12 +103,12 @@ class _InMemoryFile extends BaseVfsFile {
   void xTruncate(int size) {
     final file = vfs.fileData[path];
 
-    final result = Uint8List(size);
-    if (file != null) {
-      result.setRange(0, min(size, file.length), file);
+    if (file == null) {
+      vfs.fileData[path] = GrowingByteBuffer();
+      // TODO: grow?
+    } else {
+      file.truncate(size);
     }
-
-    vfs.fileData[path] = result;
   }
 
   @override
@@ -117,19 +118,12 @@ class _InMemoryFile extends BaseVfsFile {
 
   @override
   void xWrite(Uint8List buffer, int fileOffset) {
-    final file = vfs.fileData[path] ?? Uint8List(0);
-    final increasedSize = fileOffset + buffer.length - file.length;
-
-    if (increasedSize <= 0) {
-      // Can write directy
-      file.setRange(fileOffset, fileOffset + buffer.length, buffer);
-    } else {
-      // We need to grow the file first
-      final newFile = Uint8List(file.length + increasedSize)
-        ..setAll(0, file)
-        ..setAll(fileOffset, buffer);
+    var file = vfs.fileData[path];
 
-      vfs.fileData[path] = newFile;
+    if (file == null) {
+      file = GrowingByteBuffer();
+      vfs.fileData[path] = file;
     }
+    file.write(buffer, fileOffset);
   }
 }