From 6b4f3e367189926ff44fc10ef971615ea6ab5f33 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 17 Oct 2024 18:07:45 +0200 Subject: [PATCH 1/2] Add a GrowingByteBuffer for VFS implementations. --- sqlite3/lib/src/wasm/vfs/growing_buffer.dart | 81 ++++++++++++++++++++ sqlite3/lib/src/wasm/vfs/indexed_db.dart | 14 +++- sqlite3/lib/src/wasm/vfs/memory.dart | 34 ++++---- 3 files changed, 105 insertions(+), 24 deletions(-) create mode 100644 sqlite3/lib/src/wasm/vfs/growing_buffer.dart diff --git a/sqlite3/lib/src/wasm/vfs/growing_buffer.dart b/sqlite3/lib/src/wasm/vfs/growing_buffer.dart new file mode 100644 index 00000000..b6b6e853 --- /dev/null +++ b/sqlite3/lib/src/wasm/vfs/growing_buffer.dart @@ -0,0 +1,81 @@ +import 'dart:typed_data'; + +/// A utility class that manages a growing byte buffer. +/// It dynamically increases its capacity in factors of 2 when needed. +class GrowingByteBuffer { + Uint8List _buffer; + int _capacity; + int _length = 0; + + /// Creates a [GrowingByteBuffer] with an optional initial capacity. + /// The default initial capacity is 1024 bytes. + GrowingByteBuffer([int initialCapacity = 1024]) + : _capacity = initialCapacity, + _buffer = Uint8List(initialCapacity); + + /// Adds [data] to the buffer, expanding its capacity if necessary. + void add(Uint8List data) { + _ensureCapacity(_length + data.length); + _buffer.setRange(_length, _length + data.length, data); + _length += data.length; + } + + /// Writes [data] into the buffer starting at [offset], expanding capacity if necessary. + /// If the write operation extends beyond the current length, the length is updated accordingly. + void write(Uint8List data, int offset) { + if (offset < 0) { + throw ArgumentError("Offset must be non-negative"); + } + int endPosition = offset + data.length; + _ensureCapacity(endPosition); + _buffer.setRange(offset, endPosition, data); + if (endPosition > _length) { + _length = endPosition; + } + } + + /// Truncates the buffer to a specific [newSize]. + /// If [newSize] is less than the current length, the buffer is truncated. + /// If [newSize] is greater than the current length, the buffer length is extended with zeros. + void truncate(int newSize) { + if (newSize < 0) { + throw ArgumentError("newSize must be non-negative"); + } + _ensureCapacity(newSize); + if (newSize > _length) { + // Zero out the new extended area + _buffer.fillRange(_length, newSize, 0); + } + _length = newSize; + } + + /// Returns a [Uint8List] containing the data up to the current length. + Uint8List toUint8List() { + return Uint8List.view(_buffer.buffer, 0, _length); + } + + /// Ensures that the buffer has enough capacity to hold [requiredCapacity] bytes. + void _ensureCapacity(int requiredCapacity) { + if (requiredCapacity > _capacity) { + // Double the capacity until it is sufficient. + int newCapacity = _capacity; + while (newCapacity < requiredCapacity) { + newCapacity *= 2; + } + // Allocate a new buffer and copy existing data. + print('resizing to $newCapacity'); + Uint8List newBuffer = Uint8List(newCapacity); + newBuffer.setRange(0, _length, _buffer); + _buffer = newBuffer; + _capacity = newCapacity; + } + } + + /// Returns the current length of the data in the buffer. + int get length => _length; + + /// Resets the buffer length to zero without altering its capacity. + void reset() { + _length = 0; + } +} diff --git a/sqlite3/lib/src/wasm/vfs/indexed_db.dart b/sqlite3/lib/src/wasm/vfs/indexed_db.dart index 34bd641a..7577a0f4 100644 --- a/sqlite3/lib/src/wasm/vfs/indexed_db.dart +++ b/sqlite3/lib/src/wasm/vfs/indexed_db.dart @@ -8,6 +8,7 @@ import 'dart:math'; import 'dart:typed_data'; import 'package:meta/meta.dart'; +import 'package:sqlite3/src/wasm/vfs/growing_buffer.dart'; import 'package:web/web.dart' as web; import '../../constants.dart'; @@ -533,7 +534,11 @@ final class IndexedDbFileSystem extends BaseVirtualFileSystem { final name = entry.key; final fileId = entry.value; - _memory.fileData[name] = await _asynchronous.readFully(fileId); + final buffer = GrowingByteBuffer(); + final data = await _asynchronous.readFully(fileId); + buffer.add(data); + + _memory.fileData[name] = buffer; } } @@ -642,7 +647,7 @@ class _IndexedDbFile implements VirtualFileSystemFile { void xWrite(Uint8List buffer, int fileOffset) { vfs._checkClosed(); - final previousContent = vfs._memory.fileData[path] ?? Uint8List(0); + final previousContent = vfs._memory.fileData[path] ?? GrowingByteBuffer(); memoryFile.xWrite(buffer, fileOffset); if (!vfs._inMemoryOnlyFiles.contains(path)) { @@ -651,8 +656,9 @@ class _IndexedDbFile implements VirtualFileSystemFile { final copy = Uint8List(buffer.length); copy.setAll(0, buffer); - vfs._submitWork(_WriteFileWorkItem(vfs, path, previousContent) - ..writes.add(_OffsetAndBuffer(fileOffset, copy))); + vfs._submitWork( + _WriteFileWorkItem(vfs, path, previousContent.toUint8List()) + ..writes.add(_OffsetAndBuffer(fileOffset, copy))); } } } diff --git a/sqlite3/lib/src/wasm/vfs/memory.dart b/sqlite3/lib/src/wasm/vfs/memory.dart index 236b9cf5..6ea1c27e 100644 --- a/sqlite3/lib/src/wasm/vfs/memory.dart +++ b/sqlite3/lib/src/wasm/vfs/memory.dart @@ -2,13 +2,14 @@ import 'dart:math'; import 'dart:typed_data'; import 'package:path/path.dart' as p; +import 'growing_buffer.dart'; import '../../constants.dart'; import '../../vfs.dart'; import 'utils.dart'; final class InMemoryFileSystem extends BaseVirtualFileSystem { - final Map fileData = {}; + final Map fileData = {}; InMemoryFileSystem({super.name = 'dart-memory', super.random}); @@ -34,7 +35,7 @@ final class InMemoryFileSystem extends BaseVirtualFileSystem { final create = flags & SqlFlag.SQLITE_OPEN_CREATE; if (create != 0) { - fileData[pathStr] = Uint8List(0); + fileData[pathStr] = GrowingByteBuffer(); } else { throw VfsException(SqlError.SQLITE_CANTOPEN); } @@ -69,7 +70,7 @@ class _InMemoryFile extends BaseVfsFile { if (file == null || file.length <= offset) return 0; final available = min(buffer.length, file.length - offset); - buffer.setRange(0, available, file, offset); + buffer.setRange(0, available, file.toUint8List(), offset); return available; } @@ -102,12 +103,12 @@ class _InMemoryFile extends BaseVfsFile { void xTruncate(int size) { final file = vfs.fileData[path]; - final result = Uint8List(size); - if (file != null) { - result.setRange(0, min(size, file.length), file); + if (file == null) { + vfs.fileData[path] = GrowingByteBuffer(); + // TODO: grow? + } else { + file.truncate(size); } - - vfs.fileData[path] = result; } @override @@ -117,19 +118,12 @@ class _InMemoryFile extends BaseVfsFile { @override void xWrite(Uint8List buffer, int fileOffset) { - final file = vfs.fileData[path] ?? Uint8List(0); - final increasedSize = fileOffset + buffer.length - file.length; - - if (increasedSize <= 0) { - // Can write directy - file.setRange(fileOffset, fileOffset + buffer.length, buffer); - } else { - // We need to grow the file first - final newFile = Uint8List(file.length + increasedSize) - ..setAll(0, file) - ..setAll(fileOffset, buffer); + var file = vfs.fileData[path]; - vfs.fileData[path] = newFile; + if (file == null) { + file = GrowingByteBuffer(); + vfs.fileData[path] = file; } + file.write(buffer, fileOffset); } } From 2b1a3b95b56bc75fd2a7682374f712b8a83c36cc Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 18 Oct 2024 09:36:47 +0200 Subject: [PATCH 2/2] Cleanup. --- ...rowing_buffer.dart => dynamic_buffer.dart} | 20 ++++++++++++------- sqlite3/lib/src/wasm/vfs/indexed_db.dart | 6 +++--- sqlite3/lib/src/wasm/vfs/memory.dart | 12 +++++------ 3 files changed, 22 insertions(+), 16 deletions(-) rename sqlite3/lib/src/wasm/vfs/{growing_buffer.dart => dynamic_buffer.dart} (83%) diff --git a/sqlite3/lib/src/wasm/vfs/growing_buffer.dart b/sqlite3/lib/src/wasm/vfs/dynamic_buffer.dart similarity index 83% rename from sqlite3/lib/src/wasm/vfs/growing_buffer.dart rename to sqlite3/lib/src/wasm/vfs/dynamic_buffer.dart index b6b6e853..c970edaa 100644 --- a/sqlite3/lib/src/wasm/vfs/growing_buffer.dart +++ b/sqlite3/lib/src/wasm/vfs/dynamic_buffer.dart @@ -2,16 +2,20 @@ import 'dart:typed_data'; /// A utility class that manages a growing byte buffer. /// It dynamically increases its capacity in factors of 2 when needed. -class GrowingByteBuffer { +class DynamicBuffer { Uint8List _buffer; int _capacity; int _length = 0; - /// Creates a [GrowingByteBuffer] with an optional initial capacity. + /// Creates a [DynamicBuffer] with an optional initial capacity. /// The default initial capacity is 1024 bytes. - GrowingByteBuffer([int initialCapacity = 1024]) + DynamicBuffer([int initialCapacity = 1024]) : _capacity = initialCapacity, - _buffer = Uint8List(initialCapacity); + _buffer = Uint8List(initialCapacity) { + if (initialCapacity < 1) { + throw ArgumentError("initialCapacity must be positive"); + } + } /// Adds [data] to the buffer, expanding its capacity if necessary. void add(Uint8List data) { @@ -26,7 +30,7 @@ class GrowingByteBuffer { if (offset < 0) { throw ArgumentError("Offset must be non-negative"); } - int endPosition = offset + data.length; + final endPosition = offset + data.length; _ensureCapacity(endPosition); _buffer.setRange(offset, endPosition, data); if (endPosition > _length) { @@ -35,6 +39,9 @@ class GrowingByteBuffer { } /// Truncates the buffer to a specific [newSize]. + /// + /// No memory is freed when using [truncate]. + /// /// If [newSize] is less than the current length, the buffer is truncated. /// If [newSize] is greater than the current length, the buffer length is extended with zeros. void truncate(int newSize) { @@ -49,7 +56,7 @@ class GrowingByteBuffer { _length = newSize; } - /// Returns a [Uint8List] containing the data up to the current length. + /// Returns a [Uint8List] view containing the data up to the current length. Uint8List toUint8List() { return Uint8List.view(_buffer.buffer, 0, _length); } @@ -63,7 +70,6 @@ class GrowingByteBuffer { newCapacity *= 2; } // Allocate a new buffer and copy existing data. - print('resizing to $newCapacity'); Uint8List newBuffer = Uint8List(newCapacity); newBuffer.setRange(0, _length, _buffer); _buffer = newBuffer; diff --git a/sqlite3/lib/src/wasm/vfs/indexed_db.dart b/sqlite3/lib/src/wasm/vfs/indexed_db.dart index 7577a0f4..02d54639 100644 --- a/sqlite3/lib/src/wasm/vfs/indexed_db.dart +++ b/sqlite3/lib/src/wasm/vfs/indexed_db.dart @@ -8,7 +8,7 @@ import 'dart:math'; import 'dart:typed_data'; import 'package:meta/meta.dart'; -import 'package:sqlite3/src/wasm/vfs/growing_buffer.dart'; +import 'package:sqlite3/src/wasm/vfs/dynamic_buffer.dart'; import 'package:web/web.dart' as web; import '../../constants.dart'; @@ -534,7 +534,7 @@ final class IndexedDbFileSystem extends BaseVirtualFileSystem { final name = entry.key; final fileId = entry.value; - final buffer = GrowingByteBuffer(); + final buffer = DynamicBuffer(); final data = await _asynchronous.readFully(fileId); buffer.add(data); @@ -647,7 +647,7 @@ class _IndexedDbFile implements VirtualFileSystemFile { void xWrite(Uint8List buffer, int fileOffset) { vfs._checkClosed(); - final previousContent = vfs._memory.fileData[path] ?? GrowingByteBuffer(); + final previousContent = vfs._memory.fileData[path] ?? DynamicBuffer(); memoryFile.xWrite(buffer, fileOffset); if (!vfs._inMemoryOnlyFiles.contains(path)) { diff --git a/sqlite3/lib/src/wasm/vfs/memory.dart b/sqlite3/lib/src/wasm/vfs/memory.dart index 6ea1c27e..0b5225cb 100644 --- a/sqlite3/lib/src/wasm/vfs/memory.dart +++ b/sqlite3/lib/src/wasm/vfs/memory.dart @@ -2,14 +2,14 @@ import 'dart:math'; import 'dart:typed_data'; import 'package:path/path.dart' as p; -import 'growing_buffer.dart'; +import 'dynamic_buffer.dart'; import '../../constants.dart'; import '../../vfs.dart'; import 'utils.dart'; final class InMemoryFileSystem extends BaseVirtualFileSystem { - final Map fileData = {}; + final Map fileData = {}; InMemoryFileSystem({super.name = 'dart-memory', super.random}); @@ -35,7 +35,7 @@ final class InMemoryFileSystem extends BaseVirtualFileSystem { final create = flags & SqlFlag.SQLITE_OPEN_CREATE; if (create != 0) { - fileData[pathStr] = GrowingByteBuffer(); + fileData[pathStr] = DynamicBuffer(); } else { throw VfsException(SqlError.SQLITE_CANTOPEN); } @@ -104,8 +104,8 @@ class _InMemoryFile extends BaseVfsFile { final file = vfs.fileData[path]; if (file == null) { - vfs.fileData[path] = GrowingByteBuffer(); - // TODO: grow? + vfs.fileData[path] = DynamicBuffer(); + vfs.fileData[path]!.truncate(size); } else { file.truncate(size); } @@ -121,7 +121,7 @@ class _InMemoryFile extends BaseVfsFile { var file = vfs.fileData[path]; if (file == null) { - file = GrowingByteBuffer(); + file = DynamicBuffer(); vfs.fileData[path] = file; } file.write(buffer, fileOffset);