Package maintenance, prepare encryption and dart2wasm support

melos.yaml

@@ -41,7 +41,7 @@ scripts:
 
   test:
     description: Run tests in a specific package.
-    run: dart test -p chrome,vm
+    run: dart test -p chrome,vm --compilers dart2js,dart2wasm
     exec:
       concurrency: 1
     packageFilters:

packages/drift_sqlite_async/lib/drift_sqlite_async.dart

@@ -1,4 +1,2 @@
-library drift_sqlite_async;
-
 export './src/connection.dart';
 export './src/executor.dart';

packages/drift_sqlite_async/lib/src/executor.dart

@@ -1,7 +1,7 @@
 import 'dart:async';
 
 import 'package:drift/backends.dart';
-import 'package:drift/src/runtime/query_builder/query_builder.dart';
+import 'package:drift/drift.dart';
 import 'package:sqlite_async/sqlite3_common.dart';
 import 'package:sqlite_async/sqlite_async.dart';
 
@@ -19,6 +19,7 @@ class _SqliteAsyncDelegate extends _SqliteAsyncQueryDelegate
 
   _SqliteAsyncDelegate(this.db) : super(db, db.writeLock);
 
+  @override
   bool isInTransaction = false; // unused
 
   @override

packages/drift_sqlite_async/pubspec.yaml

@@ -16,10 +16,12 @@ environment:
 dependencies:
   drift: ">=2.19.0 <3.0.0"
   sqlite_async: ^0.11.0
+
 dev_dependencies:
   build_runner: ^2.4.8
   drift_dev: ">=2.19.0 <3.0.0"
   glob: ^2.1.2
+  lints: ^5.1.1
   sqlite3: ^2.4.0
   test: ^1.25.2
   test_api: ^0.7.0

packages/drift_sqlite_async/test/basic_test.dart

@@ -1,5 +1,7 @@
 // TODO
 @TestOn('!browser')
+library;
+
 import 'dart:async';
 
 import 'package:drift/drift.dart';

packages/drift_sqlite_async/test/db_test.dart

@@ -1,5 +1,7 @@
 // TODO
 @TestOn('!browser')
+library;
+
 import 'package:drift/drift.dart';
 import 'package:sqlite_async/sqlite_async.dart';
 import 'package:test/test.dart';

packages/drift_sqlite_async/test/generated/database.dart

@@ -21,7 +21,7 @@ class TodoDatabase extends _$TodoDatabase {
 }
 
 class TodosMigrationDatabase extends TodoDatabase {
-  TodosMigrationDatabase(SqliteConnection db) : super(db);
+  TodosMigrationDatabase(super.db);
 
   @override
   MigrationStrategy get migration {

packages/drift_sqlite_async/test/migration_test.dart

@@ -1,4 +1,6 @@
 @TestOn('!browser')
+library;
+
 import 'package:sqlite_async/sqlite_async.dart';
 import 'package:test/test.dart';
 

packages/sqlite_async/analysis_options.yaml

@@ -0,0 +1 @@
+include: package:lints/recommended.yaml

packages/sqlite_async/lib/src/web/database.dart

@@ -1,8 +1,10 @@
 import 'dart:async';
 import 'dart:js_interop';
+import 'dart:js_interop_unsafe';
 
 import 'package:sqlite3/common.dart';
 import 'package:sqlite3_web/sqlite3_web.dart';
+import 'package:sqlite3_web/protocol_utils.dart' as proto;
 import 'package:sqlite_async/sqlite_async.dart';
 import 'package:sqlite_async/src/utils/shared_utils.dart';
 import 'package:sqlite_async/src/web/database/broadcast_updates.dart';
@@ -256,9 +258,15 @@ class _ExclusiveTransactionContext extends _ExclusiveContext {
     // JavaScript object. This is the converted into a Dart ResultSet.
     return await wrapSqliteException(() async {
       var res = await _database._database.customRequest(CustomDatabaseMessage(
-          CustomDatabaseMessageKind.executeInTransaction, sql, parameters));
-      var result =
-          Map<String, dynamic>.from((res as JSObject).dartify() as Map);
+              CustomDatabaseMessageKind.executeInTransaction, sql, parameters))
+          as JSObject;
+
+      if (res.has('format') && (res['format'] as JSNumber).toDartInt == 2) {
+        // Newer workers use a serialization format more efficient than dartify().
+        return proto.deserializeResultSet(res['r'] as JSObject);
+      }
+
+      var result = Map<String, dynamic>.from(res.dartify() as Map);
       final columnNames = [
         for (final entry in result['columnNames']) entry as String
       ];

packages/sqlite_async/lib/src/web/protocol.dart

@@ -3,6 +3,7 @@
 library;
 
 import 'dart:js_interop';
+import 'package:sqlite3_web/protocol_utils.dart' as proto;
 
 enum CustomDatabaseMessageKind {
   requestSharedLock,
@@ -19,15 +20,24 @@ extension type CustomDatabaseMessage._raw(JSObject _) implements JSObject {
     required JSString rawKind,
     JSString rawSql,
     JSArray rawParameters,
+    JSArrayBuffer typeInfo,
   });
 
   factory CustomDatabaseMessage(CustomDatabaseMessageKind kind,
       [String? sql, List<Object?> parameters = const []]) {
-    final rawSql = sql?.toJS ?? ''.toJS;
-    final rawParameters =
-        <JSAny?>[for (final parameter in parameters) parameter.jsify()].toJS;
+    final rawSql = (sql ?? '').toJS;
+    // Serializing parameters this way is backwards-compatible with dartify()
+    // on the other end, but a bit more efficient while also suppporting sound
+    // communcation between dart2js workers and dart2wasm clients.
+    // Older workers ignore the typeInfo, but that's not a problem.
+    final (rawParameters, typeInfo) = proto.serializeParameters(parameters);
+
     return CustomDatabaseMessage._(
-        rawKind: kind.name.toJS, rawSql: rawSql, rawParameters: rawParameters);
+      rawKind: kind.name.toJS,
+      rawSql: rawSql,
+      rawParameters: rawParameters,
+      typeInfo: typeInfo,
+    );
   }
 
   external JSString get rawKind;
@@ -36,6 +46,9 @@ extension type CustomDatabaseMessage._raw(JSObject _) implements JSObject {
 
   external JSArray get rawParameters;
 
+  /// Not set in earlier versions of this package.
+  external JSArrayBuffer? get typeInfo;
+
   CustomDatabaseMessageKind get kind {
     return CustomDatabaseMessageKind.values.byName(rawKind.toDart);
   }

packages/sqlite_async/lib/src/web/web_sqlite_open_factory.dart

@@ -15,7 +15,7 @@ Map<String, FutureOr<WebSqlite>> webSQLiteImplementations = {};
 /// Web implementation of [AbstractDefaultSqliteOpenFactory]
 class DefaultSqliteOpenFactory
     extends AbstractDefaultSqliteOpenFactory<CommonDatabase>
-    implements WebSqliteOpenFactory {
+    with WebSqliteOpenFactory {
   late final Future<WebSqlite> _initialized = Future.sync(() {
     final cacheKey = sqliteOptions.webSqliteOptions.wasmUri +
         sqliteOptions.webSqliteOptions.workerUri;
@@ -45,9 +45,8 @@ class DefaultSqliteOpenFactory
     );
   }
 
-  @override
-
   /// This is currently not supported on web
+  @override
   CommonDatabase openDB(SqliteOpenOptions options) {
     throw UnimplementedError(
         'Direct access to CommonDatabase is not available on web.');
@@ -61,7 +60,7 @@ class DefaultSqliteOpenFactory
   /// Due to being asynchronous, the under laying CommonDatabase is not accessible
   Future<SqliteConnection> openConnection(SqliteOpenOptions options) async {
     final workers = await _initialized;
-    final connection = await workers.connectToRecommended(path);
+    final connection = await connectToWorker(workers, path);
 
     // When the database is accessed through a shared worker, we implement
     // mutexes over custom messages sent through the shared worker. In other

packages/sqlite_async/lib/src/web/worker/throttled_common_database.dart

@@ -107,6 +107,7 @@ class ThrottledCommonDatabase extends CommonDatabase {
   @override
   VoidPredicate? get commitFilter => _db.commitFilter;
 
+  @override
   set commitFilter(VoidPredicate? filter) => _db.commitFilter = filter;
 
   @override

packages/sqlite_async/lib/src/web/worker/worker_utils.dart

@@ -1,8 +1,12 @@
 import 'dart:js_interop';
+import 'dart:js_interop_unsafe';
 
+import 'package:meta/meta.dart';
 import 'package:mutex/mutex.dart';
 import 'package:sqlite3/wasm.dart';
 import 'package:sqlite3_web/sqlite3_web.dart';
+import 'package:sqlite3_web/protocol_utils.dart' as proto;
+
 import 'throttled_common_database.dart';
 
 import '../protocol.dart';
@@ -12,9 +16,9 @@ import '../protocol.dart';
 /// can be extended to perform custom requests.
 base class AsyncSqliteController extends DatabaseController {
   @override
-  Future<WorkerDatabase> openDatabase(
-      WasmSqlite3 sqlite3, String path, String vfs) async {
-    final db = sqlite3.open(path, vfs: vfs);
+  Future<WorkerDatabase> openDatabase(WasmSqlite3 sqlite3, String path,
+      String vfs, JSAny? additionalData) async {
+    final db = openUnderlying(sqlite3, path, vfs, additionalData);
 
     // Register any custom functions here if needed
 
@@ -23,6 +27,18 @@ base class AsyncSqliteController extends DatabaseController {
     return AsyncSqliteDatabase(database: throttled);
   }
 
+  /// Opens a database with the `sqlite3` package that will be wrapped in a
+  /// [ThrottledCommonDatabase] for [openDatabase].
+  @visibleForOverriding
+  CommonDatabase openUnderlying(
+    WasmSqlite3 sqlite3,
+    String path,
+    String vfs,
+    JSAny? additionalData,
+  ) {
+    return sqlite3.open(path, vfs: vfs);
+  }
+
   @override
   Future<JSAny?> handleCustomRequest(
       ClientConnection connection, JSAny? request) {
@@ -61,25 +77,32 @@ class AsyncSqliteDatabase extends WorkerDatabase {
         return database.autocommit.toJS;
       case CustomDatabaseMessageKind.executeInTransaction:
         final sql = message.rawSql.toDart;
-        final parameters = [
-          for (final raw in (message.rawParameters).toDart) raw.dartify()
-        ];
+        final hasTypeInfo = message.typeInfo.isDefinedAndNotNull;
+        final parameters = proto.deserializeParameters(
+            message.rawParameters, message.typeInfo);
         if (database.autocommit) {
           throw SqliteException(0,
               "Transaction rolled back by earlier statement. Cannot execute: $sql");
         }
-        var res = database.select(sql, parameters);
 
-        var dartMap = resultSetToMap(res);
-
-        var jsObject = dartMap.jsify();
+        var res = database.select(sql, parameters);
+        if (hasTypeInfo) {
+          // If the client is sending a request that has parameters with type
+          // information, it will also support a newer serialization format for
+          // result sets.
+          return JSObject()
+            ..['format'] = 2.toJS
+            ..['r'] = proto.serializeResultSet(res);
+        } else {
+          var dartMap = resultSetToMap(res);
+          var jsObject = dartMap.jsify();
+          return jsObject;
+        }
 
-        return jsObject;
       case CustomDatabaseMessageKind.executeBatchInTransaction:
         final sql = message.rawSql.toDart;
-        final parameters = [
-          for (final raw in (message.rawParameters).toDart) raw.dartify()
-        ];
+        final parameters = proto.deserializeParameters(
+            message.rawParameters, message.typeInfo);
         if (database.autocommit) {
           throw SqliteException(0,
               "Transaction rolled back by earlier statement. Cannot execute: $sql");

packages/sqlite_async/lib/web.dart

@@ -2,7 +2,7 @@
 ///
 /// These expose methods allowing database instances to be shared across web
 /// workers.
-library sqlite_async.web;
+library;
 
 import 'package:sqlite3_web/sqlite3_web.dart';
 import 'package:web/web.dart';
@@ -31,15 +31,29 @@ typedef WebDatabaseEndpoint = ({
 ///
 /// The [DefaultSqliteOpenFactory] class implements this interface only when
 /// compiling for the web.
-abstract interface class WebSqliteOpenFactory
+abstract mixin class WebSqliteOpenFactory
     implements SqliteOpenFactory<CommonDatabase> {
   /// Opens a [WebSqlite] instance for the given [options].
   ///
   /// This method can be overriden in scenarios where the way [WebSqlite] is
   /// opened needs to be customized. Implementers should be aware that the
   /// result of this method is cached and will be re-used by the open factory
   /// when provided with the same [options] again.
-  Future<WebSqlite> openWebSqlite(WebSqliteOptions options);
+  Future<WebSqlite> openWebSqlite(WebSqliteOptions options) async {
+    return WebSqlite.open(
+      worker: Uri.parse(options.workerUri),
+      wasmModule: Uri.parse(options.wasmUri),
+    );
+  }
+
+  /// Uses [WebSqlite] to connects to the recommended database setup for [name].
+  ///
+  /// This typically just calls [WebSqlite.connectToRecommended], but subclasses
+  /// can customize the behavior where needed.
+  Future<ConnectToRecommendedResult> connectToWorker(
+      WebSqlite sqlite, String name) {
+    return sqlite.connectToRecommended(name);
+  }
 }
 
 /// A [SqliteConnection] interface implemented by opened connections when
@@ -91,6 +105,7 @@ abstract class WebSqliteConnection implements SqliteConnection {
   /// This only has an effect when IndexedDB storage is used.
   ///
   /// See [flush] for details.
+  @override
   Future<T> writeLock<T>(Future<T> Function(SqliteWriteContext tx) callback,
       {Duration? lockTimeout, String? debugContext, bool? flush});
 
@@ -101,6 +116,7 @@ abstract class WebSqliteConnection implements SqliteConnection {
   /// This only has an effect when IndexedDB storage is used.
   ///
   /// See [flush] for details.
+  @override
   Future<T> writeTransaction<T>(
       Future<T> Function(SqliteWriteContext tx) callback,
       {Duration? lockTimeout,

packages/sqlite_async/pubspec.yaml

@@ -13,16 +13,15 @@ topics:
 
 dependencies:
   sqlite3: ^2.7.2
-  sqlite3_web: ^0.2.2
+  sqlite3_web: ^0.3.0
   async: ^2.10.0
   collection: ^1.17.0
   mutex: ^3.1.0
   meta: ^1.10.0
   web: ^1.0.0
 
 dev_dependencies:
-  dcli: ^4.0.0
-  lints: ^3.0.0
+  lints: ^5.1.1
   test: ^1.21.0
   test_api: ^0.7.0
   glob: ^2.1.1
@@ -31,6 +30,11 @@ dev_dependencies:
   shelf_static: ^1.1.2
   stream_channel: ^2.1.2
   path: ^1.9.0
+  test_descriptor: ^2.0.2
+
+dependency_overrides:
+  sqlite3_web:
+    path: /Users/simon/src/sqlite3.dart/sqlite3_web
 
 platforms:
   android: