improve isolate runner, deserialization

Author: frank06

example/lib/main.dart

@@ -28,7 +28,8 @@ void main() async {
 
     print('Using temporary directory: ${_dir.path}');
 
-    await container.read(initializeWith(adapterProviders).future);
+    container.read(adapterProviders.notifier).state = adapterProvidersMap;
+    await container.read(initializeAdapters.future);
     container.users.logLevel = 2;
     container.tasks.logLevel = 2;
 

example/lib/main.data.dart

@@ -256,13 +256,13 @@ mixin _\$${className}Adapter on Adapter<$className> {
   Map<String, RelationshipMeta> get relationshipMetas => _k${className}RelationshipMetas;
 
   @override
-  $className deserialize(map, {String? key}) {
+  $className deserializeLocal(map, {String? key}) {
     map = transformDeserialize(map);
     return internalWrapStopInit(() => $fromJson, key: key);
   }
 
   @override
-  Map<String, dynamic> serialize(model, {bool withRelationships = true}) {
+  Map<String, dynamic> serializeLocal(model, {bool withRelationships = true}) {
     final map = $toJson;
     return transformSerialize(map, withRelationships: withRelationships);
   }

example/lib/models/task.g.dart

@@ -86,17 +86,9 @@ class DataExtensionBuilder implements Builder {
 
     // imports
 
-    final isFlutter = await isDependency('flutter', b);
-    final hasPathProvider = await isDependency('path_provider', b);
     final hasFlutterRiverpod = await isDependency('flutter_riverpod', b) ||
         await isDependency('hooks_riverpod', b);
 
-    final flutterFoundationImport = isFlutter
-        ? "import 'package:flutter/foundation.dart' show kIsWeb;"
-        : '';
-    final pathProviderImport = hasPathProvider
-        ? "import 'package:path_provider/path_provider.dart';"
-        : '';
     final riverpodFlutterImport = hasFlutterRiverpod
         ? "import 'package:flutter_riverpod/flutter_riverpod.dart';"
         : '';
@@ -130,13 +122,11 @@ ${classes.map((clazz) => '  Adapter<${clazz['className']}> get ${clazz['classNam
 // ignore_for_file: directives_ordering, top_level_function_literal_block, depend_on_referenced_packages
 
 import 'package:flutter_data/flutter_data.dart';
-$flutterFoundationImport
-$pathProviderImport
 $riverpodFlutterImport
 
 $modelImports
 
-final adapterProviders = <String, Provider<Adapter<DataModelMixin>>>{
+final adapterProvidersMap = <String, Provider<Adapter<DataModelMixin>>>{
   ${classes.map((clazz) => '\'' + clazz['type']! + '\': ' + clazz['classNameLower']! + 'AdapterProvider').join(',\n')}
 };
 

example/lib/models/user.g.dart

@@ -6,7 +6,7 @@ part of flutter_data;
 ///
 ///  - Remote methods such as [_RemoteAdapter.findAll] or [_RemoteAdapter.save]
 ///  - Configuration methods and getters like [_RemoteAdapter.baseUrl] or [_RemoteAdapter.urlForFindAll]
-///  - Serialization methods like [_SerializationAdapter.serializeAsync]
+///  - Serialization methods like [_SerializationAdapter.serialize]
 ///  - Watch methods such as [_WatchAdapter.watchOneNotifier]
 ///  - Access to the [_BaseAdapter.core] for subclasses or mixins
 ///
@@ -45,6 +45,8 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
   @visibleForTesting
   final LocalStorage storage;
 
+  @protected
+  @visibleForTesting
   Database get db => storage.db;
 
   bool _stopInitialization = false;
@@ -122,7 +124,7 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
   /// Returns all models of type [T] in local storage.
   List<T> findAllLocal() {
     final result = db.select('SELECT key, data FROM $internalType');
-    return _deserializeFromResult(result);
+    return deserializeFromResult(result);
   }
 
   /// Finds many models of type [T] by [keys] in local storage.
@@ -135,13 +137,14 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
     final result = db.select(
         'SELECT key, data FROM $internalType WHERE key IN (${intKeys.map((_) => '?').join(', ')})',
         intKeys);
-    return _deserializeFromResult(result);
+    return deserializeFromResult(result);
   }
 
-  List<T> _deserializeFromResult(ResultSet result) {
+  @protected
+  List<T> deserializeFromResult(ResultSet result) {
     return result.map((r) {
       final map = Map<String, dynamic>.from(jsonDecode(r['data']));
-      return deserialize(map,
+      return deserializeLocal(map,
           key: r['key'].toString().typifyWith(internalType));
     }).toList();
   }
@@ -173,7 +176,7 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
       throw Exception("Model must be initialized:\n\n$model");
     }
     final key = model._key!.detypifyKey()!;
-    final map = serialize(model, withRelationships: false);
+    final map = serializeLocal(model, withRelationships: false);
     final data = jsonEncode(map);
     db.execute(
         'REPLACE INTO $internalType (key, data) VALUES (?, ?)', [key, data]);
@@ -183,35 +186,43 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
     return model;
   }
 
-  Future<R> _runInIsolate<R>(
-      FutureOr<R> fn(ProviderContainer container)) async {
-    final ppath = Directory(storage.path).parent.path;
-    final ip = _internalProviders!;
+  @protected
+  Future<R> runInIsolate<R>(FutureOr<R> fn(Adapter adapter)) async {
+    final _path = Directory(storage.path).parent.path;
+    final _internalProviders = ref.read(adapterProviders)!;
+    final _internalType = internalType;
 
     return await Isolate.run(() async {
       late final ProviderContainer container;
       try {
         container = ProviderContainer(
           overrides: [
             localStorageProvider.overrideWith(
-              (ref) => LocalStorage(baseDirFn: () => ppath),
+              (ref) => LocalStorage(baseDirFn: () => _path),
             ),
           ],
         );
 
-        await container.read(initializeWith(ip).future);
+        // TODO improve initializer API
+        // set providers from outer context and start initialization
+        container.read(adapterProviders.notifier).state = _internalProviders;
+        await container.read(initializeAdapters.future);
 
-        return fn(container);
+        final adapter = _internalProviders[_internalType]!;
+        return fn(container.read(adapter));
       } finally {
-        container.read(localStorageProvider).db.dispose();
+        for (final provider in container.read(adapterProviders)!.values) {
+          container.read(provider).dispose();
+        }
+        container.read(localStorageProvider).dispose();
       }
     });
   }
 
   Future<void> saveManyLocal(Iterable<DataModelMixin> models,
       {bool notify = true}) async {
-    final savedKeys = await _runInIsolate((container) async {
-      final db = container.read(localStorageProvider).db;
+    final savedKeys = await runInIsolate((adapter) async {
+      final db = adapter.db;
       final savedKeys = <String>[];
 
       final grouped = models.groupSetsBy((e) => e._adapter);
@@ -222,7 +233,7 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
             'REPLACE INTO ${adapter.internalType} (key, data) VALUES (?, ?) RETURNING key;');
         for (final model in e.value) {
           final key = model._key!.detypifyKey();
-          final map = adapter.serialize(model, withRelationships: false);
+          final map = adapter.serializeLocal(model, withRelationships: false);
           final data = jsonEncode(map);
           final result = ps.select([key, data]);
           savedKeys.add(
@@ -365,13 +376,6 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
       return model;
     }
 
-    // // (before -> after remote save)
-    // // (1) if noid -> noid => `key` is the key we want to keep
-    // // (2) if id -> noid => use autogenerated key (`key` should be the previous (derived))
-    // // so we can migrate rels
-    // // (3) if noid -> id => use derived key (`key` should be the previous (autogen'd))
-    // // so we can migrate rels
-
     if (model._key == null) {
       model._key = key ?? core.getKeyForId(internalType, model.id);
       if (model._key != key && key != null) {
@@ -411,9 +415,9 @@ abstract class _BaseAdapter<T extends DataModelMixin<T>> with _Lifecycle {
     }
   }
 
-  Map<String, dynamic> serialize(T model, {bool withRelationships = true});
+  Map<String, dynamic> serializeLocal(T model, {bool withRelationships = true});
 
-  T deserialize(Map<String, dynamic> map, {String? key});
+  T deserializeLocal(Map<String, dynamic> map, {String? key});
 
   Map<String, RelationshipMeta> get relationshipMetas;
 

lib/builders/adapter_builder.dart

@@ -210,7 +210,7 @@ mixin _RemoteAdapter<T extends DataModelMixin<T>> on _SerializationAdapter<T> {
       return model;
     }
 
-    final serialized = await serializeAsync(model);
+    final serialized = await serialize(model);
     final body = json.encode(serialized);
 
     final uri = baseUrl.asUri / urlForSave(model.id, params) & params;
@@ -499,7 +499,7 @@ mixin _RemoteAdapter<T extends DataModelMixin<T>> on _SerializationAdapter<T> {
       }
 
       print('1');
-      final data = await deserializeAsync(body as Map<String, dynamic>,
+      final data = await deserialize(body as Map<String, dynamic>,
           key: label.model!._key);
       final model = data.model!;
 
@@ -533,7 +533,7 @@ mixin _RemoteAdapter<T extends DataModelMixin<T>> on _SerializationAdapter<T> {
     }
 
     // TODO test: not properly deserializing findAll with relationship references (see example app)
-    final deserialized = await deserializeAsync(body);
+    final deserialized = await deserialize(body);
 
     if (isFindAll || (isCustom && deserialized.model == null)) {
       await _saveDeserialized(deserialized);

lib/builders/data_library_builder.dart

@@ -5,9 +5,9 @@ mixin _SerializationAdapter<T extends DataModelMixin<T>> on _BaseAdapter<T> {
   /// as a [Map<String, dynamic>] ready to be JSON-encoded.
   @protected
   @visibleForTesting
-  Future<Map<String, dynamic>> serializeAsync(T model,
+  Future<Map<String, dynamic>> serialize(T model,
       {bool withRelationships = true}) async {
-    final map = serialize(model, withRelationships: withRelationships);
+    final map = serializeLocal(model, withRelationships: withRelationships);
 
     // essentially converts keys to IDs
     for (final key in relationshipMetas.keys) {
@@ -24,24 +24,30 @@ mixin _SerializationAdapter<T extends DataModelMixin<T>> on _BaseAdapter<T> {
     return map;
   }
 
-  /// Returns a [DeserializedData] object when deserializing a given [data].
-  ///
-  @protected
-  @visibleForTesting
-  Future<DeserializedData<T>> deserializeAsync(Object? data,
-      {String? key}) async {
-    final result = DeserializedData<T>([], included: []);
-
-    Future<Object?> _processIdAndAddInclude(id, Adapter? adapter) async {
-      if (id is Map && adapter != null) {
-        final data = await adapter.deserializeAsync(id as Map<String, dynamic>);
-        result.included
-          ..add(data.model as DataModelMixin<DataModelMixin>)
-          ..addAll(data.included);
-        id = data.model!.id;
+  // Future<DeserializedData<T>> deserialize(Object? data,
+  //     {String? key, async = true}) async {
+  //   final z = await runInIsolate((container) async {
+
+  //     return <DataModelMixin>[];
+  //   });
+  //   return DeserializedData([]);
+  // }
+
+  (List<DataModelMixin>, List<DataModelMixin>) _deserialize(
+      Adapter adapter, Object? data,
+      {String? key}) {
+    final result = (<DataModelMixin>[], <DataModelMixin>[]);
+
+    Object? _processIdAndAddInclude(id, String relType) {
+      final relAdapter = adapter.adapters[relType];
+      if (id is Map && relAdapter != null) {
+        final data =
+            relAdapter._deserialize(relAdapter, id as Map<String, dynamic>);
+        result.$2.addAll([...data.$1, ...data.$2]);
+        id = data.$1.first.id;
       }
-      if (id != null && adapter != null) {
-        return core.getKeyForId(adapter.internalType, id);
+      if (id != null && relAdapter != null) {
+        return relAdapter.core.getKeyForId(relAdapter.internalType, id);
       }
       return null;
     }
@@ -62,7 +68,7 @@ mixin _SerializationAdapter<T extends DataModelMixin<T>> on _BaseAdapter<T> {
         // - process includes
         // - transform ids into keys to pass to the local deserializer
         for (final mapKey in mapIn.keys) {
-          final metadata = relationshipMetas[mapKey];
+          final metadata = adapter.relationshipMetas[mapKey];
 
           if (metadata != null) {
             final relType = metadata.type;
@@ -72,15 +78,17 @@ mixin _SerializationAdapter<T extends DataModelMixin<T>> on _BaseAdapter<T> {
             }
 
             if (metadata.kind == 'BelongsTo') {
-              final key = await _processIdAndAddInclude(
-                  mapIn[mapKey], adapters[relType]!);
+              // NOTE: when _process was async, a sqlite bug
+              // appears when awaiting it (db turns to inMemory and closed)
+              // and leaving everything sync works for now
+              final key = _processIdAndAddInclude(mapIn[mapKey], relType);
               if (key != null) mapOut[mapKey] = key;
             }
 
             if (metadata.kind == 'HasMany') {
               mapOut[mapKey] = [
                 for (final id in (mapIn[mapKey] as Iterable))
-                  await _processIdAndAddInclude(id, adapters[relType]!)
+                  _processIdAndAddInclude(id, relType)
               ].nonNulls;
             }
           } else {
@@ -90,14 +98,26 @@ mixin _SerializationAdapter<T extends DataModelMixin<T>> on _BaseAdapter<T> {
         }
 
         // Force key only if this is a single-model deserialization
-        final model = deserialize(mapOut,
-            key: key != null && data.length == 1 ? key : null);
-        result.models.add(model);
+        final model = adapter.deserializeLocal(mapOut,
+            key: (key != null && data.length == 1) ? key : null);
+        result.$1.add(model as DataModelMixin);
       }
     }
-
     return result;
   }
+
+  /// Returns a [DeserializedData] object when deserializing a given [data].
+  ///
+  @protected
+  @visibleForTesting
+  Future<DeserializedData<T>> deserialize(Object? data,
+      {String? key, async = true}) async {
+    final record = async
+        ? await runInIsolate(
+            (adapter) => adapter._deserialize(adapter, data, key: key))
+        : _deserialize(this as Adapter, data, key: key);
+    return DeserializedData<T>(record.$1.cast<T>(), included: record.$2);
+  }
 }
 
 /// A utility class used to return deserialized main [models] AND [included] models.