diff --git a/demos/benchmarks/ios/Podfile.lock b/demos/benchmarks/ios/Podfile.lock index e9e1beec..c285523d 100644 --- a/demos/benchmarks/ios/Podfile.lock +++ b/demos/benchmarks/ios/Podfile.lock @@ -3,10 +3,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - "sqlite3 (3.46.1+1)": - "sqlite3/common (= 3.46.1+1)" - "sqlite3/common (3.46.1+1)" @@ -50,8 +50,8 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 sqlite3: 0bb0e6389d824e40296f531b858a2a0b71c0d2fb sqlite3_flutter_libs: 9379996d65aa23dcda7585a5b58766cebe0aa042 diff --git a/demos/django-todolist/ios/Podfile.lock b/demos/django-todolist/ios/Podfile.lock index 84722073..257da83e 100644 --- a/demos/django-todolist/ios/Podfile.lock +++ b/demos/django-todolist/ios/Podfile.lock @@ -3,10 +3,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -57,8 +57,8 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/django-todolist/macos/Podfile.lock b/demos/django-todolist/macos/Podfile.lock index 67953800..e129448a 100644 --- a/demos/django-todolist/macos/Podfile.lock +++ b/demos/django-todolist/macos/Podfile.lock @@ -3,10 +3,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.1.6) + - powersync-sqlite-core (0.3.9) - powersync_flutter_libs (0.0.1): - FlutterMacOS - - powersync-sqlite-core (~> 0.1.6) + - powersync-sqlite-core (~> 0.3.8) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -55,13 +55,13 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: FlutterMacOS: 8f6f14fa908a6fb3fba0cd85dbd81ec4b251fb24 - path_provider_foundation: 2b6b4c569c0fb62ec74538f866245ac84301af46 - powersync-sqlite-core: 4c38c8f470f6dca61346789fd5436a6826d1e3dd - powersync_flutter_libs: 1eb1c6790a72afe08e68d4cc489d71ab61da32ee - shared_preferences_foundation: fcdcbc04712aee1108ac7fda236f363274528f78 + path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 + powersync-sqlite-core: 7515d321eb8e3c08b5259cdadb9d19b1876fe13a + powersync_flutter_libs: 330d8309223a121ec15a7334d9edc105053e5f82 + shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 292c3e1bfe89f64e51ea7fc7dab9182a017c8630 - sqlite3_flutter_libs: 5ca46c1a04eddfbeeb5b16566164aa7ad1616e7b + sqlite3_flutter_libs: 03311aede9d32fb2d24e32bebb8cd01c3b2e6239 PODFILE CHECKSUM: 236401fc2c932af29a9fcf0e97baeeb2d750d367 -COCOAPODS: 1.15.2 +COCOAPODS: 1.16.2 diff --git a/demos/django-todolist/macos/Runner/AppDelegate.swift b/demos/django-todolist/macos/Runner/AppDelegate.swift index d53ef643..b3c17614 100644 --- a/demos/django-todolist/macos/Runner/AppDelegate.swift +++ b/demos/django-todolist/macos/Runner/AppDelegate.swift @@ -1,9 +1,13 @@ import Cocoa import FlutterMacOS -@NSApplicationMain +@main class AppDelegate: FlutterAppDelegate { override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { return true } + + override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool { + return true + } } diff --git a/demos/django-todolist/macos/Runner/Configs/AppInfo.xcconfig b/demos/django-todolist/macos/Runner/Configs/AppInfo.xcconfig index 797d44b3..d00b6f29 100644 --- a/demos/django-todolist/macos/Runner/Configs/AppInfo.xcconfig +++ b/demos/django-todolist/macos/Runner/Configs/AppInfo.xcconfig @@ -8,7 +8,7 @@ PRODUCT_NAME = PowerSync Django Demo // The application's bundle identifier -PRODUCT_BUNDLE_IDENTIFIER = co.powersync.demotodolist +PRODUCT_BUNDLE_IDENTIFIER = co.powersync.demotodolist.django // The copyright displayed in application information PRODUCT_COPYRIGHT = Copyright © 2023 Journey Mobile, Inc. All rights reserved. diff --git a/demos/django-todolist/pubspec.lock b/demos/django-todolist/pubspec.lock index 33ee1fe1..1c6f8a3d 100644 --- a/demos/django-todolist/pubspec.lock +++ b/demos/django-todolist/pubspec.lock @@ -310,21 +310,21 @@ packages: path: "../../packages/powersync" relative: true source: path - version: "1.11.2" + version: "1.11.3" powersync_core: dependency: "direct overridden" description: path: "../../packages/powersync_core" relative: true source: path - version: "1.1.2" + version: "1.1.3" powersync_flutter_libs: dependency: "direct overridden" description: path: "../../packages/powersync_flutter_libs" relative: true source: path - version: "0.4.4" + version: "0.4.5" pub_semver: dependency: transitive description: diff --git a/demos/firebase-nodejs-todolist/ios/Podfile.lock b/demos/firebase-nodejs-todolist/ios/Podfile.lock index 236f21aa..2cc24cac 100644 --- a/demos/firebase-nodejs-todolist/ios/Podfile.lock +++ b/demos/firebase-nodejs-todolist/ios/Podfile.lock @@ -58,10 +58,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - RecaptchaInterop (100.0.0) - shared_preferences_foundation (0.0.1): - Flutter @@ -149,8 +149,8 @@ SPEC CHECKSUMS: GoogleUtilities: 26a3abef001b6533cf678d3eb38fd3f614b7872d GTMSessionFetcher: 257ead9ba8e15a2d389d79496e02b9cc5dd0c62c path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 RecaptchaInterop: 7d1a4a01a6b2cb1610a47ef3f85f0c411434cb21 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 diff --git a/demos/supabase-anonymous-auth/ios/Podfile.lock b/demos/supabase-anonymous-auth/ios/Podfile.lock index ac661d6b..633f5e81 100644 --- a/demos/supabase-anonymous-auth/ios/Podfile.lock +++ b/demos/supabase-anonymous-auth/ios/Podfile.lock @@ -5,10 +5,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -68,8 +68,8 @@ SPEC CHECKSUMS: app_links: 3da4c36b46cac3bf24eb897f1a6ce80bda109874 Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/supabase-edge-function-auth/ios/Podfile.lock b/demos/supabase-edge-function-auth/ios/Podfile.lock index ac661d6b..633f5e81 100644 --- a/demos/supabase-edge-function-auth/ios/Podfile.lock +++ b/demos/supabase-edge-function-auth/ios/Podfile.lock @@ -5,10 +5,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -68,8 +68,8 @@ SPEC CHECKSUMS: app_links: 3da4c36b46cac3bf24eb897f1a6ce80bda109874 Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/supabase-simple-chat/ios/Podfile.lock b/demos/supabase-simple-chat/ios/Podfile.lock index a25eeb9d..0534655c 100644 --- a/demos/supabase-simple-chat/ios/Podfile.lock +++ b/demos/supabase-simple-chat/ios/Podfile.lock @@ -5,10 +5,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -68,8 +68,8 @@ SPEC CHECKSUMS: app_links: 3da4c36b46cac3bf24eb897f1a6ce80bda109874 Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/supabase-todolist-drift/ios/Podfile.lock b/demos/supabase-todolist-drift/ios/Podfile.lock index 933d99fc..2548039b 100644 --- a/demos/supabase-todolist-drift/ios/Podfile.lock +++ b/demos/supabase-todolist-drift/ios/Podfile.lock @@ -7,10 +7,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -74,8 +74,8 @@ SPEC CHECKSUMS: camera_avfoundation: 04b44aeb14070126c6529e5ab82cc7c9fca107cf Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/supabase-todolist-optional-sync/ios/Podfile.lock b/demos/supabase-todolist-optional-sync/ios/Podfile.lock index 756d4114..e68081bc 100644 --- a/demos/supabase-todolist-optional-sync/ios/Podfile.lock +++ b/demos/supabase-todolist-optional-sync/ios/Podfile.lock @@ -7,10 +7,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -73,8 +73,8 @@ SPEC CHECKSUMS: camera_avfoundation: 7262a4e34c2e028f6aa5fb523ae74c9b74d3bd76 Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 0bb0e6389d824e40296f531b858a2a0b71c0d2fb sqlite3_flutter_libs: 9379996d65aa23dcda7585a5b58766cebe0aa042 diff --git a/demos/supabase-todolist/README.md b/demos/supabase-todolist/README.md index 542aa14d..55241dde 100644 --- a/demos/supabase-todolist/README.md +++ b/demos/supabase-todolist/README.md @@ -29,6 +29,26 @@ Create a new PowerSync instance, connecting to the database of the Supabase proj Then deploy the following sync rules: +```yaml +bucket_definitions: + user_lists: + priority: 1 + parameters: select id as list_id from lists where owner_id = request.user_id() + data: + - select * from lists where id = bucket.list_id + + user_todos: + parameters: select id as list_id from lists where owner_id = request.user_id() + data: + - select * from todos where list_id = bucket.list_id +``` + +**Note**: These rules showcase [prioritized sync](https://docs.powersync.com/usage/use-case-examples/prioritized-sync), +by syncing a user's lists with a higher priority than the items within a list (todos). This can be +useful to keep the list overview page reactive during a large sync cycle affecting many +rows in the `user_todos` bucket. The two buckets can also be unified into a single one if +priorities are not important (the app will work without changes): + ```yaml bucket_definitions: user_lists: diff --git a/demos/supabase-todolist/ios/Podfile.lock b/demos/supabase-todolist/ios/Podfile.lock index ddc83040..a3ff04f2 100644 --- a/demos/supabase-todolist/ios/Podfile.lock +++ b/demos/supabase-todolist/ios/Podfile.lock @@ -7,10 +7,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS @@ -74,8 +74,8 @@ SPEC CHECKSUMS: camera_avfoundation: 04b44aeb14070126c6529e5ab82cc7c9fca107cf Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 sqlite3_flutter_libs: 3c323550ef3b928bc0aa9513c841e45a7d242832 diff --git a/demos/supabase-todolist/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/demos/supabase-todolist/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index 5e31d3d3..c53e2b31 100644 --- a/demos/supabase-todolist/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/demos/supabase-todolist/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -48,6 +48,7 @@ ignoresPersistentStateOnLaunch = "NO" debugDocumentVersioning = "YES" debugServiceExtension = "internal" + enableGPUValidationMode = "1" allowLocationSimulation = "YES"> diff --git a/demos/supabase-todolist/lib/widgets/lists_page.dart b/demos/supabase-todolist/lib/widgets/lists_page.dart index 142d9e9f..d60cb9f5 100644 --- a/demos/supabase-todolist/lib/widgets/lists_page.dart +++ b/demos/supabase-todolist/lib/widgets/lists_page.dart @@ -1,6 +1,6 @@ -import 'dart:async'; - import 'package:flutter/material.dart'; +import 'package:powersync/powersync.dart'; +import 'package:powersync_flutter_demo/powersync.dart'; import './list_item.dart'; import './list_item_dialog.dart'; @@ -41,61 +41,36 @@ class ListsPage extends StatelessWidget { } } -class ListsWidget extends StatefulWidget { +final class ListsWidget extends StatelessWidget { const ListsWidget({super.key}); - @override - State createState() { - return _ListsWidgetState(); - } -} - -class _ListsWidgetState extends State { - List _data = []; - bool hasSynced = false; - StreamSubscription? _subscription; - StreamSubscription? _syncStatusSubscription; - - _ListsWidgetState(); - - @override - void initState() { - super.initState(); - final stream = TodoList.watchListsWithStats(); - _subscription = stream.listen((data) { - if (!context.mounted) { - return; - } - setState(() { - _data = data; - }); - }); - _syncStatusSubscription = TodoList.watchSyncStatus().listen((status) { - if (!context.mounted) { - return; - } - setState(() { - hasSynced = status.hasSynced ?? false; - }); - }); - } - - @override - void dispose() { - super.dispose(); - _subscription?.cancel(); - _syncStatusSubscription?.cancel(); - } - @override Widget build(BuildContext context) { - return !hasSynced - ? const Text("Busy with sync...") - : ListView( - padding: const EdgeInsets.symmetric(vertical: 8.0), - children: _data.map((list) { - return ListItemWidget(list: list); - }).toList(), + return FutureBuilder( + future: db.waitForFirstSync(priority: _listsPriority), + builder: (context, snapshot) { + if (snapshot.connectionState == ConnectionState.done) { + return StreamBuilder( + stream: TodoList.watchListsWithStats(), + builder: (context, snapshot) { + if (snapshot.data case final todoLists?) { + return ListView( + padding: const EdgeInsets.symmetric(vertical: 8.0), + children: todoLists.map((list) { + return ListItemWidget(list: list); + }).toList(), + ); + } else { + return const CircularProgressIndicator(); + } + }, ); + } else { + return const Text('Busy with sync...'); + } + }, + ); } + + static final _listsPriority = BucketPriority(1); } diff --git a/demos/supabase-todolist/lib/widgets/todo_list_page.dart b/demos/supabase-todolist/lib/widgets/todo_list_page.dart index e36e1867..70dde161 100644 --- a/demos/supabase-todolist/lib/widgets/todo_list_page.dart +++ b/demos/supabase-todolist/lib/widgets/todo_list_page.dart @@ -79,11 +79,20 @@ class TodoListWidgetState extends State { @override Widget build(BuildContext context) { - return ListView( - padding: const EdgeInsets.symmetric(vertical: 8.0), - children: _data.map((todo) { - return TodoItemWidget(todo: todo); - }).toList(), + return StreamBuilder( + stream: TodoList.watchSyncStatus().map((e) => e.hasSynced), + builder: (context, snapshot) { + if (snapshot.data ?? false) { + return const Text('Busy with sync'); + } + + return ListView( + padding: const EdgeInsets.symmetric(vertical: 8.0), + children: _data.map((todo) { + return TodoItemWidget(todo: todo); + }).toList(), + ); + }, ); } } diff --git a/demos/supabase-todolist/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/demos/supabase-todolist/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index 992a778b..943aed19 100644 --- a/demos/supabase-todolist/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/demos/supabase-todolist/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -59,6 +59,7 @@ ignoresPersistentStateOnLaunch = "NO" debugDocumentVersioning = "YES" debugServiceExtension = "internal" + enableGPUValidationMode = "1" allowLocationSimulation = "YES"> diff --git a/demos/supabase-todolist/pubspec.lock b/demos/supabase-todolist/pubspec.lock index c6d85781..969bb7bd 100644 --- a/demos/supabase-todolist/pubspec.lock +++ b/demos/supabase-todolist/pubspec.lock @@ -478,28 +478,28 @@ packages: path: "../../packages/powersync" relative: true source: path - version: "1.11.2" + version: "1.11.3" powersync_attachments_helper: dependency: "direct main" description: path: "../../packages/powersync_attachments_helper" relative: true source: path - version: "0.6.18" + version: "0.6.18+1" powersync_core: dependency: "direct overridden" description: path: "../../packages/powersync_core" relative: true source: path - version: "1.1.2" + version: "1.1.3" powersync_flutter_libs: dependency: "direct overridden" description: path: "../../packages/powersync_flutter_libs" relative: true source: path - version: "0.4.4" + version: "0.4.5" pub_semver: dependency: transitive description: diff --git a/demos/supabase-trello/ios/Podfile.lock b/demos/supabase-trello/ios/Podfile.lock index 51d75873..7c22cdc7 100644 --- a/demos/supabase-trello/ios/Podfile.lock +++ b/demos/supabase-trello/ios/Podfile.lock @@ -41,10 +41,10 @@ PODS: - path_provider_foundation (0.0.1): - Flutter - FlutterMacOS - - powersync-sqlite-core (0.3.10) + - powersync-sqlite-core (0.3.11) - powersync_flutter_libs (0.0.1): - Flutter - - powersync-sqlite-core (~> 0.3.10) + - powersync-sqlite-core (~> 0.3.11) - SDWebImage (5.20.0): - SDWebImage/Core (= 5.20.0) - SDWebImage/Core (5.20.0) @@ -122,8 +122,8 @@ SPEC CHECKSUMS: Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 image_picker_ios: 7fe1ff8e34c1790d6fff70a32484959f563a928a path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564 - powersync-sqlite-core: c7045a44bb1040485ba46b42a3b0acaed424a5cd - powersync_flutter_libs: f33ed290b2813a13809ea4f0e6ae6621f3de8218 + powersync-sqlite-core: 63f9e7adb74044ab7786e4f60e86994d13dcc7a9 + powersync_flutter_libs: 86c1c5214cbeba9d555798de5aece225ad1f5971 SDWebImage: 73c6079366fea25fa4bb9640d5fb58f0893facd8 shared_preferences_foundation: 9e1978ff2562383bd5676f64ec4e9aa8fa06a6f7 sqlite3: 4922312598b67e1825c6a6c821296dcbf6783046 diff --git a/packages/powersync_core/lib/src/bucket_storage.dart b/packages/powersync_core/lib/src/bucket_storage.dart index 9df64f17..d79e4c05 100644 --- a/packages/powersync_core/lib/src/bucket_storage.dart +++ b/packages/powersync_core/lib/src/bucket_storage.dart @@ -55,7 +55,7 @@ class BucketStorage { await _updateBucket2( tx, jsonEncode({ - 'buckets': [b] + 'buckets': [b], })); } // No need to flush - the data is not directly visible to the user either way. @@ -101,9 +101,9 @@ class BucketStorage { return false; } - Future syncLocalDatabase( - Checkpoint checkpoint) async { - final r = await validateChecksums(checkpoint); + Future syncLocalDatabase(Checkpoint checkpoint, + {int? forPriority}) async { + final r = await validateChecksums(checkpoint, priority: forPriority); if (!r.checkpointValid) { for (String b in r.checkpointFailures ?? []) { @@ -111,13 +111,16 @@ class BucketStorage { } return r; } - final bucketNames = [for (final c in checkpoint.checksums) c.bucket]; + final bucketNames = [ + for (final c in checkpoint.checksums) + if (forPriority == null || c.priority <= forPriority) c.bucket + ]; await writeTransaction((tx) async { await tx.execute( "UPDATE ps_buckets SET last_op = ? WHERE name IN (SELECT json_each.value FROM json_each(?))", [checkpoint.lastOpId, jsonEncode(bucketNames)]); - if (checkpoint.writeCheckpoint != null) { + if (forPriority == null && checkpoint.writeCheckpoint != null) { await tx.execute( "UPDATE ps_buckets SET last_op = ? WHERE name = '\$local'", [checkpoint.writeCheckpoint]); @@ -125,7 +128,8 @@ class BucketStorage { // Not flushing here - the flush will happen in the next step }, flush: false); - final valid = await updateObjectsFromBuckets(checkpoint); + final valid = await updateObjectsFromBuckets(checkpoint, + forPartialPriority: forPriority); if (!valid) { return SyncLocalDatabaseResult(ready: false); } @@ -135,11 +139,25 @@ class BucketStorage { return SyncLocalDatabaseResult(ready: true); } - Future updateObjectsFromBuckets(Checkpoint checkpoint) async { + Future updateObjectsFromBuckets(Checkpoint checkpoint, + {int? forPartialPriority}) async { return writeTransaction((tx) async { - await tx.execute( - "INSERT INTO powersync_operations(op, data) VALUES(?, ?)", - ['sync_local', '']); + await tx + .execute("INSERT INTO powersync_operations(op, data) VALUES(?, ?)", [ + 'sync_local', + forPartialPriority != null + ? jsonEncode({ + 'priority': forPartialPriority, + // If we're at a partial checkpoint, we should only publish the + // buckets at the completed priority levels. + 'buckets': [ + for (final desc in checkpoint.checksums) + // Note that higher priorities are encoded as smaller values + if (desc.priority <= forPartialPriority) desc.bucket, + ], + }) + : null, + ]); final rs = await tx.execute('SELECT last_insert_rowid() as result'); final result = rs[0]['result']; if (result == 1) { @@ -154,10 +172,12 @@ class BucketStorage { }, flush: true); } - Future validateChecksums( - Checkpoint checkpoint) async { - final rs = await select("SELECT powersync_validate_checkpoint(?) as result", - [jsonEncode(checkpoint)]); + Future validateChecksums(Checkpoint checkpoint, + {int? priority}) async { + final rs = + await select("SELECT powersync_validate_checkpoint(?) as result", [ + jsonEncode({...checkpoint.toJson(priority: priority)}) + ]); final result = jsonDecode(rs[0]['result'] as String) as Map; if (result['valid'] as bool) { diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 39bf8014..564e8f5a 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -121,27 +121,66 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { Future _updateHasSynced() async { // Query the database to see if any data has been synced. - final result = - await database.get('SELECT powersync_last_synced_at() as synced_at'); - final timestamp = result['synced_at'] as String?; - final hasSynced = timestamp != null; + final result = await database.getAll( + 'SELECT priority, last_synced_at FROM ps_sync_state ORDER BY priority;', + ); + const prioritySentinel = 2147483647; + var hasSynced = false; + DateTime? lastCompleteSync; + final priorityStatusEntries = []; + + DateTime parseDateTime(String sql) { + return DateTime.parse('${sql}Z').toLocal(); + } + + for (final row in result) { + final priority = row.columnAt(0) as int; + final lastSyncedAt = parseDateTime(row.columnAt(1) as String); + + if (priority == prioritySentinel) { + hasSynced = true; + lastCompleteSync = lastSyncedAt; + } else { + priorityStatusEntries.add(( + hasSynced: true, + lastSyncedAt: lastSyncedAt, + priority: BucketPriority(priority) + )); + } + } if (hasSynced != currentStatus.hasSynced) { - final lastSyncedAt = - timestamp == null ? null : DateTime.parse('${timestamp}Z').toLocal(); - final status = - SyncStatus(hasSynced: hasSynced, lastSyncedAt: lastSyncedAt); + final status = SyncStatus( + hasSynced: hasSynced, + lastSyncedAt: lastCompleteSync, + priorityStatusEntries: priorityStatusEntries, + ); setStatus(status); } } - /// Returns a [Future] which will resolve once the first full sync has completed. - Future waitForFirstSync() async { - if (currentStatus.hasSynced ?? false) { + /// Returns a [Future] which will resolve once at least one full sync cycle + /// has completed (meaninng that the first consistent checkpoint has been + /// reached across all buckets). + /// + /// When [priority] is null (the default), this method waits for the first + /// full sync checkpoint to complete. When set to a [BucketPriority] however, + /// it completes once all buckets within that priority (as well as those in + /// higher priorities) have been synchronized at least once. + Future waitForFirstSync({BucketPriority? priority}) async { + bool matches(SyncStatus status) { + if (priority == null) { + return status.hasSynced == true; + } else { + return status.statusForPriority(priority).hasSynced == true; + } + } + + if (matches(currentStatus)) { return; } await for (final result in statusStream) { - if (result.hasSynced ?? false) { + if (matches(result)) { break; } } @@ -187,7 +226,10 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { await disconnect(); // Now we can close the database await database.close(); - await statusStreamController.close(); + + // If there are paused subscriptionso n the status stream, don't delay + // closing the database because of that. + unawaited(statusStreamController.close()); } /// Connect to the PowerSync service, and keep the databases in sync. diff --git a/packages/powersync_core/lib/src/setup_web.dart b/packages/powersync_core/lib/src/setup_web.dart index e4d63d2b..b3ad99c9 100644 --- a/packages/powersync_core/lib/src/setup_web.dart +++ b/packages/powersync_core/lib/src/setup_web.dart @@ -135,7 +135,7 @@ bool coreVersionIsInRange(String tag) { // Sets the range of powersync core version that is compatible with the sqlite3 version // We're a little more selective in the versions chosen here than the range // we're compatible with. - VersionConstraint constraint = VersionConstraint.parse('>=0.3.0 <0.4.0'); + VersionConstraint constraint = VersionConstraint.parse('>=0.3.10 <0.4.0'); List parts = tag.split('-'); String powersyncPart = parts[1]; diff --git a/packages/powersync_core/lib/src/streaming_sync.dart b/packages/powersync_core/lib/src/streaming_sync.dart index 045cc8af..716b38bf 100644 --- a/packages/powersync_core/lib/src/streaming_sync.dart +++ b/packages/powersync_core/lib/src/streaming_sync.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'dart:convert' as convert; +import 'package:collection/collection.dart'; import 'package:http/http.dart' as http; import 'package:powersync_core/src/abort_controller.dart'; import 'package:powersync_core/src/exceptions.dart'; @@ -118,6 +119,7 @@ class StreamingSyncImplementation implements StreamingSync { // Now close the client in all cases not covered above _client.close(); + _statusStreamController.close(); } bool get aborted { @@ -273,58 +275,79 @@ class StreamingSyncImplementation implements StreamingSync { return body['data']['write_checkpoint'] as String; } + void _updateStatusForPriority(SyncPriorityStatus completed) { + // All status entries with a higher priority can be deleted since this + // partial sync includes them. + _updateStatus(priorityStatusEntries: [ + for (final entry in lastStatus.priorityStatusEntries) + if (entry.priority < completed.priority) entry, + completed + ]); + } + /// Update sync status based on any non-null parameters. /// To clear errors, use [_noError] instead of null. - void _updateStatus( - {DateTime? lastSyncedAt, - bool? hasSynced, - bool? connected, - bool? connecting, - bool? downloading, - bool? uploading, - Object? uploadError, - Object? downloadError}) { + void _updateStatus({ + DateTime? lastSyncedAt, + bool? hasSynced, + bool? connected, + bool? connecting, + bool? downloading, + bool? uploading, + Object? uploadError, + Object? downloadError, + List? priorityStatusEntries, + }) { final c = connected ?? lastStatus.connected; var newStatus = SyncStatus( - connected: c, - connecting: !c && (connecting ?? lastStatus.connecting), - lastSyncedAt: lastSyncedAt ?? lastStatus.lastSyncedAt, - hasSynced: hasSynced ?? lastStatus.hasSynced, - downloading: downloading ?? lastStatus.downloading, - uploading: uploading ?? lastStatus.uploading, - uploadError: uploadError == _noError - ? null - : (uploadError ?? lastStatus.uploadError), - downloadError: downloadError == _noError - ? null - : (downloadError ?? lastStatus.downloadError)); - lastStatus = newStatus; - _statusStreamController.add(newStatus); + connected: c, + connecting: !c && (connecting ?? lastStatus.connecting), + lastSyncedAt: lastSyncedAt ?? lastStatus.lastSyncedAt, + hasSynced: hasSynced ?? lastStatus.hasSynced, + downloading: downloading ?? lastStatus.downloading, + uploading: uploading ?? lastStatus.uploading, + uploadError: uploadError == _noError + ? null + : (uploadError ?? lastStatus.uploadError), + downloadError: downloadError == _noError + ? null + : (downloadError ?? lastStatus.downloadError), + priorityStatusEntries: + priorityStatusEntries ?? lastStatus.priorityStatusEntries, + ); + + if (!_statusStreamController.isClosed) { + lastStatus = newStatus; + _statusStreamController.add(newStatus); + } } - Future streamingSyncIteration( - {AbortController? abortController}) async { - adapter.startSession(); + Future<(List, Map)> + _collectLocalBucketState() async { final bucketEntries = await adapter.getBucketStates(); - Map initialBucketStates = {}; + final initialRequests = [ + for (final entry in bucketEntries) BucketRequest(entry.bucket, entry.opId) + ]; + final localDescriptions = { + for (final entry in bucketEntries) entry.bucket: null + }; + + return (initialRequests, localDescriptions); + } - for (final entry in bucketEntries) { - initialBucketStates[entry.bucket] = entry.opId; - } + Future streamingSyncIteration( + {AbortController? abortController}) async { + adapter.startSession(); - final List buckets = []; - for (var entry in initialBucketStates.entries) { - buckets.add(BucketRequest(entry.key, entry.value)); - } + var (bucketRequests, bucketMap) = await _collectLocalBucketState(); Checkpoint? targetCheckpoint; Checkpoint? validatedCheckpoint; Checkpoint? appliedCheckpoint; - var bucketSet = Set.from(initialBucketStates.keys); var requestStream = streamingSyncRequest( - StreamingSyncRequest(buckets, syncParameters, clientId!)); + StreamingSyncRequest(bucketRequests, syncParameters, clientId!)); var merged = addBroadcast(requestStream, _localPingController.stream); @@ -343,13 +366,16 @@ class StreamingSyncImplementation implements StreamingSync { switch (line) { case Checkpoint(): targetCheckpoint = line; - final Set bucketsToDelete = {...bucketSet}; - final Set newBuckets = {}; + final Set bucketsToDelete = {...bucketMap.keys}; + final Map newBuckets = {}; for (final checksum in line.checksums) { - newBuckets.add(checksum.bucket); + newBuckets[checksum.bucket] = ( + name: checksum.bucket, + priority: checksum.priority, + ); bucketsToDelete.remove(checksum.bucket); } - bucketSet = newBuckets; + bucketMap = newBuckets; await adapter.removeBuckets([...bucketsToDelete]); _updateStatus(downloading: true); case StreamingSyncCheckpointComplete(): @@ -365,13 +391,46 @@ class StreamingSyncImplementation implements StreamingSync { } else { appliedCheckpoint = targetCheckpoint; + final now = DateTime.now(); _updateStatus( - downloading: false, - downloadError: _noError, - lastSyncedAt: DateTime.now()); + downloading: false, + downloadError: _noError, + lastSyncedAt: now, + priorityStatusEntries: [ + if (appliedCheckpoint.checksums.isNotEmpty) + ( + hasSynced: true, + lastSyncedAt: now, + priority: maxBy( + appliedCheckpoint.checksums + .map((cs) => BucketPriority(cs.priority)), + (priority) => priority, + compare: BucketPriority.comparator, + )!, + ) + ], + ); } validatedCheckpoint = targetCheckpoint; + case StreamingSyncCheckpointPartiallyComplete(:final bucketPriority): + final result = await adapter.syncLocalDatabase(targetCheckpoint!, + forPriority: bucketPriority); + if (!result.checkpointValid) { + // This means checksums failed. Start again with a new checkpoint. + // TODO: better back-off + // await new Promise((resolve) => setTimeout(resolve, 50)); + return false; + } else if (!result.ready) { + // Checksums valid, but need more data for a consistent checkpoint. + // Continue waiting. + } else { + _updateStatusForPriority(( + priority: BucketPriority(bucketPriority), + lastSyncedAt: DateTime.now(), + hasSynced: true, + )); + } case StreamingSyncCheckpointDiff(): // TODO: It may be faster to just keep track of the diff, instead of // the entire checkpoint @@ -398,7 +457,8 @@ class StreamingSyncImplementation implements StreamingSync { writeCheckpoint: diff.writeCheckpoint); targetCheckpoint = newCheckpoint; - bucketSet = Set.from(newBuckets.keys); + bucketMap = newBuckets.map((name, checksum) => + MapEntry(name, (name: name, priority: checksum.priority))); await adapter.removeBuckets(diff.removedBuckets); adapter.setTargetCheckpoint(targetCheckpoint); case SyncDataBatch(): diff --git a/packages/powersync_core/lib/src/sync_status.dart b/packages/powersync_core/lib/src/sync_status.dart index f04e7300..3d883757 100644 --- a/packages/powersync_core/lib/src/sync_status.dart +++ b/packages/powersync_core/lib/src/sync_status.dart @@ -1,4 +1,6 @@ -class SyncStatus { +import 'package:collection/collection.dart'; + +final class SyncStatus { /// true if currently connected. /// /// This means the PowerSync connection is ready to download, and @@ -38,15 +40,19 @@ class SyncStatus { /// Cleared on the next successful data download. final Object? downloadError; - const SyncStatus( - {this.connected = false, - this.connecting = false, - this.lastSyncedAt, - this.hasSynced, - this.downloading = false, - this.uploading = false, - this.downloadError, - this.uploadError}); + final List priorityStatusEntries; + + const SyncStatus({ + this.connected = false, + this.connecting = false, + this.lastSyncedAt, + this.hasSynced, + this.downloading = false, + this.uploading = false, + this.downloadError, + this.uploadError, + this.priorityStatusEntries = const [], + }); @override bool operator ==(Object other) { @@ -58,7 +64,9 @@ class SyncStatus { other.downloadError == downloadError && other.uploadError == uploadError && other.lastSyncedAt == lastSyncedAt && - other.hasSynced == hasSynced); + other.hasSynced == hasSynced && + _statusEquality.equals( + other.priorityStatusEntries, priorityStatusEntries)); } SyncStatus copyWith({ @@ -70,6 +78,7 @@ class SyncStatus { Object? downloadError, DateTime? lastSyncedAt, bool? hasSynced, + List? priorityStatusEntries, }) { return SyncStatus( connected: connected ?? this.connected, @@ -80,6 +89,8 @@ class SyncStatus { downloadError: downloadError ?? this.downloadError, lastSyncedAt: lastSyncedAt ?? this.lastSyncedAt, hasSynced: hasSynced ?? this.hasSynced, + priorityStatusEntries: + priorityStatusEntries ?? this.priorityStatusEntries, ); } @@ -88,18 +99,88 @@ class SyncStatus { return downloadError ?? uploadError; } + /// Returns information for [lastSyncedAt] and [hasSynced] information at a + /// partial sync priority, or `null` if the status for that priority is + /// unknown. + /// + /// The information returned may be more generic than requested. For instance, + /// a fully-completed sync cycle (as expressed by [lastSyncedAt]) necessarily + /// includes all buckets across all priorities. So, if no further partial + /// checkpoints have been received since that complete sync, + /// [statusForPriority] may return information for that complete sync. + /// Similarly, requesting the sync status for priority `1` may return + /// information extracted from the lower priority `2` since each partial sync + /// in priority `2` necessarily includes a consistent view over data in + /// priority `1`. + SyncPriorityStatus statusForPriority(BucketPriority priority) { + assert(priorityStatusEntries.isSortedByCompare( + (e) => e.priority, BucketPriority.comparator)); + + for (final known in priorityStatusEntries) { + // Lower-priority buckets are synchronized after higher-priority buckets, + // and since priorityStatusEntries is sorted we look for the first entry + // that doesn't have a higher priority. + if (known.priority <= priority) { + return known; + } + } + + // If we have a complete sync, that necessarily includes all priorities. + return ( + priority: priority, + hasSynced: hasSynced, + lastSyncedAt: lastSyncedAt + ); + } + @override int get hashCode { - return Object.hash(connected, downloading, uploading, connecting, - uploadError, downloadError, lastSyncedAt); + return Object.hash( + connected, + downloading, + uploading, + connecting, + uploadError, + downloadError, + lastSyncedAt, + _statusEquality.hash(priorityStatusEntries)); } @override String toString() { return "SyncStatus"; } + + static const _statusEquality = ListEquality(); +} + +/// The priority of a PowerSync bucket. +extension type const BucketPriority._(int priorityNumber) { + static const _highest = 0; + + factory BucketPriority(int i) { + assert(i >= _highest); + return BucketPriority._(i); + } + + bool operator >(BucketPriority other) => comparator(this, other) > 0; + bool operator >=(BucketPriority other) => comparator(this, other) >= 0; + bool operator <(BucketPriority other) => comparator(this, other) < 0; + bool operator <=(BucketPriority other) => comparator(this, other) <= 0; + + /// A [Comparator] instance suitable for comparing [BucketPriority] values. + static int comparator(BucketPriority a, BucketPriority b) => + -a.priorityNumber.compareTo(b.priorityNumber); } +/// Partial information about the synchronization status for buckets within a +/// priority. +typedef SyncPriorityStatus = ({ + BucketPriority priority, + DateTime? lastSyncedAt, + bool? hasSynced, +}); + /// Stats of the local upload queue. class UploadQueueStats { /// Number of records in the upload queue. diff --git a/packages/powersync_core/lib/src/sync_types.dart b/packages/powersync_core/lib/src/sync_types.dart index e5d4ab28..968b53d4 100644 --- a/packages/powersync_core/lib/src/sync_types.dart +++ b/packages/powersync_core/lib/src/sync_types.dart @@ -17,6 +17,9 @@ sealed class StreamingSyncLine { } else if (line.containsKey('checkpoint_complete')) { return StreamingSyncCheckpointComplete.fromJson( line['checkpoint_complete'] as Map); + } else if (line.containsKey('partial_checkpoint_complete')) { + return StreamingSyncCheckpointPartiallyComplete.fromJson( + line['partial_checkpoint_complete'] as Map); } else if (line.containsKey('data')) { return SyncDataBatch([ SyncBucketData.fromJson(line['data'] as Map), @@ -127,19 +130,27 @@ final class Checkpoint extends StreamingSyncLine { .map((b) => BucketChecksum.fromJson(b as Map)) .toList(); - Map toJson() { + Map toJson({int? priority}) { return { 'last_op_id': lastOpId, 'write_checkpoint': writeCheckpoint, 'buckets': checksums - .map((c) => {'bucket': c.bucket, 'checksum': c.checksum}) + .where((c) => priority == null || c.priority <= priority) + .map((c) => { + 'bucket': c.bucket, + 'checksum': c.checksum, + 'priority': c.priority, + }) .toList(growable: false) }; } } -final class BucketChecksum { +typedef BucketDescription = ({String name, int priority}); + +class BucketChecksum { final String bucket; + final int priority; final int checksum; /// Count is informational only @@ -148,12 +159,17 @@ final class BucketChecksum { const BucketChecksum( {required this.bucket, + required this.priority, required this.checksum, this.count, this.lastOpId}); BucketChecksum.fromJson(Map json) : bucket = json['bucket'] as String, + // Use the default priority (3) as a fallback if the server doesn't send + // priorities. This value is arbitrary though, it won't get used since + // servers not sending priorities also won't send partial checkpoints. + priority = json['priority'] as int? ?? 3, checksum = json['checksum'] as int, count = json['count'] as int?, lastOpId = json['last_op_id'] as String?; @@ -196,6 +212,19 @@ final class StreamingSyncCheckpointComplete extends StreamingSyncLine { : lastOpId = json['last_op_id'] as String; } +/// Sent after all the [SyncBucketData] messages for a given priority within a +/// checkpoint have been sent. +final class StreamingSyncCheckpointPartiallyComplete extends StreamingSyncLine { + String lastOpId; + int bucketPriority; + + StreamingSyncCheckpointPartiallyComplete(this.lastOpId, this.bucketPriority); + + StreamingSyncCheckpointPartiallyComplete.fromJson(Map json) + : lastOpId = json['last_op_id'] as String, + bucketPriority = json['priority'] as int; +} + /// Sent as a periodic ping to keep the connection alive and to notify the /// client about the remaining lifetime of the JWT. /// diff --git a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart index d5281e3d..8a064e1e 100644 --- a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart +++ b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart @@ -157,6 +157,7 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { required bool? hasSyned, required String? uploadError, required String? downloadError, + required JSArray? priorityStatusEntries, }); factory SerializedSyncStatus.from(SyncStatus status) { @@ -169,6 +170,14 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { hasSyned: status.hasSynced, uploadError: status.uploadError?.toString(), downloadError: status.downloadError?.toString(), + priorityStatusEntries: [ + for (final entry in status.priorityStatusEntries) + [ + entry.priority.priorityNumber.toJS, + entry.lastSyncedAt?.microsecondsSinceEpoch.toJS, + entry.hasSynced?.toJS, + ].toJS + ].toJS, ); } @@ -180,6 +189,7 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { external bool? hasSynced; external String? uploadError; external String? downloadError; + external JSArray? priorityStatusEntries; SyncStatus asSyncStatus() { return SyncStatus( @@ -193,6 +203,20 @@ extension type SerializedSyncStatus._(JSObject _) implements JSObject { hasSynced: hasSynced, uploadError: uploadError, downloadError: downloadError, + priorityStatusEntries: priorityStatusEntries?.toDart.map((e) { + final [rawPriority, rawSynced, rawHasSynced, ...] = + (e as JSArray).toDart; + final syncedMillis = (rawSynced as JSNumber?)?.toDartInt; + + return ( + priority: BucketPriority((rawPriority as JSNumber).toDartInt), + lastSyncedAt: syncedMillis != null + ? DateTime.fromMicrosecondsSinceEpoch(syncedMillis) + : null, + hasSynced: (rawHasSynced as JSBoolean?)?.toDart, + ); + }).toList() ?? + const [], ); } } diff --git a/packages/powersync_core/test/bucket_storage_test.dart b/packages/powersync_core/test/bucket_storage_test.dart index 95d2e58c..d1b29a3f 100644 --- a/packages/powersync_core/test/bucket_storage_test.dart +++ b/packages/powersync_core/test/bucket_storage_test.dart @@ -39,6 +39,15 @@ const removeAsset1_4 = OplogEntry( const removeAsset1_5 = OplogEntry( opId: '5', op: OpType.remove, rowType: 'assets', rowId: 'O1', checksum: 5); +BucketChecksum checksum( + {required String bucket, required int checksum, int priority = 1}) { + return BucketChecksum(bucket: bucket, priority: priority, checksum: checksum); +} + +SyncDataBatch syncDataBatch(List data) { + return SyncDataBatch(data); +} + void main() { group('Bucket Storage Tests', () { late PowerSyncDatabase powersync; @@ -88,7 +97,7 @@ void main() { expect(await bucketStorage.getBucketStates(), equals([])); expect(await bucketStorage.hasCompletedSync(), equals(false)); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -101,7 +110,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); await expectAsset1_3(); @@ -119,7 +128,7 @@ void main() { }); test('should get an object from multiple buckets', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_3], @@ -128,8 +137,8 @@ void main() { ])); await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 3), - BucketChecksum(bucket: 'bucket2', checksum: 3) + checksum(bucket: 'bucket1', checksum: 3), + checksum(bucket: 'bucket2', checksum: 3) ])); await expectAsset1_3(); @@ -140,14 +149,14 @@ void main() { // In this case, there are two different versions in the different buckets. // While we should not get this with our server implementation, the client still specifies this behaviour: // The largest op_id wins. - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3]), SyncBucketData(bucket: 'bucket2', data: [putAsset1_1]) ])); await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 3), - BucketChecksum(bucket: 'bucket2', checksum: 1) + checksum(bucket: 'bucket1', checksum: 3), + checksum(bucket: 'bucket2', checksum: 1) ])); await expectAsset1_3(); @@ -155,14 +164,14 @@ void main() { test('should ignore a remove from one bucket', () async { // When we have 1 PUT and 1 REMOVE, the object must be kept. - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3]), SyncBucketData(bucket: 'bucket2', data: [putAsset1_3, removeAsset1_4]) ])); await syncLocalChecked(Checkpoint(lastOpId: '4', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 3), - BucketChecksum(bucket: 'bucket2', checksum: 7) + checksum(bucket: 'bucket1', checksum: 3), + checksum(bucket: 'bucket2', checksum: 7) ])); await expectAsset1_3(); @@ -170,70 +179,70 @@ void main() { test('should remove when removed from all buckets', () async { // When we only have REMOVE left for an object, it must be deleted. - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3, removeAsset1_5]), SyncBucketData(bucket: 'bucket2', data: [putAsset1_3, removeAsset1_4]) ])); await syncLocalChecked(Checkpoint(lastOpId: '5', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 8), - BucketChecksum(bucket: 'bucket2', checksum: 7) + checksum(bucket: 'bucket1', checksum: 8), + checksum(bucket: 'bucket2', checksum: 7) ])); await expectNoAssets(); }); test('put then remove', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3]), ])); await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 3), + checksum(bucket: 'bucket1', checksum: 3), ])); await expectAsset1_3(); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [removeAsset1_5]) ])); await syncLocalChecked(Checkpoint(lastOpId: '5', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 8), + checksum(bucket: 'bucket1', checksum: 8), ])); await expectNoAssets(); }); test('blank remove', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3, removeAsset1_4]), ])); await syncLocalChecked(Checkpoint(lastOpId: '4', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 7), + checksum(bucket: 'bucket1', checksum: 7), ])); await expectNoAssets(); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [removeAsset1_5]) ])); await syncLocalChecked(Checkpoint(lastOpId: '5', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 12), + checksum(bucket: 'bucket1', checksum: 12), ])); await expectNoAssets(); }); test('put | put remove', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_1]), ])); await syncLocalChecked(Checkpoint(lastOpId: '1', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 1), + checksum(bucket: 'bucket1', checksum: 1), ])); expect( @@ -243,13 +252,13 @@ void main() { {'id': 'O1', 'description': 'bar', 'make': null} ])); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [putAsset1_3]), SyncBucketData(bucket: 'bucket1', data: [removeAsset1_5]) ])); await syncLocalChecked(Checkpoint(lastOpId: '5', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 9), + checksum(bucket: 'bucket1', checksum: 9), ])); await expectNoAssets(); @@ -275,13 +284,13 @@ void main() { rowId: 'O1', checksum: 5); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset1_3, put4]), ])); await syncLocalChecked(Checkpoint(lastOpId: '4', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 8), + checksum(bucket: 'bucket1', checksum: 8), ])); expect( @@ -291,12 +300,12 @@ void main() { {'id': 'O1', 'description': 'B', 'make': null} ])); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [remove5]), ])); await syncLocalChecked(Checkpoint(lastOpId: '5', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 13), + checksum(bucket: 'bucket1', checksum: 13), ])); await expectAsset1_3(); @@ -304,15 +313,15 @@ void main() { test('should fail checksum validation', () async { // Simple checksum validation - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3]), ])); var result = await bucketStorage .syncLocalDatabase(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 10), - BucketChecksum(bucket: 'bucket2', checksum: 1) + checksum(bucket: 'bucket1', checksum: 10), + checksum(bucket: 'bucket2', checksum: 1) ])); expect( result, @@ -325,7 +334,7 @@ void main() { }); test('should delete buckets', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_3], @@ -340,7 +349,7 @@ void main() { // The delete only takes effect after syncLocal. await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 3), + checksum(bucket: 'bucket1', checksum: 3), ])); // Bucket is deleted, but object is still present in other buckets. @@ -354,7 +363,7 @@ void main() { test('should delete and re-create buckets', () async { // Save some data - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1], @@ -365,7 +374,7 @@ void main() { await bucketStorage.removeBuckets(['bucket1']); // Save some data again - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset1_3], @@ -375,7 +384,7 @@ void main() { await bucketStorage.removeBuckets(['bucket1']); // Final save of data - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset1_3], @@ -384,7 +393,7 @@ void main() { // Check that the data is there await syncLocalChecked(Checkpoint(lastOpId: '3', checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 4), + checksum(bucket: 'bucket1', checksum: 4), ])); await expectAsset1_3(); @@ -395,14 +404,14 @@ void main() { }); test('should handle MOVE', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [OplogEntry(opId: '1', op: OpType.move, checksum: 1)], ), ])); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_3], @@ -411,14 +420,14 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 4)])); + checksums: [checksum(bucket: 'bucket1', checksum: 4)])); await expectAsset1_3(); }); test('should handle CLEAR', () async { // Save some data - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1], @@ -427,10 +436,10 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '1', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 1)])); + checksums: [checksum(bucket: 'bucket1', checksum: 1)])); // CLEAR, then save new data - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [ @@ -449,7 +458,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', // 2 + 3. 1 is replaced with 2. - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 5)])); + checksums: [checksum(bucket: 'bucket1', checksum: 5)])); await expectNoAsset1(); expect( @@ -472,7 +481,7 @@ void main() { await powersync.initialize(); bucketStorage = BucketStorage(powersync); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -481,7 +490,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); await expectLater(() async { await powersync.execute('SELECT * FROM assets'); @@ -499,7 +508,7 @@ void main() { }); test('should remove types', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -508,7 +517,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); await expectAsset1_3(); @@ -537,7 +546,7 @@ void main() { // Test compacting behaviour. // This test relies heavily on internals, and will have to be updated when the compact implementation is updated. - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, removeAsset1_4]) ])); @@ -545,14 +554,14 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 7)])); + checksums: [checksum(bucket: 'bucket1', checksum: 7)])); await bucketStorage.forceCompact(); await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 7)])); + checksums: [checksum(bucket: 'bucket1', checksum: 7)])); final stats = await powersync.execute( 'SELECT row_type as type, row_id as id, count(*) as count FROM ps_oplog GROUP BY row_type, row_id ORDER BY row_type, row_id'); @@ -564,7 +573,7 @@ void main() { }); test('should compact with checksum wrapping', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [ OplogEntry( opId: '1', @@ -593,18 +602,14 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 2147483642) - ])); + checksums: [checksum(bucket: 'bucket1', checksum: 2147483642)])); await bucketStorage.forceCompact(); await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [ - BucketChecksum(bucket: 'bucket1', checksum: 2147483642) - ])); + checksums: [checksum(bucket: 'bucket1', checksum: 2147483642)])); final stats = await powersync.execute( 'SELECT row_type as type, row_id as id, count(*) as count FROM ps_oplog GROUP BY row_type, row_id ORDER BY row_type, row_id'); @@ -616,7 +621,7 @@ void main() { }); test('should compact with checksum wrapping (2)', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData(bucket: 'bucket1', data: [ OplogEntry( opId: '1', @@ -638,14 +643,14 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: -3)])); + checksums: [checksum(bucket: 'bucket1', checksum: -3)])); await bucketStorage.forceCompact(); await syncLocalChecked(Checkpoint( lastOpId: '4', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: -3)])); + checksums: [checksum(bucket: 'bucket1', checksum: -3)])); final stats = await powersync.execute( 'SELECT row_type as type, row_id as id, count(*) as count FROM ps_oplog GROUP BY row_type, row_id ORDER BY row_type, row_id'); @@ -658,7 +663,7 @@ void main() { test('should not sync local db with pending crud - server removed', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -667,7 +672,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local save powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); @@ -681,7 +686,7 @@ void main() { final result = await bucketStorage.syncLocalDatabase(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect(result, equals(SyncLocalDatabaseResult(ready: false))); final batch = await bucketStorage.getCrudBatch(); @@ -694,7 +699,7 @@ void main() { final result3 = await bucketStorage.syncLocalDatabase(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect(result3, equals(SyncLocalDatabaseResult(ready: false))); // The data must still be present locally. @@ -705,14 +710,14 @@ void main() { ])); await bucketStorage.saveSyncData( - SyncDataBatch([SyncBucketData(bucket: 'bucket1', data: [])])); + syncDataBatch([SyncBucketData(bucket: 'bucket1', data: [])])); // Now we have synced the data back (or lack of data in this case), // so we can do a local sync. await syncLocalChecked(Checkpoint( lastOpId: '5', writeCheckpoint: '5', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Since the object was not in the sync response, it is deleted. expect(await powersync.execute('SELECT id FROM assets WHERE id = \'O3\''), @@ -722,7 +727,7 @@ void main() { test( 'should not sync local db with pending crud when more crud is added (1)', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -732,7 +737,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local save powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); @@ -746,11 +751,11 @@ void main() { final result3 = await bucketStorage.syncLocalDatabase(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect(result3, equals(SyncLocalDatabaseResult(ready: false))); await bucketStorage.saveSyncData( - SyncDataBatch([SyncBucketData(bucket: 'bucket1', data: [])])); + syncDataBatch([SyncBucketData(bucket: 'bucket1', data: [])])); // Add more data before syncLocalDatabase. powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O4']); @@ -758,14 +763,14 @@ void main() { final result4 = await bucketStorage.syncLocalDatabase(Checkpoint( lastOpId: '5', writeCheckpoint: '5', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect(result4, equals(SyncLocalDatabaseResult(ready: false))); }); test( 'should not sync local db with pending crud when more crud is added (2)', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -775,7 +780,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local save await powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); @@ -788,7 +793,7 @@ void main() { return '4'; }); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [], @@ -798,13 +803,13 @@ void main() { final result4 = await bucketStorage.syncLocalDatabase(Checkpoint( lastOpId: '5', writeCheckpoint: '5', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect(result4, equals(SyncLocalDatabaseResult(ready: false))); }); test('should not sync local db with pending crud - update on server', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -814,7 +819,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local save powersync.execute('INSERT INTO assets(id) VALUES(?)', ['O3']); @@ -824,7 +829,7 @@ void main() { return '4'; }); - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [ @@ -842,7 +847,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '5', writeCheckpoint: '5', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 11)])); + checksums: [checksum(bucket: 'bucket1', checksum: 11)])); expect( await powersync @@ -853,7 +858,7 @@ void main() { }); test('should revert a failing insert', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -863,7 +868,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local insert, later rejected by server await powersync.execute( @@ -885,7 +890,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect( await powersync @@ -894,7 +899,7 @@ void main() { }); test('should revert a failing delete', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -904,7 +909,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local delete, later rejected by server await powersync.execute('DELETE FROM assets WHERE id = ?', ['O2']); @@ -924,7 +929,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect( await powersync @@ -935,7 +940,7 @@ void main() { }); test('should revert a failing update', () async { - await bucketStorage.saveSyncData(SyncDataBatch([ + await bucketStorage.saveSyncData(syncDataBatch([ SyncBucketData( bucket: 'bucket1', data: [putAsset1_1, putAsset2_2, putAsset1_3], @@ -945,7 +950,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '3', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); // Local update, later rejected by server await powersync.execute( @@ -968,7 +973,7 @@ void main() { await syncLocalChecked(Checkpoint( lastOpId: '3', writeCheckpoint: '4', - checksums: [BucketChecksum(bucket: 'bucket1', checksum: 6)])); + checksums: [checksum(bucket: 'bucket1', checksum: 6)])); expect( await powersync diff --git a/packages/powersync_core/test/in_memory_sync_test.dart b/packages/powersync_core/test/in_memory_sync_test.dart index ee5a0fc1..258d80fe 100644 --- a/packages/powersync_core/test/in_memory_sync_test.dart +++ b/packages/powersync_core/test/in_memory_sync_test.dart @@ -75,11 +75,15 @@ void main() { final status = await waitForConnection(); syncService.addLine({ - 'checkpoint': Checkpoint( - lastOpId: '0', - writeCheckpoint: null, - checksums: [BucketChecksum(bucket: 'bkt', checksum: 0)], - ) + 'checkpoint': { + 'last_op_id': '0', + 'buckets': [ + { + 'bucket': 'bkt', + 'checksum': 0, + } + ], + }, }); await expectLater(status, emits(isSyncStatus(downloading: true))); @@ -94,6 +98,12 @@ void main() { // is should reconstruct hasSynced from the database. await independentDb.initialize(); expect(independentDb.currentStatus.hasSynced, isTrue); + // A complete sync also means that all partial syncs have completed + expect( + independentDb.currentStatus + .statusForPriority(BucketPriority(3)) + .hasSynced, + isTrue); }); test('can save independent buckets in same transaction', () async { @@ -104,8 +114,8 @@ void main() { lastOpId: '0', writeCheckpoint: null, checksums: [ - BucketChecksum(bucket: 'a', checksum: 0), - BucketChecksum(bucket: 'b', checksum: 0), + BucketChecksum(bucket: 'a', checksum: 0, priority: 3), + BucketChecksum(bucket: 'b', checksum: 0, priority: 3), ], ) }); @@ -155,6 +165,122 @@ void main() { // because the messages were received in quick succession. expect(commits, 1); }); + + group('partial sync', () { + test('updates sync state incrementally', () async { + final status = await waitForConnection(); + + final checksums = [ + for (var prio = 0; prio <= 3; prio++) + BucketChecksum( + bucket: 'prio$prio', priority: prio, checksum: 10 + prio) + ]; + syncService.addLine({ + 'checkpoint': Checkpoint( + lastOpId: '4', + writeCheckpoint: null, + checksums: checksums, + ) + }); + var operationId = 1; + + void addRow(int priority) { + syncService.addLine({ + 'data': { + 'bucket': 'prio$priority', + 'data': [ + { + 'checksum': priority + 10, + 'data': {'name': 'test', 'email': 'email'}, + 'op': 'PUT', + 'op_id': '${operationId++}', + 'object_id': 'prio$priority', + 'object_type': 'customers' + } + ] + } + }); + } + + // Receiving the checkpoint sets the state to downloading + await expectLater( + status, emits(isSyncStatus(downloading: true, hasSynced: false))); + + // Emit partial sync complete for each priority but the last. + for (var prio = 0; prio < 3; prio++) { + addRow(prio); + syncService.addLine({ + 'partial_checkpoint_complete': { + 'last_op_id': operationId.toString(), + 'priority': prio, + } + }); + + await expectLater( + status, + emits(isSyncStatus(downloading: true, hasSynced: false).having( + (e) => e.statusForPriority(BucketPriority(0)).hasSynced, + 'status for $prio', + isTrue, + )), + ); + + await database.waitForFirstSync(priority: BucketPriority(prio)); + expect(await database.getAll('SELECT * FROM customers'), + hasLength(prio + 1)); + } + + // Complete the sync + addRow(3); + syncService.addLine({ + 'checkpoint_complete': {'last_op_id': operationId.toString()} + }); + + await expectLater( + status, emits(isSyncStatus(downloading: false, hasSynced: true))); + await database.waitForFirstSync(); + expect(await database.getAll('SELECT * FROM customers'), hasLength(4)); + }); + + test('remembers last partial sync state', () async { + final status = await waitForConnection(); + + syncService.addLine({ + 'checkpoint': Checkpoint( + lastOpId: '0', + writeCheckpoint: null, + checksums: [ + BucketChecksum(bucket: 'bkt', priority: 1, checksum: 0) + ], + ) + }); + await expectLater(status, emits(isSyncStatus(downloading: true))); + + syncService.addLine({ + 'partial_checkpoint_complete': { + 'last_op_id': '0', + 'priority': 1, + } + }); + await database.waitForFirstSync(priority: BucketPriority(1)); + expect(database.currentStatus.hasSynced, isFalse); + + final independentDb = factory.wrapRaw(raw); + await independentDb.initialize(); + expect(independentDb.currentStatus.hasSynced, isFalse); + // Completing a sync for prio 1 implies a completed sync for prio 0 + expect( + independentDb.currentStatus + .statusForPriority(BucketPriority(0)) + .hasSynced, + isTrue); + expect( + independentDb.currentStatus + .statusForPriority(BucketPriority(3)) + .hasSynced, + isFalse); + }); + }); }); } diff --git a/packages/powersync_core/test/sync_types_test.dart b/packages/powersync_core/test/sync_types_test.dart index fa4e71fa..18e06931 100644 --- a/packages/powersync_core/test/sync_types_test.dart +++ b/packages/powersync_core/test/sync_types_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:powersync_core/src/sync_status.dart'; import 'package:powersync_core/src/sync_types.dart'; import 'package:test/test.dart'; @@ -214,5 +215,12 @@ void main() { }); } }); + + test('bucket priority comparisons', () { + expect(BucketPriority(0) < BucketPriority(3), isFalse); + expect(BucketPriority(0) > BucketPriority(3), isTrue); + expect(BucketPriority(0) >= BucketPriority(3), isTrue); + expect(BucketPriority(0) >= BucketPriority(0), isTrue); + }); }); } diff --git a/packages/powersync_core/test/utils/native_test_utils.dart b/packages/powersync_core/test/utils/native_test_utils.dart index 915af2a0..65916a5f 100644 --- a/packages/powersync_core/test/utils/native_test_utils.dart +++ b/packages/powersync_core/test/utils/native_test_utils.dart @@ -2,7 +2,7 @@ import 'dart:async'; import 'dart:ffi'; import 'dart:io'; import 'package:powersync_core/powersync_core.dart'; -import 'package:sqlite3/sqlite3.dart'; +import 'package:powersync_core/sqlite3.dart'; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite3/open.dart' as sqlite_open; diff --git a/packages/powersync_core/test/utils/web_test_utils.dart b/packages/powersync_core/test/utils/web_test_utils.dart index 678af784..3a74a448 100644 --- a/packages/powersync_core/test/utils/web_test_utils.dart +++ b/packages/powersync_core/test/utils/web_test_utils.dart @@ -3,7 +3,7 @@ import 'dart:js_interop'; import 'package:logging/logging.dart'; import 'package:powersync_core/powersync_core.dart'; -import 'package:sqlite3/wasm.dart'; +import 'package:sqlite_async/sqlite3_wasm.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; import 'package:web/web.dart' show Blob, BlobPropertyBag; @@ -62,9 +62,9 @@ class TestUtils extends AbstractTestUtils { Future cleanDb({required String path}) async {} @override - Future testFactory( + Future testFactory( {String? path, - String? sqlitePath, + String sqlitePath = '', SqliteOptions options = const SqliteOptions.defaults()}) async { await _isInitialized; diff --git a/packages/powersync_flutter_libs/android/build.gradle b/packages/powersync_flutter_libs/android/build.gradle index 613d3b52..3a1a85a9 100644 --- a/packages/powersync_flutter_libs/android/build.gradle +++ b/packages/powersync_flutter_libs/android/build.gradle @@ -50,5 +50,5 @@ android { } dependencies { - implementation 'co.powersync:powersync-sqlite-core:0.3.10' + implementation 'co.powersync:powersync-sqlite-core:0.3.11' } diff --git a/packages/powersync_flutter_libs/ios/powersync_flutter_libs.podspec b/packages/powersync_flutter_libs/ios/powersync_flutter_libs.podspec index 2e5ab535..4f32ee2c 100644 --- a/packages/powersync_flutter_libs/ios/powersync_flutter_libs.podspec +++ b/packages/powersync_flutter_libs/ios/powersync_flutter_libs.podspec @@ -22,7 +22,7 @@ A new Flutter FFI plugin project. s.dependency 'Flutter' s.platform = :ios, '11.0' - s.dependency "powersync-sqlite-core", "~> 0.3.10" + s.dependency "powersync-sqlite-core", "~> 0.3.11" # Flutter.framework does not contain a i386 slice. s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' } diff --git a/packages/powersync_flutter_libs/macos/powersync_flutter_libs.podspec b/packages/powersync_flutter_libs/macos/powersync_flutter_libs.podspec index 54e251b4..24632626 100644 --- a/packages/powersync_flutter_libs/macos/powersync_flutter_libs.podspec +++ b/packages/powersync_flutter_libs/macos/powersync_flutter_libs.podspec @@ -21,7 +21,7 @@ A new Flutter FFI plugin project. s.source_files = 'Classes/**/*' s.dependency 'FlutterMacOS' - s.dependency "powersync-sqlite-core", "~> 0.3.10" + s.dependency "powersync-sqlite-core", "~> 0.3.11" s.platform = :osx, '10.11' s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES' } diff --git a/scripts/download_core_binary_demos.dart b/scripts/download_core_binary_demos.dart index e5c53990..77d8f47d 100644 --- a/scripts/download_core_binary_demos.dart +++ b/scripts/download_core_binary_demos.dart @@ -3,7 +3,7 @@ import 'dart:io'; final coreUrl = - 'https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v0.3.10'; + 'https://github.com/powersync-ja/powersync-sqlite-core/releases/download/v0.3.11'; void main() async { final powersyncLibsLinuxPath = "packages/powersync_flutter_libs/linux";