Allow tracking changed values only

simolus3 · simolus3 · commit cfcadb84026c · 2025-04-09T15:34:04.000+02:00
diff --git a/crates/core/src/diff.rs b/crates/core/src/diff.rs
@@ -19,11 +19,24 @@ fn powersync_diff_impl(
 ) -> Result<String, SQLiteError> {
     let data_old = args[0].text();
     let data_new = args[1].text();
+    let ignore_removed = args.get(2).map_or(false, |v| v.int() != 0);
 
-    diff_objects(data_old, data_new)
+    diff_objects_with_options(data_old, data_new, ignore_removed)
 }
 
-pub fn diff_objects(data_old: &str, data_new: &str) -> Result<String, SQLiteError> {
+fn diff_objects(data_old: &str, data_new: &str) -> Result<String, SQLiteError> {
+    diff_objects_with_options(data_old, data_new, false)
+}
+
+/// Returns a JSON object containing entries from [data_new] that are not present in [data_old].
+///
+/// When [ignore_removed_columns] is set, columns that are present in [data_old] but not in
+/// [data_new] will not be present in the returned object. Otherwise, they will be set to `null`.
+fn diff_objects_with_options(
+    data_old: &str,
+    data_new: &str,
+    ignore_removed_columns: bool,
+) -> Result<String, SQLiteError> {
     let v_new: json::Value = json::from_str(data_new)?;
     let v_old: json::Value = json::from_str(data_old)?;
 
@@ -38,9 +51,11 @@ pub fn diff_objects(data_old: &str, data_new: &str) -> Result<String, SQLiteErro
         }
 
         // Add missing nulls to left
-        for key in right.keys() {
-            if !left.contains_key(key) {
-                left.insert(key.clone(), json::Value::Null);
+        if !ignore_removed_columns {
+            for key in right.keys() {
+                if !left.contains_key(key) {
+                    left.insert(key.clone(), json::Value::Null);
+                }
             }
         }
 
@@ -75,6 +90,17 @@ pub fn register(db: *mut sqlite::sqlite3) -> Result<(), ResultCode> {
         None,
     )?;
 
+    db.create_function_v2(
+        "powersync_diff",
+        3,
+        sqlite::UTF8 | sqlite::DETERMINISTIC,
+        None,
+        Some(powersync_diff),
+        None,
+        None,
+        None,
+    )?;
+
     Ok(())
 }
 
diff --git a/crates/core/src/schema/table_info.rs b/crates/core/src/schema/table_info.rs
@@ -27,7 +27,8 @@ impl TableInfo {
         json_extract(?1, '$.local_only'),
         json_extract(?1, '$.insert_only'),
         json_extract(?1, '$.include_old'),
-        json_extract(?1, '$.include_metadata')",
+        json_extract(?1, '$.include_metadata'),
+        json_extract(?1, '$.include_old_only_when_changed')",
         )?;
         statement.bind_text(1, data, sqlite::Destructor::STATIC)?;
 
@@ -42,11 +43,17 @@ impl TableInfo {
             let local_only = statement.column_int(2) != 0;
             let insert_only = statement.column_int(3) != 0;
             let include_metadata = statement.column_int(5) != 0;
+            let include_old_only_when_changed = statement.column_int(6) != 0;
 
             let mut flags = TableInfoFlags::default();
             flags = flags.set_flag(TableInfoFlags::LOCAL_ONLY, local_only);
             flags = flags.set_flag(TableInfoFlags::INSERT_ONLY, insert_only);
             flags = flags.set_flag(TableInfoFlags::INCLUDE_METADATA, include_metadata);
+            flags = flags.set_flag(
+                TableInfoFlags::INCLUDE_OLD_ONLY_IF_CHANGED,
+                include_old_only_when_changed,
+            );
+
             flags
         };
 
@@ -88,6 +95,7 @@ impl TableInfoFlags {
     pub const LOCAL_ONLY: u32 = 1;
     pub const INSERT_ONLY: u32 = 2;
     pub const INCLUDE_METADATA: u32 = 4;
+    pub const INCLUDE_OLD_ONLY_IF_CHANGED: u32 = 8;
 
     pub const fn local_only(self) -> bool {
         self.0 & Self::LOCAL_ONLY != 0
@@ -101,6 +109,10 @@ impl TableInfoFlags {
         self.0 & Self::INCLUDE_METADATA != 0
     }
 
+    pub const fn include_old_only_if_changed(self) -> bool {
+        self.0 & Self::INCLUDE_OLD_ONLY_IF_CHANGED != 0
+    }
+
     const fn with_flag(self, flag: u32) -> Self {
         Self(self.0 | flag)
     }
diff --git a/crates/core/src/views.rs b/crates/core/src/views.rs
@@ -224,21 +224,32 @@ fn powersync_trigger_update_sql_impl(
     let json_fragment_new = json_object_fragment("NEW", &mut columns.names_iter())?;
     let json_fragment_old = json_object_fragment("OLD", &mut columns.names_iter())?;
 
-    let old_fragment: Cow<'static, str> = match &table_info.diff_include_old {
-        None => "".into(),
-        Some(DiffIncludeOld::ForAllColumns) => format!(", 'old', {:}", json_fragment_old).into(),
+    let mut old_values_fragment = match &table_info.diff_include_old {
+        None => None,
+        Some(DiffIncludeOld::ForAllColumns) => Some(json_fragment_old.clone()),
         Some(DiffIncludeOld::OnlyForColumns { columns }) => {
             let mut iterator = columns.iter();
             let mut columns =
                 streaming_iterator::from_fn(|| -> Option<Result<&str, ResultCode>> {
                     Some(Ok(iterator.next()?.as_str()))
                 });
 
-            let json_fragment = json_object_fragment("OLD", &mut columns)?;
-            format!(", 'old', {:}", json_fragment).into()
+            Some(json_object_fragment("OLD", &mut columns)?)
         }
     };
 
+    if table_info.flags.include_old_only_if_changed() {
+        // We're setting ignore_removed_columns here because values that are present in the new
+        // values but not in the old values should not lead to a null entry.
+        old_values_fragment = old_values_fragment
+            .map(|f| format!("json(powersync_diff({json_fragment_new}, {f}, TRUE))"));
+    }
+
+    let old_fragment: Cow<'static, str> = match old_values_fragment {
+        Some(f) => format!(", 'old', {f}").into(),
+        None => "".into(),
+    };
+
     let metadata_fragment = if table_info.flags.include_metadata() {
         ", 'metadata', NEW._metadata"
     } else {
@@ -247,38 +258,37 @@ fn powersync_trigger_update_sql_impl(
 
     return if !local_only && !insert_only {
         let trigger = format!("\
-CREATE TRIGGER {:}
-INSTEAD OF UPDATE ON {:}
+CREATE TRIGGER {trigger_name}
+INSTEAD OF UPDATE ON {quoted_name}
 FOR EACH ROW
 BEGIN
   SELECT CASE
   WHEN (OLD.id != NEW.id)
   THEN RAISE (FAIL, 'Cannot update id')
   END;
-  UPDATE {:}
-      SET data = {:}
+  UPDATE {internal_name}
+      SET data = {json_fragment_new}
       WHERE id = NEW.id;
   INSERT INTO powersync_crud_(data) VALUES(json_object('op', 'PATCH', 'type', {:}, 'id', NEW.id, 'data', json(powersync_diff({:}, {:})){:}{:}));
-  INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES({:}, NEW.id);
-  INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) VALUES('$local', 0, {:});
-END", trigger_name, quoted_name, internal_name, json_fragment_new, type_string, json_fragment_old, json_fragment_new, old_fragment, metadata_fragment, type_string, MAX_OP_ID);
+  INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES({type_string}, NEW.id);
+  INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) VALUES('$local', 0, {MAX_OP_ID});
+END", type_string, json_fragment_old, json_fragment_new, old_fragment, metadata_fragment);
         Ok(trigger)
     } else if local_only {
         let trigger = format!(
             "\
-CREATE TRIGGER {:}
-INSTEAD OF UPDATE ON {:}
+CREATE TRIGGER {trigger_name}
+INSTEAD OF UPDATE ON {quoted_name}
 FOR EACH ROW
 BEGIN
   SELECT CASE
   WHEN (OLD.id != NEW.id)
   THEN RAISE (FAIL, 'Cannot update id')
   END;
-  UPDATE {:}
-      SET data = {:}
+  UPDATE {internal_name}
+      SET data = {json_fragment_new}
       WHERE id = NEW.id;
-END",
-            trigger_name, quoted_name, internal_name, json_fragment_new
+END"
         );
         Ok(trigger)
     } else if insert_only {
diff --git a/dart/test/crud_test.dart b/dart/test/crud_test.dart
@@ -249,7 +249,8 @@ void main() {
 
       void insertThenUpdate() {
         db
-          ..execute('insert into test (id, name) values (?, ?)', ['id', 'test'])
+          ..execute('insert into test (id, name, name2) values (?, ?, ?)',
+              ['id', 'name', 'name2'])
           ..execute('delete from ps_crud')
           ..execute('update test set name = name || ?', ['.']);
       }
@@ -276,8 +277,8 @@ void main() {
 
         final [row] = db.select('select data from ps_crud');
         final op = jsonDecode(row[0] as String);
-        expect(op['data'], {'name': 'test.'});
-        expect(op['old'], {'name': 'test', 'name2': null});
+        expect(op['data'], {'name': 'name.'});
+        expect(op['old'], {'name': 'name', 'name2': 'name2'});
       });
 
       test('can be enabled for some columns', () {
@@ -288,8 +289,34 @@ void main() {
 
         final [row] = db.select('select data from ps_crud');
         final op = jsonDecode(row[0] as String);
-        expect(op['data'], {'name': 'test.'});
-        expect(op['old'], {'name': 'test'});
+        expect(op['data'], {'name': 'name.'});
+        expect(op['old'], {'name': 'name'});
+      });
+
+      test('can track changed values only', () {
+        createTable({
+          'include_old': true,
+          'include_old_only_when_changed': true,
+        });
+        insertThenUpdate();
+
+        final [row] = db.select('select data from ps_crud');
+        final op = jsonDecode(row[0] as String);
+        expect(op['data'], {'name': 'name.'});
+        expect(op['old'], {'name': 'name'});
+      });
+
+      test('combined column filter and only tracking changes', () {
+        createTable({
+          'include_old': ['name2'],
+          'include_old_only_when_changed': true,
+        });
+        insertThenUpdate();
+
+        final [row] = db.select('select data from ps_crud');
+        final op = jsonDecode(row[0] as String);
+        expect(op['data'], {'name': 'name.'});
+        expect(op['old'], {});
       });
     });
   });
