Add new schema options

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## unreleased
+
+* Add `includeOld` option on `Table` which sets `CrudEntry.oldData` to previous values on updates.
+* Add `includeMetadata` option on `Table` which adds a `_metadata` column that can be used for updates.
+  The configured metadata is available through `CrudEntry.metadata`.
+* Add `ignoreEmptyUpdate` option which skips creating CRUD entries for updates that don't change any values.
+
 ## 1.0.0-BETA32
 
 * Added `onChange` method to the PowerSync client. This allows for observing table changes.

core/src/commonIntegrationTest/kotlin/com/powersync/CrudTest.kt

@@ -0,0 +1,91 @@
+package com.powersync
+
+import com.powersync.db.schema.Column
+import com.powersync.db.schema.IncludeOldOptions
+import com.powersync.db.schema.Schema
+import com.powersync.db.schema.Table
+import com.powersync.testutils.databaseTest
+import io.kotest.matchers.shouldBe
+import kotlin.test.Test
+
+class CrudTest {
+    @Test
+    fun includeMetadata() =
+        databaseTest {
+            database.updateSchema(Schema(Table("lists", listOf(Column.text("name")), includeMetadata = true)))
+
+            database.execute("INSERT INTO lists (id, name, _metadata) VALUES (uuid(), ?, ?)", listOf("entry", "so meta"))
+            val batch = database.getNextCrudTransaction()
+            batch!!.crud[0].metadata shouldBe "so meta"
+        }
+
+    @Test
+    fun includeOldValues() =
+        databaseTest {
+            database.updateSchema(
+                Schema(Table("lists", listOf(Column.text("name"), Column.text("content")), includeOld = IncludeOldOptions())),
+            )
+
+            database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content"))
+            database.execute("DELETE FROM ps_crud")
+            database.execute("UPDATE lists SET name = ?", listOf("new name"))
+
+            val batch = database.getNextCrudTransaction()
+            batch!!.crud[0].oldData shouldBe mapOf("name" to "entry", "content" to "content")
+        }
+
+    @Test
+    fun includeOldValuesWithFilter() =
+        databaseTest {
+            database.updateSchema(
+                Schema(
+                    Table(
+                        "lists",
+                        listOf(Column.text("name"), Column.text("content")),
+                        includeOld = IncludeOldOptions(columnFilter = listOf("name")),
+                    ),
+                ),
+            )
+
+            database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content"))
+            database.execute("DELETE FROM ps_crud")
+            database.execute("UPDATE lists SET name = ?, content = ?", listOf("new name", "new content"))
+
+            val batch = database.getNextCrudTransaction()
+            batch!!.crud[0].oldData shouldBe mapOf("name" to "entry")
+        }
+
+    @Test
+    fun includeOldValuesWhenChanged() =
+        databaseTest {
+            database.updateSchema(
+                Schema(
+                    Table(
+                        "lists",
+                        listOf(Column.text("name"), Column.text("content")),
+                        includeOld = IncludeOldOptions(onlyWhenChanged = true),
+                    ),
+                ),
+            )
+
+            database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content"))
+            database.execute("DELETE FROM ps_crud")
+            database.execute("UPDATE lists SET name = ?", listOf("new name"))
+
+            val batch = database.getNextCrudTransaction()
+            batch!!.crud[0].oldData shouldBe mapOf("name" to "entry")
+        }
+
+    @Test
+    fun ignoreEmptyUpdate() =
+        databaseTest {
+            database.updateSchema(Schema(Table("lists", listOf(Column.text("name"), Column.text("content")), ignoreEmptyUpdate = true)))
+
+            database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content"))
+            database.execute("DELETE FROM ps_crud")
+            database.execute("UPDATE lists SET name = ?", listOf("entry"))
+
+            val batch = database.getNextCrudTransaction()
+            batch shouldBe null
+        }
+}

core/src/commonMain/kotlin/com/powersync/db/crud/CrudEntry.kt

@@ -37,6 +37,7 @@ public data class CrudEntry(
      * This may change in the future.
      */
     val transactionId: Int?,
+    val metadata: String? = null,
     /**
      * Data associated with the change.
      *
@@ -47,6 +48,7 @@ public data class CrudEntry(
      * For DELETE, this is null.
      */
     val opData: Map<String, String?>?,
+    val oldData: Map<String, String?>? = null,
 ) {
     public companion object {
         public fun fromRow(row: CrudRow): CrudEntry {
@@ -61,6 +63,11 @@ public data class CrudEntry(
                     },
                 table = data["type"]!!.jsonPrimitive.content,
                 transactionId = row.txId,
+                metadata = data["metadata"]?.jsonPrimitive?.content,
+                oldData =
+                    data["old"]?.jsonObject?.mapValues { (_, value) ->
+                        value.jsonPrimitive.contentOrNull
+                    },
             )
         }
     }

core/src/commonMain/kotlin/com/powersync/db/schema/Table.kt

@@ -1,14 +1,18 @@
 package com.powersync.db.schema
 
+import com.powersync.db.crud.CrudEntry
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.buildJsonArray
 
 private const val MAX_AMOUNT_OF_COLUMNS = 1999
 
 /**
  * A single table in the schema.
  */
-public data class Table constructor(
+public data class Table(
     /**
      * The synced table name, matching sync rules.
      */
@@ -33,6 +37,22 @@ public data class Table constructor(
      * Override the name for the view
      */
     private val viewNameOverride: String? = null,
+    /**
+     *  Whether to add a hidden `_metadata` column that will be enabled for updates to attach custom
+     *  information about writes that will be reported through [CrudEntry.metadata].
+     */
+    val includeMetadata: Boolean = false,
+    /**
+     * When set to a non-null value, track old values of columns for [CrudEntry.oldData].
+     *
+     * See [IncludeOldOptions] for details.
+     */
+    val includeOld: IncludeOldOptions? = null,
+    /**
+     * Whether an `UPDATE` statement that doesn't change any values should be ignored when creating
+     * CRUD entries.
+     */
+    val ignoreEmptyUpdate: Boolean = false,
 ) {
     init {
         /**
@@ -81,6 +101,9 @@ public data class Table constructor(
             name: String,
             columns: List<Column>,
             viewName: String? = null,
+            ignoreEmptyUpdate: Boolean = false,
+            includeMetadata: Boolean = false,
+            includeOld: IncludeOldOptions? = null,
         ): Table =
             Table(
                 name,
@@ -89,6 +112,9 @@ public data class Table constructor(
                 localOnly = false,
                 insertOnly = true,
                 viewNameOverride = viewName,
+                ignoreEmptyUpdate = ignoreEmptyUpdate,
+                includeMetadata = includeMetadata,
+                includeOld = includeOld,
             )
     }
 
@@ -135,6 +161,13 @@ public data class Table constructor(
             throw AssertionError("Invalid characters in view name: $viewNameOverride")
         }
 
+        check(!localOnly || !includeMetadata) {
+            "Can't track metadata for local-only tables."
+        }
+        check(!localOnly || includeOld == null) {
+            "Can't track old values for local-only tables."
+        }
+
         val columnNames = mutableSetOf("id")
         for (column in columns) {
             when {
@@ -185,6 +218,26 @@ public data class Table constructor(
         get() = viewNameOverride ?: name
 }
 
+/**
+ * Options to include old values in [CrudEntry.oldData] for update statements.
+ *
+ * These options are enabled by passing them to a non-local [Table] constructor.
+ */
+public data class IncludeOldOptions(
+    /**
+     * A filter of column names for which updates should be tracked.
+     *
+     * When set to a non-null value, columns not included in this list will not appear in
+     * [CrudEntry.oldData]. By default, all columns are included.
+     */
+    val columnFilter: List<String>? = null,
+    /**
+     * Whether to only include old values when they were changed by an update, instead of always
+     * including all old values,
+     */
+    val onlyWhenChanged: Boolean = false,
+)
+
 @Serializable
 internal data class SerializableTable(
     var name: String,
@@ -196,16 +249,39 @@ internal data class SerializableTable(
     val insertOnly: Boolean = false,
     @SerialName("view_name")
     val viewName: String? = null,
+    @SerialName("ignore_empty_update")
+    val ignoreEmptyUpdate: Boolean = false,
+    @SerialName("include_metadata")
+    val includeMetadata: Boolean = false,
+    @SerialName("include_old")
+    val includeOld: JsonElement = JsonPrimitive(false),
+    @SerialName("include_old_only_when_changed")
+    val includeOldOnlyWhenChanged: Boolean = false,
 )
 
 internal fun Table.toSerializable(): SerializableTable =
     with(this) {
         SerializableTable(
-            name,
-            columns.map { it.toSerializable() },
-            indexes.map { it.toSerializable() },
-            localOnly,
-            insertOnly,
-            viewName,
+            name = name,
+            columns = columns.map { it.toSerializable() },
+            indexes = indexes.map { it.toSerializable() },
+            localOnly = localOnly,
+            insertOnly = insertOnly,
+            viewName = viewName,
+            ignoreEmptyUpdate = ignoreEmptyUpdate,
+            includeMetadata = includeMetadata,
+            includeOld =
+                includeOld?.let {
+                    if (it.columnFilter != null) {
+                        buildJsonArray {
+                            for (column in it.columnFilter) {
+                                add(JsonPrimitive(column))
+                            }
+                        }
+                    } else {
+                        JsonPrimitive(true)
+                    }
+                } ?: JsonPrimitive(false),
+            includeOldOnlyWhenChanged = includeOld?.onlyWhenChanged ?: false,
         )
     }

core/src/commonTest/kotlin/com/powersync/db/schema/TableTest.kt

@@ -1,5 +1,14 @@
 package com.powersync.db.schema
 
+import com.powersync.utils.JsonUtil
+import io.kotest.assertions.throwables.shouldThrow
+import io.kotest.matchers.shouldBe
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.boolean
+import kotlinx.serialization.json.encodeToJsonElement
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonPrimitive
+import kotlinx.serialization.serializer
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
@@ -180,4 +189,44 @@ class TableTest {
 
         assertEquals(exception.message, "users: id column is automatically added, custom id columns are not supported")
     }
+
+    @Test
+    fun testValidationLocalOnlyWithMetadata() {
+        val table = Table("foo", listOf(Column.text("bar")), localOnly = true, includeMetadata = true)
+
+        val exception = shouldThrow<IllegalStateException> { table.validate() }
+        exception.message shouldBe "Can't track metadata for local-only tables."
+    }
+
+    @Test
+    fun testValidationLocalOnlyWithIncludeOld() {
+        val table = Table("foo", listOf(Column.text("bar")), localOnly = true, includeOld = IncludeOldOptions())
+
+        val exception = shouldThrow<IllegalStateException> { table.validate() }
+        exception.message shouldBe "Can't track old values for local-only tables."
+    }
+
+    @Test
+    fun handlesOptions() {
+        fun serialize(table: Table): JsonObject =
+            JsonUtil.json.encodeToJsonElement(serializer<SerializableTable>(), table.toSerializable()) as JsonObject
+
+        serialize(Table("foo", emptyList(), includeMetadata = true))["include_metadata"]!!.jsonPrimitive.boolean shouldBe true
+        serialize(Table("foo", emptyList(), ignoreEmptyUpdate = true))["ignore_empty_update"]!!.jsonPrimitive.boolean shouldBe true
+
+        serialize(Table("foo", emptyList(), includeOld = IncludeOldOptions())).let {
+            it["include_old"]!!.jsonPrimitive.boolean shouldBe true
+            it["include_old_only_when_changed"]!!.jsonPrimitive.boolean shouldBe false
+        }
+
+        serialize(Table("foo", emptyList(), includeOld = IncludeOldOptions(columnFilter = listOf("foo", "bar")))).let {
+            it["include_old"]!!.jsonArray.map { e -> e.jsonPrimitive.content } shouldBe listOf("foo", "bar")
+            it["include_old_only_when_changed"]!!.jsonPrimitive.boolean shouldBe false
+        }
+
+        serialize(Table("foo", emptyList(), includeOld = IncludeOldOptions(onlyWhenChanged = true))).let {
+            it["include_old"]!!.jsonPrimitive.boolean shouldBe true
+            it["include_old_only_when_changed"]!!.jsonPrimitive.boolean shouldBe true
+        }
+    }
 }

gradle/libs.versions.toml

@@ -17,7 +17,7 @@ kotlinx-datetime = "0.6.2"
 kotlinx-io = "0.5.4"
 ktor = "3.0.1"
 uuid = "0.8.2"
-powersync-core = "0.3.12"
+powersync-core = "0.3.13"
 sqlite-jdbc = "3.49.1.0"
 sqliter = "1.3.1"
 turbine = "1.2.0"