Add new schema options

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## unreleased
+
+* Add `trackPreviousValues` option on `Table` which sets `CrudEntry.oldData` to previous values on updates.
+* Add `trackMetadata` option on `Table` which adds a `_metadata` column that can be used for updates.
+  The configured metadata is available through `CrudEntry.metadata`.
+* Add `ignoreEmptyUpdates` option which skips creating CRUD entries for updates that don't change any values.
+
 ## 1.0.1
 
 * [Internal] Version bump for broken Swift release pipeline

core/src/commonIntegrationTest/kotlin/com/powersync/CrudTest.kt

@@ -0,0 +1,93 @@
+package com.powersync
+
+import com.powersync.db.schema.Column
+import com.powersync.db.schema.Schema
+import com.powersync.db.schema.Table
+import com.powersync.db.schema.TrackPreviousValuesOptions
+import com.powersync.testutils.databaseTest
+import io.kotest.matchers.shouldBe
+import kotlin.test.Test
+
+class CrudTest {
+    @Test
+    fun includeMetadata() =
+        databaseTest {
+            database.updateSchema(Schema(Table("lists", listOf(Column.text("name")), trackMetadata = true)))
+
+            database.execute("INSERT INTO lists (id, name, _metadata) VALUES (uuid(), ?, ?)", listOf("entry", "so meta"))
+            val batch = database.getNextCrudTransaction()
+            batch!!.crud[0].metadata shouldBe "so meta"
+        }
+
+    @Test
+    fun includeOldValues() =
+        databaseTest {
+            database.updateSchema(
+                Schema(
+                    Table("lists", listOf(Column.text("name"), Column.text("content")), trackPreviousValues = TrackPreviousValuesOptions()),
+                ),
+            )
+
+            database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content"))
+            database.execute("DELETE FROM ps_crud")
+            database.execute("UPDATE lists SET name = ?", listOf("new name"))
+
+            val batch = database.getNextCrudTransaction()
+            batch!!.crud[0].oldData shouldBe mapOf("name" to "entry", "content" to "content")
+        }
+
+    @Test
+    fun includeOldValuesWithFilter() =
+        databaseTest {
+            database.updateSchema(
+                Schema(
+                    Table(
+                        "lists",
+                        listOf(Column.text("name"), Column.text("content")),
+                        trackPreviousValues = TrackPreviousValuesOptions(columnFilter = listOf("name")),
+                    ),
+                ),
+            )
+
+            database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content"))
+            database.execute("DELETE FROM ps_crud")
+            database.execute("UPDATE lists SET name = ?, content = ?", listOf("new name", "new content"))
+
+            val batch = database.getNextCrudTransaction()
+            batch!!.crud[0].oldData shouldBe mapOf("name" to "entry")
+        }
+
+    @Test
+    fun includeOldValuesWhenChanged() =
+        databaseTest {
+            database.updateSchema(
+                Schema(
+                    Table(
+                        "lists",
+                        listOf(Column.text("name"), Column.text("content")),
+                        trackPreviousValues = TrackPreviousValuesOptions(onlyWhenChanged = true),
+                    ),
+                ),
+            )
+
+            database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content"))
+            database.execute("DELETE FROM ps_crud")
+            database.execute("UPDATE lists SET name = ?", listOf("new name"))
+
+            val batch = database.getNextCrudTransaction()
+            batch!!.crud[0].oldData shouldBe mapOf("name" to "entry")
+        }
+
+    @Test
+    fun ignoreEmptyUpdate() =
+        databaseTest {
+            database.updateSchema(Schema(Table("lists", listOf(Column.text("name"), Column.text("content")), ignoreEmptyUpdates = true)))
+
+            database.execute("INSERT INTO lists (id, name, content) VALUES (uuid(), ?, ?)", listOf("entry", "content"))
+            database.execute("DELETE FROM ps_crud")
+            database.execute("UPDATE lists SET name = ?", listOf("entry"))
+
+            val batch = database.getNextCrudTransaction()
+            batch shouldBe null
+        }
+}

core/src/commonMain/kotlin/com/powersync/db/crud/CrudEntry.kt

@@ -1,5 +1,7 @@
 package com.powersync.db.crud
 
+import com.powersync.PowerSyncDatabase
+import com.powersync.db.schema.Table
 import com.powersync.utils.JsonUtil
 import kotlinx.serialization.json.contentOrNull
 import kotlinx.serialization.json.jsonObject
@@ -37,6 +39,15 @@ public data class CrudEntry(
      * This may change in the future.
      */
     val transactionId: Int?,
+    /**
+     * User-defined metadata that can be attached to writes.
+     *
+     * This is the value the `_metadata` column had when the write to the database was made,
+     * allowing backend connectors to e.g. identify a write and treat it specially.
+     *
+     * Note that the `_metadata` column is only available when [Table.trackMetadata] is enabled.
+     */
+    val metadata: String? = null,
     /**
      * Data associated with the change.
      *
@@ -47,6 +58,13 @@ public data class CrudEntry(
      * For DELETE, this is null.
      */
     val opData: Map<String, String?>?,
+    /**
+     * Previous values before this change.
+     *
+     * These values can be tracked for `UPDATE` statements when [Table.trackPreviousValues] is
+     * enabled.
+     */
+    val oldData: Map<String, String?>? = null,
 ) {
     public companion object {
         public fun fromRow(row: CrudRow): CrudEntry {
@@ -61,6 +79,11 @@ public data class CrudEntry(
                     },
                 table = data["type"]!!.jsonPrimitive.content,
                 transactionId = row.txId,
+                metadata = data["metadata"]?.jsonPrimitive?.content,
+                oldData =
+                    data["old"]?.jsonObject?.mapValues { (_, value) ->
+                        value.jsonPrimitive.contentOrNull
+                    },
             )
         }
     }

core/src/commonMain/kotlin/com/powersync/db/schema/Table.kt

@@ -1,14 +1,18 @@
 package com.powersync.db.schema
 
+import com.powersync.db.crud.CrudEntry
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.buildJsonArray
 
 private const val MAX_AMOUNT_OF_COLUMNS = 1999
 
 /**
  * A single table in the schema.
  */
-public data class Table constructor(
+public data class Table(
     /**
      * The synced table name, matching sync rules.
      */
@@ -33,6 +37,22 @@ public data class Table constructor(
      * Override the name for the view
      */
     private val viewNameOverride: String? = null,
+    /**
+     *  Whether to add a hidden `_metadata` column that will be enabled for updates to attach custom
+     *  information about writes that will be reported through [CrudEntry.metadata].
+     */
+    val trackMetadata: Boolean = false,
+    /**
+     * When set to a non-null value, track old values of columns for [CrudEntry.oldData].
+     *
+     * See [TrackPreviousValuesOptions] for details.
+     */
+    val trackPreviousValues: TrackPreviousValuesOptions? = null,
+    /**
+     * Whether an `UPDATE` statement that doesn't change any values should be ignored when creating
+     * CRUD entries.
+     */
+    val ignoreEmptyUpdates: Boolean = false,
 ) {
     init {
         /**
@@ -81,6 +101,9 @@ public data class Table constructor(
             name: String,
             columns: List<Column>,
             viewName: String? = null,
+            ignoreEmptyUpdates: Boolean = false,
+            trackMetadata: Boolean = false,
+            trackPreviousValues: TrackPreviousValuesOptions? = null,
         ): Table =
             Table(
                 name,
@@ -89,6 +112,9 @@ public data class Table constructor(
                 localOnly = false,
                 insertOnly = true,
                 viewNameOverride = viewName,
+                ignoreEmptyUpdates = ignoreEmptyUpdates,
+                trackMetadata = trackMetadata,
+                trackPreviousValues = trackPreviousValues,
             )
     }
 
@@ -135,6 +161,13 @@ public data class Table constructor(
             throw AssertionError("Invalid characters in view name: $viewNameOverride")
         }
 
+        check(!localOnly || !trackMetadata) {
+            "Can't track metadata for local-only tables."
+        }
+        check(!localOnly || trackPreviousValues == null) {
+            "Can't track old values for local-only tables."
+        }
+
         val columnNames = mutableSetOf("id")
         for (column in columns) {
             when {
@@ -185,6 +218,26 @@ public data class Table constructor(
         get() = viewNameOverride ?: name
 }
 
+/**
+ * Options to include old values in [CrudEntry.oldData] for update statements.
+ *
+ * These options are enabled by passing them to a non-local [Table] constructor.
+ */
+public data class TrackPreviousValuesOptions(
+    /**
+     * A filter of column names for which updates should be tracked.
+     *
+     * When set to a non-null value, columns not included in this list will not appear in
+     * [CrudEntry.oldData]. By default, all columns are included.
+     */
+    val columnFilter: List<String>? = null,
+    /**
+     * Whether to only include old values when they were changed by an update, instead of always
+     * including all old values,
+     */
+    val onlyWhenChanged: Boolean = false,
+)
+
 @Serializable
 internal data class SerializableTable(
     var name: String,
@@ -196,16 +249,39 @@ internal data class SerializableTable(
     val insertOnly: Boolean = false,
     @SerialName("view_name")
     val viewName: String? = null,
+    @SerialName("ignore_empty_update")
+    val ignoreEmptyUpdate: Boolean = false,
+    @SerialName("include_metadata")
+    val includeMetadata: Boolean = false,
+    @SerialName("include_old")
+    val includeOld: JsonElement = JsonPrimitive(false),
+    @SerialName("include_old_only_when_changed")
+    val includeOldOnlyWhenChanged: Boolean = false,
 )
 
 internal fun Table.toSerializable(): SerializableTable =
     with(this) {
         SerializableTable(
-            name,
-            columns.map { it.toSerializable() },
-            indexes.map { it.toSerializable() },
-            localOnly,
-            insertOnly,
-            viewName,
+            name = name,
+            columns = columns.map { it.toSerializable() },
+            indexes = indexes.map { it.toSerializable() },
+            localOnly = localOnly,
+            insertOnly = insertOnly,
+            viewName = viewName,
+            ignoreEmptyUpdate = ignoreEmptyUpdates,
+            includeMetadata = trackMetadata,
+            includeOld =
+                trackPreviousValues?.let {
+                    if (it.columnFilter != null) {
+                        buildJsonArray {
+                            for (column in it.columnFilter) {
+                                add(JsonPrimitive(column))
+                            }
+                        }
+                    } else {
+                        JsonPrimitive(true)
+                    }
+                } ?: JsonPrimitive(false),
+            includeOldOnlyWhenChanged = trackPreviousValues?.onlyWhenChanged ?: false,
         )
     }

core/src/commonTest/kotlin/com/powersync/db/schema/TableTest.kt

@@ -1,5 +1,14 @@
 package com.powersync.db.schema
 
+import com.powersync.utils.JsonUtil
+import io.kotest.assertions.throwables.shouldThrow
+import io.kotest.matchers.shouldBe
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.boolean
+import kotlinx.serialization.json.encodeToJsonElement
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonPrimitive
+import kotlinx.serialization.serializer
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
@@ -180,4 +189,44 @@ class TableTest {
 
         assertEquals(exception.message, "users: id column is automatically added, custom id columns are not supported")
     }
+
+    @Test
+    fun testValidationLocalOnlyWithMetadata() {
+        val table = Table("foo", listOf(Column.text("bar")), localOnly = true, trackMetadata = true)
+
+        val exception = shouldThrow<IllegalStateException> { table.validate() }
+        exception.message shouldBe "Can't track metadata for local-only tables."
+    }
+
+    @Test
+    fun testValidationLocalOnlyWithIncludeOld() {
+        val table = Table("foo", listOf(Column.text("bar")), localOnly = true, trackPreviousValues = TrackPreviousValuesOptions())
+
+        val exception = shouldThrow<IllegalStateException> { table.validate() }
+        exception.message shouldBe "Can't track old values for local-only tables."
+    }
+
+    @Test
+    fun handlesOptions() {
+        fun serialize(table: Table): JsonObject =
+            JsonUtil.json.encodeToJsonElement(serializer<SerializableTable>(), table.toSerializable()) as JsonObject
+
+        serialize(Table("foo", emptyList(), trackMetadata = true))["include_metadata"]!!.jsonPrimitive.boolean shouldBe true
+        serialize(Table("foo", emptyList(), ignoreEmptyUpdates = true))["ignore_empty_update"]!!.jsonPrimitive.boolean shouldBe true
+
+        serialize(Table("foo", emptyList(), trackPreviousValues = TrackPreviousValuesOptions())).let {
+            it["include_old"]!!.jsonPrimitive.boolean shouldBe true
+            it["include_old_only_when_changed"]!!.jsonPrimitive.boolean shouldBe false
+        }
+
+        serialize(Table("foo", emptyList(), trackPreviousValues = TrackPreviousValuesOptions(columnFilter = listOf("foo", "bar")))).let {
+            it["include_old"]!!.jsonArray.map { e -> e.jsonPrimitive.content } shouldBe listOf("foo", "bar")
+            it["include_old_only_when_changed"]!!.jsonPrimitive.boolean shouldBe false
+        }
+
+        serialize(Table("foo", emptyList(), trackPreviousValues = TrackPreviousValuesOptions(onlyWhenChanged = true))).let {
+            it["include_old"]!!.jsonPrimitive.boolean shouldBe true
+            it["include_old_only_when_changed"]!!.jsonPrimitive.boolean shouldBe true
+        }
+    }
 }