Update Schema

CHANGELOG.md

@@ -7,6 +7,9 @@
 * Improved concurrent SQLite connection support accross various platforms. All platforms now use a single write connection and multiple read connections for concurrent read queries.
 * Added the ability to open a SQLite database given a custom `dbDirectory` path. This is currently not supported on iOS due to internal driver restrictions.
 * Internaly improved the linking of SQLite for iOS.
+* Enabled Full Text Search on iOS platforms.
+* Added the ability to update the schema for existing PowerSync clients.
+* Fixed bug where local only, insert only and view name overrides were not applied for schema tables.
 * The Android SQLite driver now uses the [Xerial JDBC library](https://github.com/xerial/sqlite-jdbc). This removes the requirement for users to add the jitpack Maven repository to their projects.
 ```diff
 // settings.gradle.kts example

core/src/commonIntegrationTest/kotlin/com/powersync/DatabaseTest.kt

@@ -267,7 +267,10 @@ class DatabaseTest {
 
             // Any new readLocks should throw
             val exception = assertFailsWith<PowerSyncException> { database.readLock {} }
-            assertEquals(expected = "Cannot process connection pool request", actual = exception.message)
+            assertEquals(
+                expected = "Cannot process connection pool request",
+                actual = exception.message,
+            )
             // Release the lock
             pausedLock.complete(Unit)
             lockJob.await()
@@ -344,4 +347,79 @@ class DatabaseTest {
                 }
             assertEquals(expected = 0, actual = count)
         }
+
+    @Test
+    fun localOnlyCRUD() =
+        runTest {
+            database.updateSchema(
+                schema =
+                    Schema(
+                        UserRow.table.copy(
+                            // Updating the existing "users" view to localOnly causes an error
+                            // no such table: main.ps_data_local__users.
+                            // Perhaps this is a bug in the core extension
+                            name = "local_users",
+                            localOnly = true,
+                        ),
+                    ),
+            )
+
+            database.execute(
+                """
+                    INSERT INTO
+                       local_users (id, name, email)
+                    VALUES
+                     (uuid(), "one", "[email protected]")
+                    """,
+            )
+
+            val count = database.get("SELECT COUNT(*) FROM local_users") { it.getLong(0)!! }
+            assertEquals(actual = count, expected = 1)
+
+            // No CRUD entries should be present for local only tables
+            val crudItems = database.getAll("SELECT id from ps_crud") { it.getLong(0)!! }
+            assertEquals(actual = crudItems.size, expected = 0)
+        }
+
+    @Test
+    fun insertOnlyCRUD() =
+        runTest {
+            database.updateSchema(schema = Schema(UserRow.table.copy(insertOnly = true)))
+
+            database.execute(
+                """
+                    INSERT INTO
+                       users (id, name, email)
+                    VALUES
+                     (uuid(), "one", "[email protected]")
+                    """,
+            )
+
+            val crudItems = database.getAll("SELECT id from ps_crud") { it.getLong(0)!! }
+            assertEquals(actual = crudItems.size, expected = 1)
+
+            val count = database.get("SELECT COUNT(*) from users") { it.getLong(0)!! }
+            assertEquals(actual = count, expected = 0)
+        }
+
+    @Test
+    fun viewOverride() =
+        runTest {
+            database.updateSchema(schema = Schema(UserRow.table.copy(viewNameOverride = "people")))
+
+            database.execute(
+                """
+                    INSERT INTO
+                       people (id, name, email)
+                    VALUES
+                     (uuid(), "one", "[email protected]")
+                    """,
+            )
+
+            val crudItems = database.getAll("SELECT id from ps_crud") { it.getLong(0)!! }
+            assertEquals(actual = crudItems.size, expected = 1)
+
+            val count = database.get("SELECT COUNT(*) from people") { it.getLong(0)!! }
+            assertEquals(actual = count, expected = 1)
+        }
 }

core/src/commonMain/kotlin/com/powersync/PowerSyncDatabase.kt

@@ -5,6 +5,7 @@ import com.powersync.connectors.PowerSyncBackendConnector
 import com.powersync.db.Queries
 import com.powersync.db.crud.CrudBatch
 import com.powersync.db.crud.CrudTransaction
+import com.powersync.db.schema.Schema
 import com.powersync.sync.SyncStatus
 import com.powersync.utils.JsonParam
 import kotlin.coroutines.cancellation.CancellationException
@@ -36,6 +37,15 @@ public interface PowerSyncDatabase : Queries {
      */
     public val currentStatus: SyncStatus
 
+    /**
+     * Replace the schema with a new version. This is for advanced use cases - typically the schema
+     * should just be specified once in the constructor.
+     *
+     * Cannot be used while connected - this should only be called before connect.
+     */
+    @Throws(PowerSyncException::class, CancellationException::class)
+    public suspend fun updateSchema(schema: Schema)
+
     /**
      * Suspend function that resolves when the first sync has occurred
      */

core/src/commonMain/kotlin/com/powersync/db/PowerSyncDatabaseImpl.kt

@@ -3,6 +3,7 @@ package com.powersync.db
 import co.touchlab.kermit.Logger
 import com.powersync.DatabaseDriverFactory
 import com.powersync.PowerSyncDatabase
+import com.powersync.PowerSyncException
 import com.powersync.bucket.BucketPriority
 import com.powersync.bucket.BucketStorage
 import com.powersync.bucket.BucketStorageImpl
@@ -14,6 +15,7 @@ import com.powersync.db.crud.CrudTransaction
 import com.powersync.db.internal.InternalDatabaseImpl
 import com.powersync.db.internal.InternalTable
 import com.powersync.db.schema.Schema
+import com.powersync.db.schema.toSerializable
 import com.powersync.sync.PriorityStatusEntry
 import com.powersync.sync.SyncStatus
 import com.powersync.sync.SyncStatusData
@@ -50,7 +52,7 @@ import kotlinx.serialization.encodeToString
  * All changes to local tables are automatically recorded, whether connected or not. Once connected, the changes are uploaded.
  */
 internal class PowerSyncDatabaseImpl(
-    val schema: Schema,
+    var schema: Schema,
     val scope: CoroutineScope,
     val factory: DatabaseDriverFactory,
     private val dbFilename: String,
@@ -113,21 +115,25 @@ internal class PowerSyncDatabaseImpl(
                 tx.getOptional("SELECT powersync_init()") {}
             }
 
-            applySchema()
+            updateSchema(schema)
             updateHasSynced()
         }
     }
 
-    private suspend fun applySchema() {
-        val schemaJson = JsonUtil.json.encodeToString(schema)
-
-        internalDb.writeTransaction { tx ->
-            tx.getOptional(
-                "SELECT powersync_replace_schema(?);",
-                listOf(schemaJson),
-            ) {}
+    override suspend fun updateSchema(schema: Schema) =
+        runWrappedSuspending {
+            mutex.withLock {
+                if (this.syncStream != null) {
+                    throw PowerSyncException(
+                        "Cannot update schema while connected",
+                        cause = Exception("PowerSync client is already connected"),
+                    )
+                }
+                val schemaJson = JsonUtil.json.encodeToString(schema.toSerializable())
+                internalDb.updateSchema(schemaJson)
+                this.schema = schema
+            }
         }
-    }
 
     override suspend fun connect(
         connector: PowerSyncBackendConnector,

core/src/commonMain/kotlin/com/powersync/db/internal/ConnectionPool.kt

@@ -56,6 +56,33 @@ internal class ConnectionPool(
         }
     }
 
+    suspend fun <R> withAllConnections(action: suspend (connections: List<TransactorDriver>) -> R): R {
+        val obtainedConnections = mutableListOf<Pair<TransactorDriver, CompletableDeferred<Unit>>>()
+
+        try {
+            /**
+             * This attempts to receive (all) the number of available connections.
+             * This creates a hold for each connection received. This means that subsequent
+             * receive operations must return unique connections until all the available connections
+             * have a hold.
+             */
+            repeat(connections.size) {
+                try {
+                    obtainedConnections.add(available.receive())
+                } catch (e: PoolClosedException) {
+                    throw PowerSyncException(
+                        message = "Cannot process connection pool request",
+                        cause = e,
+                    )
+                }
+            }
+
+            return action(obtainedConnections.map { it.first })
+        } finally {
+            obtainedConnections.forEach { it.second.complete(Unit) }
+        }
+    }
+
     suspend fun close() {
         available.cancel(PoolClosedException)
         connections.joinAll()

core/src/commonMain/kotlin/com/powersync/db/internal/InternalDatabase.kt

@@ -6,5 +6,7 @@ import kotlinx.coroutines.flow.SharedFlow
 internal interface InternalDatabase : Queries {
     fun updatesOnTables(): SharedFlow<Set<String>>
 
+    suspend fun updateSchema(schemaJson: String): Unit
+
     suspend fun close(): Unit
 }

core/src/commonMain/kotlin/com/powersync/db/internal/InternalDatabaseImpl.kt

@@ -66,6 +66,26 @@ internal class InternalDatabaseImpl(
             context.execute(sql, parameters)
         }
 
+    override suspend fun updateSchema(schemaJson: String) {
+        withContext(dbContext) {
+            runWrappedSuspending {
+                // First get a lock on all read connections
+                readPool.withAllConnections { readConnections ->
+                    // Then get access to the write connection
+                    writeTransaction { tx ->
+                        tx.getOptional(
+                            "SELECT powersync_replace_schema(?);",
+                            listOf(schemaJson),
+                        ) {}
+                    }
+
+                    // Update the schema on all read connections
+                    readConnections.forEach { it.driver.getAll("pragma table_info('sqlite_master')") {} }
+                }
+            }
+        }
+    }
+
     override suspend fun <RowType : Any> get(
         sql: String,
         parameters: List<Any?>?,

core/src/commonMain/kotlin/com/powersync/db/schema/Column.kt

@@ -3,18 +3,8 @@ package com.powersync.db.schema
 import kotlinx.serialization.Serializable
 
 /** A single column in a table schema. */
-@Serializable
 public data class Column(
-    /** Name of the column. */
     val name: String,
-    /** Type of the column.
-     *
-     * If the underlying data does not match this type,
-     * it is cast automatically.
-     *
-     * For details on the cast, see:
-     * https://www.sqlite.org/lang_expr.html#castexpr
-     */
     val type: ColumnType,
 ) {
     public companion object {
@@ -28,3 +18,11 @@ public data class Column(
         public fun real(name: String): Column = Column(name, ColumnType.REAL)
     }
 }
+
+@Serializable
+internal data class SerializableColumn(
+    val name: String,
+    val type: ColumnType,
+)
+
+internal fun Column.toSerializable(): SerializableColumn = with(this) { SerializableColumn(name, type) }

core/src/commonMain/kotlin/com/powersync/db/schema/Index.kt

@@ -2,7 +2,6 @@ package com.powersync.db.schema
 
 import kotlinx.serialization.Serializable
 
-@Serializable
 public data class Index(
     /**
      * Descriptive name of the index.
@@ -44,3 +43,17 @@ public data class Index(
         return """CREATE INDEX "${fullName(table)}" ON "${table.internalName}"($fields)"""
     }
 }
+
+@Serializable
+internal data class SerializableIndex(
+    val name: String,
+    val columns: List<SerializableIndexColumn>,
+)
+
+internal fun Index.toSerializable(): SerializableIndex =
+    with(this) {
+        SerializableIndex(
+            name,
+            columns.map { it.toSerializable() },
+        )
+    }

core/src/commonMain/kotlin/com/powersync/db/schema/IndexedColumn.kt

@@ -6,21 +6,10 @@ import kotlinx.serialization.Serializable
 /**
  * Describes an indexed column.
  */
-@Serializable
 public data class IndexedColumn(
-    /**
-     * Name of the column to index.
-     */
-    @SerialName("name")
     val column: String,
-    /**
-     * Whether this column is stored in ascending order in the index.
-     */
-    private val ascending: Boolean = true,
+    val ascending: Boolean = true,
     private var columnDefinition: Column? = null,
-    /**
-     * The column definition type
-     */
     var type: ColumnType? = null,
 ) {
     public companion object {
@@ -46,4 +35,17 @@ public data class IndexedColumn(
     }
 }
 
+@Serializable
+internal data class SerializableIndexColumn(
+    @SerialName("name")
+    val column: String,
+    val type: ColumnType?,
+    val ascending: Boolean,
+)
+
+internal fun IndexedColumn.toSerializable(): SerializableIndexColumn =
+    with(this) {
+        SerializableIndexColumn(column, type, ascending)
+    }
+
 internal fun mapColumn(column: Column): String = "CAST(json_extract(data, ${column.name}) as ${column.type})"

core/src/commonMain/kotlin/com/powersync/db/schema/Schema.kt

@@ -2,7 +2,6 @@ package com.powersync.db.schema
 
 import kotlinx.serialization.Serializable
 
-@Serializable
 public data class Schema(
     val tables: List<Table>,
 ) {
@@ -22,3 +21,29 @@ public data class Schema(
         }
     }
 }
+
+/**
+ * A small note on the use of Serializable.
+ * Using Serializable on public classes has an affect on the Object C headers for the Swift
+ * SDK. The use causes:
+ * An extra Objective-C Companion class for each of these types,
+ * and Kotlin/Native having to export a bunch of the KotlinX Serialization classes and protocols.
+ *
+ * The actual requirements of serialization are quite standard and relatively small
+ * in our use case. The implementation here declares a public data class for users to interact with
+ * and an internal Serializable data class. Instances provided by consumers of the SDK are converted
+ * to the serializable version then passed for serialization.
+ *
+ * An alternative would be to provide a custom serializer for each class. This approach has not been
+ * implemented since we can use the built-in serialization methods with the internal serializable
+ * classes.
+ */
+@Serializable
+internal data class SerializableSchema(
+    val tables: List<SerializableTable>,
+)
+
+internal fun Schema.toSerializable(): SerializableSchema =
+    with(this) {
+        SerializableSchema(tables.map { it.toSerializable() })
+    }